Batch scrape web pages
Queue a batch scrape job and poll the returned operation URL for per-URL results.
POST
/
api
/
v1
/
platform
/
scrapers
/
web
/
scrape
/
batch
Batch scrape web pages
curl --request POST \
--url https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape/batch \
--header 'Content-Type: application/json' \
--header 'X-Access-Key: <api-key>' \
--header 'X-Secret-Key: <api-key>' \
--data '
{
"urls": [
"<string>"
],
"items": [
{
"url": "https://example.com/page",
"renderJs": false,
"maxAgeMs": 0,
"maxAgeDays": 0
}
],
"renderJs": false,
"maxAgeMs": 0,
"maxAgeDays": 0,
"maxConcurrency": 3
}
'const options = {
method: 'POST',
headers: {
'X-Access-Key': '<api-key>',
'X-Secret-Key': '<api-key>',
'Content-Type': 'application/json'
},
body: JSON.stringify({
urls: ['<string>'],
items: [{url: 'https://example.com/page', renderJs: false, maxAgeMs: 0, maxAgeDays: 0}],
renderJs: false,
maxAgeMs: 0,
maxAgeDays: 0,
maxConcurrency: 3
})
};
fetch('https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape/batch', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));import requests
url = "https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape/batch"
payload = {
"urls": ["<string>"],
"items": [
{
"url": "https://example.com/page",
"renderJs": False,
"maxAgeMs": 0,
"maxAgeDays": 0
}
],
"renderJs": False,
"maxAgeMs": 0,
"maxAgeDays": 0,
"maxConcurrency": 3
}
headers = {
"X-Access-Key": "<api-key>",
"X-Secret-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text){
"data": {
"id": "<string>",
"operationId": "<string>",
"status": "queued",
"pollUrl": "<string>"
},
"meta": {
"requestId": "<string>"
}
}{
"statusCode": 400,
"error": "Bad Request",
"message": "Validation failed",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b",
"errors": [
{
"property": "body.query",
"message": "query must be a string",
"constraints": {
"isString": "query must be a string"
}
}
]
}{
"statusCode": 401,
"error": "Unauthorized",
"message": "Authentication is required.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 402,
"error": "Payment Required",
"message": "Insufficient credits.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 500,
"error": "Internal Server Error",
"message": "An unexpected error occurred.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 503,
"error": "Service Unavailable",
"message": "Service temporarily unavailable.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}POST /api/v1/platform/scrapers/web/scrape/batch
Queue multiple web page scrapes and poll for per-URL results.Use batch scrape when you have many URLs and do not need the results in the initial HTTP response. The endpoint returns an operation ID immediately; poll
GET /api/operations/:id until the operation reaches a terminal status.
Example
curl -X POST https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape/batch \
-H "Authorization: Bearer <key>" \
-H "Content-Type: application/json" \
-H "Idempotency-Key: batch-2026-05-22-001" \
-d '{"urls": ["https://example.com/pricing", "https://example.com/about"], "maxConcurrency": 2}'
Authorizations
Headers
Body
application/json
⌘I
Batch scrape web pages
curl --request POST \
--url https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape/batch \
--header 'Content-Type: application/json' \
--header 'X-Access-Key: <api-key>' \
--header 'X-Secret-Key: <api-key>' \
--data '
{
"urls": [
"<string>"
],
"items": [
{
"url": "https://example.com/page",
"renderJs": false,
"maxAgeMs": 0,
"maxAgeDays": 0
}
],
"renderJs": false,
"maxAgeMs": 0,
"maxAgeDays": 0,
"maxConcurrency": 3
}
'const options = {
method: 'POST',
headers: {
'X-Access-Key': '<api-key>',
'X-Secret-Key': '<api-key>',
'Content-Type': 'application/json'
},
body: JSON.stringify({
urls: ['<string>'],
items: [{url: 'https://example.com/page', renderJs: false, maxAgeMs: 0, maxAgeDays: 0}],
renderJs: false,
maxAgeMs: 0,
maxAgeDays: 0,
maxConcurrency: 3
})
};
fetch('https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape/batch', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));import requests
url = "https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape/batch"
payload = {
"urls": ["<string>"],
"items": [
{
"url": "https://example.com/page",
"renderJs": False,
"maxAgeMs": 0,
"maxAgeDays": 0
}
],
"renderJs": False,
"maxAgeMs": 0,
"maxAgeDays": 0,
"maxConcurrency": 3
}
headers = {
"X-Access-Key": "<api-key>",
"X-Secret-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text){
"data": {
"id": "<string>",
"operationId": "<string>",
"status": "queued",
"pollUrl": "<string>"
},
"meta": {
"requestId": "<string>"
}
}{
"statusCode": 400,
"error": "Bad Request",
"message": "Validation failed",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b",
"errors": [
{
"property": "body.query",
"message": "query must be a string",
"constraints": {
"isString": "query must be a string"
}
}
]
}{
"statusCode": 401,
"error": "Unauthorized",
"message": "Authentication is required.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 402,
"error": "Payment Required",
"message": "Insufficient credits.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 500,
"error": "Internal Server Error",
"message": "An unexpected error occurred.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 503,
"error": "Service Unavailable",
"message": "Service temporarily unavailable.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}