Batch scrape web pages
Queue a batch scrape job and poll the returned operation URL for per-URL results.
POST
/
api
/
v1
/
platform
/
scrapers
/
web
/
scrape
/
batch
Batch scrape web pages
curl --request POST \
--url https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape/batch \
--header 'Content-Type: application/json' \
--header 'X-Access-Key: <api-key>' \
--header 'X-Secret-Key: <api-key>' \
--data '
{
"urls": [
"<string>"
],
"items": [
{
"url": "https://example.com/page",
"renderJs": false,
"maxAgeMs": 0,
"maxAgeDays": 0
}
],
"renderJs": false,
"maxAgeMs": 0,
"maxAgeDays": 0,
"maxConcurrency": 3
}
'const options = {
method: 'POST',
headers: {
'X-Access-Key': '<api-key>',
'X-Secret-Key': '<api-key>',
'Content-Type': 'application/json'
},
body: JSON.stringify({
urls: ['<string>'],
items: [{url: 'https://example.com/page', renderJs: false, maxAgeMs: 0, maxAgeDays: 0}],
renderJs: false,
maxAgeMs: 0,
maxAgeDays: 0,
maxConcurrency: 3
})
};
fetch('https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape/batch', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));import requests
url = "https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape/batch"
payload = {
"urls": ["<string>"],
"items": [
{
"url": "https://example.com/page",
"renderJs": False,
"maxAgeMs": 0,
"maxAgeDays": 0
}
],
"renderJs": False,
"maxAgeMs": 0,
"maxAgeDays": 0,
"maxConcurrency": 3
}
headers = {
"X-Access-Key": "<api-key>",
"X-Secret-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text){
"data": {
"id": "<string>",
"operationId": "<string>",
"status": "queued",
"pollUrl": "<string>"
},
"meta": {
"requestId": "<string>"
}
}{
"statusCode": 400,
"error": "Bad Request",
"message": "Validation failed",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b",
"errors": [
{
"property": "body.query",
"message": "query must be a string",
"constraints": {
"isString": "query must be a string"
}
}
]
}{
"statusCode": 401,
"error": "Unauthorized",
"message": "Authentication is required.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 402,
"error": "Payment Required",
"message": "Insufficient credits.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 500,
"error": "Internal Server Error",
"message": "An unexpected error occurred.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 503,
"error": "Service Unavailable",
"message": "Service temporarily unavailable.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}POST /api/v1/platform/scrapers/web/scrape/batch
Queue multiple web page scrapes and poll for per-URL results.Use batch scrape when you have many URLs and do not need the results in the initial HTTP response. The endpoint returns an operation ID immediately; poll
GET /api/operations/:id until the operation reaches a terminal status.
Example
curl -X POST https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape/batch \
-H "Authorization: Bearer <key>" \
-H "Content-Type: application/json" \
-H "Idempotency-Key: batch-2026-05-22-001" \
-d '{"urls": ["https://example.com/pricing", "https://example.com/about"], "maxConcurrency": 2}'
Authorizations
The public API key from the Credentials page.
The API secret shown when the credential is created.
Headers
Optional. Reusing the same key for the same organization returns the existing queued batch operation.
Body
application/json
URLs to scrape using the top-level renderJs and cache freshness options.
Per-item scrape requests. Use either items or urls, not both.
Show child attributes
Show child attributes
Render JavaScript before scraping for urls entries.
Required range:
0 <= x <= 2592000000Required range:
0 <= x <= 30Maximum number of batch items processed concurrently.
Required range:
1 <= x <= 10⌘I
Batch scrape web pages
curl --request POST \
--url https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape/batch \
--header 'Content-Type: application/json' \
--header 'X-Access-Key: <api-key>' \
--header 'X-Secret-Key: <api-key>' \
--data '
{
"urls": [
"<string>"
],
"items": [
{
"url": "https://example.com/page",
"renderJs": false,
"maxAgeMs": 0,
"maxAgeDays": 0
}
],
"renderJs": false,
"maxAgeMs": 0,
"maxAgeDays": 0,
"maxConcurrency": 3
}
'const options = {
method: 'POST',
headers: {
'X-Access-Key': '<api-key>',
'X-Secret-Key': '<api-key>',
'Content-Type': 'application/json'
},
body: JSON.stringify({
urls: ['<string>'],
items: [{url: 'https://example.com/page', renderJs: false, maxAgeMs: 0, maxAgeDays: 0}],
renderJs: false,
maxAgeMs: 0,
maxAgeDays: 0,
maxConcurrency: 3
})
};
fetch('https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape/batch', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));import requests
url = "https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape/batch"
payload = {
"urls": ["<string>"],
"items": [
{
"url": "https://example.com/page",
"renderJs": False,
"maxAgeMs": 0,
"maxAgeDays": 0
}
],
"renderJs": False,
"maxAgeMs": 0,
"maxAgeDays": 0,
"maxConcurrency": 3
}
headers = {
"X-Access-Key": "<api-key>",
"X-Secret-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text){
"data": {
"id": "<string>",
"operationId": "<string>",
"status": "queued",
"pollUrl": "<string>"
},
"meta": {
"requestId": "<string>"
}
}{
"statusCode": 400,
"error": "Bad Request",
"message": "Validation failed",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b",
"errors": [
{
"property": "body.query",
"message": "query must be a string",
"constraints": {
"isString": "query must be a string"
}
}
]
}{
"statusCode": 401,
"error": "Unauthorized",
"message": "Authentication is required.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 402,
"error": "Payment Required",
"message": "Insufficient credits.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 500,
"error": "Internal Server Error",
"message": "An unexpected error occurred.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 503,
"error": "Service Unavailable",
"message": "Service temporarily unavailable.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}