Crawl website
Queue an asynchronous website crawl. Poll the returned operation URL for status and results.
POST
/
api
/
v1
/
platform
/
scrapers
/
web
/
crawl
Crawl website
curl --request POST \
--url https://developer.thehog.ai/api/v1/platform/scrapers/web/crawl \
--header 'Content-Type: application/json' \
--header 'X-Access-Key: <api-key>' \
--header 'X-Secret-Key: <api-key>' \
--data '
{
"url": "https://example.com",
"limit": 5,
"maxPages": 5,
"maxDepth": 1,
"sameDomainOnly": true,
"instructions": "<string>"
}
'const options = {
method: 'POST',
headers: {
'X-Access-Key': '<api-key>',
'X-Secret-Key': '<api-key>',
'Content-Type': 'application/json'
},
body: JSON.stringify({
url: 'https://example.com',
limit: 5,
maxPages: 5,
maxDepth: 1,
sameDomainOnly: true,
instructions: '<string>'
})
};
fetch('https://developer.thehog.ai/api/v1/platform/scrapers/web/crawl', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));import requests
url = "https://developer.thehog.ai/api/v1/platform/scrapers/web/crawl"
payload = {
"url": "https://example.com",
"limit": 5,
"maxPages": 5,
"maxDepth": 1,
"sameDomainOnly": True,
"instructions": "<string>"
}
headers = {
"X-Access-Key": "<api-key>",
"X-Secret-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text){
"data": {
"id": "<string>",
"operationId": "<string>",
"status": "queued",
"pollUrl": "<string>"
},
"meta": {
"requestId": "<string>"
}
}{
"statusCode": 400,
"error": "Bad Request",
"message": "Validation failed",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b",
"errors": [
{
"property": "body.query",
"message": "query must be a string",
"constraints": {
"isString": "query must be a string"
}
}
]
}{
"statusCode": 401,
"error": "Unauthorized",
"message": "Authentication is required.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 402,
"error": "Payment Required",
"message": "Insufficient credits.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 403,
"error": "Forbidden",
"message": "Error 403.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 500,
"error": "Internal Server Error",
"message": "An unexpected error occurred.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 503,
"error": "Service Unavailable",
"message": "Service temporarily unavailable.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}POST /api/v1/platform/scrapers/web/crawl
Crawl a website and return discovered page content.Crawl a website from a starting URL with an optional page limit.
Example
curl -X POST https://developer.thehog.ai/api/v1/platform/scrapers/web/crawl \
-H "X-Access-Key: ak_xxxxxxxxxxxxxxxx" \
-H "X-Secret-Key: sk_xxxxxxxxxxxxxxxx" \
-H "Content-Type: application/json" \
-d '{"url": "https://example.com", "limit": 5}'
Authorizations
The public API key from the Credentials page.
The API secret shown when the credential is created.
Headers
Optional. Reusing the same key for the same organization returns the existing queued crawl operation.
Body
application/json
Domain or URL to crawl
Example:
"https://example.com"
Required range:
1 <= x <= 50Maximum number of pages to crawl. Overrides limit when set.
Required range:
1 <= x <= 50Maximum link depth from the starting URL.
Required range:
0 <= x <= 5Restrict discovered links to the starting hostname.
Instructions for content extraction
⌘I
Crawl website
curl --request POST \
--url https://developer.thehog.ai/api/v1/platform/scrapers/web/crawl \
--header 'Content-Type: application/json' \
--header 'X-Access-Key: <api-key>' \
--header 'X-Secret-Key: <api-key>' \
--data '
{
"url": "https://example.com",
"limit": 5,
"maxPages": 5,
"maxDepth": 1,
"sameDomainOnly": true,
"instructions": "<string>"
}
'const options = {
method: 'POST',
headers: {
'X-Access-Key': '<api-key>',
'X-Secret-Key': '<api-key>',
'Content-Type': 'application/json'
},
body: JSON.stringify({
url: 'https://example.com',
limit: 5,
maxPages: 5,
maxDepth: 1,
sameDomainOnly: true,
instructions: '<string>'
})
};
fetch('https://developer.thehog.ai/api/v1/platform/scrapers/web/crawl', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));import requests
url = "https://developer.thehog.ai/api/v1/platform/scrapers/web/crawl"
payload = {
"url": "https://example.com",
"limit": 5,
"maxPages": 5,
"maxDepth": 1,
"sameDomainOnly": True,
"instructions": "<string>"
}
headers = {
"X-Access-Key": "<api-key>",
"X-Secret-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text){
"data": {
"id": "<string>",
"operationId": "<string>",
"status": "queued",
"pollUrl": "<string>"
},
"meta": {
"requestId": "<string>"
}
}{
"statusCode": 400,
"error": "Bad Request",
"message": "Validation failed",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b",
"errors": [
{
"property": "body.query",
"message": "query must be a string",
"constraints": {
"isString": "query must be a string"
}
}
]
}{
"statusCode": 401,
"error": "Unauthorized",
"message": "Authentication is required.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 402,
"error": "Payment Required",
"message": "Insufficient credits.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 403,
"error": "Forbidden",
"message": "Error 403.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 500,
"error": "Internal Server Error",
"message": "An unexpected error occurred.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 503,
"error": "Service Unavailable",
"message": "Service temporarily unavailable.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}