Scrape web page
Fetch a web page and return readable text plus requested markdown, HTML, links, metadata, or schema-guided JSON.
POST
/
api
/
v1
/
platform
/
scrapers
/
web
/
scrape
Scrape web page
curl --request POST \
--url https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape \
--header 'Content-Type: application/json' \
--header 'X-Access-Key: <api-key>' \
--header 'X-Secret-Key: <api-key>' \
--data '
{
"url": "https://example.com/page",
"renderJs": false,
"formats": [
"text"
],
"jsonSchema": {},
"instructions": "<string>",
"maxAgeMs": 0,
"maxAgeDays": 0
}
'const options = {
method: 'POST',
headers: {
'X-Access-Key': '<api-key>',
'X-Secret-Key': '<api-key>',
'Content-Type': 'application/json'
},
body: JSON.stringify({
url: 'https://example.com/page',
renderJs: false,
formats: ['text'],
jsonSchema: {},
instructions: '<string>',
maxAgeMs: 0,
maxAgeDays: 0
})
};
fetch('https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));import requests
url = "https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape"
payload = {
"url": "https://example.com/page",
"renderJs": False,
"formats": ["text"],
"jsonSchema": {},
"instructions": "<string>",
"maxAgeMs": 0,
"maxAgeDays": 0
}
headers = {
"X-Access-Key": "<api-key>",
"X-Secret-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text){
"data": {
"url": "<string>",
"text": "<string>",
"statusCode": 123,
"markdown": "<string>",
"html": "<string>",
"links": [
"<string>"
],
"metadata": {
"title": "<string>",
"description": "<string>",
"canonicalUrl": "<string>",
"finalUrl": "<string>",
"contentType": "<string>",
"statusCode": 123,
"textLength": 123,
"responseBytes": 123,
"capture": {
"mode": "<string>",
"completionReason": "<string>",
"scrollsCompleted": 123,
"maxScrolls": 123,
"expansionClicks": 123,
"maxExpansionClicks": 123,
"durationMs": 123
}
},
"json": {}
},
"meta": {
"requestId": "<string>"
}
}{
"statusCode": 400,
"error": "Bad Request",
"message": "Validation failed",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b",
"errors": [
{
"property": "body.query",
"message": "query must be a string",
"constraints": {
"isString": "query must be a string"
}
}
]
}{
"statusCode": 401,
"error": "Unauthorized",
"message": "Authentication is required.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 402,
"error": "Payment Required",
"message": "Insufficient credits.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 500,
"error": "Internal Server Error",
"message": "An unexpected error occurred.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 502,
"error": "Bad Gateway",
"message": "An unexpected error occurred.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 503,
"error": "Service Unavailable",
"message": "Service temporarily unavailable.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 504,
"error": "Gateway Timeout",
"message": "An unexpected error occurred.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}POST /api/v1/platform/scrapers/web/scrape
Scrape a single web page and return its content.Scrape a single web page and return readable content. By default the endpoint returns the backward-compatible
url, text, and statusCode fields. Use
formats when you need markdown, HTML, discovered links, page metadata, or
schema-guided JSON extraction.
Example
curl -X POST https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape \
-H "X-Access-Key: ak_xxxxxxxxxxxxxxxx" \
-H "X-Secret-Key: sk_xxxxxxxxxxxxxxxx" \
-H "Content-Type: application/json" \
-d '{"url": "https://example.com/pricing", "renderJs": true, "formats": ["markdown", "metadata"]}'
Formats
formats is optional and defaults to ["text"].
| Format | Returns |
|---|---|
text | Readable text. This is always included for backward compatibility. |
markdown | Markdown-friendly page content for agents and LLM workflows. |
html | Fetched or rendered HTML when available. |
links | Normalized links discovered on the page. |
metadata | Page title, description, canonical URL, final URL, status code, content type, and size metadata when available. |
json | Data extracted into your provided JSON Schema. |
data. If a
requested format is unavailable, the field may be null or an empty array.
Schema-guided JSON
Requestjson only when you provide jsonSchema. The schema defines the
returned keys and shape; page content is mapped into that schema and validated
before the response is returned. The root schema must be an object; use array
properties for repeated data such as comments, products, or plans.
curl -X POST https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape \
-H "X-Access-Key: ak_xxxxxxxxxxxxxxxx" \
-H "X-Secret-Key: sk_xxxxxxxxxxxxxxxx" \
-H "Content-Type: application/json" \
-d '{
"url": "https://example.com/pricing",
"formats": ["markdown", "json"],
"jsonSchema": {
"type": "object",
"additionalProperties": false,
"required": ["plans"],
"properties": {
"plans": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["name", "price"],
"properties": {
"name": { "type": "string" },
"price": { "type": "string" }
}
}
}
}
},
"instructions": "Extract public pricing plans visible on the page."
}'
jsonSchema and instructions are only valid when formats includes json.
Schema-guided extraction performs additional model work and may use more credits
than a plain text or markdown scrape.Authorizations
The public API key from the Credentials page.
The API secret shown when the credential is created.
Body
application/json
URL to scrape
Example:
"https://example.com/page"
Render JavaScript before scraping
Output formats to return. Defaults to ["text"]. Request "json" only with jsonSchema.
Available options:
text, markdown, html, links, metadata, json JSON Schema used for schema-guided extraction when formats includes "json".
Additional extraction instructions used only when formats includes "json".
Maximum string length:
8000Maximum accepted cache age in milliseconds. Use 0 or omit to force a fresh scrape.
Required range:
0 <= x <= 2592000000Maximum accepted cache age in days. Use 0 or omit to force a fresh scrape. Ignored when maxAgeMs is provided.
Required range:
0 <= x <= 30⌘I
Scrape web page
curl --request POST \
--url https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape \
--header 'Content-Type: application/json' \
--header 'X-Access-Key: <api-key>' \
--header 'X-Secret-Key: <api-key>' \
--data '
{
"url": "https://example.com/page",
"renderJs": false,
"formats": [
"text"
],
"jsonSchema": {},
"instructions": "<string>",
"maxAgeMs": 0,
"maxAgeDays": 0
}
'const options = {
method: 'POST',
headers: {
'X-Access-Key': '<api-key>',
'X-Secret-Key': '<api-key>',
'Content-Type': 'application/json'
},
body: JSON.stringify({
url: 'https://example.com/page',
renderJs: false,
formats: ['text'],
jsonSchema: {},
instructions: '<string>',
maxAgeMs: 0,
maxAgeDays: 0
})
};
fetch('https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));import requests
url = "https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape"
payload = {
"url": "https://example.com/page",
"renderJs": False,
"formats": ["text"],
"jsonSchema": {},
"instructions": "<string>",
"maxAgeMs": 0,
"maxAgeDays": 0
}
headers = {
"X-Access-Key": "<api-key>",
"X-Secret-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text){
"data": {
"url": "<string>",
"text": "<string>",
"statusCode": 123,
"markdown": "<string>",
"html": "<string>",
"links": [
"<string>"
],
"metadata": {
"title": "<string>",
"description": "<string>",
"canonicalUrl": "<string>",
"finalUrl": "<string>",
"contentType": "<string>",
"statusCode": 123,
"textLength": 123,
"responseBytes": 123,
"capture": {
"mode": "<string>",
"completionReason": "<string>",
"scrollsCompleted": 123,
"maxScrolls": 123,
"expansionClicks": 123,
"maxExpansionClicks": 123,
"durationMs": 123
}
},
"json": {}
},
"meta": {
"requestId": "<string>"
}
}{
"statusCode": 400,
"error": "Bad Request",
"message": "Validation failed",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b",
"errors": [
{
"property": "body.query",
"message": "query must be a string",
"constraints": {
"isString": "query must be a string"
}
}
]
}{
"statusCode": 401,
"error": "Unauthorized",
"message": "Authentication is required.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 402,
"error": "Payment Required",
"message": "Insufficient credits.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 500,
"error": "Internal Server Error",
"message": "An unexpected error occurred.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 502,
"error": "Bad Gateway",
"message": "An unexpected error occurred.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 503,
"error": "Service Unavailable",
"message": "Service temporarily unavailable.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 504,
"error": "Gateway Timeout",
"message": "An unexpected error occurred.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}