Raw schema
{
"type": "object",
"properties": {
"url": {
"type": "string",
"format": "uri",
"description": "The page URL to scrape (optional only when you pass `html` to convert)"
},
"format": {
"type": "string",
"enum": [
"markdown",
"html",
"text"
],
"description": "Output format (default markdown)"
},
"formats": {
"type": "array",
"items": {
"type": "string",
"enum": [
"markdown",
"html",
"text"
]
},
"maxItems": 3,
"description": "Additional formats to return together in payload.formats, e.g. ['markdown','text']"
},
"include_links": {
"type": "boolean",
"description": "Return all de-duplicated absolute page links in payload.links"
},
"content_mode": {
"type": "string",
"enum": [
"smart",
"article",
"full"
],
"description": "smart (default): whole page minus nav/footer/cookie chrome. article: Readability main article only (news/blogs). full: entire body as-is."
},
"engine": {
"type": "string",
"enum": [
"auto",
"tls",
"fetch",
"render"
],
"description": "auto (default): TLS tier, escalate to browser on block. tls: never escalate — exactly what a pure HTTP bot (no JS) sees, right for SEO checks. render: force browser."
},
"render": {
"type": "boolean",
"description": "Force the headless browser (JS execution)"
},
"mode": {
"type": "string",
"enum": [
"summary"
],
"description": "summary: return only metadata (title, description, canonical, contentLength, status, engine, bytes) with no page content — use this when auditing pages instead of reading them"
},
"country": {
"type": "string",
"minLength": 2,
"maxLength": 2,
"description": "ISO country code for the proxy exit, e.g. 'us'"
},
"ai_prompt": {
"type": "string",
"description": "Natural-language instruction — the LLM turns the page into structured JSON"
},
"ai_schema": {
"type": "object",
"additionalProperties": {},
"description": "JSON Schema for deterministic AI extraction; returned under payload.ai.data"
},
"extract": {
"type": "object",
"additionalProperties": {},
"description": "Structured-extraction schema: { field: \"css selector\" | { selector, attr, all, fns } }. `fns` is a transform pipeline run on the value — e.g. { \"price\": { \"selector\": \".price\", \"fns\": [\"amount_from_string\"] } } returns a number, not text. Functions: amount_from_string, amount_range_from_string, convert_to_float/int/str, trim, lower, upper, {regex_search|regex_find_all: \"pat\"}, {replace:{from,to}}, {join:\",\"}, {select_nth:0}, length, unique, max, min, average, product."
},
"app_state": {
"anyOf": [
{
"type": "boolean"
},
{
"type": "string",
"enum": [
"auto",
"raw"
]
}
],
"description": "Mine the page's own hydration state (Next.js __NEXT_DATA__, Nuxt, embedded JSON islands) into payload.metadata.appState. This is where SPAs keep the real data — prices behind a picker, stock, download counts, listings — even when the DOM shows only a shell, so it often answers the question without a browser render. true/'auto': pruned to the informative parts (recommended). 'raw': the complete blobs, up to 512KB."
},
"parser": {
"type": "object",
"properties": {
"include": {
"type": "array",
"items": {
"type": "string"
},
"maxItems": 25
},
"exclude": {
"type": "array",
"items": {
"type": "string"
},
"maxItems": 25
},
"keep": {
"type": "array",
"items": {
"type": "string"
},
"maxItems": 25
}
},
"additionalProperties": false,
"description": "Your own parsing rules, as CSS selector lists — use these when you know the page and don't want to rely on heuristics. include: keep ONLY these subtrees (targeted extraction, e.g. ['article.post']). exclude: delete site-specific chrome we kept. keep: protect a section (sidebar, dialog, form) that smart mode would strip."
},
"reveal_hidden": {
"type": "boolean",
"description": "Render tier only: before capturing, open <details>/accordions and click through every tab, appending each revealed panel to the page. Use it for tabbed code samples or spec accordions where a plain render captures only the visible variant."
},
"xhr": {
"type": "boolean",
"description": "Record the page's XHR/fetch traffic (URL, method, status, response body) into payload.xhr. Forces a browser render. An SPA's own JSON API is usually far cleaner than its DOM — use this to DISCOVER the API, then fetch_resource to return it directly."
},
"fetch_resource": {
"type": "string",
"maxLength": 500,
"description": "Regex matched against the page's network requests: the first matching response's BODY becomes the result instead of the page HTML (e.g. '/api/products' to get an SPA's JSON directly). Forces a render. Fails with 504 if nothing matches."
},
"preset_id": {
"type": "string",
"description": "Run a stored parser preset (see save_parser_preset) instead of passing `extract` selectors. Results land in payload.data exactly the same way, and the run is scored so the preset can detect decay and self-heal."
},
"actions": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": {}
},
"maxItems": 20,
"description": "Ordered browser interactions before capture (forces a render). Each is one object: {\"click\":\"#sel\"}, {\"clickText\":\"Accept\"} (click by visible text — dismiss a consent wall without knowing its CSS), {\"type\":{\"selector\":\"#q\",\"text\":\"shoes\"}}, {\"scroll\":\"bottom\"}, {\"wait\":1000}, {\"waitForSelector\":\".results\"}. Add \"optional\":true to skip a miss, or \"timeoutMs\":N to bound one action."
},
"frontmatter": {
"type": "boolean",
"description": "Prepend YAML front-matter (title, url, canonical, description, author, date) so the markdown is self-contained for RAG/Obsidian pipelines"
},
"links_mode": {
"type": "string",
"enum": [
"inline",
"footnote",
"strip"
],
"description": "Link rendering. inline (default): [text](url). footnote: URLs moved to a numbered reference list at the end. strip: keep only the link text — cuts 30-48% of the tokens on link-dense pages when you only need the prose."
},
"toc": {
"type": "boolean",
"description": "Prepend a table of contents built from the page headings"
},
"max_tokens": {
"type": "integer",
"minimum": 200,
"maximum": 2000000,
"description": "Cap the markdown at ~this many tokens, cutting at a section boundary (never inside a table or code block) and noting how much was omitted"
},
"query": {
"type": "string",
"maxLength": 512,
"description": "What you are looking for on the page. Keeps only the relevant sections (BM25 scoring over blocks, headings preserved) — the way to read one fact off a huge page without spending its whole token budget."
},
"highlights": {
"type": "integer",
"minimum": 1,
"maximum": 20,
"description": "With `query`: also return the N most relevant passages in payload.highlights"
},
"chunk": {
"type": "object",
"properties": {
"by": {
"type": "string",
"enum": [
"heading",
"sentence",
"tokens"
]
},
"size": {
"type": "integer",
"minimum": 1,
"maximum": 100000
},
"overlap": {
"type": "integer",
"minimum": 0,
"maximum": 100000
}
},
"additionalProperties": false,
"description": "Segment the output into payload.chunks[] for RAG/vector-DB ingestion — each chunk carries its heading path and token count. Fences and tables are never split."
},
"images_mode": {
"type": "string",
"enum": [
"inline",
"alt",
"strip"
],
"description": "inline (default) keeps ; 'alt' keeps only alt text; 'strip' removes images"
},
"summary_sections": {
"type": "boolean",
"description": "Append 'Links on this page' / 'Images on this page' sections — handy when deciding the next hop"
},
"html": {
"type": "string",
"description": "Convert HTML you already have instead of fetching: no proxy bandwidth is used, and the full parser pipeline still applies. Pass `url` too if you want relative links absolutized."
},
"content_modes": {
"type": "array",
"items": {
"type": "string",
"enum": [
"smart",
"article",
"full"
]
},
"maxItems": 3,
"description": "Return several content scopes from ONE fetch under payload.contents (e.g. compare smart vs full)"
},
"cookies": {
"type": "object",
"additionalProperties": {
"type": "string"
},
"description": "Cookies to send as name→value — the simple way to scrape behind a login"
}
},
"additionalProperties": false,
"$schema": "http://json-schema.org/draft-07/schema#"
}