Skip to main content
POST
/
v1
/
extract
Webit v2 Realtime extract Endpoint
curl --request POST \
  --url https://gateway.webit.live/v1/extract \
  --header 'Content-Type: application/json' \
  --data '
{
  "debug_options": {
    "collect_har": true,
    "record_screen": true,
    "verbose": true,
    "trace": true,
    "no_retry_mode": true,
    "upload_engine_logs": true,
    "solve_captcha": true,
    "show_cursor": true,
    "with_proxy_usage": true,
    "redact": true
  },
  "url": "https://example.com/page",
  "cookies": [],
  "parse_options": {
    "merge_dynamic": true
  },
  "parse": true,
  "dynamic_parser": {
    "myParser": {
      "option1": "value1"
    }
  },
  "parser": {
    "myParser": {
      "option1": "value1"
    }
  },
  "type": "generic",
  "method": "GET",
  "referrer_type": "no-referrer",
  "expected_status_codes": [
    200,
    201
  ],
  "headers": {
    "User-Agent": "CustomBot/1.0",
    "Accept-Language": "en-US"
  },
  "raw_headers": true,
  "request_timeout": 30000,
  "client_timeout": 25000,
  "return_response_headers_as_header": true,
  "format": "json",
  "skill": "dynamic-content",
  "http2": true,
  "ip6": false,
  "is_xhr": true,
  "no_html": false,
  "export_userbrowser": false,
  "save_userbrowser": false,
  "native_mode": "requester",
  "driver": "vx8",
  "disable_ip_check": false,
  "template": {
    "name": "<string>",
    "params": {}
  },
  "markdown": false,
  "consent_header": true,
  "skip_ubct": false,
  "userbrowser_creation_template_rendered": {
    "id": "<string>",
    "allowed_parameter_names": [
      "<string>"
    ],
    "render_flow_rendered": [
      {}
    ]
  },
  "query_template": {
    "id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
    "api_type": "WEB",
    "params": {},
    "pagination": {
      "next_page_params": {}
    }
  },
  "render": true,
  "render_options": {
    "wait_until": "networkidle2",
    "render_type": "load",
    "headless": true,
    "timeout": 30000,
    "userbrowser": true,
    "connector_type": "webit-cdp",
    "hackium_configuration": {
      "collect_logs": false,
      "enable_verbose_logs": false,
      "enable_sniffer": false,
      "do_not_fix_math_salt": false,
      "enable_document_element_spoof": false,
      "enable_key_ordering": false,
      "enable_document_has_focus": false,
      "enable_fake_navigation_history": false
    },
    "include_iframes": true,
    "browser_engine": "chrome",
    "fingerprint_id": "fp-abc123",
    "disabled_resources": [
      "image",
      "stylesheet"
    ],
    "adblock": true,
    "cache": false,
    "blocked_domains": [
      "ads.example.com",
      "tracker.com"
    ],
    "with_performance_metrics": true,
    "no_accept_encoding": true,
    "override_permissions": true,
    "store_local_storage": true,
    "load_local_storage": true,
    "local_storage_keys_to_load": [
      "authToken",
      "userId"
    ],
    "enable_2captcha": true,
    "mouse_strategy": "linear",
    "typing_strategy": "simple",
    "typing_interval": 100,
    "random_header_order": true,
    "extensions": [
      "extension-id-1",
      "extension-id-2"
    ]
  },
  "network_capture": [
    {
      "status_code": 349.5,
      "method": "GET",
      "url": {
        "value": "<string>",
        "type": "exact"
      },
      "resource_type": [
        "document",
        "script",
        "xhr",
        "fetch"
      ],
      "validation": false,
      "wait_for_requests_count": 0,
      "wait_for_requests_count_timeout": 150000
    }
  ],
  "render_flow": [
    {
      "wait": {
        "delay": 2000
      }
    },
    {
      "click": {
        "selector": "#load-more",
        "timeout": 5000
      }
    }
  ],
  "session": {
    "id": "<string>",
    "timeout": 1,
    "retry": false,
    "prefetch_userbrowser": false
  },
  "tag": "campaign-2024-q1",
  "metadata": {
    "source": "web-app",
    "pipeline_execution_id": 12345,
    "execution_id": "exec-abc123",
    "endpoint": "/api/v2/scrape",
    "definition_id": 456,
    "definition_name": "product-scraper",
    "template_id": 789,
    "template_name": "e-commerce-template",
    "account_name": "acme-corp",
    "flowit_task_id": "task-xyz789",
    "input_id": "input-123",
    "query_template_id": "template-qry-001"
  },
  "locale": "en-US",
  "country": "US",
  "device": "desktop",
  "proxy_provider": "brightdata",
  "proxy_providers": {
    "brightdata": 70,
    "oxylabs": 30
  },
  "browser": "chrome",
  "os": "windows",
  "no_userbrowser": false,
  "state": "CA",
  "city": "Los Angeles"
}
'
{
  "id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "status": 123
}

Body

application/json
​
debug_options
object
required

Debug and troubleshooting options for the request

​
url
string<uri>
required

Target URL to scrape

Example:

"https://example.com/page"

​
cookies

Browser cookies as array of cookie objects

Example:
[]
​
parse_options
object

Configuration options for parsing behavior

​
parse
boolean
default:false

Whether to parse the response content

Example:

true

​
dynamic_parser
object

Custom parser configuration as a key-value map

Example:
{ "myParser": { "option1": "value1" } }
​
parser

Custom parser configuration as a key-value map

Example:
{ "myParser": { "option1": "value1" } }
​
type
string
default:generic

Type of query or scraping template

Example:

"generic"

​
method
enum<string>
default:GET

HTTP method for the request

Available options:
GET,
POST,
PUT,
PATCH,
DELETE
Example:

"GET"

​
referrer_type

Referrer policy for the request

Available options:
random,
no-referer,
same-origin
Example:

"no-referrer"

​
expected_status_codes
integer[]

Expected HTTP status codes for successful requests

Required range: -9007199254740991 <= x <= 9007199254740991
Example:
[200, 201]
​
headers
object

Custom HTTP headers to include in the request

Example:
{
  "User-Agent": "CustomBot/1.0",
  "Accept-Language": "en-US"
}
​
raw_headers
boolean
default:false

Whether to return raw HTTP headers in response

Example:

true

​
request_timeout
number

Request timeout in milliseconds

Required range: x >= 1
Example:

30000

​
client_timeout
number

Client-side timeout in milliseconds

Required range: x >= 1
Example:

25000

​
return_response_headers_as_header
boolean

Whether to return response headers in HTTP headers

Example:

true

​
format
enum<string>
default:json

Response format

Available options:
json,
html,
csv,
raw,
json-lines,
markdown
Example:

"json"

​
skill

Skills or capabilities required for the request

Example:

"dynamic-content"

​
http2
boolean
default:false

Whether to use HTTP/2 protocol

Example:

true

​
ip6
boolean

Whether to use IPv6 for the request

Example:

false

​
is_xhr
boolean
default:false

Whether to emulate XMLHttpRequest behavior

Example:

true

​
no_html
boolean
default:false

Whether to exclude HTML from the response

Example:

false

​
export_userbrowser
boolean
default:false

Whether to export the userbrowser session

Example:

false

​
save_userbrowser
boolean
default:false

Whether to save the userbrowser session for reuse

Example:

false

​
native_mode
enum<string>

Native execution mode

Available options:
requester,
apm,
direct
Example:

"requester"

​
driver
enum<string>

Browser driver to use

Available options:
vx6,
vx8,
vx8-pro,
vx10,
vx10-pro,
vx12,
vx12-pro
Example:

"vx8"

​
disable_ip_check
boolean

Whether to disable IP address validation

Example:

false

​
template
object

Userbrowser creation template configuration

​
markdown
boolean

Whether to return response in Markdown format

Example:

false

​
consent_header
boolean

Whether to automatically handle cookie consent headers

​
skip_ubct
boolean

Whether to skip userbrowser creation template processing

Example:

false

​
userbrowser_creation_template_rendered
object

Pre-rendered userbrowser creation template configuration

​
query_template
object

Query template configuration for structured data extraction

​
render
boolean

Whether to render JavaScript content using a browser

Example:

true

​
render_options
object
​
network_capture
object[]

Filters for capturing network traffic

​
render_flow
object[]

Array of actions to perform during browser rendering

Example:
[
  { "wait": { "delay": 2000 } },
  {
    "click": { "selector": "#load-more", "timeout": 5000 }
  }
]
​
session
object
​
tag
string

User-defined tag for request identification

Example:

"campaign-2024-q1"

​
metadata
object

Structured metadata about the request execution context

​
locale
default:en

Locale for browser language and region settings

Available options:
aa-DJ,
aa-ER,
aa-ET,
af,
af-NA,
af-ZA,
ak,
ak-GH,
am,
am-ET,
an-ES,
ar,
ar-AE,
ar-BH,
ar-DZ,
ar-EG,
ar-IN,
ar-IQ,
ar-JO,
ar-KW,
ar-LB,
ar-LY,
ar-MA,
ar-OM,
ar-QA,
ar-SA,
ar-SD,
ar-SY,
ar-TN,
ar-YE,
as,
as-IN,
asa,
asa-TZ,
ast-ES,
az,
az-AZ,
az-Cyrl,
az-Cyrl-AZ,
az-Latn,
az-Latn-AZ,
be,
be-BY,
bem,
bem-ZM,
ber-DZ,
ber-MA,
bez,
bez-TZ,
bg,
bg-BG,
bho-IN,
bm,
bm-ML,
bn,
bn-BD,
bn-IN,
bo,
bo-CN,
bo-IN,
br-FR,
brx-IN,
bs,
bs-BA,
byn-ER,
ca,
ca-AD,
ca-ES,
ca-FR,
ca-IT,
cgg,
cgg-UG,
chr,
chr-US,
crh-UA,
cs,
cs-CZ,
csb-PL,
cv-RU,
cy,
cy-GB,
da,
da-DK,
dav,
dav-KE,
de,
de-AT,
de-BE,
de-CH,
de-DE,
de-LI,
de-LU,
dv-MV,
dz-BT,
ebu,
ebu-KE,
ee,
ee-GH,
ee-TG,
el,
el-CY,
el-GR,
en,
en-AG,
en-AS,
en-AU,
en-BE,
en-BW,
en-BZ,
en-CA,
en-DK,
en-GB,
en-GU,
en-HK,
en-IE,
en-IN,
en-JM,
en-MH,
en-MP,
en-MT,
en-MU,
en-NA,
en-NG,
en-NZ,
en-PH,
en-PK,
en-SG,
en-TT,
en-UM,
en-US,
en-VI,
en-ZA,
en-ZM,
en-ZW,
eo,
es,
es-419,
es-AR,
es-BO,
es-CL,
es-CO,
es-CR,
es-CU,
es-DO,
es-EC,
es-ES,
es-GQ,
es-GT,
es-HN,
es-MX,
es-NI,
es-PA,
es-PE,
es-PR,
es-PY,
es-SV,
es-US,
es-UY,
es-VE,
et,
et-EE,
eu,
eu-ES,
fa,
fa-AF,
fa-IR,
ff,
ff-SN,
fi,
fi-FI,
fil,
fil-PH,
fo,
fo-FO,
fr,
fr-BE,
fr-BF,
fr-BI,
fr-BJ,
fr-BL,
fr-CA,
fr-CD,
fr-CF,
fr-CG,
fr-CH,
fr-CI,
fr-CM,
fr-DJ,
fr-FR,
fr-GA,
fr-GN,
fr-GP,
fr-GQ,
fr-KM,
fr-LU,
fr-MC,
fr-MF,
fr-MG,
fr-ML,
fr-MQ,
fr-NE,
fr-RE,
fr-RW,
fr-SN,
fr-TD,
fr-TG,
fur-IT,
fy-DE,
fy-NL,
ga,
ga-IE,
gd-GB,
gez-ER,
gez-ET,
gl,
gl-ES,
gsw,
gsw-CH,
gu,
gu-IN,
guz,
guz-KE,
gv,
gv-GB,
ha,
ha-Latn,
ha-Latn-GH,
ha-Latn-NE,
ha-Latn-NG,
ha-NG,
haw,
haw-US,
he,
he-IL,
hi,
hi-IN,
hne-IN,
hr,
hr-HR,
hsb-DE,
ht-HT,
hu,
hu-HU,
hy,
hy-AM,
id,
id-ID,
ig,
ig-NG,
ii,
ii-CN,
ik-CA,
is,
is-IS,
it,
it-CH,
it-IT,
iu-CA,
iw-IL,
ja,
ja-JP,
jmc,
jmc-TZ,
ka,
ka-GE,
kab,
kab-DZ,
kam,
kam-KE,
kde,
kde-TZ,
kea,
kea-CV,
khq,
khq-ML,
ki,
ki-KE,
kk,
kk-Cyrl,
kk-Cyrl-KZ,
kk-KZ,
kl,
kl-GL,
kln,
kln-KE,
km,
km-KH,
kn,
kn-IN,
ko,
ko-KR,
kok,
kok-IN,
ks-IN,
ku-TR,
kw,
kw-GB,
ky-KG,
lag,
lag-TZ,
lb-LU,
lg,
lg-UG,
li-BE,
li-NL,
lij-IT,
lo-LA,
lt,
lt-LT,
luo,
luo-KE,
luy,
luy-KE,
lv,
lv-LV,
mag-IN,
mai-IN,
mas,
mas-KE,
mas-TZ,
mer,
mer-KE,
mfe,
mfe-MU,
mg,
mg-MG,
mhr-RU,
mi-NZ,
mk,
mk-MK,
ml,
ml-IN,
mn-MN,
mr,
mr-IN,
ms,
ms-BN,
ms-MY,
mt,
mt-MT,
my,
my-MM,
nan-TW,
naq,
naq-NA,
nb,
nb-NO,
nd,
nd-ZW,
nds-DE,
nds-NL,
ne,
ne-IN,
ne-NP,
nl,
nl-AW,
nl-BE,
nl-NL,
nn,
nn-NO,
nr-ZA,
nso-ZA,
nyn,
nyn-UG,
oc-FR,
om,
om-ET,
om-KE,
or,
or-IN,
os-RU,
pa,
pa-Arab,
pa-Arab-PK,
pa-Guru,
pa-Guru-IN,
pa-IN,
pa-PK,
pap-AN,
pl,
pl-PL,
ps,
ps-AF,
pt,
pt-BR,
pt-GW,
pt-MZ,
pt-PT,
rm,
rm-CH,
ro,
ro-MD,
ro-RO,
rof,
rof-TZ,
ru,
ru-MD,
ru-RU,
ru-UA,
rw,
rw-RW,
rwk,
rwk-TZ,
sa-IN,
saq,
saq-KE,
sc-IT,
sd-IN,
se-NO,
seh,
seh-MZ,
ses,
ses-ML,
sg,
sg-CF,
shi,
shi-Latn,
shi-Latn-MA,
shi-Tfng,
shi-Tfng-MA,
shs-CA,
si,
si-LK,
sid-ET,
sk,
sk-SK,
sl,
sl-SI,
sn,
sn-ZW,
so,
so-DJ,
so-ET,
so-KE,
so-SO,
sq,
sq-AL,
sq-MK,
sr,
sr-Cyrl,
sr-Cyrl-BA,
sr-Cyrl-ME,
sr-Cyrl-RS,
sr-Latn,
sr-Latn-BA,
sr-Latn-ME,
sr-Latn-RS,
sr-ME,
sr-RS,
ss-ZA,
st-ZA,
sv,
sv-FI,
sv-SE,
sw,
sw-KE,
sw-TZ,
ta,
ta-IN,
ta-LK,
te,
te-IN,
teo,
teo-KE,
teo-UG,
tg-TJ,
th,
th-TH,
ti,
ti-ER,
ti-ET,
tig-ER,
tk-TM,
tl-PH,
tn-ZA,
to,
to-TO,
tr,
tr-CY,
tr-TR,
ts-ZA,
tt-RU,
tzm,
tzm-Latn,
tzm-Latn-MA,
ug-CN,
uk,
uk-UA,
unm-US,
ur,
ur-IN,
ur-PK,
uz,
uz-Arab,
uz-Arab-AF,
uz-Cyrl,
uz-Cyrl-UZ,
uz-Latn,
uz-Latn-UZ,
uz-UZ,
ve-ZA,
vi,
vi-VN,
vun,
vun-TZ,
wa-BE,
wae-CH,
wal-ET,
wo-SN,
xh-ZA,
xog,
xog-UG,
yi-US,
yo,
yo-NG,
yue-HK,
zh,
zh-CN,
zh-HK,
zh-Hans,
zh-Hans-CN,
zh-Hans-HK,
zh-Hans-MO,
zh-Hans-SG,
zh-Hant,
zh-Hant-HK,
zh-Hant-MO,
zh-Hant-TW,
zh-SG,
zh-TW,
zu,
zu-ZA
Example:

"en-US"

​
country
default:ALL

Country code for geolocation and proxy selection

Available options:
AD,
AE,
AF,
AG,
AI,
AL,
AM,
AO,
AQ,
AR,
AS,
AT,
AU,
AW,
AX,
AZ,
BA,
BB,
BD,
BE,
BF,
BG,
BH,
BI,
BJ,
BL,
BM,
BN,
BO,
BQ,
BR,
BS,
BT,
BV,
BW,
BY,
BZ,
CA,
CC,
CD,
CF,
CG,
CH,
CI,
CK,
CL,
CM,
CN,
CO,
CR,
CU,
CV,
CW,
CX,
CY,
CZ,
DE,
DJ,
DK,
DM,
DO,
DZ,
EC,
EE,
EG,
EH,
ER,
ES,
ET,
FI,
FJ,
FK,
FM,
FO,
FR,
GA,
GB,
GD,
GE,
GF,
GG,
GH,
GI,
GL,
GM,
GN,
GP,
GQ,
GR,
GS,
GT,
GU,
GW,
GY,
HK,
HM,
HN,
HR,
HT,
HU,
ID,
IE,
IL,
IM,
IN,
IO,
IQ,
IR,
IS,
IT,
JE,
JM,
JO,
JP,
KE,
KG,
KH,
KI,
KM,
KN,
KP,
KR,
KW,
KY,
KZ,
LA,
LB,
LC,
LI,
LK,
LR,
LS,
LT,
LU,
LV,
LY,
MA,
MC,
MD,
ME,
MF,
MG,
MH,
MK,
ML,
MM,
MN,
MO,
MP,
MQ,
MR,
MS,
MT,
MU,
MV,
MW,
MX,
MY,
MZ,
NA,
NC,
NE,
NF,
NG,
NI,
NL,
NO,
NP,
NR,
NU,
NZ,
OM,
PA,
PE,
PF,
PG,
PH,
PK,
PL,
PM,
PN,
PR,
PS,
PT,
PW,
PY,
QA,
RE,
RO,
RS,
RU,
RW,
SA,
SB,
SC,
SD,
SE,
SG,
SH,
SI,
SJ,
SK,
SL,
SM,
SN,
SO,
SR,
SS,
ST,
SV,
SX,
SY,
SZ,
TC,
TD,
TF,
TG,
TH,
TJ,
TK,
TL,
TM,
TN,
TO,
TR,
TT,
TV,
TW,
TZ,
UA,
UG,
UM,
US,
UY,
UZ,
VA,
VC,
VE,
VG,
VI,
VN,
VU,
WF,
WS,
XK,
YE,
YT,
ZA,
ZM,
ZW
Example:

"US"

​
device
enum<string>
default:desktop

Device type for browser emulation

Available options:
desktop,
mobile,
tablet
Example:

"desktop"

​
proxy_provider
default:proxit

Proxy provider to use for the request

Available options:
brightdata,
oxylabs,
smartproxy,
proxit,
proxit_preprod,
local,
rayobyte,
always,
oculusproxies,
froxy,
packetstream,
911proxy,
direct911proxy,
thesocialproxy,
thesocialproxy2,
nimble-isp,
nimble-isp-mobile,
proxit-linux,
proxit-macos,
proxit-windows,
proxit-rental,
ipfoxy,
brightup,
research
Example:

"brightdata"

​
proxy_providers
object

Weighted distribution of proxy providers

Example:
{ "brightdata": 70, "oxylabs": 30 }
​
browser

Browser type to emulate

Available options:
chrome,
firefox
Example:

"chrome"

​
os
enum<string>

Operating system to emulate

Available options:
windows,
mac os,
linux,
android,
ios
Example:

"windows"

​
no_userbrowser
boolean

Whether to disable browser-based rendering

Example:

false

​
state
enum<string>

US state for geolocation (only valid when country is US)

Available options:
AL,
AK,
AS,
AZ,
AR,
CA,
CO,
CT,
DE,
DC,
FL,
GA,
GU,
HI,
ID,
IL,
IN,
IA,
KS,
KY,
LA,
ME,
MD,
MA,
MI,
MN,
MS,
MO,
MT,
NE,
NV,
NH,
NJ,
NM,
NY,
NC,
ND,
MP,
OH,
OK,
OR,
PA,
PR,
RI,
SC,
SD,
TN,
TX,
UT,
VT,
VA,
VI,
WA,
WV,
WI,
WY
Example:

"CA"

​
city
string

City for geolocation

Example:

"Los Angeles"

Response

200 - application/json

Successful Response

​
id
string<uuid>
required
​
status
number
required