curl --request GET \
  --url https://gateway.webit.live/v1/crawl

{
  "data": [
    {
      "id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
      "accountName": "<string>",
      "url": "<string>",
      "options": {
        "url": "https://example.com",
        "sitemap": "include",
        "crawl_entire_domain": false,
        "limit": 100,
        "max_discovery_depth": 3,
        "ignore_query_parameters": false,
        "allow_external_links": false,
        "allow_subdomains": false,
        "name": "The best crawl ever",
        "exclude_paths": [
          "/exclude-this-path",
          "/and-this-path"
        ],
        "include_paths": [
          "/include-this-path",
          "/and-this-path"
        ],
        "callback": {
          "url": "https://example.com/webhook",
          "headers": {
            "X-Custom-Header": "value"
          },
          "metadata": {
            "crawlId": "12345"
          },
          "events": [
            "page"
          ]
        },
        "extract_options": {
          "debug_options": {},
          "parse_options": {},
          "parse": true,
          "url": "https://example.com/page",
          "type": "generic",
          "method": "GET",
          "headers": {
            "User-Agent": "CustomBot/1.0",
            "Accept-Language": "en-US"
          },
          "raw_headers": true,
          "format": "json",
          "http2": true,
          "is_xhr": true,
          "no_html": false,
          "export_userbrowser": false,
          "save_userbrowser": false,
          "session": {
            "retry": false,
            "prefetch_userbrowser": false
          },
          "locale": "en-US",
          "country": "US",
          "device": "desktop",
          "proxy_provider": "brightdata",
          "cookies": [],
          "dynamic_parser": {
            "myParser": {
              "option1": "value1"
            }
          },
          "parser": {
            "myParser": {
              "option1": "value1"
            }
          },
          "referrer_type": "no-referrer",
          "expected_status_codes": [
            200,
            201
          ],
          "request_timeout": 30000,
          "client_timeout": 25000,
          "return_response_headers_as_header": true,
          "skill": "dynamic-content",
          "ip6": false,
          "native_mode": "requester",
          "driver": "vx8",
          "disable_ip_check": false,
          "template": {
            "name": "<string>",
            "params": {}
          },
          "markdown": false,
          "consent_header": true,
          "skip_ubct": false,
          "userbrowser_creation_template_rendered": {
            "id": "<string>",
            "allowed_parameter_names": [
              "<string>"
            ],
            "render_flow_rendered": [
              {}
            ]
          },
          "query_template": {
            "id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
            "api_type": "WEB",
            "params": {},
            "pagination": {
              "next_page_params": {}
            }
          },
          "render": true,
          "render_options": {},
          "network_capture": [
            {
              "validation": false,
              "wait_for_requests_count": 0,
              "status_code": 349.5,
              "method": "GET",
              "url": {
                "type": "exact",
                "value": "<string>"
              },
              "resource_type": [
                "document",
                "script",
                "xhr",
                "fetch"
              ],
              "wait_for_requests_count_timeout": 150000
            }
          ],
          "render_flow": [
            {
              "wait": {
                "delay": 2000
              }
            },
            {
              "click": {
                "selector": "#load-more",
                "timeout": 5000
              }
            }
          ],
          "tag": "campaign-2024-q1",
          "metadata": {
            "source": "web-app",
            "pipeline_execution_id": 12345,
            "execution_id": "exec-abc123",
            "endpoint": "/api/v2/scrape",
            "definition_id": 456,
            "definition_name": "product-scraper",
            "template_id": 789,
            "template_name": "e-commerce-template",
            "account_name": "acme-corp",
            "flowit_task_id": "task-xyz789",
            "input_id": "input-123",
            "query_template_id": "template-qry-001"
          },
          "proxy_providers": {
            "brightdata": 70,
            "oxylabs": 30
          },
          "browser": "chrome",
          "os": "windows",
          "no_userbrowser": false,
          "state": "CA",
          "city": "Los Angeles"
        }
      }
    }
  ],
  "pagination": {
    "hasNext": true,
    "nextCursor": "<string>",
    "total": 123
  }
}