Get the page data./scrape

Authorization: Bearer rtrvr_your_api_key

curl -X POST https://api.rtrvr.ai/scrape \
  -H "Authorization: Bearer YOUR_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "urls": ["https://example.com/blog/ai-trends-2025"]
  }'

interface UserSettings {
  extractionConfig: {
    maxParallelTabs?: number;
    /** Minimum wait inside totalBudgetMs; default 0. */
    pageLoadDelay?: number;
    /** Initial read deadline: default 15000, bounded to 1500–30000 ms. */
    totalBudgetMs?: number;
    makeNewTabsActive?: boolean;
    writeRowProcessingTime?: boolean;
    disableAutoScroll?: boolean;
    /**
     * When true, only text content is returned from scrapes.
     * The accessibility tree + elementLinkRecord are omitted.
     * When false (the default), HTML pages return tree + elementLinkRecord
     * and no content.
     */
    onlyTextContent?: boolean;
  };

  // Proxy Configuration
  proxyConfig: {
    mode: 'none' | 'custom' | 'default' | 'device';
    customProxies?: ProxySettings[];
    selectedProxyId?: string;
    selectedProxyName?: string;
    selectedDeviceId?: string;
    // Managed proxy exit (mode 'default'): ISO country, optional ISO 3166-2
    // region part ("NY", "25") and English city name.
    location?: { country: string; region?: string; city?: string };
  };
}

interface ScrapeApiRequest {
    /**
     * Unique caller id for this scrape. Save it before dispatch to recover
     * results or cancel after a lost response. Reusing it returns HTTP 409.
     */
    trajectoryId?: string;

    /**
     * One or more absolute URLs to load in the browser.
     * Must be a non-empty array of non-empty strings.
     */
    urls: string[];

    /**
     * Optional per-request settings override.
     * Other settings merge with your profile; omitted proxyConfig uses the profile default, or the managed proxy if none is set.
     *
     * Use extraction-related settings if you only want text content and don't need
     * the accessibility tree + elementLinkRecord.
     */
    settings?: Partial<UserSettings>;

    /**
     * Response size control for API callers.
     */
    response?: {
      /**
       * Max bytes allowed for the inline JSON response.
       * If the full response exceeds this, tabs remain inline as preview content,
       * and a StorageReference is returned under metadata.responseRef for full payload download.
       * Default: 1MB (1048576 bytes)
       */
      inlineOutputMaxBytes?: number;
    };

    /**
     * Optional execution options.
     * Set options.ui.emitEvents=true to write progress events for SSE/polling clients.
     * If omitted/false, no execution event stream is written.
     */
    options?: {
      ui?: {
        emitEvents?: boolean;
      };
    };

    /**
     * Webhooks to call when the scrape completes, fails, or is cancelled.
     */
    webhooks?: WebhookSubscription[];
  }

interface WebhookSubscription {
    /** The URL to POST to */
    url: string;
    /** Events to subscribe to. Defaults to all scrape events. */
    events?: WebhookEvent[];
    /** Optional custom headers */
    headers?: Record<string, string>;
    /** Optional auth (bearer or basic) */
    auth?: { type: "bearer"; token: string } | { type: "basic"; username: string; password: string };
    /** Optional secret for HMAC signing (X-Rtrvr-Signature header) */
    secret?: string;
    /** Timeout for webhook delivery (default: 8000ms) */
    timeoutMs?: number;
    /** Retry policy (default: { mode: "default" }) */
    retry?: { mode: "default" | "none" };
  }

type WebhookEvent =
    | "rtrvr.scrape.succeeded"
    | "rtrvr.scrape.failed"
    | "rtrvr.scrape.cancelled";

curl -X POST https://api.rtrvr.ai/scrape \
  -H "Authorization: Bearer YOUR_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "urls": ["https://example.com/page1", "https://example.com/page2"],
    "webhooks": [
      {
        "url": "https://your-server.com/webhooks/scrape",
        "events": ["rtrvr.scrape.succeeded", "rtrvr.scrape.failed"],
        "secret": "whsec_your_signing_secret",
        "headers": { "X-Custom-Header": "my-value" }
      }
    ]
  }'

{
  "id": "whd_abc123...",          // unique delivery id
  "event": "rtrvr.scrape.succeeded",
  "createdAt": "2025-01-15T10:30:00.000Z",
  "data": {
    "trajectoryId": "traj_xyz...",
    "success": true,
    "tabs": [...],
    "usageData": {...}
  }
}

X-Rtrvr-Signature: t=1705312200,v1=5257a869e7ecebeda32affa62cdca3fa51cad7e77a0e56ff536d0ce8e108d8bd

import crypto from 'crypto';

function verifyWebhookSignature(payload, signature, secret) {
  const [tPart, vPart] = signature.split(',');
  const timestamp = tPart.split('=')[1];
  const receivedSig = vPart.split('=')[1];

  // Recreate the signed payload
  const signedPayload = `${timestamp}.${JSON.stringify(payload)}`;
  const expectedSig = crypto
    .createHmac('sha256', secret)
    .update(signedPayload)
    .digest('hex');

  // Timing-safe comparison
  return crypto.timingSafeEqual(
    Buffer.from(receivedSig),
    Buffer.from(expectedSig)
  );
}

interface ScrapedTab {
  tabId: number;
  url: string;
  title: string;
  contentType: string;
  status: "success" | "error";
  error?: string;

  /**
   * The page's visible text. HTML pages fill it when
   * extractionConfig.onlyTextContent is true, or with a text fallback when
   * the tree could not be read. PDFs and other documents return their text here.
   */
  content?: string;

  /**
   * The accessibility tree as a plain-text outline, not JSON. Do not JSON.parse it.
   * One element per line, indented by depth: "[role] text", then markers such
   * as [id=N]. Lines of plain page text start with "[]". Link lines carry their
   * URL inline:
   *   [link] Learn more [id=8 href="https://iana.org/help/example-domains"]
   * Omitted when onlyTextContent is true.
   */
  tree?: string;

  /**
   * Element id -> href for the link elements in 'tree', e.g. { "8": "https://..." }.
   * The same URLs already appear inline in 'tree'.
   * Only present when 'tree' is present.
   */
  elementLinkRecord?: Record<number, string>;
}

interface ScrapeUsageData {
  totalCredits: number;
  browserCredits: number;
  proxyCredits: number;
  totalUsd: number;
  requestDurationMs: number;
  proxyPageLoads: number;
  proxyTabsDataFetches: number;
  usingBillableProxy: boolean;
}

interface ScrapeApiResponse {
  success: boolean;
  status: "success" | "error";
  trajectoryId: string;

  tabs?: ScrapedTab[];
  usageData: ScrapeUsageData;

  // Where the managed proxy exited, e.g. { country: "IT", city: "Milan", timezone: "Europe/Rome" }
  egress?: { country: string; countries?: string[]; region?: string; city?: string; timezone?: string;
             requested?: { country: string; region?: string; city?: string }; relaxed?: boolean };

  metadata?: {
    taskRef?: string;
    inlineOutputMaxBytes: number;
    durationMs: number;
    outputTooLarge?: boolean;
    responseRef?: StorageReference;
  };

  error?: string;
}

[] This domain is for use in documentation examples without needing permission. ...
[link] Learn more [id=8 href="https://iana.org/help/example-domains"]
[title] Example Domain

GET /executions?limit=50
GET /executions/YOUR_TRAJECTORY_ID
POST /scrape/cancel
Content-Type: application/json

{"trajectoryId":"YOUR_TRAJECTORY_ID"}

# Basic scrape using profile defaults
curl -X POST https://api.rtrvr.ai/scrape \
  -H "Authorization: Bearer YOUR_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "urls": ["https://example.com/blog/ai-trends-2025"],
    "response": { "inlineOutputMaxBytes": 1048576 }
  }'

# With per-request settings override
curl -X POST https://api.rtrvr.ai/scrape \
  -H "Authorization: Bearer YOUR_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "urls": [
      "https://example.com/blog/ai-trends-2025",
      "https://example.com/pricing"
    ],
    "settings": {
      "extractionConfig": {
        "onlyTextContent": true
      },
      "proxyConfig": {
        "mode": "default"
      }
    },
    "response": {
      "inlineOutputMaxBytes": 1048576
    }
  }'