scrapfly-webhooks

Receive and verify Scrapfly webhooks. Use when setting up Scrapfly webhook handlers for async scrape, extraction, screenshot, or crawler jobs, debugging X-Scrapfly-Webhook-Signature verification, or routing on X-Scrapfly-Webhook-Resource-Type.

Jump to install

Repository: hookdeck/webhook-skills
Last source activity: May 15, 2026 at 15:00
Detected SKILL.md language: English
Stars: 79
Forks: 11

Install options

The review-first prompt is selected by default. You can switch to a direct command or download a local copy.

Review the source files

Read SKILL.md and any companion files shown by SkillsMP before deciding whether to install.

Back to search Back to installation

const crypto = require('crypto');

function verifyScrapflySignature(rawBody, signatureHeader, secret) {
  if (!signatureHeader || !secret) return false;

  // Scrapfly emits uppercase hex
  const expected = crypto
    .createHmac('sha256', secret)
    .update(rawBody)
    .digest('hex')
    .toUpperCase();

  // Accept either casing — Scrapfly also sends an X-...-Lowercase variant
  const received = signatureHeader.toUpperCase();

  try {
    return crypto.timingSafeEqual(
      Buffer.from(received, 'hex'),
      Buffer.from(expected, 'hex')
    );
  } catch {
    return false;
  }
}

const express = require('express');
const app = express();

// CRITICAL: Use express.raw() — Scrapfly signs the raw body bytes
app.post('/webhooks/scrapfly',
  express.raw({ type: '*/*' }),
  (req, res) => {
    const signature = req.headers['x-scrapfly-webhook-signature'];
    const resourceType = req.headers['x-scrapfly-webhook-resource-type'];
    const jobId = req.headers['x-scrapfly-webhook-job-id'];
    const webhookId = req.headers['x-scrapfly-webhook-id'];

    if (!verifyScrapflySignature(req.body, signature, process.env.SCRAPFLY_WEBHOOK_SECRET)) {
      console.error('Scrapfly signature verification failed');
      return res.status(401).send('Invalid signature');
    }

    console.log(`Scrapfly ${resourceType} webhook (job ${jobId}, id ${webhookId})`);

    // CRITICAL: dispatch BEFORE JSON.parse — Screenshot API deliveries carry
    // raw image bytes (JPEG/PNG/WebP/GIF) regardless of the Content-Type you
    // configured in the Scrapfly dashboard. Content-Type is whatever you
    // picked (application/json by default; application/msgpack is also an
    // option). JSON.parse on a binary body throws after the signature
    // has already verified.
    if (resourceType === 'screenshot') {
      console.log(`Screenshot received: ${req.body.length} bytes (binary)`);
      // req.body is the raw image. Persist it to storage and return 200.
      return res.status(200).send('OK');
    }

    // Remaining resource types deliver JSON payloads.
    const payload = JSON.parse(req.body.toString());

    switch (resourceType) {
      case 'scrape':
        // Scrape API places the fetched URL at result.url; the webhook overlay's
        // context only carries `webhook` and `job` sub-objects.
        console.log('Scrape result:', payload.result?.status_code, payload.result?.url);
        break;
      case 'extraction':
        // Extraction body shape: { content_type, data: {...}, context: {...} }.
        // Extracted fields live at payload.data, NOT payload.result.data.
        console.log('Extraction result:', payload.content_type, payload.data);
        break;
      default:
        // Crawler API uses event names in the body
        if (payload.event) {
          console.log(`Crawler event: ${payload.event}`, payload.payload);
        } else {
          console.log('Unhandled resource type:', resourceType);
        }
    }

    res.status(200).send('OK');
  }
);

import hmac
import hashlib

def verify_scrapfly_signature(raw_body: bytes, signature_header: str, secret: str) -> bool:
    if not signature_header or not secret:
        return False

    expected = hmac.new(
        secret.encode('utf-8'),
        raw_body,
        hashlib.sha256,
    ).hexdigest().upper()

    # Compare case-insensitively (Scrapfly also sends a lowercase header)
    return hmac.compare_digest(expected, signature_header.upper())

Event	Description
`crawler_started`	Crawl job began
`crawler_url_visited`	A URL was successfully fetched
`crawler_url_discovered`	A new URL was queued
`crawler_url_skipped`	A URL was skipped (filters, dedupe, ...)
`crawler_url_failed`	A URL fetch failed
`crawler_stopped`	Crawl stopped (limit reached)
`crawler_cancelled`	Crawl cancelled by user
`crawler_finished`	Crawl finished naturally

Header	Description
`X-Scrapfly-Webhook-Signature`	HMAC-SHA256 of the raw body, uppercase hex
`X-Scrapfly-Webhook-Signature-Lowercase`	Same signature, lowercase hex
`X-Scrapfly-Webhook-Id`	Unique webhook delivery identifier
`X-Scrapfly-Webhook-Name`	Name of the configured webhook
`X-Scrapfly-Webhook-Resource-Type`	`scrape`, `extraction`, or `screenshot`
`X-Scrapfly-Webhook-Job-Id`	Unique job identifier (use for reconciliation)
`X-Scrapfly-Webhook-Env`	Environment (`test` or `live`)
`X-Scrapfly-Webhook-Project`	Project name
`X-Scrapfly-Crawl-Event-Name`	Crawler API event name (e.g. `crawler_finished`)

SCRAPFLY_WEBHOOK_SECRET=your_signing_secret_here   # From the Scrapfly dashboard

# Express / Next.js (port 3000)
npx hookdeck-cli listen 3000 scrapfly --path /webhooks/scrapfly

# FastAPI (port 8000)
npx hookdeck-cli listen 8000 scrapfly --path /webhooks/scrapfly

// Generated with: scrapfly-webhooks skill
// https://github.com/hookdeck/webhook-skills