firecrawl-load-scale

name	firecrawl-load-scale
description	Load test and scale Firecrawl scraping pipelines with concurrency control and batching. Use when testing scraping throughput, planning capacity for large crawl jobs, or optimizing concurrent scrape performance. Trigger with phrases like "firecrawl load test", "firecrawl scale", "firecrawl throughput", "firecrawl capacity", "firecrawl concurrent".
allowed-tools	Read, Write, Edit, Bash(node:*), Bash(npm:*)
version	1.11.0
license	MIT
author	Jeremy Longshore <[e-mail]>
tags	["saas","firecrawl","testing","performance","scaling"]
compatibility	Designed for Claude Code

Firecrawl Load & Scale

Overview

Load test and scale Firecrawl scraping pipelines. Firecrawl's rate limits are per-plan (RPM and concurrent connections), so scaling means maximizing throughput within those limits using batch scraping, async crawls, and queue-based request management.

Rate Limits by Plan

Plan	Scrape RPM	Concurrent Crawls	Max Batch Size
Free	10	2	10
Hobby	20	3	50
Standard	50	5	100
Growth	100	10	100
Scale	500+	50+	100

Instructions

Step 1: Measure Baseline Throughput

import FirecrawlApp from "@mendable/firecrawl-js";

const firecrawl = new FirecrawlApp({
  apiKey: process.env.FIRECRAWL_API_KEY!,
});

async function measureThroughput(urls: string[], concurrency: number) {
  const start = Date.now();
  const results: Array<{ url: string; durationMs: number; success: boolean; chars: number }> = [];

  
   ( i = ; i < urls.; i += concurrency) {
     batch = urls.(i, i + concurrency);
     batchResults =  .(
      batch.( url => {
         t0 = .();
         {
           result =  firecrawl.(url, { : [] });
           { url, : .() - t0, : , : result.?. ||  };
        }  {
           { url, : .() - t0, : , :  };
        }
      })
    );
    results.(...batchResults);
  }

   totalMs = .() - start;
   succeeded = results.( r.).;

  .();
  .();
  .();
  .();
  .();
  .();

   results;
}