Batch Processing Guide

curl -X POST https://crawlforge.dev/api/v1/tools/batch_scrape \
  -H "X-API-Key: cf_test_YOUR_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "urls": [
      { "url": "https://example.com/page1" },
      { "url": "https://example.com/page2" },
      { "url": "https://example.com/page3" }
    ],
    "batch_config": { "concurrency": 5 }
  }'

// Chunk array into batches of 50
function chunkArray<T>(array: T[], size: number): T[][] {
  const chunks: T[][] = [];
  for (let i = 0; i < array.length; i += size) {
    chunks.push(array.slice(i, i + size));
  }
  return chunks;
}

// Process all URLs in batches
async function processBatches(urls: string[]) {
  const batches = chunkArray(urls, 50); // Max 50 URLs per batch
  const allResults: any[] = [];

  for (let i = 0; i < batches.length; i++) {
    const batch = batches[i];
    console.log(`Processing batch ${i + 1}/${batches.length}...`);

    const response = await fetch('https://crawlforge.dev/api/v1/tools/batch_scrape', {
      method: 'POST',
      headers: {
        'X-API-Key': process.env.CRAWLFORGE_API_KEY!,
        'Content-Type': 'application/json',
      },
      body: JSON.stringify({
        urls: batch.map(url => ({ url })),
        batch_config: { concurrency: 8 },
      }),
    });

    const data = await response.json();
    allResults.push(...data.data.results);

    // Wait between batches to respect rate limits
    if (i < batches.length - 1) {
      await new Promise(resolve => setTimeout(resolve, 2000));
    }
  }

  return allResults;
}

// Usage
const urls = [...]; // 500 URLs
const results = await processBatches(urls);
console.log(`Scraped ${results.length} total pages`);

interface BatchResult {
  successful: any[];
  failed: { url: string; error: string }[];
}

async function batchScrapeWithRetry(
  urls: string[],
  maxRetries = 3
): Promise<BatchResult> {
  const successful: any[] = [];
  const failed: { url: string; error: string }[] = [];
  let remainingUrls = [...urls];
  let retries = 0;

  while (remainingUrls.length > 0 && retries <= maxRetries) {
    try {
      const response = await fetch('https://crawlforge.dev/api/v1/tools/batch_scrape', {
        method: 'POST',
        headers: {
          'X-API-Key': process.env.CRAWLFORGE_API_KEY!,
          'Content-Type': 'application/json',
        },
        body: JSON.stringify({
          urls: remainingUrls.map(url => ({ url })),
          batch_config: { concurrency: 5 },
        }),
      });

      const data = await response.json();

      // Each result carries its own status: 'success' | 'failed' | 'skipped'.
      // 'skipped' means the request's time budget ran out before that URL was
      // started — those are not charged, so they are always worth retrying.
      const batchSuccessful = data.data.results.filter((r: any) => r.status === 'success');
      const batchUnfinished = data.data.results.filter((r: any) => r.status !== 'success');

      successful.push(...batchSuccessful);

      // Only retry the URLs that did not come back successful
      remainingUrls = batchUnfinished.map((r: any) => r.url);

      if (remainingUrls.length > 0) {
        console.log(`Retrying ${remainingUrls.length} failed URLs...`);
        retries++;
        await new Promise(resolve => setTimeout(resolve, 2000 * retries));
      }

    } catch (error) {
      console.error('Batch request failed:', error);
      retries++;

      if (retries > maxRetries) {
        // Mark all remaining URLs as failed
        failed.push(...remainingUrls.map(url => ({
          url,
          error: String(error)
        })));
        break;
      }

      await new Promise(resolve => setTimeout(resolve, 2000 * retries));
    }
  }

  return { successful, failed };
}

// Usage
const { successful, failed } = await batchScrapeWithRetry(urls);
console.log(`Success: ${successful.length}, Failed: ${failed.length}`);

// Save failed URLs for manual review
if (failed.length > 0) {
  fs.writeFileSync('failed-urls.json', JSON.stringify(failed, null, 2));
}