How to Use CrawlForge with OpenAI Agents SDK

pip install openai-agents
# or for the TypeScript/Node.js SDK:
npm install @openai/agents-sdk dotenv

# .env
OPENAI_API_KEY=sk-xxxxx
CRAWLFORGE_API_KEY=cf_live_xxxxx

// lib/crawlforge-tools.ts
import { tool } from '@openai/agents-sdk';
import { z } from 'zod';

const CRAWLFORGE_BASE = 'https://crawlforge.dev/api/v1/tools';

async function callCrawlForge(toolName: string, params: Record<string, unknown>) {
  const response = await fetch(`${CRAWLFORGE_BASE}/${toolName}`, {
    method: 'POST',
    headers: {
      'Authorization': `Bearer ${process.env.CRAWLFORGE_API_KEY}`,
      'Content-Type': 'application/json',
    },
    body: JSON.stringify(params),
  });

  if (!response.ok) {
    throw new Error(`CrawlForge error: ${response.status} ${response.statusText}`);
  }

  return response.json();
}

// Search the web (5 credits)
export const searchWebTool = tool({
  name: 'search_web',
  description: 'Search Google and return top results with titles, URLs, and snippets.',
  parameters: z.object({
    query: z.string().describe('Search query'),
    limit: z.number().default(5).describe('Max results to return'),
  }),
  execute: async ({ query, limit }) => {
    return callCrawlForge('search_web', { query, limit });
  },
});

// Extract page content (2 credits)
export const extractContentTool = tool({
  name: 'extract_content',
  description: 'Extract the main readable content from a web page URL.',
  parameters: z.object({
    url: z.string().url().describe('Full URL to extract content from'),
  }),
  execute: async ({ url }) => {
    return callCrawlForge('extract_content', { url });
  },
});

// Structured scraping (2 credits)
export const scrapeStructuredTool = tool({
  name: 'scrape_structured',
  description: 'Extract structured data from a web page using CSS selectors.',
  parameters: z.object({
    url: z.string().url().describe('URL to scrape'),
    selectors: z.record(z.string()).describe('Map of field names to CSS selectors'),
  }),
  execute: async ({ url, selectors }) => {
    return callCrawlForge('scrape_structured', { url, selectors });
  },
});

// Fetch raw URL (1 credit)
export const fetchUrlTool = tool({
  name: 'fetch_url',
  description: 'Fetch raw HTML content from a URL. Cheapest option for simple retrieval.',
  parameters: z.object({
    url: z.string().url().describe('URL to fetch'),
  }),
  execute: async ({ url }) => {
    return callCrawlForge('fetch_url', { url });
  },
});

// agents/researcher.ts
import { Agent, run } from '@openai/agents-sdk';
import {
  searchWebTool,
  extractContentTool,
  scrapeStructuredTool,
} from '../lib/crawlforge-tools';

const researchAgent = new Agent({
  name: 'Web Researcher',
  model: 'gpt-4o',
  instructions: `You are a thorough web researcher. When asked about a topic:
1. Search the web for relevant, recent sources
2. Read the top 2-3 results to gather comprehensive information
3. Synthesize findings into a clear, cited summary
4. Always mention the URLs you sourced data from

Use search_web to find sources, then extract_content to read them.
Prefer extract_content over fetch_url when you need readable text.`,
  tools: [searchWebTool, extractContentTool, scrapeStructuredTool],
});

// Run the agent
async function research(topic: string) {
  const result = await run(researchAgent, {
    messages: [{ role: 'user', content: topic }],
  });

  console.log(result.finalOutput);
  return result;
}

// Example usage
await research('What are the latest trends in web scraping regulation in 2026?');

// agents/extractor.ts
import { Agent, run } from '@openai/agents-sdk';
import { scrapeStructuredTool, fetchUrlTool } from '../lib/crawlforge-tools';

const extractorAgent = new Agent({
  name: 'Data Extractor',
  model: 'gpt-4o',
  instructions: `You are a data extraction specialist. When given a URL and a data request:
1. Determine the best CSS selectors to extract the requested data
2. Use scrape_structured to pull the data
3. Return the results in clean JSON format

For simple JSON APIs, use fetch_url instead (it costs 1 credit vs 2).`,
  tools: [scrapeStructuredTool, fetchUrlTool],
});

async function extractData(url: string, description: string) {
  const result = await run(extractorAgent, {
    messages: [{
      role: 'user',
      content: `Extract from ${url}: ${description}`,
    }],
  });

  return result.finalOutput;
}

// Extract pricing data
await extractData(
  'https://stripe.com/pricing',
  'All plan names, prices, and key features'
);

// agents/pipeline.ts
import { Agent, run } from '@openai/agents-sdk';
import {
  searchWebTool,
  extractContentTool,
  scrapeStructuredTool,
} from '../lib/crawlforge-tools';

const collectorAgent = new Agent({
  name: 'Data Collector',
  model: 'gpt-4o',
  instructions: `You collect raw data from the web. Search for sources,
extract their content, and pass the raw data to the analyst.
Focus on gathering data, not analyzing it.`,
  tools: [searchWebTool, extractContentTool, scrapeStructuredTool],
  handoff_description: 'Collects raw web data for analysis',
});

const analystAgent = new Agent({
  name: 'Data Analyst',
  model: 'gpt-4o',
  instructions: `You analyze data collected by the Data Collector.
Identify patterns, compare data points, and produce actionable insights.
Always structure your output with clear sections and data tables.`,
  tools: [], // No web tools needed -- works with collected data
  handoffs: [collectorAgent], // Can request more data if needed
});

const orchestrator = new Agent({
  name: 'Research Orchestrator',
  model: 'gpt-4o',
  instructions: `You manage research projects. Delegate data collection to
the Data Collector and analysis to the Data Analyst. Ensure the final
output answers the user's question completely.`,
  handoffs: [collectorAgent, analystAgent],
});

// Run the multi-agent pipeline
const result = await run(orchestrator, {
  messages: [{
    role: 'user',
    content: 'Compare the pricing and features of the top 3 web scraping APIs in 2026',
  }],
});

console.log(result.finalOutput);