How to Use CrawlForge with Anthropic Claude API

npm install @anthropic-ai/sdk dotenv

# .env
ANTHROPIC_API_KEY=sk-ant-xxxxx
CRAWLFORGE_API_KEY=cf_live_xxxxx

// lib/tool-definitions.ts
import Anthropic from '@anthropic-ai/sdk';

export const crawlforgeTools: Anthropic.Tool[] = [
  {
    name: 'search_web',
    description: 'Search Google and return top results. Use when you need to find web pages about a topic. Returns titles, URLs, and snippets. Costs 5 credits.',
    input_schema: {
      type: 'object' as const,
      properties: {
        query: {
          type: 'string',
          description: 'The search query',
        },
        limit: {
          type: 'number',
          description: 'Maximum number of results (default: 5)',
        },
      },
      required: ['query'],
    },
  },
  {
    name: 'extract_content',
    description: 'Extract the main readable content from a web page. Returns clean text without navigation, ads, or boilerplate. Use when you need to read a specific URL. Costs 2 credits.',
    input_schema: {
      type: 'object' as const,
      properties: {
        url: {
          type: 'string',
          description: 'The full URL to extract content from',
        },
      },
      required: ['url'],
    },
  },
  {
    name: 'scrape_structured',
    description: 'Extract specific data from a page using CSS selectors. Use when you need structured data like prices, names, or lists. Costs 2 credits.',
    input_schema: {
      type: 'object' as const,
      properties: {
        url: {
          type: 'string',
          description: 'The URL to scrape',
        },
        selectors: {
          type: 'object',
          description: 'Map of field names to CSS selectors',
          additionalProperties: { type: 'string' },
        },
      },
      required: ['url', 'selectors'],
    },
  },
  {
    name: 'fetch_url',
    description: 'Fetch raw HTML from a URL. Cheapest option at 1 credit. Use for APIs or when you need the raw HTML.',
    input_schema: {
      type: 'object' as const,
      properties: {
        url: {
          type: 'string',
          description: 'The URL to fetch',
        },
      },
      required: ['url'],
    },
  },
  {
    name: 'extract_metadata',
    description: 'Get page metadata including title, description, Open Graph tags, and other meta information. Costs 1 credit.',
    input_schema: {
      type: 'object' as const,
      properties: {
        url: {
          type: 'string',
          description: 'The URL to extract metadata from',
        },
      },
      required: ['url'],
    },
  },
];

// lib/agent.ts
import Anthropic from '@anthropic-ai/sdk';
import { crawlforgeTools } from './tool-definitions';

const client = new Anthropic();
const CRAWLFORGE_BASE = 'https://crawlforge.dev/api/v1/tools';

// Execute a CrawlForge tool
async function executeTool(
  name: string,
  input: Record<string, unknown>
): Promise<string> {
  const response = await fetch(`${CRAWLFORGE_BASE}/${name}`, {
    method: 'POST',
    headers: {
      'Authorization': `Bearer ${process.env.CRAWLFORGE_API_KEY}`,
      'Content-Type': 'application/json',
    },
    body: JSON.stringify(input),
  });

  if (!response.ok) {
    return JSON.stringify({ error: `HTTP ${response.status}: ${response.statusText}` });
  }

  const data = await response.json();
  return JSON.stringify(data);
}

// Run Claude with CrawlForge tools
export async function askClaude(prompt: string): Promise<string> {
  const messages: Anthropic.MessageParam[] = [
    { role: 'user', content: prompt },
  ];

  // Tool use loop: keep going until Claude gives a final text response
  while (true) {
    const response = await client.messages.create({
      model: 'claude-sonnet-4-20250514',
      max_tokens: 4096,
      tools: crawlforgeTools,
      messages,
    });

    // Check if Claude wants to use a tool
    if (response.stop_reason === 'tool_use') {
      // Add Claude's response to message history
      messages.push({ role: 'assistant', content: response.content });

      // Execute each tool call and collect results
      const toolResults: Anthropic.ToolResultBlockParam[] = [];

      for (const block of response.content) {
        if (block.type === 'tool_use') {
          const result = await executeTool(
            block.name,
            block.input as Record<string, unknown>
          );
          toolResults.push({
            type: 'tool_result',
            tool_use_id: block.id,
            content: result,
          });
        }
      }

      // Return tool results to Claude
      messages.push({ role: 'user', content: toolResults });
    } else {
      // Claude gave a final text response
      const textBlock = response.content.find(b => b.type === 'text');
      return textBlock ? textBlock.text : '';
    }
  }
}

// Usage
const answer = await askClaude(
  'What are the top 3 trending repositories on GitHub right now?'
);
console.log(answer);

// research-assistant.ts
import { askClaude } from './lib/agent';

async function researchTopic(topic: string, depth: 'quick' | 'thorough' = 'quick') {
  const systemPrompt = depth === 'thorough'
    ? `Research this topic thoroughly. Search for at least 3 different sources,
       read each one, and provide a comprehensive summary with citations.
       Topic: ${topic}`
    : `Quickly answer this question using web search if needed.
       Be concise and cite your source. Topic: ${topic}`;

  console.log(`Researching: ${topic} (mode: ${depth})`);
  const result = await askClaude(systemPrompt);
  console.log(result);
  return result;
}

// Quick research: ~7 credits (search + 1 extract)
await researchTopic('What is the current version of Next.js?', 'quick');

// Thorough research: ~11-15 credits (search + 3 extracts)
await researchTopic(
  'Compare the performance of Bun vs Node.js for HTTP servers in 2026',
  'thorough'
);

// lib/streaming-agent.ts
import Anthropic from '@anthropic-ai/sdk';
import { crawlforgeTools } from './tool-definitions';

const client = new Anthropic();
const CRAWLFORGE_BASE = 'https://crawlforge.dev/api/v1/tools';

async function executeTool(
  name: string,
  input: Record<string, unknown>
): Promise<string> {
  const response = await fetch(`${CRAWLFORGE_BASE}/${name}`, {
    method: 'POST',
    headers: {
      'Authorization': `Bearer ${process.env.CRAWLFORGE_API_KEY}`,
      'Content-Type': 'application/json',
    },
    body: JSON.stringify(input),
  });
  return JSON.stringify(await response.json());
}

export async function streamWithTools(prompt: string) {
  const messages: Anthropic.MessageParam[] = [
    { role: 'user', content: prompt },
  ];

  while (true) {
    const stream = client.messages.stream({
      model: 'claude-sonnet-4-20250514',
      max_tokens: 4096,
      tools: crawlforgeTools,
      messages,
    });

    // Collect streamed text
    let hasToolUse = false;

    stream.on('text', (text) => {
      process.stdout.write(text); // Stream to terminal in real time
    });

    const response = await stream.finalMessage();

    if (response.stop_reason === 'tool_use') {
      hasToolUse = true;
      messages.push({ role: 'assistant', content: response.content });

      const toolResults: Anthropic.ToolResultBlockParam[] = [];
      for (const block of response.content) {
        if (block.type === 'tool_use') {
          console.log(`\n[Calling ${block.name}...]\n`);
          const result = await executeTool(
            block.name,
            block.input as Record<string, unknown>
          );
          toolResults.push({
            type: 'tool_result',
            tool_use_id: block.id,
            content: result,
          });
        }
      }

      messages.push({ role: 'user', content: toolResults });
    }

    if (!hasToolUse) break;
  }
}

// Usage: streams text to stdout as Claude generates it
await streamWithTools('What is CrawlForge MCP and how does it compare to Firecrawl?');