How to Use CrawlForge with Vercel AI SDK

npm install ai @ai-sdk/anthropic zod dotenv

# .env.local
ANTHROPIC_API_KEY=sk-ant-xxxxx
CRAWLFORGE_API_KEY=cf_live_xxxxx

// lib/crawlforge.ts
const CRAWLFORGE_BASE = 'https://crawlforge.dev/api/v1/tools';

export async function callCrawlForge<T>(
  toolName: string,
  params: Record<string, unknown>
): Promise<T> {
  const response = await fetch(`${CRAWLFORGE_BASE}/${toolName}`, {
    method: 'POST',
    headers: {
      'Authorization': `Bearer ${process.env.CRAWLFORGE_API_KEY}`,
      'Content-Type': 'application/json',
    },
    body: JSON.stringify(params),
  });

  if (!response.ok) {
    throw new Error(`CrawlForge ${toolName} failed: ${response.status}`);
  }

  return response.json() as Promise<T>;
}

// app/api/chat/route.ts
import { generateText, tool } from 'ai';
import { anthropic } from '@ai-sdk/anthropic';
import { z } from 'zod';
import { callCrawlForge } from '@/lib/crawlforge';

const result = await generateText({
  model: anthropic('claude-sonnet-4-20250514'),
  tools: {
    // Fetch and extract clean content from any URL (2 credits)
    extractContent: tool({
      description: 'Extract the main content from a web page as clean text. Use this when you need to read an article, documentation page, or any web content.',
      parameters: z.object({
        url: z.string().url().describe('The URL to extract content from'),
      }),
      execute: async ({ url }) => {
        return callCrawlForge('extract_content', { url });
      },
    }),

    // Search the web (5 credits)
    searchWeb: tool({
      description: 'Search the web using Google. Returns titles, URLs, and snippets for the top results.',
      parameters: z.object({
        query: z.string().describe('The search query'),
        limit: z.number().optional().default(5).describe('Number of results'),
      }),
      execute: async ({ query, limit }) => {
        return callCrawlForge('search_web', { query, limit });
      },
    }),

    // Extract structured data with CSS selectors (2 credits)
    scrapeStructured: tool({
      description: 'Extract specific data from a web page using CSS selectors. Use when you need structured data like prices, names, or lists.',
      parameters: z.object({
        url: z.string().url().describe('The URL to scrape'),
        selectors: z.record(z.string()).describe('CSS selectors mapping field names to selectors'),
      }),
      execute: async ({ url, selectors }) => {
        return callCrawlForge('scrape_structured', { url, selectors });
      },
    }),
  },
  prompt: 'Find the current pricing for Vercel Pro plan and compare it to Netlify Pro.',
});

console.log(result.text);

// app/api/chat/route.ts
import { streamText, tool } from 'ai';
import { anthropic } from '@ai-sdk/anthropic';
import { z } from 'zod';
import { callCrawlForge } from '@/lib/crawlforge';

export async function POST(req: Request) {
  const { messages } = await req.json();

  const result = streamText({
    model: anthropic('claude-sonnet-4-20250514'),
    system: `You are a research assistant with access to live web data.
When users ask about current information, use your tools to fetch it.
Always cite the URLs you retrieved data from.`,
    messages,
    tools: {
      extractContent: tool({
        description: 'Extract main content from a URL as clean text',
        parameters: z.object({
          url: z.string().url(),
        }),
        execute: async ({ url }) =>
          callCrawlForge('extract_content', { url }),
      }),
      searchWeb: tool({
        description: 'Search the web and return top results',
        parameters: z.object({
          query: z.string(),
          limit: z.number().optional().default(5),
        }),
        execute: async ({ query, limit }) =>
          callCrawlForge('search_web', { query, limit }),
      }),
    },
    maxSteps: 5, // Allow up to 5 tool calls per response
  });

  return result.toDataStreamResponse();
}

// app/page.tsx
'use client';
import { useChat } from 'ai/react';

export default function Chat() {
  const { messages, input, handleInputChange, handleSubmit, isLoading } = useChat();

  return (
    <div className="max-w-2xl mx-auto p-4">
      {messages.map((m) => (
        <div key={m.id} className="mb-4">
          <strong>{m.role === 'user' ? 'You' : 'AI'}:</strong>
          <p>{m.content}</p>
        </div>
      ))}
      <form onSubmit={handleSubmit}>
        <input
          value={input}
          onChange={handleInputChange}
          placeholder="Ask anything about the web..."
          className="w-full p-2 border rounded"
          disabled={isLoading}
        />
      </form>
    </div>
  );
}

import { generateText, tool } from 'ai';
import { anthropic } from '@ai-sdk/anthropic';
import { z } from 'zod';
import { callCrawlForge } from '@/lib/crawlforge';

const research = await generateText({
  model: anthropic('claude-sonnet-4-20250514'),
  tools: {
    searchWeb: tool({
      description: 'Search Google for information',
      parameters: z.object({ query: z.string(), limit: z.number().default(5) }),
      execute: async ({ query, limit }) =>
        callCrawlForge('search_web', { query, limit }),
    }),
    extractContent: tool({
      description: 'Read the full content of a web page',
      parameters: z.object({ url: z.string().url() }),
      execute: async ({ url }) =>
        callCrawlForge('extract_content', { url }),
    }),
    analyzeContent: tool({
      description: 'Analyze text for topics, sentiment, and key entities',
      parameters: z.object({ text: z.string() }),
      execute: async ({ text }) =>
        callCrawlForge('analyze_content', { text }),
    }),
  },
  maxSteps: 10,
  prompt: `Research the current state of MCP server adoption in 2026.
    Search for recent articles, read the top 3, and analyze the key trends.
    Provide a summary with citations.`,
});

console.log(research.text);
// Returns a cited research summary using live web data