How to Use CrawlForge with LangGraph Agents

mkdir langgraph-scraper && cd langgraph-scraper
npm init -y
npm install @langchain/langgraph @langchain/anthropic @langchain/core zod dotenv
npm install -D typescript @types/node tsx

{
  "compilerOptions": {
    "target": "ES2022",
    "module": "ESNext",
    "moduleResolution": "bundler",
    "strict": true,
    "esModuleInterop": true,
    "outDir": "dist"
  },
  "include": ["src/**/*"]
}

CRAWLFORGE_API_KEY=cf_live_your_key_here
ANTHROPIC_API_KEY=sk-ant-your_key_here

// src/tools.ts
import { tool } from '@langchain/core/tools';
import { z } from 'zod';

const CRAWLFORGE_API = 'https://crawlforge.dev/api/v1/tools';

async function callCrawlForge(
  endpoint: string,
  params: Record<string, unknown>
): Promise<string> {
  const response = await fetch(`${CRAWLFORGE_API}/${endpoint}`, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      'Authorization': `Bearer ${process.env.CRAWLFORGE_API_KEY}`,
    },
    body: JSON.stringify(params),
  });

  if (!response.ok) {
    return JSON.stringify({ error: `HTTP ${response.status}`, endpoint });
  }

  const data = await response.json();
  return JSON.stringify(data);
}

export const searchWebTool = tool(
  async ({ query, limit }) => {
    return callCrawlForge('search_web', { query, limit });
  },
  {
    name: 'search_web',
    description: 'Search the web for information. Costs 5 credits. Use when you need to find URLs for a topic.',
    schema: z.object({
      query: z.string().describe('Search query'),
      limit: z.number().default(5).describe('Max results'),
    }),
  }
);

export const extractContentTool = tool(
  async ({ url }) => {
    return callCrawlForge('extract_content', { url });
  },
  {
    name: 'extract_content',
    description: 'Extract clean readable content from a URL. Costs 2 credits.',
    schema: z.object({
      url: z.string().describe('URL to extract content from'),
    }),
  }
);

export const scrapeStructuredTool = tool(
  async ({ url, selectors }) => {
    return callCrawlForge('scrape_structured', { url, selectors });
  },
  {
    name: 'scrape_structured',
    description: 'Extract structured data using CSS selectors. Costs 2 credits.',
    schema: z.object({
      url: z.string().describe('URL to scrape'),
      selectors: z.record(z.string()).describe('CSS selectors map'),
    }),
  }
);

export const fetchUrlTool = tool(
  async ({ url }) => {
    return callCrawlForge('fetch_url', { url });
  },
  {
    name: 'fetch_url',
    description: 'Fetch raw HTML from a URL. Cheapest option at 1 credit.',
    schema: z.object({
      url: z.string().describe('URL to fetch'),
    }),
  }
);

export const allTools = [
  searchWebTool,
  extractContentTool,
  scrapeStructuredTool,
  fetchUrlTool,
];

// src/state.ts
import { BaseMessage } from '@langchain/core/messages';
import { Annotation } from '@langchain/langgraph';

// Define the graph state
export const AgentState = Annotation.Root({
  // Conversation messages (LLM context)
  messages: Annotation<BaseMessage[]>({
    reducer: (prev, next) => [...prev, ...next],
    default: () => [],
  }),

  // URLs discovered during research
  discoveredUrls: Annotation<string[]>({
    reducer: (prev, next) => [...new Set([...prev, ...next])],
    default: () => [],
  }),

  // Content extracted from URLs
  extractedContent: Annotation<Record<string, string>>({
    reducer: (prev, next) => ({ ...prev, ...next }),
    default: () => ({}),
  }),

  // Total credits consumed
  creditsUsed: Annotation<number>({
    reducer: (prev, next) => prev + next,
    default: () => 0,
  }),

  // Current phase of the scraping pipeline
  phase: Annotation<'search' | 'extract' | 'analyze' | 'complete'>({
    reducer: (_prev, next) => next,
    default: () => 'search' as const,
  }),
});

// src/nodes.ts
import { ChatAnthropic } from '@langchain/anthropic';
import { HumanMessage, SystemMessage } from '@langchain/core/messages';
import { ToolNode } from '@langchain/langgraph/prebuilt';
import { AgentState } from './state';
import { allTools } from './tools';

const model = new ChatAnthropic({
  model: 'claude-sonnet-4-20250514',
  temperature: 0,
}).bindTools(allTools);

// Node: LLM decides which tool to call next
export async function agentNode(
  state: typeof AgentState.State
) {
  const systemPrompt = new SystemMessage(
    `You are a web research agent. Your goal is to find and extract information efficiently.
    Always prefer cheaper tools: fetch_url (1cr) > extract_content (2cr) > search_web (5cr).
    Track credits used. Stop when you have enough information or reach 20 credits.`
  );

  const response = await model.invoke([systemPrompt, ...state.messages]);

  return { messages: [response] };
}

// Node: Execute tool calls
export const toolNode = new ToolNode(allTools);

// Node: Analyze extracted content and decide next step
export async function analyzeNode(
  state: typeof AgentState.State
) {
  const extractedCount = Object.keys(state.extractedContent).length;

  if (extractedCount >= 3 || state.creditsUsed >= 20) {
    return { phase: 'complete' as const };
  }

  return { phase: 'extract' as const };
}

// src/graph.ts
import { StateGraph, END } from '@langchain/langgraph';
import { AgentState } from './state';
import { agentNode, toolNode, analyzeNode } from './nodes';
import { AIMessage } from '@langchain/core/messages';

// Determine if the agent wants to use a tool or is finished
function shouldContinue(state: typeof AgentState.State) {
  const lastMessage = state.messages[state.messages.length - 1];

  // If the LLM returned tool calls, route to tool execution
  if (
    lastMessage instanceof AIMessage &&
    lastMessage.tool_calls &&
    lastMessage.tool_calls.length > 0
  ) {
    return 'tools';
  }

  // Otherwise, analyze what we have
  return 'analyze';
}

// Determine if we should continue scraping or wrap up
function shouldFinish(state: typeof AgentState.State) {
  if (state.phase === 'complete') {
    return 'end';
  }
  return 'agent';
}

// Build the graph
const workflow = new StateGraph(AgentState)
  // Add nodes
  .addNode('agent', agentNode)
  .addNode('tools', toolNode)
  .addNode('analyze', analyzeNode)

  // Set entry point
  .addEdge('__start__', 'agent')

  // Agent -> tools (if tool call) or analyze (if no tool call)
  .addConditionalEdges('agent', shouldContinue, {
    tools: 'tools',
    analyze: 'analyze',
  })

  // Tools -> agent (return results to LLM)
  .addEdge('tools', 'agent')

  // Analyze -> agent (continue) or end (done)
  .addConditionalEdges('analyze', shouldFinish, {
    agent: 'agent',
    end: END,
  });

export const app = workflow.compile();

// src/index.ts
import 'dotenv/config';
import { HumanMessage } from '@langchain/core/messages';
import { app } from './graph';

async function main() {
  const result = await app.invoke({
    messages: [
      new HumanMessage(
        'Research the top 3 MCP server implementations for web scraping. ' +
        'Find their websites, extract their key features, and compare pricing.'
      ),
    ],
  });

  // Print final state
  console.log('--- Research Complete ---');
  console.log('Credits used:', result.creditsUsed);
  console.log('URLs discovered:', result.discoveredUrls.length);
  console.log('Pages extracted:', Object.keys(result.extractedContent).length);
  console.log('\nFinal response:');
  console.log(result.messages[result.messages.length - 1].content);
}

main().catch(console.error);

npx tsx src/index.ts