How to Use CrawlForge with Mastra AI Agents

npm create mastra@latest my-web-agent
cd my-web-agent

# Install additional dependencies
npm install dotenv node-fetch

CRAWLFORGE_API_KEY=cf_live_your_key_here
ANTHROPIC_API_KEY=sk-ant-your_key_here

// src/tools/crawlforge.ts
import { createTool } from '@mastra/core';
import { z } from 'zod';

const CRAWLFORGE_BASE = 'https://crawlforge.dev/api/v1/tools';

async function callCrawlForge(tool: string, params: Record<string, unknown>) {
  const response = await fetch(`${CRAWLFORGE_BASE}/${tool}`, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      'Authorization': `Bearer ${process.env.CRAWLFORGE_API_KEY}`,
    },
    body: JSON.stringify(params),
  });

  if (!response.ok) {
    throw new Error(`CrawlForge ${tool} failed: ${response.status}`);
  }

  return response.json();
}

// Extract clean content from a URL (2 credits)
export const extractContent = createTool({
  id: 'crawlforge-extract-content',
  description: 'Extract clean, readable content from any web page. Costs 2 credits.',
  inputSchema: z.object({
    url: z.string().url().describe('The URL to extract content from'),
  }),
  execute: async ({ context }) => {
    return callCrawlForge('extract_content', { url: context.url });
  },
});

// Search the web (5 credits)
export const searchWeb = createTool({
  id: 'crawlforge-search-web',
  description: 'Search the web using Google. Returns titles, URLs, and snippets. Costs 5 credits.',
  inputSchema: z.object({
    query: z.string().describe('Search query string'),
    limit: z.number().optional().default(10).describe('Max results to return'),
  }),
  execute: async ({ context }) => {
    return callCrawlForge('search_web', {
      query: context.query,
      limit: context.limit,
    });
  },
});

// Fetch raw HTML (1 credit)
export const fetchUrl = createTool({
  id: 'crawlforge-fetch-url',
  description: 'Fetch raw HTML content from a URL. Cheapest option at 1 credit.',
  inputSchema: z.object({
    url: z.string().url().describe('The URL to fetch'),
  }),
  execute: async ({ context }) => {
    return callCrawlForge('fetch_url', { url: context.url });
  },
});

// Extract structured data with CSS selectors (2 credits)
export const scrapeStructured = createTool({
  id: 'crawlforge-scrape-structured',
  description: 'Extract data using CSS selectors. Costs 2 credits.',
  inputSchema: z.object({
    url: z.string().url().describe('The URL to scrape'),
    selectors: z.record(z.string()).describe('CSS selectors mapping field names to selectors'),
  }),
  execute: async ({ context }) => {
    return callCrawlForge('scrape_structured', {
      url: context.url,
      selectors: context.selectors,
    });
  },
});

// src/agents/researcher.ts
import { Agent } from '@mastra/core';
import { searchWeb, extractContent, fetchUrl } from '../tools/crawlforge';

export const researcherAgent = new Agent({
  name: 'Web Researcher',
  instructions: `You are a web research agent. When asked to research a topic:
    1. Use search_web to find relevant pages (5 credits)
    2. Use extract_content on the top 2-3 results (2 credits each)
    3. Synthesize findings into a structured summary

    Always report the total credits used.
    Prefer fetch_url (1 credit) over extract_content (2 credits) when you only need raw HTML.
    Never use search_web if the user provides a specific URL.`,
  model: {
    provider: 'ANTHROPIC',
    name: 'claude-sonnet-4-20250514',
  },
  tools: {
    searchWeb,
    extractContent,
    fetchUrl,
  },
});

// src/index.ts
import { researcherAgent } from './agents/researcher';

async function main() {
  const result = await researcherAgent.generate(
    'Research the current state of MCP protocol adoption. ' +
    'Which companies are using it and for what use cases?'
  );

  console.log(result.text);
  // Output: Structured research summary with sources
  // Credits used: ~11 (1 search + 3 extractions)
}

main();

// src/workflows/pricing-monitor.ts
import { Workflow, Step } from '@mastra/core';
import { searchWeb, scrapeStructured } from '../tools/crawlforge';

const findCompetitors = new Step({
  id: 'find-competitors',
  execute: async ({ context }) => {
    // Search for competitors (5 credits)
    const results = await searchWeb.execute({
      context: { query: `${context.product} pricing plans`, limit: 5 },
    });
    return { urls: results.results.map((r: { link: string }) => r.link) };
  },
});

const extractPricing = new Step({
  id: 'extract-pricing',
  execute: async ({ context }) => {
    const pricingData = [];

    // Extract pricing from each competitor (2 credits each)
    for (const url of context.urls.slice(0, 3)) {
      try {
        const data = await scrapeStructured.execute({
          context: {
            url,
            selectors: {
              planNames: '.plan-name, .pricing-tier h3',
              prices: '.price, .plan-price',
              features: '.feature-list li, .plan-features li',
            },
          },
        });
        pricingData.push({ url, ...data });
      } catch (error) {
        pricingData.push({ url, error: 'Extraction failed' });
      }
    }

    return { pricingData };
    // Total: 5 + (3 * 2) = 11 credits
  },
});

export const pricingMonitorWorkflow = new Workflow({
  name: 'Pricing Monitor',
  steps: [findCompetitors, extractPricing],
});

// src/tools/resilient-crawlforge.ts
import { createTool } from '@mastra/core';
import { z } from 'zod';

const CRAWLFORGE_BASE = 'https://crawlforge.dev/api/v1/tools';

async function callWithRetry(
  tool: string,
  params: Record<string, unknown>,
  maxRetries = 2
): Promise<unknown> {
  let lastError: Error | null = null;

  for (let attempt = 0; attempt <= maxRetries; attempt++) {
    try {
      const response = await fetch(`${CRAWLFORGE_BASE}/${tool}`, {
        method: 'POST',
        headers: {
          'Content-Type': 'application/json',
          'Authorization': `Bearer ${process.env.CRAWLFORGE_API_KEY}`,
        },
        body: JSON.stringify(params),
      });

      if (response.status === 429) {
        // Rate limited -- wait and retry
        const retryAfter = parseInt(response.headers.get('Retry-After') || '2');
        await new Promise(resolve => setTimeout(resolve, retryAfter * 1000));
        continue;
      }

      if (!response.ok) {
        throw new Error(`HTTP ${response.status}: ${await response.text()}`);
      }

      return response.json();
    } catch (error) {
      lastError = error as Error;
      if (attempt < maxRetries) {
        await new Promise(resolve => setTimeout(resolve, 1000 * (attempt + 1)));
      }
    }
  }

  throw lastError;
}

export const resilientExtractContent = createTool({
  id: 'crawlforge-extract-content-resilient',
  description: 'Extract content with automatic retry on failure. 2 credits per successful call.',
  inputSchema: z.object({
    url: z.string().url(),
  }),
  execute: async ({ context }) => {
    return callWithRetry('extract_content', { url: context.url });
  },
});