Automate SEO Audits with CrawlForge MCP

import { Client } from '@modelcontextprotocol/sdk/client/index.js';

const client = new Client({
  name: 'seo-auditor',
  version: '1.0.0',
});

interface SiteAuditConfig {
  url: string;
  maxPages: number;
  maxDepth: number;
}

async function crawlSite(config: SiteAuditConfig) {
  // First, get the sitemap for a complete URL list
  const siteMap = await client.callTool({
    name: 'map_site',
    arguments: {
      url: config.url,
      max_urls: config.maxPages,
      include_sitemap: true,
      include_metadata: true,
      group_by_path: true,
    },
  });

  const mapResult = JSON.parse(siteMap.content[0].text);

  // Then crawl to discover pages not in the sitemap
  const crawlResult = await client.callTool({
    name: 'crawl_deep',
    arguments: {
      url: config.url,
      max_pages: config.maxPages,
      max_depth: config.maxDepth,
      extract_content: true,
      respect_robots: true,
    },
  });

  const crawled = JSON.parse(crawlResult.content[0].text);

  // Combine and deduplicate
  const allUrls = new Set([
    ...mapResult.urls,
    ...crawled.pages.map((p: { url: string }) => p.url),
  ]);

  // Flag pages in crawl but missing from sitemap
  const missingFromSitemap = crawled.pages
    .filter((p: { url: string }) => !mapResult.urls.includes(p.url))
    .map((p: { url: string }) => p.url);

  return {
    totalPages: allUrls.size,
    urls: Array.from(allUrls),
    missingFromSitemap,
    crawlDepthDistribution: crawled.depthDistribution,
  };
}

interface MetaIssue {
  url: string;
  type: 'error' | 'warning';
  message: string;
}

async function auditMetadata(urls: string[]): Promise<MetaIssue[]> {
  const issues: MetaIssue[] = [];

  // Batch process metadata extraction
  const batchResult = await client.callTool({
    name: 'batch_scrape',
    arguments: {
      urls: urls.map(url => ({ url })),
      includeMetadata: true,
      maxConcurrency: 15,
    },
  });

  const results = JSON.parse(batchResult.content[0].text);

  for (const page of results.results) {
    const meta = page.metadata;
    const url = page.url;

    // Title checks
    if (!meta?.title) {
      issues.push({ url, type: 'error', message: 'Missing title tag' });
    } else if (meta.title.length > 60) {
      issues.push({ url, type: 'warning', message: `Title too long (${meta.title.length} chars, max 60)` });
    } else if (meta.title.length < 30) {
      issues.push({ url, type: 'warning', message: `Title too short (${meta.title.length} chars, min 30)` });
    }

    // Description checks
    if (!meta?.description) {
      issues.push({ url, type: 'error', message: 'Missing meta description' });
    } else if (meta.description.length > 155) {
      issues.push({ url, type: 'warning', message: `Description too long (${meta.description.length} chars)` });
    }

    // Open Graph checks
    if (!meta?.ogTitle) {
      issues.push({ url, type: 'warning', message: 'Missing og:title' });
    }
    if (!meta?.ogImage) {
      issues.push({ url, type: 'warning', message: 'Missing og:image' });
    }

    // Canonical check
    if (!meta?.canonical) {
      issues.push({ url, type: 'warning', message: 'Missing canonical URL' });
    }
  }

  return issues;
}

interface ContentScore {
  url: string;
  wordCount: number;
  readabilityScore: number;
  topTopics: string[];
  sentiment: string;
  issues: string[];
}

async function analyzePageContent(url: string): Promise<ContentScore> {
  // First extract clean content
  const content = await client.callTool({
    name: 'extract_content',
    arguments: { url },
  });

  const extracted = JSON.parse(content.content[0].text);

  // Then analyze it
  const analysis = await client.callTool({
    name: 'analyze_content',
    arguments: {
      text: extracted.content,
    },
  });

  const result = JSON.parse(analysis.content[0].text);
  const issues: string[] = [];

  // Flag thin content (under 300 words)
  if (result.wordCount < 300) {
    issues.push(`Thin content: only ${result.wordCount} words (recommend 800+)`);
  }

  // Flag low readability
  if (result.readabilityScore < 40) {
    issues.push('Content may be too complex for general audience');
  }

  return {
    url,
    wordCount: result.wordCount,
    readabilityScore: result.readabilityScore,
    topTopics: result.topics || [],
    sentiment: result.sentiment || 'neutral',
    issues,
  };
}

interface LinkIssue {
  sourceUrl: string;
  targetUrl: string;
  type: 'broken' | 'redirect' | 'external-nofollow';
  statusCode?: number;
}

async function auditLinks(urls: string[]): Promise<LinkIssue[]> {
  const issues: LinkIssue[] = [];

  for (const url of urls) {
    const linksResult = await client.callTool({
      name: 'extract_links',
      arguments: {
        url,
        base_url: urls[0], // Use site root as base
      },
    });

    const links = JSON.parse(linksResult.content[0].text);

    // Check each internal link
    for (const link of links.internal || []) {
      try {
        const check = await client.callTool({
          name: 'fetch_url',
          arguments: { url: link.href, timeout: 5000 },
        });

        const response = JSON.parse(check.content[0].text);

        if (response.statusCode >= 400) {
          issues.push({
            sourceUrl: url,
            targetUrl: link.href,
            type: 'broken',
            statusCode: response.statusCode,
          });
        } else if (response.statusCode >= 300) {
          issues.push({
            sourceUrl: url,
            targetUrl: link.href,
            type: 'redirect',
            statusCode: response.statusCode,
          });
        }
      } catch {
        issues.push({
          sourceUrl: url,
          targetUrl: link.href,
          type: 'broken',
        });
      }
    }
  }

  return issues;
}

interface SEOAuditReport {
  siteUrl: string;
  auditDate: string;
  totalPages: number;
  overallScore: number;
  criticalIssues: number;
  warnings: number;
  sections: {
    metadata: MetaIssue[];
    content: ContentScore[];
    links: LinkIssue[];
    sitemapGaps: string[];
  };
}

async function generateReport(siteUrl: string): Promise<SEOAuditReport> {
  // Run all audit steps
  const site = await crawlSite({
    url: siteUrl,
    maxPages: 200,
    maxDepth: 4,
  });

  const metaIssues = await auditMetadata(site.urls);
  const linkIssues = await auditLinks(site.urls.slice(0, 50)); // Top 50 pages

  // Analyze top pages by importance
  const topPages = site.urls.slice(0, 20);
  const contentScores: ContentScore[] = [];
  for (const url of topPages) {
    contentScores.push(await analyzePageContent(url));
  }

  const criticalCount = metaIssues.filter(i => i.type === 'error').length
    + linkIssues.filter(i => i.type === 'broken').length;

  const warningCount = metaIssues.filter(i => i.type === 'warning').length
    + contentScores.reduce((sum, s) => sum + s.issues.length, 0);

  // Score: start at 100, deduct for issues
  const score = Math.max(0, 100 - (criticalCount * 5) - (warningCount * 2));

  return {
    siteUrl,
    auditDate: new Date().toISOString(),
    totalPages: site.totalPages,
    overallScore: score,
    criticalIssues: criticalCount,
    warnings: warningCount,
    sections: {
      metadata: metaIssues,
      content: contentScores,
      links: linkIssues,
      sitemapGaps: site.missingFromSitemap,
    },
  };
}