Content Migration Automation with CrawlForge

import { Client } from '@modelcontextprotocol/sdk/client/index.js';

const client = new Client({
  name: 'content-migrator',
  version: '1.0.0',
});

interface SiteInventory {
  pages: Array<{
    url: string;
    path: string;
    depth: number;
    type: 'page' | 'post' | 'category' | 'media' | 'other';
  }>;
  totalPages: number;
  pathGroups: Record<string, number>;
}

async function inventorySite(siteUrl: string): Promise<SiteInventory> {
  const mapResult = await client.callTool({
    name: 'map_site',
    arguments: {
      url: siteUrl,
      max_urls: 2000,
      include_sitemap: true,
      group_by_path: true,
      include_metadata: true,
    },
  });

  const result = JSON.parse(mapResult.content[0].text);

  // Classify pages by URL pattern
  const pages = result.urls.map((url: string) => {
    const path = new URL(url).pathname;
    let type: 'page' | 'post' | 'category' | 'media' | 'other' = 'other';

    if (path.match(/\/\d{4}\/\d{2}\//)) type = 'post';       // WordPress date URLs
    else if (path.match(/\/blog\//)) type = 'post';
    else if (path.match(/\/category\//)) type = 'category';
    else if (path.match(/\.(pdf|doc|jpg|png)/)) type = 'media';
    else type = 'page';

    return { url, path, depth: path.split('/').length - 1, type };
  });

  return {
    pages,
    totalPages: pages.length,
    pathGroups: result.groupedByPath || {},
  };
}

interface MigratedPage {
  sourceUrl: string;
  slug: string;
  title: string;
  content: string;        // Clean markdown or HTML
  excerpt: string;
  metadata: {
    title: string;
    description: string;
    ogImage: string;
    canonical: string;
    publishedAt: string;
    author: string;
  };
  internalLinks: string[];
  images: string[];
}

async function extractPage(url: string): Promise<MigratedPage> {
  // Extract clean content (readability-optimized)
  const contentResult = await client.callTool({
    name: 'extract_content',
    arguments: { url },
  });

  const content = JSON.parse(contentResult.content[0].text);

  // Extract metadata separately for completeness
  const metaResult = await client.callTool({
    name: 'extract_metadata',
    arguments: { url },
  });

  const meta = JSON.parse(metaResult.content[0].text);

  // Extract links for internal link mapping
  const linksResult = await client.callTool({
    name: 'extract_links',
    arguments: { url },
  });

  const links = JSON.parse(linksResult.content[0].text);
  const internalLinks = (links.internal || []).map(
    (l: { href: string }) => l.href
  );

  // Extract image URLs from content
  const imageRegex = /!\[.*?\]\((.*?)\)/g;
  const images: string[] = [];
  let match;
  while ((match = imageRegex.exec(content.content)) !== null) {
    images.push(match[1]);
  }

  const slug = new URL(url).pathname
    .replace(/^\//,'')
    .replace(/\/$/,'')
    .replace(/\//g, '-');

  return {
    sourceUrl: url,
    slug,
    title: meta.title || content.title || '',
    content: content.content,
    excerpt: meta.description || content.excerpt || '',
    metadata: {
      title: meta.title || '',
      description: meta.description || '',
      ogImage: meta.ogImage || '',
      canonical: meta.canonical || url,
      publishedAt: meta.publishedDate || '',
      author: meta.author || '',
    },
    internalLinks,
    images,
  };
}

type UrlMap = Map<string, string>; // old URL -> new slug

async function batchExtract(
  inventory: SiteInventory
): Promise<{ pages: MigratedPage[]; urlMap: UrlMap }> {
  const pages: MigratedPage[] = [];
  const urlMap: UrlMap = new Map();

  // Process in batches of 20
  const batchSize = 20;
  const urls = inventory.pages
    .filter(p => p.type === 'page' || p.type === 'post')
    .map(p => p.url);

  for (let i = 0; i < urls.length; i += batchSize) {
    const batch = urls.slice(i, i + batchSize);
    console.log(
      `Processing batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(urls.length / batchSize)}`
    );

    // Extract each page in the batch
    for (const url of batch) {
      try {
        const page = await extractPage(url);
        pages.push(page);
        urlMap.set(url, page.slug);
      } catch (error) {
        console.error(`Failed to extract ${url}:`, error);
      }
    }
  }

  return { pages, urlMap };
}

function transformForTarget(
  page: MigratedPage,
  urlMap: UrlMap,
  targetFormat: 'mdx' | 'contentful' | 'sanity'
): Record<string, unknown> {
  // Rewrite internal links to new URL structure
  let content = page.content;
  for (const [oldUrl, newSlug] of urlMap) {
    content = content.replace(
      new RegExp(oldUrl.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'),
      `/${newSlug}`
    );
  }

  switch (targetFormat) {
    case 'mdx':
      return {
        slug: page.slug,
        title: page.title,
        description: page.metadata.description,
        date: page.metadata.publishedAt || new Date().toISOString(),
        content: content, // Already markdown from extract_content
      };

    case 'contentful':
      return {
        fields: {
          title: { 'en-US': page.title },
          slug: { 'en-US': page.slug },
          body: { 'en-US': content },
          description: { 'en-US': page.metadata.description },
          publishedAt: { 'en-US': page.metadata.publishedAt },
        },
        contentType: 'blogPost',
      };

    case 'sanity':
      return {
        _type: 'post',
        title: page.title,
        slug: { current: page.slug },
        body: content,
        excerpt: page.excerpt,
        publishedAt: page.metadata.publishedAt,
      };

    default:
      return { ...page, content };
  }
}

interface ValidationResult {
  totalPages: number;
  successCount: number;
  failedCount: number;
  warnings: Array<{
    slug: string;
    issue: string;
  }>;
}

function validateMigration(
  original: SiteInventory,
  migrated: MigratedPage[]
): ValidationResult {
  const warnings: Array<{ slug: string; issue: string }> = [];

  for (const page of migrated) {
    // Check for empty content
    if (!page.content || page.content.trim().length < 50) {
      warnings.push({
        slug: page.slug,
        issue: 'Content appears empty or very short',
      });
    }

    // Check for missing metadata
    if (!page.metadata.title) {
      warnings.push({ slug: page.slug, issue: 'Missing title' });
    }
    if (!page.metadata.description) {
      warnings.push({ slug: page.slug, issue: 'Missing meta description' });
    }

    // Check for broken internal links
    for (const link of page.internalLinks) {
      const isLinkMigrated = migrated.some(
        p => p.sourceUrl === link || link.includes(p.slug)
      );
      if (!isLinkMigrated) {
        warnings.push({
          slug: page.slug,
          issue: `Internal link may be broken: ${link}`,
        });
      }
    }
  }

  const contentPages = original.pages.filter(
    p => p.type === 'page' || p.type === 'post'
  );

  return {
    totalPages: contentPages.length,
    successCount: migrated.length,
    failedCount: contentPages.length - migrated.length,
    warnings,
  };
}