面向 AI 训练数据流水线的网页抓取

import { Client } from '@modelcontextprotocol/sdk/client/index.js';

const client = new Client({
  name: 'training-data-pipeline',
  version: '1.0.0',
});

interface CrawlSource {
  domain: string;
  category: string;
  maxPages: number;
  includePatterns: string[];
  excludePatterns: string[];
}

async function crawlSource(source: CrawlSource) {
  const result = await client.callTool({
    name: 'crawl_deep',
    arguments: {
      url: `https://${source.domain}`,
      max_pages: source.maxPages,
      max_depth: 3,
      extract_content: false, // Just discover URLs first
      respect_robots: true,
      include_patterns: source.includePatterns,
      exclude_patterns: source.excludePatterns,
    },
  });

  const crawled = JSON.parse(result.content[0].text);

  return {
    domain: source.domain,
    category: source.category,
    urls: crawled.pages.map((p: { url: string }) => p.url),
    totalDiscovered: crawled.totalPages,
  };
}

// Example: Crawl technical documentation sites
const sources: CrawlSource[] = [
  {
    domain: 'docs.python.org',
    category: 'programming',
    maxPages: 500,
    includePatterns: ['/3/library/', '/3/tutorial/'],
    excludePatterns: ['/2.7/', '/genindex'],
  },
  {
    domain: 'developer.mozilla.org',
    category: 'web-development',
    maxPages: 500,
    includePatterns: ['/en-US/docs/Web/'],
    excludePatterns: ['/Users/', '/search'],
  },
];

interface ExtractedContent {
  url: string;
  title: string;
  content: string;
  wordCount: number;
  category: string;
}

async function extractBatch(
  urls: string[],
  category: string
): Promise<ExtractedContent[]> {
  const results: ExtractedContent[] = [];

  // Process in batches of 25 for efficiency
  const batchSize = 25;
  for (let i = 0; i < urls.length; i += batchSize) {
    const batch = urls.slice(i, i + batchSize);

    const batchResult = await client.callTool({
      name: 'batch_scrape',
      arguments: {
        urls: batch.map(url => ({ url })),
        formats: ['text'],
        includeMetadata: true,
        maxConcurrency: 10,
      },
    });

    const parsed = JSON.parse(batchResult.content[0].text);

    for (const page of parsed.results) {
      if (page.status === 'success' && page.content) {
        const wordCount = page.content.split(/\s+/).length;

        // Skip pages with too little content
        if (wordCount < 200) continue;

        results.push({
          url: page.url,
          title: page.metadata?.title || '',
          content: page.content,
          wordCount,
          category,
        });
      }
    }
  }

  return results;
}

interface QualityScore {
  url: string;
  readability: number;
  topicRelevance: number;
  contentDepth: number;
  overallScore: number;
  passesFilter: boolean;
}

async function scoreContentQuality(
  item: ExtractedContent,
  targetTopics: string[]
): Promise<QualityScore> {
  const analysis = await client.callTool({
    name: 'analyze_content',
    arguments: {
      text: item.content,
    },
  });

  const result = JSON.parse(analysis.content[0].text);

  // Calculate topic relevance based on overlap
  const detectedTopics = (result.topics || []).map(
    (t: string) => t.toLowerCase()
  );
  const topicOverlap = targetTopics.filter(t =>
    detectedTopics.some((dt: string) => dt.includes(t.toLowerCase()))
  ).length;
  const topicRelevance = topicOverlap / targetTopics.length;

  // Content depth: word count normalized (800+ words = 1.0)
  const contentDepth = Math.min(item.wordCount / 800, 1.0);

  // Readability: normalize to 0-1 scale
  const readability = (result.readabilityScore || 50) / 100;

  // Weighted overall score
  const overallScore =
    topicRelevance * 0.4 + contentDepth * 0.3 + readability * 0.3;

  return {
    url: item.url,
    readability,
    topicRelevance,
    contentDepth,
    overallScore,
    passesFilter: overallScore >= 0.5, // Minimum quality threshold
  };
}

// JSONL format for fine-tuning (OpenAI compatible)
interface TrainingRecord {
  messages: Array<{
    role: 'system' | 'user' | 'assistant';
    content: string;
  }>;
  metadata: {
    source: string;
    category: string;
    quality_score: number;
  };
}

function structureForFineTuning(
  items: ExtractedContent[],
  scores: QualityScore[]
): TrainingRecord[] {
  return items
    .filter((_, i) => scores[i].passesFilter)
    .map((item, i) => ({
      messages: [
        {
          role: 'system' as const,
          content: `You are a knowledgeable assistant specializing in ${item.category}.`,
        },
        {
          role: 'user' as const,
          content: `Explain the following topic in detail: ${item.title}`,
        },
        {
          role: 'assistant' as const,
          content: item.content,
        },
      ],
      metadata: {
        source: item.url,
        category: item.category,
        quality_score: scores[i].overallScore,
      },
    }));
}

// For embedding generation (simpler format)
interface EmbeddingRecord {
  text: string;
  metadata: {
    source: string;
    title: string;
    chunk_index: number;
  };
}

function structureForEmbeddings(
  items: ExtractedContent[],
  chunkSize: number = 512
): EmbeddingRecord[] {
  const records: EmbeddingRecord[] = [];

  for (const item of items) {
    // Split into chunks for embedding models
    const words = item.content.split(/\s+/);
    const chunks = [];
    for (let i = 0; i < words.length; i += chunkSize) {
      chunks.push(words.slice(i, i + chunkSize).join(' '));
    }

    chunks.forEach((chunk, index) => {
      records.push({
        text: chunk,
        metadata: {
          source: item.url,
          title: item.title,
          chunk_index: index,
        },
      });
    });
  }

  return records;
}

async function buildTrainingDataset(
  sources: CrawlSource[],
  targetTopics: string[],
  outputFormat: 'fine-tuning' | 'embeddings'
) {
  console.log(`Starting pipeline for ${sources.length} sources...`);

  let allContent: ExtractedContent[] = [];

  // Stage 1: Crawl all sources
  for (const source of sources) {
    console.log(`Crawling ${source.domain}...`);
    const crawled = await crawlSource(source);
    console.log(`  Found ${crawled.urls.length} pages`);

    // Stage 2: Extract content
    const extracted = await extractBatch(crawled.urls, crawled.category);
    console.log(`  Extracted ${extracted.length} quality pages`);
    allContent = allContent.concat(extracted);
  }

  // Stage 3: Quality scoring
  console.log(`\nScoring ${allContent.length} pages for quality...`);
  const scores: QualityScore[] = [];
  for (const item of allContent) {
    scores.push(await scoreContentQuality(item, targetTopics));
  }

  const passing = scores.filter(s => s.passesFilter).length;
  console.log(`  ${passing}/${allContent.length} passed quality filter`);

  // Stage 4: Structure output
  if (outputFormat === 'fine-tuning') {
    const records = structureForFineTuning(allContent, scores);
    console.log(`\nGenerated ${records.length} training records`);
    return records;
  } else {
    const filtered = allContent.filter((_, i) => scores[i].passesFilter);
    const records = structureForEmbeddings(filtered);
    console.log(`\nGenerated ${records.length} embedding chunks`);
    return records;
  }
}