网页抓取：2026 年 Python 对比 MCP

// Python equivalent in TypeScript for comparison
// Using node-fetch + cheerio (mirrors requests + BeautifulSoup pattern)
import * as cheerio from 'cheerio';

async function extractArticle(url: string): Promise<string> {
  const response = await fetch(url);
  const html = await response.text();
  const $ = cheerio.load(html);

  // Remove scripts, styles, nav, footer
  $('script, style, nav, footer, header, aside').remove();

  // Try common article selectors
  const selectors = ['article', '.post-content', '.article-body', 'main'];
  for (const selector of selectors) {
    const text = $(selector).text().trim();
    if (text.length > 200) return text;
  }

  // Fallback to body
  return $('body').text().trim();
}

const text = await extractArticle('https://techcrunch.com/2026/04/01/ai-agents');
console.log(text); // Often includes nav text, ads, related articles

// With CrawlForge MCP -- Claude handles this automatically when you say:
// "Extract the main article text from this TechCrunch post"

// Behind the scenes, Claude calls extract_content:
const result = await crawlforge.extract_content({
  url: 'https://techcrunch.com/2026/04/01/ai-agents'
});

console.log(result.content); // Clean article text, no nav/ads/boilerplate

// Python-style scraping with cheerio
import * as cheerio from 'cheerio';

interface Product {
  name: string;
  price: string;
  rating: string;
}

async function scrapeProducts(url: string): Promise<Product[]> {
  const response = await fetch(url, {
    headers: {
      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
    }
  });
  const html = await response.text();
  const $ = cheerio.load(html);
  const products: Product[] = [];

  $('.product-card').each((_, element) => {
    products.push({
      name: $(element).find('.product-title').text().trim(),
      price: $(element).find('.price').text().trim(),
      rating: $(element).find('.rating').attr('data-score') || 'N/A'
    });
  });

  return products;
}

// CrawlForge scrape_structured with CSS selectors
const products = await crawlforge.scrape_structured({
  url: 'https://store.example.com/products',
  selectors: {
    name: '.product-card .product-title',
    price: '.product-card .price',
    rating: '.product-card .rating'
  }
});

console.log(products);
// [{ name: "Widget Pro", price: "$29.99", rating: "4.5" }, ...]

// Multi-page crawl with manual pagination
import * as cheerio from 'cheerio';

interface SearchResult {
  title: string;
  url: string;
  snippet: string;
}

async function crawlDocs(baseUrl: string, pages: number): Promise<SearchResult[]> {
  const results: SearchResult[] = [];

  for (let page = 1; page <= pages; page++) {
    const url = `${baseUrl}?page=${page}`;
    const response = await fetch(url);
    const html = await response.text();
    const $ = cheerio.load(html);

    $('.search-result').each((_, el) => {
      results.push({
        title: $(el).find('h3 a').text().trim(),
        url: $(el).find('h3 a').attr('href') || '',
        snippet: $(el).find('.snippet').text().trim()
      });
    });

    // Respectful delay between requests
    await new Promise(resolve => setTimeout(resolve, 1000));
  }

  return results;
}

// 5 pages * 1 second delay = minimum 5 seconds
const results = await crawlDocs('https://docs.example.com/search?q=auth', 5);

// CrawlForge crawl_deep handles pagination and parallel fetching
const results = await crawlforge.crawl_deep({
  url: 'https://docs.example.com/search?q=auth',
  max_depth: 2,
  max_pages: 50,
  include_patterns: ['/docs/'],
  extract_content: true,
  concurrency: 5 // 5 parallel requests
});

console.log(results.pages.length); // Up to 50 pages crawled
console.log(results.pages[0].content); // Clean extracted content per page

// Requires Playwright or Puppeteer for JS rendering
import { chromium } from 'playwright';

async function scrapeSPA(url: string) {
  const browser = await chromium.launch({ headless: true });
  const page = await browser.newPage();

  await page.goto(url, { waitUntil: 'networkidle' });

  // Wait for dynamic content to render
  await page.waitForSelector('.product-grid', { timeout: 10000 });

  const products = await page.evaluate(() => {
    return Array.from(document.querySelectorAll('.product-card')).map(card => ({
      name: card.querySelector('.name')?.textContent?.trim() || '',
      price: card.querySelector('.price')?.textContent?.trim() || ''
    }));
  });

  await browser.close();
  return products;
}

// CrawlForge scrape_with_actions handles JS rendering + interaction
const result = await crawlforge.scrape_with_actions({
  url: 'https://react-store.example.com/products',
  actions: [
    { type: 'wait', selector: '.product-grid', timeout: 10000 },
    { type: 'scroll', selector: 'body' } // Trigger lazy loading
  ],
  extractionOptions: {
    selectors: {
      name: '.product-card .name',
      price: '.product-card .price'
    }
  }
});

// Use CrawlForge for extraction, Python/TypeScript for pipeline logic
import { CrawlForge } from '@crawlforge/sdk';

const cf = new CrawlForge({ apiKey: process.env.CRAWLFORGE_API_KEY });

// Step 1: Use CrawlForge to extract clean content
const extracted = await cf.extractContent({
  url: 'https://competitor.com/pricing'
});

// Step 2: Process with your own pipeline
const pricing = parsePricingMarkdown(extracted.content);
await database.upsert('competitor_pricing', pricing);
await notifySlack(`Updated pricing data: ${pricing.plans.length} plans found`);