大规模提取电商产品数据

import { Client } from '@modelcontextprotocol/sdk/client/index.js';

const client = new Client({
  name: 'ecommerce-extractor',
  version: '1.0.0',
});

interface ProductPageDiscovery {
  domain: string;
  productUrls: string[];
  categoryUrls: string[];
  totalPages: number;
}

async function discoverProducts(
  siteUrl: string,
  maxPages: number = 500
): Promise<ProductPageDiscovery> {
  const crawlResult = await client.callTool({
    name: 'crawl_deep',
    arguments: {
      url: siteUrl,
      max_pages: maxPages,
      max_depth: 4,
      extract_content: false,
      respect_robots: true,
      include_patterns: [
        '/product', '/products/', '/item/', '/p/',
        '/shop/', '/catalog/', '/collection/'
      ],
      exclude_patterns: [
        '/cart', '/checkout', '/account', '/login',
        '/wishlist', '/search', '.css', '.js', '.png', '.jpg'
      ],
    },
  });

  const crawled = JSON.parse(crawlResult.content[0].text);

  const productUrls = crawled.pages
    .map((p: { url: string }) => p.url)
    .filter((url: string) =>
      url.match(/\/products?\/|/item\/|\/p\//)
    );

  const categoryUrls = crawled.pages
    .map((p: { url: string }) => p.url)
    .filter((url: string) =>
      url.match(/\/collections?\/|\/category\/|\/shop\//)
    );

  return {
    domain: new URL(siteUrl).hostname,
    productUrls,
    categoryUrls,
    totalPages: crawled.totalPages,
  };
}

interface ProductData {
  url: string;
  name: string;
  price: string;
  currency: string;
  description: string;
  images: string[];
  availability: string;
  sku: string;
  brand: string;
  category: string;
  rating: string;
  reviewCount: string;
  extractedAt: string;
}

// Selector presets for common e-commerce platforms
const PLATFORM_SELECTORS: Record<string, Record<string, string>> = {
  shopify: {
    name: '.product-single__title, h1.product__title',
    price: '.product__price, .price-item--regular',
    description: '.product-single__description, .product__description',
    images: '.product-single__photo img, .product__media img',
    availability: '.product-form__inventory, [data-availability]',
    sku: '[data-product-sku], .product-single__sku',
    brand: '.product-single__vendor, .product__vendor',
  },
  woocommerce: {
    name: '.product_title, h1.entry-title',
    price: '.woocommerce-Price-amount, .price ins .amount',
    description: '.woocommerce-product-details__short-description, #tab-description',
    images: '.woocommerce-product-gallery img',
    availability: '.stock, .availability',
    sku: '.sku',
    brand: '.posted_in a',
  },
  generic: {
    name: 'h1, [itemprop="name"]',
    price: '[itemprop="price"], .price, .product-price',
    description: '[itemprop="description"], .product-description',
    images: '.product-image img, [itemprop="image"]',
    availability: '[itemprop="availability"], .availability',
    sku: '[itemprop="sku"]',
    brand: '[itemprop="brand"], .brand',
  },
};

async function extractProduct(
  url: string,
  platform: string = 'generic'
): Promise<ProductData> {
  const selectors = PLATFORM_SELECTORS[platform] || PLATFORM_SELECTORS.generic;

  const result = await client.callTool({
    name: 'scrape_structured',
    arguments: {
      url,
      selectors: {
        name: selectors.name,
        price: selectors.price,
        description: selectors.description,
        images: selectors.images,
        availability: selectors.availability,
        sku: selectors.sku,
        brand: selectors.brand,
        rating: '[itemprop="ratingValue"], .star-rating',
        reviewCount: '[itemprop="reviewCount"], .review-count',
        category: '.breadcrumb a, [itemprop="category"]',
      },
    },
  });

  const data = JSON.parse(result.content[0].text);

  return {
    url,
    name: data.name || '',
    price: data.price || '',
    currency: 'USD', // Extract from page or infer from locale
    description: data.description || '',
    images: Array.isArray(data.images) ? data.images : [data.images].filter(Boolean),
    availability: data.availability || 'Unknown',
    sku: data.sku || '',
    brand: data.brand || '',
    category: data.category || '',
    rating: data.rating || '',
    reviewCount: data.reviewCount || '',
    extractedAt: new Date().toISOString(),
  };
}

async function extractDynamicProduct(url: string): Promise<ProductData> {
  const result = await client.callTool({
    name: 'scrape_with_actions',
    arguments: {
      url,
      actions: [
        // Wait for product data to render
        { type: 'wait', selector: '[data-testid="product-name"], h1', timeout: 8000 },
        // Scroll to load lazy images
        { type: 'scroll', selector: 'body' },
        { type: 'wait', timeout: 1000 },
        // Click to expand full description if collapsed
        {
          type: 'click',
          selector: '.read-more, .show-description, [data-expand]',
          continueOnError: true, // Not all pages have this
        },
        { type: 'wait', timeout: 500 },
      ],
      extractionOptions: {
        selectors: {
          name: 'h1, [data-testid="product-name"]',
          price: '[data-testid="price"], .price',
          description: '.description, [data-testid="description"]',
          images: '.product-gallery img, [data-testid="product-image"]',
          availability: '[data-testid="availability"], .stock-status',
          rating: '[data-testid="rating"], .rating-value',
        },
        includeMetadata: true,
        includeImages: true,
      },
      continueOnActionError: true,
    },
  });

  const data = JSON.parse(result.content[0].text);

  return {
    url,
    name: data.extracted?.name || data.metadata?.title || '',
    price: data.extracted?.price || '',
    currency: 'USD',
    description: data.extracted?.description || '',
    images: data.extracted?.images || [],
    availability: data.extracted?.availability || 'Unknown',
    sku: '',
    brand: '',
    category: '',
    rating: data.extracted?.rating || '',
    reviewCount: '',
    extractedAt: new Date().toISOString(),
  };
}

async function batchExtractProducts(
  urls: string[],
  platform: string = 'generic'
): Promise<ProductData[]> {
  const selectors = PLATFORM_SELECTORS[platform] || PLATFORM_SELECTORS.generic;
  const products: ProductData[] = [];
  const batchSize = 25;

  for (let i = 0; i < urls.length; i += batchSize) {
    const batch = urls.slice(i, i + batchSize);

    console.log(
      `Batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(urls.length / batchSize)}: ${batch.length} URLs`
    );

    const result = await client.callTool({
      name: 'batch_scrape',
      arguments: {
        urls: batch.map(url => ({
          url,
          selectors: {
            name: selectors.name,
            price: selectors.price,
            description: selectors.description,
            availability: selectors.availability,
            sku: selectors.sku,
            brand: selectors.brand,
          },
        })),
        maxConcurrency: 10,
        includeMetadata: true,
        delayBetweenRequests: 200, // Respectful crawling
      },
    });

    const batchResult = JSON.parse(result.content[0].text);

    for (const page of batchResult.results) {
      if (page.status === 'success') {
        products.push({
          url: page.url,
          name: page.data?.name || page.metadata?.title || '',
          price: page.data?.price || '',
          currency: 'USD',
          description: page.data?.description || '',
          images: [],
          availability: page.data?.availability || 'Unknown',
          sku: page.data?.sku || '',
          brand: page.data?.brand || '',
          category: '',
          rating: '',
          reviewCount: '',
          extractedAt: new Date().toISOString(),
        });
      }
    }
  }

  return products;
}

async function extractProtectedProduct(url: string): Promise<ProductData> {
  // Configure stealth browsing context
  const stealthConfig = await client.callTool({
    name: 'stealth_mode',
    arguments: {
      operation: 'create_context',
      stealthConfig: {
        level: 'advanced',
        hideWebDriver: true,
        randomizeFingerprint: true,
        simulateHumanBehavior: true,
        antiDetection: {
          cloudflareBypass: true,
          hideAutomation: true,
        },
        fingerprinting: {
          canvasNoise: true,
          webglSpoofing: true,
          audioContextSpoofing: true,
        },
      },
      urlToTest: url,
    },
  });

  const context = JSON.parse(stealthConfig.content[0].text);

  // Now use scrape_with_actions within the stealth context
  const result = await client.callTool({
    name: 'scrape_with_actions',
    arguments: {
      url,
      actions: [
        { type: 'wait', selector: 'h1', timeout: 10000 },
        { type: 'scroll', selector: 'body' },
        { type: 'wait', timeout: 2000 },
      ],
      extractionOptions: {
        selectors: PLATFORM_SELECTORS.generic,
        includeMetadata: true,
      },
    },
  });

  const data = JSON.parse(result.content[0].text);

  return {
    url,
    name: data.extracted?.name || '',
    price: data.extracted?.price || '',
    currency: 'USD',
    description: data.extracted?.description || '',
    images: [],
    availability: data.extracted?.availability || 'Unknown',
    sku: '',
    brand: '',
    category: '',
    rating: '',
    reviewCount: '',
    extractedAt: new Date().toISOString(),
  };
}