用 CrawlForge 构建 lead enrichment 引擎

import { Client } from '@modelcontextprotocol/sdk/client/index.js';

const client = new Client({
  name: 'lead-enrichment',
  version: '1.0.0',
});

interface CompanyProfile {
  domain: string;
  name: string;
  description: string;
  industry: string;
  topics: string[];
  socialLinks: {
    twitter?: string;
    linkedin?: string;
    github?: string;
  };
  estimatedSize: string;
}

async function analyzeCompanyWebsite(
  domain: string
): Promise<CompanyProfile> {
  // Extract main page content
  const contentResult = await client.callTool({
    name: 'extract_content',
    arguments: { url: `https://${domain}` },
  });

  const content = JSON.parse(contentResult.content[0].text);

  // Get metadata including social links
  const metaResult = await client.callTool({
    name: 'extract_metadata',
    arguments: { url: `https://${domain}` },
  });

  const meta = JSON.parse(metaResult.content[0].text);

  // Analyze content for industry and topics
  const analysis = await client.callTool({
    name: 'analyze_content',
    arguments: { text: content.content },
  });

  const analyzed = JSON.parse(analysis.content[0].text);

  // Try to get team page for company size estimation
  let estimatedSize = 'Unknown';
  try {
    const teamPage = await client.callTool({
      name: 'extract_content',
      arguments: { url: `https://${domain}/about` },
    });
    const teamContent = JSON.parse(teamPage.content[0].text);
    // Simple heuristic: count mentions of team-related terms
    const teamMentions = (teamContent.content.match(/team|employee|people|staff/gi) || []).length;
    if (teamMentions > 20) estimatedSize = '100+';
    else if (teamMentions > 10) estimatedSize = '50-100';
    else if (teamMentions > 5) estimatedSize = '10-50';
    else estimatedSize = '1-10';
  } catch {
    // About page might not exist; that is fine
  }

  return {
    domain,
    name: meta.title?.split(/[|\-]/)[0]?.trim() || domain,
    description: meta.description || analyzed.summary || '',
    industry: analyzed.topics?.[0] || 'Unknown',
    topics: analyzed.topics || [],
    socialLinks: {
      twitter: meta.twitterHandle,
      linkedin: meta.linkedinUrl,
      github: meta.githubUrl,
    },
    estimatedSize,
  };
}

interface TechStack {
  frontend: string[];
  analytics: string[];
  marketing: string[];
  infrastructure: string[];
  cms: string[];
}

async function detectTechStack(domain: string): Promise<TechStack> {
  const result = await client.callTool({
    name: 'fetch_url',
    arguments: {
      url: `https://${domain}`,
      timeout: 10000,
    },
  });

  const page = JSON.parse(result.content[0].text);
  const html = page.body || '';
  const headers = page.headers || {};

  const stack: TechStack = {
    frontend: [],
    analytics: [],
    marketing: [],
    infrastructure: [],
    cms: [],
  };

  // Frontend framework detection
  if (html.includes('__next') || html.includes('_next/static')) {
    stack.frontend.push('Next.js');
  }
  if (html.includes('__nuxt')) stack.frontend.push('Nuxt.js');
  if (html.includes('react')) stack.frontend.push('React');
  if (html.includes('vue')) stack.frontend.push('Vue.js');
  if (html.includes('tailwindcss') || html.includes('tailwind')) {
    stack.frontend.push('Tailwind CSS');
  }

  // Analytics detection
  if (html.includes('gtag') || html.includes('google-analytics')) {
    stack.analytics.push('Google Analytics');
  }
  if (html.includes('segment.com') || html.includes('analytics.js')) {
    stack.analytics.push('Segment');
  }
  if (html.includes('mixpanel')) stack.analytics.push('Mixpanel');
  if (html.includes('hotjar')) stack.analytics.push('Hotjar');

  // Marketing tools
  if (html.includes('hubspot')) stack.marketing.push('HubSpot');
  if (html.includes('intercom')) stack.marketing.push('Intercom');
  if (html.includes('drift')) stack.marketing.push('Drift');
  if (html.includes('mailchimp')) stack.marketing.push('Mailchimp');

  // Infrastructure
  const server = headers['server'] || headers['x-powered-by'] || '';
  if (server.includes('Vercel') || html.includes('vercel')) {
    stack.infrastructure.push('Vercel');
  }
  if (server.includes('cloudflare') || headers['cf-ray']) {
    stack.infrastructure.push('Cloudflare');
  }
  if (html.includes('amazonaws')) stack.infrastructure.push('AWS');

  // CMS
  if (html.includes('wp-content')) stack.cms.push('WordPress');
  if (html.includes('shopify')) stack.cms.push('Shopify');
  if (html.includes('webflow')) stack.cms.push('Webflow');

  return stack;
}

interface CompanyIntel {
  recentNews: Array<{ title: string; url: string; snippet: string }>;
  fundingInfo: string;
  competitorMentions: string[];
}

async function gatherIntelligence(
  companyName: string,
  domain: string
): Promise<CompanyIntel> {
  // Search for recent company news
  const newsResult = await client.callTool({
    name: 'search_web',
    arguments: {
      query: `"${companyName}" OR "${domain}" news funding`,
      limit: 10,
      time_range: 'month',
    },
  });

  const news = JSON.parse(newsResult.content[0].text);

  return {
    recentNews: (news.results || []).slice(0, 5).map(
      (r: { title: string; url: string; snippet: string }) => ({
        title: r.title,
        url: r.url,
        snippet: r.snippet,
      })
    ),
    fundingInfo: news.results?.find(
      (r: { title: string }) =>
        r.title.toLowerCase().includes('funding') ||
        r.title.toLowerCase().includes('raises') ||
        r.title.toLowerCase().includes('series')
    )?.snippet || 'No recent funding news found',
    competitorMentions: news.results
      ?.filter((r: { title: string }) =>
        r.title.toLowerCase().includes('vs') ||
        r.title.toLowerCase().includes('alternative')
      )
      .map((r: { title: string }) => r.title) || [],
  };
}

interface Lead {
  email: string;
  name: string;
  company?: string;
}

interface EnrichedLead extends Lead {
  companyProfile: CompanyProfile;
  techStack: TechStack;
  intelligence: CompanyIntel;
  enrichedAt: string;
  qualityScore: number;
}

function extractDomain(email: string): string {
  return email.split('@')[1];
}

function calculateLeadScore(enriched: Omit<EnrichedLead, 'qualityScore'>): number {
  let score = 0;

  // Company has a real website with content
  if (enriched.companyProfile.description.length > 50) score += 20;

  // Company uses modern tech stack (potential product fit)
  if (enriched.techStack.frontend.length > 0) score += 15;

  // Company uses analytics/marketing tools (signals budget)
  if (enriched.techStack.analytics.length > 0) score += 10;
  if (enriched.techStack.marketing.length > 0) score += 15;

  // Recent news/activity
  if (enriched.intelligence.recentNews.length > 0) score += 10;

  // Funding signals
  if (!enriched.intelligence.fundingInfo.includes('No recent')) score += 20;

  // Company size signals
  const size = enriched.companyProfile.estimatedSize;
  if (size === '50-100' || size === '100+') score += 10;

  return Math.min(score, 100);
}

async function enrichLeads(leads: Lead[]): Promise<EnrichedLead[]> {
  const enriched: EnrichedLead[] = [];

  for (const lead of leads) {
    const domain = lead.company
      ? lead.company.toLowerCase().replace(/\s+/g, '') + '.com'
      : extractDomain(lead.email);

    // Skip free email providers
    if (['gmail.com', 'yahoo.com', 'hotmail.com', 'outlook.com'].includes(domain)) {
      continue;
    }

    try {
      console.log(`Enriching ${lead.email} (${domain})...`);

      const companyProfile = await analyzeCompanyWebsite(domain);
      const techStack = await detectTechStack(domain);
      const intelligence = await gatherIntelligence(
        companyProfile.name,
        domain
      );

      const partial = {
        ...lead,
        companyProfile,
        techStack,
        intelligence,
        enrichedAt: new Date().toISOString(),
      };

      enriched.push({
        ...partial,
        qualityScore: calculateLeadScore(partial),
      });
    } catch (error) {
      console.error(`Failed to enrich ${lead.email}:`, error);
    }
  }

  // Sort by quality score (highest first)
  return enriched.sort((a, b) => b.qualityScore - a.qualityScore);
}