Build a Research Agent with CrawlForge Deep Research

import { Client } from '@modelcontextprotocol/sdk/client/index.js';

const client = new Client({
  name: 'research-agent',
  version: '1.0.0',
});

interface ResearchConfig {
  topic: string;
  approach: 'broad' | 'focused' | 'academic' | 'current_events' | 'comparative';
  maxSources: number;
  maxDepth: number;
  credibilityThreshold: number;
  sourceTypes: Array<'academic' | 'news' | 'government' | 'commercial' | 'blog'>;
  timeLimit: number;
}

// Preset configurations for common research types
const RESEARCH_PRESETS: Record<string, Partial<ResearchConfig>> = {
  market: {
    approach: 'broad',
    maxSources: 30,
    maxDepth: 5,
    credibilityThreshold: 0.4,
    sourceTypes: ['news', 'commercial', 'blog'],
    timeLimit: 90000, // 90 seconds
  },
  technical: {
    approach: 'focused',
    maxSources: 20,
    maxDepth: 3,
    credibilityThreshold: 0.6,
    sourceTypes: ['academic', 'blog', 'commercial'],
    timeLimit: 60000,
  },
  competitive: {
    approach: 'comparative',
    maxSources: 25,
    maxDepth: 4,
    credibilityThreshold: 0.5,
    sourceTypes: ['commercial', 'news', 'blog'],
    timeLimit: 120000,
  },
  academic: {
    approach: 'academic',
    maxSources: 40,
    maxDepth: 5,
    credibilityThreshold: 0.7,
    sourceTypes: ['academic', 'government'],
    timeLimit: 120000,
  },
};

interface ResearchResult {
  synthesis: string;
  sources: Array<{
    url: string;
    title: string;
    credibilityScore: number;
    relevantFindings: string[];
  }>;
  conflicts: Array<{
    topic: string;
    positions: Array<{ source: string; claim: string }>;
  }>;
  confidence: number;
  queryExpansions: string[];
}

async function runResearch(config: ResearchConfig): Promise<ResearchResult> {
  const result = await client.callTool({
    name: 'deep_research',
    arguments: {
      topic: config.topic,
      maxDepth: config.maxDepth,
      maxUrls: config.maxSources,
      enableSourceVerification: true,
      enableConflictDetection: true,
      enableSynthesis: true,
      researchApproach: config.approach,
      credibilityThreshold: config.credibilityThreshold,
      sourceTypes: config.sourceTypes,
      timeLimit: config.timeLimit,
      outputFormat: 'comprehensive',
      queryExpansion: {
        enableSynonyms: true,
        enableContextual: true,
        maxVariations: 8,
      },
    },
  });

  return JSON.parse(result.content[0].text);
}

// Example: Research a market question
const marketResearch = await runResearch({
  topic: 'MCP server adoption trends in enterprise AI development 2026',
  ...RESEARCH_PRESETS.market,
} as ResearchConfig);

console.log(`Research complete: ${marketResearch.sources.length} sources analyzed`);
console.log(`Confidence level: ${marketResearch.confidence}`);
console.log(`Conflicts found: ${marketResearch.conflicts.length}`);

async function validateFindings(
  research: ResearchResult,
  topN: number = 5
): Promise<{
  validated: ResearchResult;
  additionalInsights: string[];
}> {
  const additionalInsights: string[] = [];

  // Get full content from top sources
  const topSources = research.sources
    .sort((a, b) => b.credibilityScore - a.credibilityScore)
    .slice(0, topN);

  for (const source of topSources) {
    // Extract full content for deeper analysis
    const fullContent = await client.callTool({
      name: 'extract_content',
      arguments: { url: source.url },
    });

    const content = JSON.parse(fullContent.content[0].text);

    // Check for PDF references in the source
    if (content.content.includes('.pdf')) {
      const pdfUrls = content.content.match(
        /https?:\/\/[^\s]+\.pdf/g
      ) || [];

      for (const pdfUrl of pdfUrls.slice(0, 2)) {
        try {
          const doc = await client.callTool({
            name: 'process_document',
            arguments: {
              source: pdfUrl,
              sourceType: 'pdf_url',
            },
          });

          const docResult = JSON.parse(doc.content[0].text);
          additionalInsights.push(
            `From ${pdfUrl}: ${docResult.summary || docResult.content?.slice(0, 200)}`
          );
        } catch {
          // PDF might be behind auth; skip
        }
      }
    }
  }

  return {
    validated: research,
    additionalInsights,
  };
}

interface ResearchReport {
  title: string;
  executiveSummary: string;
  keyFindings: string[];
  detailedAnalysis: string;
  conflictsAndCaveats: string[];
  sources: Array<{ title: string; url: string; credibility: string }>;
  methodology: string;
  generatedAt: string;
}

async function generateReport(
  research: ResearchResult,
  reportTitle: string
): Promise<ResearchReport> {
  // Generate executive summary
  const summaryResult = await client.callTool({
    name: 'summarize_content',
    arguments: {
      text: research.synthesis,
    },
  });

  const summary = JSON.parse(summaryResult.content[0].text);

  // Extract key findings as bullet points
  const keyFindings = research.sources
    .flatMap(s => s.relevantFindings)
    .filter((finding, index, self) => self.indexOf(finding) === index)
    .slice(0, 10);

  return {
    title: reportTitle,
    executiveSummary: summary.summary,
    keyFindings,
    detailedAnalysis: research.synthesis,
    conflictsAndCaveats: research.conflicts.map(
      c => `Conflicting information on "${c.topic}": ${c.positions.length} different positions found across sources`
    ),
    sources: research.sources.map(s => ({
      title: s.title,
      url: s.url,
      credibility: s.credibilityScore > 0.7 ? 'High' : s.credibilityScore > 0.4 ? 'Medium' : 'Low',
    })),
    methodology: `Automated research using CrawlForge deep_research. ${research.sources.length} sources analyzed with ${research.queryExpansions.length} query variations. Credibility threshold: ${research.confidence}.`,
    generatedAt: new Date().toISOString(),
  };
}

interface RecurringResearch {
  id: string;
  topic: string;
  schedule: 'daily' | 'weekly' | 'monthly';
  config: ResearchConfig;
  history: ResearchReport[];
}

async function runRecurringResearch(
  research: RecurringResearch
): Promise<ResearchReport> {
  // Run the research
  const result = await runResearch(research.config);

  // Generate the report
  const report = await generateReport(
    result,
    `${research.topic} - ${new Date().toISOString().split('T')[0]}`
  );

  // Compare with previous report if available
  if (research.history.length > 0) {
    const previousReport = research.history[research.history.length - 1];
    const newFindings = report.keyFindings.filter(
      f => !previousReport.keyFindings.includes(f)
    );

    if (newFindings.length > 0) {
      console.log(`\nNew findings since last report:`);
      newFindings.forEach(f => console.log(`  - ${f}`));
    }
  }

  research.history.push(report);
  return report;
}