LangChain Integration

npm install langchain @langchain/core @langchain/openai zod

import { BaseDocumentLoader } from '@langchain/core/document_loaders/base';
import { Document } from '@langchain/core/documents';

const CRAWLFORGE_API = 'https://www.crawlforge.dev/api/v1/tools';

/** Every CrawlForge tool is a POST to /api/v1/tools/<tool_name>. */
export async function callTool<T>(
  tool: string,
  params: Record<string, unknown>,
): Promise<T> {
  const response = await fetch(`${CRAWLFORGE_API}/${tool}`, {
    method: 'POST',
    headers: {
      'X-API-Key': process.env.CRAWLFORGE_API_KEY!,
      'Content-Type': 'application/json',
    },
    body: JSON.stringify(params),
  });

  const payload = await response.json();
  if (!payload.success) {
    throw new Error(payload.error?.message ?? 'CrawlForge request failed');
  }
  return payload.data as T;
}

interface ExtractedContent {
  url: string;
  title: string;
  content: string;
  author: string | null;
  publish_date: string | null;
}

/** Loads each URL through extract_content (2 credits per URL). */
export class CrawlForgeLoader extends BaseDocumentLoader {
  constructor(private readonly urls: string[]) {
    super();
  }

  async load(): Promise<Document[]> {
    return Promise.all(
      this.urls.map(async (url) => {
        const data = await callTool<ExtractedContent>('extract_content', { url });
        return new Document({
          pageContent: data.content,
          metadata: {
            source: data.url,
            title: data.title,
            author: data.author,
            publishedAt: data.publish_date,
          },
        });
      }),
    );
  }
}

// ---------- usage ----------

const docs = await new CrawlForgeLoader([
  'https://example.com/page1',
  'https://example.com/page2',
]).load();

console.log(docs[0].pageContent);
console.log(docs[0].metadata.title);

import { ChatPromptTemplate } from '@langchain/core/prompts';
import { StringOutputParser } from '@langchain/core/output_parsers';
import { ChatOpenAI, OpenAIEmbeddings } from '@langchain/openai';
import { MemoryVectorStore } from 'langchain/vectorstores/memory';
import { CrawlForgeLoader } from './crawlforge';

// 1. Load web pages as LangChain documents (2 credits per URL)
const docs = await new CrawlForgeLoader([
  'https://example.com/doc1',
  'https://example.com/doc2',
  'https://example.com/doc3',
]).load();

// 2. Embed them into a vector store
const store = await MemoryVectorStore.fromDocuments(docs, new OpenAIEmbeddings());
const retriever = store.asRetriever({ k: 4 });

// 3. Retrieve the relevant chunks, then answer over them
const prompt = ChatPromptTemplate.fromTemplate(
  'Answer using only this context:\n\n{context}\n\nQuestion: {question}',
);
const model = new ChatOpenAI({ model: 'gpt-4o-mini' });

const question = 'What are the key points from these documents?';
const context = (await retriever.invoke(question))
  .map((doc) => doc.pageContent)
  .join('\n\n');

const answer = await prompt
  .pipe(model)
  .pipe(new StringOutputParser())
  .invoke({ context, question });

console.log(answer);

import { DynamicStructuredTool } from '@langchain/core/tools';
import { ChatOpenAI } from '@langchain/openai';
import { z } from 'zod';
import { callTool } from './crawlforge';

interface SearchResults {
  results: { title: string; url: string; snippet: string }[];
}

export const searchWeb = new DynamicStructuredTool({
  name: 'search_web',
  description: 'Search the web and return ranked results. Costs 5 credits.',
  schema: z.object({
    query: z.string().describe('What to search for'),
    limit: z.number().optional().describe('How many results, 1-100 (default 10)'),
  }),
  func: async ({ query, limit }) => {
    const data = await callTool<SearchResults>('search_web', { query, limit });
    return JSON.stringify(data.results);
  },
});

export const readPage = new DynamicStructuredTool({
  name: 'extract_content',
  description: 'Extract the main article text from one URL. Costs 2 credits.',
  schema: z.object({
    url: z.string().url().describe('The page to read'),
  }),
  func: async ({ url }) => {
    const data = await callTool<{ title: string; content: string }>(
      'extract_content',
      { url },
    );
    return `${data.title}\n\n${data.content}`;
  },
});

// ---------- usage ----------

// These are ordinary LangChain tools, so they also drop straight into an
// agent executor or a LangGraph node.
const model = new ChatOpenAI({ model: 'gpt-4o-mini', temperature: 0 })
  .bindTools([searchWeb, readPage]);

const reply = await model.invoke('Which CrawlForge plan includes 50,000 credits?');

for (const call of reply.tool_calls ?? []) {
  console.log(call.name, call.args);
}

import { ChatPromptTemplate } from '@langchain/core/prompts';
import { StringOutputParser } from '@langchain/core/output_parsers';
import { RunnableSequence } from '@langchain/core/runnables';
import { ChatOpenAI } from '@langchain/openai';
import { callTool } from './crawlforge';

interface ResearchResult {
  research_report: {
    title: string;
    sections: { executive_summary?: string; findings?: unknown };
  };
}

// deep_research costs 10 credits per call, so put it behind a chain step
// rather than calling it per retrieved chunk.
const research = async (question: string) => {
  const data = await callTool<ResearchResult>('deep_research', {
    research_query: question,
    research_scope: { depth_level: 'moderate' },
  });
  return data.research_report.sections.executive_summary ?? '';
};

const prompt = ChatPromptTemplate.fromTemplate(
  'Based on this research, answer the question.\n\n{context}\n\nQuestion: {question}',
);

const chain = RunnableSequence.from([
  {
    context: (input: { question: string }) => research(input.question),
    question: (input: { question: string }) => input.question,
  },
  prompt,
  new ChatOpenAI({ model: 'gpt-4o-mini' }),
  new StringOutputParser(),
]);

const result = await chain.invoke({
  question: 'What are the latest AI safety research findings?',
});

console.log(result);