LlamaIndex Integration

npm install llamaindex

import { Document } from 'llamaindex';

const CRAWLFORGE_API = 'https://www.crawlforge.dev/api/v1/tools';

/** Every CrawlForge tool is a POST to /api/v1/tools/<tool_name>. */
export async function callTool<T>(
  tool: string,
  params: Record<string, unknown>,
): Promise<T> {
  const response = await fetch(`${CRAWLFORGE_API}/${tool}`, {
    method: 'POST',
    headers: {
      'X-API-Key': process.env.CRAWLFORGE_API_KEY!,
      'Content-Type': 'application/json',
    },
    body: JSON.stringify(params),
  });

  const payload = await response.json();
  if (!payload.success) {
    throw new Error(payload.error?.message ?? 'CrawlForge request failed');
  }
  return payload.data as T;
}

interface ExtractedContent {
  url: string;
  title: string;
  content: string;
  author: string | null;
  publish_date: string | null;
}

/** Reads each URL through extract_content (2 credits per URL). */
export class CrawlForgeReader {
  async loadData(urls: string[]): Promise<Document[]> {
    return Promise.all(
      urls.map(async (url) => {
        const data = await callTool<ExtractedContent>('extract_content', { url });
        return new Document({
          text: data.content,
          id_: data.url,
          metadata: {
            source: data.url,
            title: data.title,
            author: data.author,
            publishedAt: data.publish_date,
          },
        });
      }),
    );
  }
}

// ---------- usage ----------

const reader = new CrawlForgeReader();

const documents = await reader.loadData([
  'https://example.com/page1',
  'https://example.com/page2',
]);

console.log(documents[0].text);
console.log(documents[0].metadata.title);

import { OpenAIEmbedding, VectorStoreIndex } from 'llamaindex';
import { CrawlForgeReader } from './crawlforge';

// 1. Load documents from the web (2 credits per URL)
const documents = await new CrawlForgeReader().loadData([
  'https://example.com/doc1',
  'https://example.com/doc2',
  'https://example.com/doc3',
]);

// 2. Build the vector index
const index = await VectorStoreIndex.fromDocuments(documents, {
  embedModel: new OpenAIEmbedding({ model: 'text-embedding-3-small' }),
});

// 3. Query it
const queryEngine = index.asQueryEngine();
const response = await queryEngine.query({
  query: 'What are the main topics covered?',
});

console.log(response.toString());

import { FunctionTool } from 'llamaindex';
import { callTool } from './crawlforge';

interface SearchResults {
  results: { title: string; url: string; snippet: string }[];
}

export const searchWeb = FunctionTool.from(
  async ({ query, limit }: { query: string; limit?: number }) => {
    const data = await callTool<SearchResults>('search_web', { query, limit });
    return JSON.stringify(data.results);
  },
  {
    name: 'search_web',
    description: 'Search the web and return ranked results. Costs 5 credits.',
    parameters: {
      type: 'object',
      properties: {
        query: { type: 'string', description: 'What to search for' },
        limit: { type: 'number', description: 'How many results, 1-100' },
      },
      required: ['query'],
    },
  },
);

export const readPage = FunctionTool.from(
  async ({ url }: { url: string }) => {
    const data = await callTool<{ title: string; content: string }>(
      'extract_content',
      { url },
    );
    return `${data.title}\n\n${data.content}`;
  },
  {
    name: 'extract_content',
    description: 'Extract the main article text from one URL. Costs 2 credits.',
    parameters: {
      type: 'object',
      properties: {
        url: { type: 'string', description: 'The page to read' },
      },
      required: ['url'],
    },
  },
);

// Pass [searchWeb, readPage] to whichever agent your llamaindex version
// exposes — the tool objects are the same either way.

import { Document } from 'llamaindex';
import { callTool } from './crawlforge';
import { CrawlForgeReader } from './crawlforge';

interface SearchResults {
  results: { title: string; url: string; snippet: string }[];
}

/**
 * Search the web, then read the top hits.
 * Costs 5 credits for the search plus 2 per page read.
 */
export class WebRetriever {
  private readonly reader = new CrawlForgeReader();

  constructor(private readonly topK = 3) {}

  async retrieve(query: string): Promise<Document[]> {
    const { results } = await callTool<SearchResults>('search_web', {
      query,
      limit: this.topK,
    });

    return this.reader.loadData(results.map((result) => result.url));
  }
}

// ---------- usage ----------

const documents = await new WebRetriever().retrieve('latest AI safety research');

console.log(`Retrieved ${documents.length} documents`);

// Feed them into an index when you want scored, chunk-level retrieval:
// const index = await VectorStoreIndex.fromDocuments(documents);

import { VectorStoreIndex } from 'llamaindex';
import { CrawlForgeReader } from './crawlforge';

const urls = [
  'https://example.com/1',
  'https://example.com/2',
  'https://example.com/3',
  'https://example.com/4',
  'https://example.com/5',
  'https://example.com/6',
];

// Cap concurrency so you stay inside your plan's requests-per-second limit
// (Free 1/s, Hobby 2/s, Professional 4/s, Business 10/s).
const CONCURRENCY = 2;
const reader = new CrawlForgeReader();

const batches = [];
for (let i = 0; i < urls.length; i += CONCURRENCY) {
  batches.push(urls.slice(i, i + CONCURRENCY));
}

const documents = [];
for (const batch of batches) {
  documents.push(...(await reader.loadData(batch)));
}

console.log(`Loaded ${documents.length} documents`);

const index = await VectorStoreIndex.fromDocuments(documents);

console.log('Index created successfully');