generate_llms_txt

curl -X POST https://crawlforge.dev/api/v1/tools/generate_llms_txt \
  -H "X-API-Key: cf_test_YOUR_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://example.com",
    "format": "both",
    "complianceLevel": "standard",
    "outputOptions": {
      "organizationName": "Example Inc.",
      "contactEmail": "ai@example.com"
    }
  }'

// npm install crawlforge-sdk
import { CrawlForge } from 'crawlforge-sdk';
import fs from 'node:fs/promises';

const client = new CrawlForge({ apiKey: process.env.CRAWLFORGE_API_KEY });

const result = await client.generateLlmsTxt({
  url: 'https://docs.example.com',
  format: 'both',
  complianceLevel: 'strict',
  analysisOptions: {
    maxDepth: 4,
    maxPages: 250,
    detectAPIs: true,
    analyzeContent: true,
  },
  outputOptions: {
    organizationName: 'Example Inc.',
    contactEmail: 'ai@example.com',
    customGuidelines: [
      'AI crawlers must respect robots.txt',
      'Cache responses for up to 24 hours',
    ],
    customRestrictions: [
      'No training on user-submitted content',
    ],
    includeAnalysis: true,
  },
});

// result.data is untyped in crawlforge-sdk 0.1 — its shape is the Response Example below.
const { llms_txt, llms_full_txt } = result.data as { llms_txt: string; llms_full_txt: string };
await fs.writeFile('public/llms.txt', llms_txt);
await fs.writeFile('public/llms-full.txt', llms_full_txt);

# pip install crawlforge
from crawlforge import CrawlForge

client = CrawlForge()  # reads CRAWLFORGE_API_KEY

result = client.generate_llms_txt(
    url='https://example.com',
    format='llms-txt',
    complianceLevel='basic',
)

# result.data is a plain dict — its shape is the Response Example below.
with open('public/llms.txt', 'w') as f:
    f.write(result.data['llms_txt'])