scrape

curl -X POST https://crawlforge.dev/api/v1/tools/scrape \
  -H "X-API-Key: cf_test_YOUR_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://example.com",
    "formats": [
      "markdown", "links", "metadata",
      { "type": "highlights", "query": "professional plan price per month" }
    ],
    "onlyMainContent": true,
    "escalate": true
  }'

// npm install crawlforge-sdk
import { CrawlForge, ToolError } from 'crawlforge-sdk';

const client = new CrawlForge({ apiKey: process.env.CRAWLFORGE_API_KEY });

try {
  const result = await client.scrape({
    url: 'https://example.com',
    formats: [
      'markdown', 'links', 'metadata',
      // 1 extra credit: the matching sentences, table rows and code blocks, verbatim, with offsets
      { type: 'highlights', query: 'professional plan price per month' },
    ],
    onlyMainContent: true,
    // Only when the plain fetch meets a bot wall: one stealth render in this same call (5 extra credits)
    escalate: true,
  });

  // result.data is untyped in crawlforge-sdk 0.1 — its shape is the Response Example below.
  const { formats } = result.data as {
    formats: { markdown: string; links: string[]; metadata: { title: string }; highlights: { text: string }[] };
  };
  console.log('Markdown:', formats.markdown);
  console.log('Links found:', formats.links.length);
  console.log('Title:', formats.metadata.title);
  // Each highlight is verbatim page text; offset/length index formats.markdown.
  console.log('Best match:', formats.highlights[0]?.text);

  // A success with warnings is a PARTIAL result — some formats came back, others didn't.
  if (result.warnings.length > 0) {
    console.warn('Partial result:', result.warnings);
  }

  console.log('Credits used:', result.creditsUsed);
  console.log('Credits remaining:', result.creditsRemaining);
} catch (err) {
  // A bot wall the stealth render could not pass either: not charged, blocked.vendor names it.
  if (err instanceof ToolError && err.blocked) {
    console.error('Blocked by', err.blocked.vendor, '— escalated:', err.escalated);
  } else {
    throw err;
  }
}

# pip install crawlforge
from crawlforge import CrawlForge, ToolError

client = CrawlForge()  # reads CRAWLFORGE_API_KEY

try:
    result = client.scrape(
        url='https://example.com',
        formats=[
            'markdown', 'links', 'metadata',
            # 1 extra credit: the matching sentences, table rows and code blocks, verbatim, with offsets
            {'type': 'highlights', 'query': 'professional plan price per month'},
        ],
        onlyMainContent=True,
        # Only when the plain fetch meets a bot wall: one stealth render in this same call (5 extra credits)
        escalate=True,
    )
except ToolError as e:
    # A bot wall the stealth render could not pass either: not charged, blocked names the vendor.
    print(f"Blocked: {e.blocked} (escalated: {e.escalated})")
    raise

# result.data is a plain dict — its shape is the Response Example below.
formats = result.data['formats']
print(f"Markdown: {formats['markdown']}")
print(f"Links found: {len(formats['links'])}")
print(f"Title: {formats['metadata']['title']}")
# Each highlight is verbatim page text; offset/length index formats['markdown'].
print(f"Best match: {formats['highlights'][0]['text']}")

# A success with warnings is a PARTIAL result — some formats came back, others didn't.
if result.warnings:
    print(f"Partial result: {result.warnings}")

print(f"Credits used: {result.credits_used}")
print(f"Credits remaining: {result.credits_remaining}")