extract_embedded_state

curl -X POST https://crawlforge.dev/api/v1/tools/extract_embedded_state \
  -H "X-API-Key: cf_test_YOUR_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://example.com/products/widget"
  }'

// npm install crawlforge-sdk
import { CrawlForge, ValidationError } from 'crawlforge-sdk';

const client = new CrawlForge({ apiKey: process.env.CRAWLFORGE_API_KEY });

try {
  const result = await client.extractEmbeddedState({
    url: 'https://example.com/products/widget',
    // Dotted keys and array indexes. Not JSONPath: no wildcards or filters.
    path: 'next_data.props.pageProps',
  });

  // result.data is untyped in crawlforge-sdk 0.1 — its shape is the Response Example below.
  const { found, data } = result.data as {
    found: { name: string; variable: string; bytes: number }[];
    data: { product: { price: number } };
  };

  // Every state source on the page, largest first, whether or not it was scoped.
  for (const source of found) {
    console.log(source.name, source.variable, source.bytes);
  }

  // The values are the site's own, so they can be used as they are.
  console.log(data.product.price);
} catch (err) {
  // A path that does not resolve is a 400 naming the keys that were available.
  if (err instanceof ValidationError) {
    console.error(err.message);
  } else {
    throw err;
  }
}

# pip install crawlforge
from crawlforge import CrawlForge

client = CrawlForge()  # reads CRAWLFORGE_API_KEY

# 1. Discover what the page carries. Nothing is truncated, so this can be big.
discovery = client.extract_embedded_state(
    url='https://example.com/products/widget',
)

# discovery.data is a plain dict — its shape is the Response Example below.
for source in discovery.data['found']:
    print(source['name'], source['variable'], source['bytes'])

# This tool's warnings live inside data, next to the state itself.
for warning in discovery.data['warnings']:
    print('warning:', warning)

# 2. Ask again for just the branch you want.
scoped = client.extract_embedded_state(
    url='https://example.com/products/widget',
    path='next_data.props.pageProps.product',
)

print(scoped.data['bytes'], 'bytes')
print(scoped.data['data'])