extract_embedded_state

curl -X POST https://crawlforge.dev/api/v1/tools/extract_embedded_state \
  -H "X-API-Key: cf_test_YOUR_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://example.com/products/widget"
  }'

const response = await fetch('https://crawlforge.dev/api/v1/tools/extract_embedded_state', {
  method: 'POST',
  headers: {
    'X-API-Key': process.env.CRAWLFORGE_API_KEY!,
    'Content-Type': 'application/json',
  },
  body: JSON.stringify({
    url: 'https://example.com/products/widget',
    // Dotted keys and array indexes. Not JSONPath: no wildcards or filters.
    path: 'next_data.props.pageProps',
  }),
});

const payload = await response.json();
if (!response.ok) {
  // A path that does not resolve is a 400 naming the keys that were available.
  throw new Error(payload.error.code + ': ' + payload.error.message);
}

const result = payload.data;

// Every state source on the page, largest first, whether or not it was scoped.
for (const source of result.found) {
  console.log(source.name, source.variable, source.bytes);
}

// The values are the site's own, so they can be used as they are.
console.log(result.data.product.price);

import os
import requests

URL = 'https://crawlforge.dev/api/v1/tools/extract_embedded_state'
HEADERS = {
    'X-API-Key': os.environ['CRAWLFORGE_API_KEY'],
    'Content-Type': 'application/json',
}

# 1. Discover what the page carries. Nothing is truncated, so this can be big.
discovery = requests.post(
    URL,
    headers=HEADERS,
    json={'url': 'https://example.com/products/widget'},
).json()

for source in discovery['data']['found']:
    print(source['name'], source['variable'], source['bytes'])

for warning in discovery['data']['warnings']:
    print('warning:', warning)

# 2. Ask again for just the branch you want.
scoped = requests.post(
    URL,
    headers=HEADERS,
    json={
        'url': 'https://example.com/products/widget',
        'path': 'next_data.props.pageProps.product',
    },
)

payload = scoped.json()
if not scoped.ok:
    error = payload['error']
    raise RuntimeError(f"{error['code']}: {error['message']}")

print(payload['data']['bytes'], 'bytes')
print(payload['data']['data'])