# CrawlForge Robots.txt
# https://www.crawlforge.dev
# Last reviewed: 2026-08-23

User-agent: *
Allow: /
Allow: /api/og
Allow: /api/logo
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

# Content usage signals (https://contentsignals.org) — CrawlForge welcomes AI
# training, AI search indexing, and AI inference/grounding on its public content.
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# --- AI Search & Retrieval Bots (drive citations in AI search results) ---

User-agent: OAI-SearchBot
Allow: /
Allow: /api/og
Allow: /api/logo
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

User-agent: ChatGPT-User
Allow: /
Allow: /api/og
Allow: /api/logo
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

User-agent: Claude-SearchBot
Allow: /
Allow: /api/og
Allow: /api/logo
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

User-agent: Claude-User
Allow: /
Allow: /api/og
Allow: /api/logo
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

User-agent: PerplexityBot
Allow: /
Allow: /api/og
Allow: /api/logo
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

User-agent: Perplexity-User
Allow: /
Allow: /api/og
Allow: /api/logo
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

User-agent: DuckAssistBot
Allow: /
Allow: /api/og
Allow: /api/logo
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

User-agent: DuckDuckBot
Allow: /
Allow: /api/og
Allow: /api/logo
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

User-agent: MistralAI-User
Allow: /
Allow: /api/og
Allow: /api/logo
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

User-agent: Applebot
Allow: /
Allow: /api/og
Allow: /api/logo
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

User-agent: Google-CloudVertexBot
Allow: /
Allow: /api/og
Allow: /api/logo
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

# --- AI Training Bots ---

User-agent: GPTBot
Allow: /
Allow: /api/og
Allow: /api/logo
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

User-agent: ClaudeBot
Allow: /
Allow: /api/og
Allow: /api/logo
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

User-agent: Amazonbot
Allow: /
Allow: /api/og
Allow: /api/logo
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

User-agent: Meta-ExternalAgent
Allow: /
Allow: /api/og
Allow: /api/logo
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

User-agent: CCBot
Allow: /
Allow: /api/og
Allow: /api/logo
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

User-agent: cohere-ai
Allow: /
Allow: /api/og
Allow: /api/logo
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

User-agent: Bytespider
Allow: /
Allow: /api/og
Allow: /api/logo
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

# --- Opt-out control tokens (not crawlers; they make no requests) ---
# A bare "Allow: /" is the opt-in signal for AI training/grounding.

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

Sitemap: https://www.crawlforge.dev/sitemap.xml

# IndexNow key for fast search engine indexation
# https://www.crawlforge.dev/b9b567a2380d9e4203de3e503333a1aa.txt

# AI-friendly content files
# See https://llmstxt.org for specification
# llms.txt: https://www.crawlforge.dev/llms.txt
# llms-ctx.txt: https://www.crawlforge.dev/llms-ctx.txt
# llms-full.txt: https://www.crawlforge.dev/llms-full.txt
# ai.txt: https://www.crawlforge.dev/ai.txt