# ==========================================================================
# robots.txt — Optimized for AI Search Citations & Answer Engine Grounding
# Generated by CiteContext (https://citecontext.com)
# ==========================================================================

User-agent: *
Allow: /
Disallow: /client/
Disallow: /dashboard
Disallow: /settings/
Disallow: /admin/
Disallow: /api/
Disallow: /auth/
Disallow: /checkout/simulate
Disallow: /checkout/success

# --- REAL-TIME AI CITATION BOTS (Required for AI Answer Engines) ---
# ChatGPT Search real-time web citations
User-agent: OAI-SearchBot
Allow: /

# ChatGPT user-initiated browsing & link fetches
User-agent: ChatGPT-User
Allow: /

# Anthropic Claude web search & source citations
User-agent: Claude-SearchBot
Allow: /

# Claude user-initiated fetches of shared links
User-agent: Claude-User
Allow: /

# Perplexity AI indexing & citation generation
User-agent: PerplexityBot
Allow: /

# Perplexity user-initiated answer fetches
User-agent: Perplexity-User
Allow: /

# Google Search & Google AI Overviews
User-agent: Googlebot
Allow: /

# Apple Intelligence & Siri search synthesis
User-agent: Applebot
Allow: /

# Meta AI answers across Facebook, WhatsApp & Instagram
User-agent: Meta-ExternalAgent
Allow: /

# Alexa & Amazon AI answer grounding
User-agent: Amazonbot
Allow: /

# --- OFFLINE AI MODEL TRAINING CRAWLERS ---
# OpenAI model training
User-agent: GPTBot
Disallow: /

# Anthropic model training
User-agent: ClaudeBot
Disallow: /

# Google Gemini training opt-out
User-agent: Google-Extended
Disallow: /

# ByteDance/TikTok model training
User-agent: Bytespider
Disallow: /

# Common Crawl dataset
User-agent: CCBot
Disallow: /

# Cohere foundation training
User-agent: cohere-ai
Disallow: /

Sitemap: https://citecontext.com/sitemap.xml
LLMs-Txt: https://citecontext.com/llms.txt