# OpenAI – ChatGPT Search and on-demand fetch
User-agent: OAI-SearchBot
Allow: /
User-agent: ChatGPT-User
Allow: /

# OpenAI general crawler (safe to allow)
User-agent: GPTBot
Allow: /

# Perplexity (search + user fetcher)
User-agent: PerplexityBot
Allow: /
User-agent: Perplexity-User
Allow: /   # note: user-triggered fetcher often ignores robots.txt anyway

# Common Crawl (many LLMs train/index from here)
User-agent: CCBot
Allow: /

# Claude / Anthropic
User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

# Training crawler — choose one
User-agent: ClaudeBot
Allow: /       # <-- if you DO want training use

# xAI / Grok (unofficial tokens seen in the wild)
User-agent: GrokAI
Allow: /
User-agent: Grok
Allow: /
User-agent: xAI
Allow: /
# Optional catch-all seen in some lists:
User-agent: GrokBot
Allow: /


Sitemap: https://www.curvedtext.io/sitemap.xml