# RackDiff robots.txt
# Content Signal spec: https://rslstandard.org/

User-Agent: *
Allow: /
Disallow: /api
Content-Signal: search=yes, ai-input=yes, ai-train=no

# Retrieval-Crawler (live AI grounding — search product visibility)
User-Agent: GPTBot
Allow: /

User-Agent: ClaudeBot
Allow: /

User-Agent: PerplexityBot
Allow: /

User-Agent: Google-Extended
Allow: /

User-Agent: OAI-SearchBot
Allow: /

User-Agent: ChatGPT-User
Allow: /

User-Agent: Claude-Web
Allow: /

User-Agent: Claude-User
Allow: /

# Training-only crawlers (no retrieval value — block to protect content)
User-Agent: CCBot
Disallow: /

User-Agent: Bytespider
Disallow: /

User-Agent: meta-externalagent
Disallow: /

User-Agent: Amazonbot
Disallow: /

User-Agent: Applebot-Extended
Disallow: /

User-Agent: cohere-ai
Disallow: /

User-Agent: Diffbot
Disallow: /

Sitemap: https://rackdiff.com/sitemap.xml