# rhnfzl.com
#
# Training crawlers are blocked; search, retrieval and assistant fetchers are
# allowed. Being findable is the point, being training data is not.
#
# Source of truth: docs/research/unknowns/ai-crawl-optout.md (section 4), with
# its post-review correction adding use=reference to the Content-Signal line.
# This hand-owned file is the ONE authority: do not also enable Cloudflare's
# managed robots.txt, the two would drift.

# Search engines, full access
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# AI search / retrieval / assistant bots, allowed: this is what being cited in
# an AI answer depends on
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Applebot
Allow: /

# AI training crawlers, blocked
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: cohere-ai
Disallow: /

# Default for everything else. ai-input=yes deliberately permits live citation
# in answer to a real question, while ai-train=no still refuses foundation
# model training.
User-agent: *
Content-Signal: ai-train=no, search=yes, ai-input=yes, use=reference
Allow: /

Sitemap: https://rhnfzl.com/sitemap-index.xml