# robots.txt — generated 2026-07-27
# Source: bot-registry.json (last_reviewed: 2026-04-17)
# Policy: permissive
# Re-generate every 180 days; next review due 2026-10-17.
# Edit data/bot-registry.json (not this file) to change rules.

# OpenAI — search — Powers ChatGPT search results (web browsing). Allow for AI-search visibility.
#   docs: https://platform.openai.com/docs/bots
User-agent: OAI-SearchBot
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /api/
Disallow: /staging/
Disallow: /tmp/
Disallow: /internal/
Allow: /

# OpenAI — training — OpenAI's training crawler. Allowing makes content eligible for future model training.
#   ^ default_recommendation: client_decision (review with client)
#   docs: https://platform.openai.com/docs/bots
User-agent: GPTBot
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /api/
Disallow: /staging/
Disallow: /tmp/
Disallow: /internal/
Allow: /

# OpenAI — user_fetch — Fetches a URL when a user explicitly asks ChatGPT about it.
#   docs: https://platform.openai.com/docs/bots
User-agent: ChatGPT-User
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /api/
Disallow: /staging/
Disallow: /tmp/
Disallow: /internal/
Allow: /

# Anthropic — search — Anthropic's search crawler for Claude with web browsing.
#   docs: https://support.anthropic.com/en/articles/8896518
User-agent: Claude-SearchBot
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /api/
Disallow: /staging/
Disallow: /tmp/
Disallow: /internal/
Allow: /

# Anthropic — training — Anthropic's training crawler.
#   ^ default_recommendation: client_decision (review with client)
#   docs: https://support.anthropic.com/en/articles/8896518
User-agent: ClaudeBot
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /api/
Disallow: /staging/
Disallow: /tmp/
Disallow: /internal/
Allow: /

# Anthropic — user_fetch — Fetches a URL when a user explicitly asks Claude about it.
#   docs: https://support.anthropic.com/en/articles/8896518
User-agent: Claude-User
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /api/
Disallow: /staging/
Disallow: /tmp/
Disallow: /internal/
Allow: /

# Perplexity — search — Perplexity's web crawler for search results.
#   docs: https://docs.perplexity.ai/guides/bots
User-agent: PerplexityBot
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /api/
Disallow: /staging/
Disallow: /tmp/
Disallow: /internal/
Allow: /

# Google — search — Standard Google search crawler. Powers Google AI Overviews and Gemini search responses.
#   docs: https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers
User-agent: Googlebot
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /api/
Disallow: /staging/
Disallow: /tmp/
Disallow: /internal/
Allow: /

# Google — training — Controls whether content is used for training Bard/Gemini and Vertex AI.
#   ^ default_recommendation: client_decision (review with client)
#   docs: https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers
User-agent: Google-Extended
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /api/
Disallow: /staging/
Disallow: /tmp/
Disallow: /internal/
Allow: /

# Microsoft — search — Microsoft Bing's search crawler. Powers Copilot (formerly Bing Chat) responses.
#   docs: https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0
User-agent: Bingbot
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /api/
Disallow: /staging/
Disallow: /tmp/
Disallow: /internal/
Allow: /

# Apple — search — Apple's search crawler. Powers Spotlight, Siri Suggestions, Safari smart search.
#   docs: https://support.apple.com/en-us/119829
User-agent: Applebot
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /api/
Disallow: /staging/
Disallow: /tmp/
Disallow: /internal/
Allow: /

# Apple — training — Controls whether content is used for Apple Intelligence training.
#   ^ default_recommendation: client_decision (review with client)
#   docs: https://support.apple.com/en-us/119829
User-agent: Applebot-Extended
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /api/
Disallow: /staging/
Disallow: /tmp/
Disallow: /internal/
Allow: /

# Common Crawl — training — Common Crawl. Feeds many open LLM training datasets used by Anthropic, Mistral, Meta research and others.
#   ^ default_recommendation: client_decision (review with client)
#   docs: https://commoncrawl.org/ccbot
User-agent: CCBot
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /api/
Disallow: /staging/
Disallow: /tmp/
Disallow: /internal/
Allow: /

# Meta — training — Meta's general-purpose crawler for AI training and tooling.
#   ^ default_recommendation: client_decision (review with client)
#   docs: https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/
User-agent: Meta-ExternalAgent
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /api/
Disallow: /staging/
Disallow: /tmp/
Disallow: /internal/
Allow: /

# Meta — link_preview — Generates link previews for Facebook and Instagram posts.
#   docs: https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/
User-agent: FacebookExternalHit
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /api/
Disallow: /staging/
Disallow: /tmp/
Disallow: /internal/
Allow: /

# Cohere — training — Cohere's training crawler.
#   ^ default_recommendation: client_decision (review with client)
#   docs: https://docs.cohere.com/
User-agent: cohere-ai
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /api/
Disallow: /staging/
Disallow: /tmp/
Disallow: /internal/
Allow: /

# You.com — search — You.com AI search crawler.
#   docs: https://about.you.com/youbot/
User-agent: YouBot
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /api/
Disallow: /staging/
Disallow: /tmp/
Disallow: /internal/
Allow: /

# ByteDance — training — ByteDance crawler. Feeds Doubao and other ByteDance AI products. Material in non-Western mobile-first markets.
#   ^ default_recommendation: client_decision (review with client)
#   docs: https://www.bytedance.com/
User-agent: Bytespider
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /api/
Disallow: /staging/
Disallow: /tmp/
Disallow: /internal/
Allow: /

# Catch-all (fallback for unrecognized bots)
User-agent: *
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /api/
Disallow: /staging/
Disallow: /tmp/
Disallow: /internal/
Allow: /

Sitemap: https://openaeo.dev/sitemap.xml