# https://benharris.ai/robots.txt
#
# Two goals at once:
#   1. Maximum visibility in search engines and AI answer engines.
#   2. No use of this site's content to train AI models.
#
# Both are possible because training crawlers and retrieval crawlers are
# separate user agents. GPTBot is not OAI-SearchBot, ClaudeBot is not
# Claude-SearchBot, and Amazonbot is not Amzn-SearchBot. Blocking the first
# of each pair opts out of training; the second still cites us in answers.
#
# Keep the groups below in sync with vendor docs when tokens change.


# ---------------------------------------------------------------------------
# AI model training: not permitted.
# Every agent below is documented as collecting content for foundation-model
# training or for datasets that are resold as training corpora.
# ---------------------------------------------------------------------------

# OpenAI: training crawler. OAI-SearchBot and ChatGPT-User stay allowed.
User-agent: GPTBot
Disallow: /

# Anthropic: training crawler, plus two legacy tokens.
# Claude-SearchBot and Claude-User stay allowed.
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
Disallow: /

# Google: usage-control token for Gemini and Vertex model training.
# It is not a crawler and does not affect Googlebot, Search ranking, or
# eligibility for AI Overviews (those cannot be opted out of separately).
User-agent: Google-Extended
Disallow: /

# Apple: usage-control token for Apple Intelligence training.
# Applebot itself stays allowed, so Siri and Spotlight still index the site.
User-agent: Applebot-Extended
Disallow: /

# Meta: training crawler. meta-externalfetcher (user-initiated) stays allowed.
# FacebookBot trains speech models; facebookexternalhit (link previews) is
# a different agent and stays allowed.
User-agent: meta-externalagent
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
Disallow: /

# Amazon: training crawler. Amzn-SearchBot (Alexa, Rufus) and Amzn-User
# do not crawl for training and stay allowed.
User-agent: Amazonbot
Disallow: /

# ByteDance: undocumented, feeds Doubao training. Often ignores robots.txt.
User-agent: Bytespider
Disallow: /

# Common Crawl: the corpus most open models are trained on.
User-agent: CCBot
Disallow: /

# Other model trainers and dataset resellers.
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: Diffbot
User-agent: Omgili
User-agent: Omgilibot
User-agent: Webzio-Extended
User-agent: ImagesiftBot
User-agent: PanguBot
User-agent: Timpibot
User-agent: SemrushBot-OCOB
User-agent: Scrapy
Disallow: /


# ---------------------------------------------------------------------------
# Everyone else: full access. Nothing here is gated or paywalled.
#
# This group is what keeps the site visible, and it deliberately covers:
#   Search        Googlebot, bingbot, Applebot, DuckDuckBot, Slurp, PetalBot
#   Answer        OAI-SearchBot, Claude-SearchBot, PerplexityBot,
#                 DuckAssistBot, Amzn-SearchBot, YouBot,
#                 Google-CloudVertexBot
#   On demand     ChatGPT-User, Claude-User, Perplexity-User, MistralAI-User,
#                 Amzn-User, meta-externalfetcher
#   Previews      Twitterbot, facebookexternalhit, LinkedInBot, Slackbot,
#                 Discordbot, WhatsApp, TelegramBot
#
# /api holds POST-only Azure Functions for the contact and newsletter forms.
# There is nothing to crawl there. Note that /404.html is intentionally left
# crawlable so its noindex tag can be read.
# ---------------------------------------------------------------------------

User-agent: *
Allow: /
Disallow: /api/

Sitemap: https://benharris.ai/sitemap-index.xml