# mehenot.com — robots.txt
#
# NOTE ON STRUCTURE: a crawler obeys exactly ONE group — the most specific one
# matching its user-agent — and ignores every other group. So a bare
# "User-agent: X / Allow: /" block does not *add* permission, it *replaces* the
# rules below and lets that bot crawl /api/, /og.png and every faceted URL.
#
# That is why the welcome list is only a comment. Every crawler we want is
# already covered by "User-agent: *", which grants access and applies our
# disallow rules at the same time. Only bots we are turning away get their own
# group.
#
# Welcome (via the * group): Googlebot · bingbot · Applebot · BraveBot ·
#   OAI-SearchBot & GPTBot (ChatGPT) · ClaudeBot & anthropic-ai (Claude) ·
#   Google-Extended (Gemini / AI Overviews) · PerplexityBot · CCBot ·
#   Meta-ExternalAgent · Amazonbot · cohere-ai · YouBot · Diffbot

User-agent: *
Allow: /
Disallow: /auth/
Disallow: /api/
Disallow: /_app/
# Rendered per request and never useful to a crawler.
Disallow: /og.png
Disallow: /og-profile.png
# Faceted/paginated URLs multiply into thousands of near-duplicate crawls.
Disallow: /*?page=
Disallow: /*?sort=
Disallow: /*?category=
Disallow: /*?q=
Disallow: /search
Crawl-delay: 10

# ─────────────────────────────────────────────────────────────────────────────
# Blocked: bulk scrapers and commercial SEO crawlers.
#
# These crawl aggressively — often the whole sitemap in one session — and send
# back no visitors and no citations. Every page they fetch is an SSR render
# billed against our compute budget. None of them power a search engine or an
# AI assistant our users actually use.
#
# Blocking these does NOT affect ChatGPT, Claude, Perplexity or Gemini: each of
# those runs its own crawler, and all of them are allowed above.
# ─────────────────────────────────────────────────────────────────────────────

# CCBot (Common Crawl) is allowed via the * group: its dataset trains most LLMs,
# so being in it improves how AI models describe mehenot from memory.

# ByteDance / TikTok. Notoriously aggressive crawl rate.
User-agent: Bytespider
Disallow: /

# SEO tool crawlers — they index us so competitors can read our backlinks.
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: SeekportBot
Disallow: /

# Data brokers / content scrapers.
User-agent: ZoominfoBot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: magpie-crawler
Disallow: /

# Sitemap
Sitemap: https://mehenot.com/sitemap.xml

# AI discoverability: https://mehenot.com/llms.txt
# Machine-readable pricing: https://mehenot.com/pricing.txt