# TurboMax AI — https://turbomaxai.com
#
# Search crawlers and AI answer-engine crawlers are BOTH welcome. Being cited by
# ChatGPT, Claude, Perplexity and Google AI Overviews is a distribution channel,
# not a threat — the product is a paid service, so an assistant explaining what
# it does and what it costs is doing our marketing for us.
#
# Note the distinction some operators miss:
#   * "search/answer" crawlers index the site so it can be cited with a link.
#   * "training" crawlers collect text for model training.
# Both are allowed here. If you ever want to keep the content out of TRAINING
# while staying citable, disallow Google-Extended, Applebot-Extended and
# CCBot below, and leave the search/answer agents allowed.

# ---------------------------------------------------------------------------
# Everyone: the defaults
# ---------------------------------------------------------------------------
User-agent: *
Allow: /

# Private, per-user or privileged surfaces. Nothing here is useful in an index,
# and all of it is auth-gated anyway — this just avoids wasting crawl budget.
Disallow: /admin
Disallow: /account
Disallow: /projects
Disallow: /api/
Disallow: /uploads/
Disallow: /?checkout=
Disallow: /*?recovery=
Disallow: /*?verified=

# ---------------------------------------------------------------------------
# AI answer engines — explicitly allowed so there is no ambiguity
# ---------------------------------------------------------------------------

# OpenAI: ChatGPT browsing + the OpenAI search index
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

# Anthropic: Claude's search + browsing
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google: Gemini / AI Overviews grounding
User-agent: Google-Extended
Allow: /

# Microsoft Copilot / Bing
User-agent: bingbot
Allow: /

# Meta AI
User-agent: meta-externalagent
Allow: /

# Apple Intelligence
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Common Crawl — feeds many downstream datasets and search tools
User-agent: CCBot
Allow: /

# Amazon
User-agent: Amazonbot
Allow: /

# You.com, Mistral, Cohere and similar
User-agent: YouBot
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: cohere-ai
Allow: /

# ---------------------------------------------------------------------------
# Not welcome: SEO scrapers that only cost bandwidth
# ---------------------------------------------------------------------------
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# ---------------------------------------------------------------------------
Sitemap: https://turbomaxai.com/sitemap.xml