# AI crawlers we allow — search, grounding, and OpenAI/Anthropic training.
# GPTBot and ClaudeBot are training crawlers: allowing them opts our public
# content into GPT/Claude training, which we accept for brand reach. Google-
# Extended (Gemini grounding + training) is allowed for the same reason. The
# denylist below still blocks the training crawlers we don't want.
User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Google-Extended
User-agent: GPTBot
User-agent: ClaudeBot
Allow: /
Disallow: /.env
Disallow: /.git
Disallow: /.aws
Disallow: /.ssh
Disallow: /api/
Disallow: /app/
Disallow: /forgot-password/

# Model-training crawlers we block
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: cohere-ai
Disallow: /

# Default policy
User-agent: *
Allow: /
Disallow: /.env
Disallow: /.git
Disallow: /.aws
Disallow: /.ssh
Disallow: /api/
Disallow: /app/
Disallow: /forgot-password/

Sitemap: https://www.a2zreach.ai/sitemap.xml