# ==========================================================
# GEO Xray Machine Intelligence Directive
# Standard: Vers 2.1 Final | March 2026
# ==========================================================

# LAYER 1: AI SEARCH & CITATION BOTS (Full Access)
# These bots drive traffic and recommend geoxray.ai to users.
# Explicitly allowing all major AI crawlers for full 12-point robots_txt_ai signal.

User-agent: GPTBot
Allow: /
Allow: /llms.txt
Allow: /robots.txt
Allow: /sitemap.xml

User-agent: ChatGPT-User
Allow: /
Allow: /llms.txt
Allow: /robots.txt
Allow: /sitemap.xml

User-agent: OAI-SearchBot
Allow: /

User-agent: Google-Extended
Allow: /
Allow: /llms.txt
Allow: /robots.txt
Allow: /sitemap.xml

User-agent: Googlebot
Allow: /

User-agent: ClaudeBot
Allow: /
Allow: /llms.txt
Allow: /robots.txt
Allow: /sitemap.xml

User-agent: Claude-Bot
Allow: /
Allow: /llms.txt
Allow: /robots.txt
Allow: /sitemap.xml

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Allow: /robots.txt
Allow: /sitemap.xml

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: anthropic-ai
Allow: /

# LAYER 1b: SCANNER & CRAWL INFRASTRUCTURE (Full Access)
# Our own scanner and Firecrawl must have unrestricted access.

User-agent: Firecrawl
Allow: /

User-agent: FirecrawlBot
Allow: /

# LAYER 2: CONTENT SCRAPING BOTS (Block)
User-agent: CCBot
Disallow: /

User-agent: ByteSpider
Disallow: /

# LAYER 3: DEFAULT ACCESS
# All other bots get full public access except admin routes.
User-agent: *
Allow: /
Allow: /llms.txt
Allow: /robots.txt
Allow: /sitemap.xml
Allow: /faq
Allow: /ssr/rating/
Allow: /ssr/top-ratings
Allow: /ratings/
Disallow: /admin-dashboard-private
Disallow: /supabase-internal

# Sitemap location
Sitemap: https://geoxray.ai/sitemap.xml