# Rankry - robots.txt
# Explicit allowances for AI search crawlers (citation bots).
# Updated: 2026-05-01

# --- Content-Signal (Cloudflare-proposed AI usage declaration) ---
# Declares stance on search indexing, AI training, and AI input grounding.
# Bots that don't recognize this directive ignore it (standard robots.txt
# behavior for unknown directives). Maximizes AI visibility for citations.

Content-Signal: search=yes, ai-train=yes, ai-input=yes

# --- AI search citation bots (explicit allow) ---
# These bots fetch live URLs to cite sources in AI-generated answers.
# Blocking = no citations from that platform.

User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /app/
Disallow: /admin/
Disallow: /sign_up
Disallow: /sign_in

# --- Training-only crawlers (blocked) ---
# These feed training data but are not citation sources.
# Blocking prevents training on our content without losing AI citations.

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

# --- All other bots (default) ---

User-agent: *
Allow: /
Disallow: /api/
Disallow: /app/
Disallow: /admin/
Disallow: /sign_up
Disallow: /sign_in

Sitemap: https://rankry.ai/sitemap.xml
Sitemap: https://rankry.ai/blog/sitemap-index.xml