# celestino.ai robots.txt
# AI crawlers welcome with rate limits. Training-only scrapers blocked.

# Default
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/

# OpenAI - ChatGPT retrieval
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /admin/
Crawl-delay: 10

# OpenAI - search
User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Crawl-delay: 5

# OpenAI - ChatGPT browsing
User-agent: ChatGPT-User
Allow: /
Disallow: /api/

# Anthropic - Claude retrieval
User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /admin/
Crawl-delay: 10

# Anthropic - general
User-agent: anthropic-ai
Allow: /
Disallow: /api/
Disallow: /admin/
Crawl-delay: 10

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /api/
Crawl-delay: 5

# Apple
User-agent: Applebot-Extended
Allow: /

# Block training-only crawlers
User-agent: Google-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: meta-externalagent
Disallow: /

Sitemap: https://celestino.ai
/sitemap.xml

# LLM-readable content (English)
# https://celestino.ai
/llms.txt - structured index for AI crawlers
# https://celestino.ai
/llms-full.txt - full content description
# LLM-readable content (Spanish / Español)
# https://celestino.ai
/es/llms.txt - índice estructurado para crawlers de IA
# https://celestino.ai
/es/llms-full.txt - contenido completo en texto plano