# robots.txt for https://rye.dev
# Last updated: May 2026

# Allow all crawlers full access to the site
User-agent: *
# Content Signals — declare AI usage preferences (contentsignals.org,
# draft-romm-aipref-contentsignals). ai-train=no: don't train on this
# content. search=yes: index for search. ai-input=yes: allow agent
# retrieval at query time (RAG, summarisation, MCP).
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /

# Sitemap location
Sitemap: https://rye.dev/sitemap-index.xml

# LLM-friendly documentation (llms.txt specification)
# See: https://llmstxt.org/
# llms.txt: Concise overview with links to key resources
# llms-full.txt: Complete content inlined for large context windows

# Disallow crawling of build artifacts and internal paths
Disallow: /_astro/
Disallow: /pagefind/
Disallow: /api/
Disallow: /admin/
Disallow: /admin
Disallow: /login

# Public agent-discovery + tool endpoints — explicitly re-allowed inside
# the /api/ disallow so RFC 9727 catalog entries remain reachable.
Allow: /api/health
Allow: /api/contact
Allow: /api/newsletter
Allow: /api/reactions
Allow: /api/mentions
Allow: /api/pulse
Allow: /api/agent-observatory
Allow: /api/webmention
Allow: /mcp
Allow: /openapi.json

# Allow search engine image crawlers
User-agent: Googlebot-Image
Allow: /images/
Allow: /og/
Allow: /screenshots/

# Allow social media crawlers for rich previews
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

# Crawl-delay for less aggressive bots (optional, not all bots respect this)
# User-agent: *
# Crawl-delay: 1