# ─── Default: allow all crawlers ───
User-agent: *
# Content Signals (contentsignals.org / draft-romm-aipref-contentsignals):
# search=yes   — index us for traditional + AI search results (we want citations)
# ai-input=yes — fine to use our pages as grounding input for AI answers (with citation)
# ai-train=no  — do not use our content to train foundation models
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /booking-confirmed/
Disallow: /book/v2/
Disallow: /book/v3/

# ─── AI Search Crawlers — allow to be cited ───
# GPTBot (ChatGPT search results + AI training)
User-agent: GPTBot
Allow: /

# ChatGPT user-facing browsing
User-agent: ChatGPT-User
Allow: /

# Google AI (Gemini, AI Overviews)
User-agent: Google-Extended
Allow: /

# Perplexity AI search
User-agent: PerplexityBot
Allow: /

# Anthropic (Claude)
User-agent: anthropic-ai
Allow: /
User-agent: Claude-Web
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /
User-agent: meta-externalagent
Allow: /

# Apple intelligence
User-agent: Applebot-Extended
Allow: /

# Microsoft Copilot / Bing Chat
User-agent: bingbot
Allow: /

# ─── Archive / corpus crawlers — allowed (Wayback Machine + Common Crawl) ───
# Common Crawl (CCBot) feeds the Wayback Machine and the corpora many AI search
# and agent systems read; blocking it removed us from all of them. Allowed for
# discoverability, consistent with our agent-readiness surface (llms.txt,
# AGENTS.md, api-catalog). Training preference is expressed via the
# Content-Signal header above (ai-train=no), not by blocking the crawler.
User-agent: CCBot
Allow: /

User-agent: GPTBot-Historical
Allow: /

# ─── Sitemaps ───
Sitemap: https://www.ellomind.com/sitemap.xml
# (The blog RSS feed at /blog/feed.xml is NOT declared as a Sitemap: it is an
# RSS document, not sitemap XML, and every blog post is already listed in
# sitemap.xml. It stays discoverable via <link rel="alternate" type="rss">.)