User-agent: *
Allow: /

# Internal site search — never index
Disallow: /search/
Disallow: /search?
Disallow: /*?q=
Disallow: /*?query=

# API endpoints — not for users
Disallow: /api/
Disallow: /_api/

# Draft pages — quality gate failures land here
Disallow: /draft/

# Embeds — used as iframes elsewhere, not standalone pages
Disallow: /embed/

# Faceted filter combinations (any combination of 2+ filter params on listings)
Disallow: /*?*filter=*&*filter=*
Disallow: /*?*sort=*
Disallow: /*?*page=*&*filter=*

# Tracking params (already canonicalized but block crawl too)
Disallow: /*?utm_*
Disallow: /*?gclid=
Disallow: /*?fbclid=
Disallow: /*?mc_*

# Print/email-friendly versions (if generated)
Disallow: /*?print=
Disallow: /*?email=

# AMP/mobile variants (reserved, not used yet)
Disallow: /amp/

# Cloudflare admin / preview URLs
Disallow: /cdn-cgi/
Disallow: /__/preview/

# Sensitive system endpoints
Disallow: /admin/
Disallow: /.well-known/
Allow: /.well-known/security.txt
Allow: /.well-known/ai-plugin.json

# Sitemap location
Sitemap: https://aicost.tools/sitemap-index.xml

# Curated LLM-friendly site index
# https://aicost.tools/llms.txt

# AI training crawlers — explicit allow (citation-positive content strategy)
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: anthropic-ai
Allow: /