User-agent: *
Allow: /

# Block private / user-specific / admin URLs
Disallow: /admin/
Disallow: /studio/
Disallow: /settings/
Disallow: /calendar/feed/
Disallow: /subscription/
Disallow: /notifications/
Disallow: /api-keys/
Disallow: /invoices/
Disallow: /storage/
Disallow: /vendor/

# Block JSON-LD API endpoints from general crawlers (AI crawlers allowed below)
Disallow: /api/

# Internal API endpoints
Disallow: /api/v1/internal/

# --- Search engines (explicitly OK) ---
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /
Crawl-delay: 5

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

# --- LLM / AI crawlers (explicit allow including /api/) ---
User-agent: GPTBot            # OpenAI
Allow: /
Allow: /api/

User-agent: ChatGPT-User      # ChatGPT browsing user agent token
Allow: /
Allow: /api/

User-agent: ClaudeBot         # Anthropic
Allow: /
Allow: /api/

User-agent: Claude-Web        # Anthropic web crawler
Allow: /
Allow: /api/

User-agent: PerplexityBot     # Perplexity
Allow: /
Allow: /api/

User-agent: Google-Extended   # Google AI data-use control
Allow: /
Allow: /api/

User-agent: Applebot-Extended # Apple AI data-use control
Allow: /
Allow: /api/

User-agent: CCBot             # Common Crawl (feeds many AI systems)
Allow: /
Allow: /api/

# --- Block aggressive scrapers ---
User-agent: Bytespider
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# --- AI Discovery Files ---
# LLMs.txt files (lightweight and full content reference for AI)
Allow: /llms.txt
Allow: /llms-full.txt

Sitemap: https://www.quodat.com/index_sitemap.xml