User-agent: *
Allow: /

# Allow all main pages to be indexed
Allow: /pricing
Allow: /login
Allow: /account
Allow: /privacy-policy
Allow: /terms-of-service
Allow: /refund-policy
Allow: /sitemap

# Disallow API endpoints and authentication callbacks
Disallow: /api/
Disallow: /auth/callback

# Disallow audio files to reduce bandwidth usage
Disallow: /audio/

# ================================
# AI Crawler Specific Rules - Optimized Based on Real Data
# ================================

# OpenAI Crawlers
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /pricing
Allow: /privacy-policy
Allow: /terms-of-service
Allow: /refund-policy
Disallow: /api/
Disallow: /auth/callback

# Anthropic Crawlers (Most responsive to optimization based on data)
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Anthropic-AI
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /pricing
Allow: /privacy-policy
Allow: /terms-of-service
Allow: /refund-policy
Disallow: /api/
Disallow: /auth/callback

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/

# Google AI (Bard/Gemini)
User-agent: GoogleOther
User-agent: Google-Extended
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/

# ByteDance AI
User-agent: Bytespider
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/

# Amazon AI
User-agent: Amazonbot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/

# Apple AI
User-agent: Applebot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/

# DuckDuckGo AI
User-agent: DuckAssistBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/

# Meta AI
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/

# Sitemap location
Sitemap: https://white-noise-generator.com/sitemap.xml

# AI-specific content guidance
# Guide AI crawlers to structured content files
# llms.txt: Concise version for quick understanding
# llms-full.txt: Detailed version providing complete information

# Crawl delay (server-friendly)
Crawl-delay: 1