# AI Agent Robots.txt
# Specific directives for AI agents and crawlers

# OpenAI GPT Agents
User-agent: GPTBot
Allow: /
Allow: /pricing
Allow: /about
Allow: /contact
Allow: /api-docs
Crawl-delay: 1

# ChatGPT Plugin/API
User-agent: ChatGPT-User
Allow: /
Allow: /pricing
Allow: /about
Allow: /contact
Crawl-delay: 1

# Claude (Anthropic)
User-agent: Claude-Web
Allow: /
Allow: /pricing
Allow: /about
Allow: /contact
Crawl-delay: 1

# Google Bard/Gemini
User-agent: Google-Extended
Allow: /
Allow: /pricing
Allow: /about
Allow: /contact
Crawl-delay: 1

# Bing AI/Copilot
User-agent: BingPreview
Allow: /
Allow: /pricing
Allow: /about
Allow: /contact
Crawl-delay: 1

# General AI Agents
User-agent: AI-Agent
Allow: /
Allow: /pricing
Allow: /about
Allow: /contact
Allow: /api-docs
Crawl-delay: 1

# Machine Learning Crawlers
User-agent: ML-Crawler
Allow: /
Allow: /pricing
Allow: /about
Allow: /contact
Crawl-delay: 2

# Automation Tools
User-agent: Automation-Bot
Allow: /
Allow: /pricing
Allow: /about
Allow: /contact
Crawl-delay: 1

# Research Bots
User-agent: Research-Bot
Allow: /
Allow: /pricing
Allow: /about
Allow: /contact
Crawl-delay: 1

# All other agents (fallback)
User-agent: *
Allow: /
Allow: /about
Allow: /pricing
Allow: /contact
Allow: /login
Allow: /signup
Allow: /terms
Allow: /privacy

# Disallow private/protected pages from all agents
Disallow: /account
Disallow: /billing
Disallow: /profile

# AI Agent specific resources
Allow: /ai-robots.txt
Allow: /ai-sitemap.xml
Allow: /api-docs
Allow: /.well-known/ai-plugin.json

# Sitemap locations for AI agents
Sitemap: https://aiformaudit.com/sitemap.xml
Sitemap: https://aiformaudit.com/ai-sitemap.xml

# AI Agent discovery endpoint
AI-Discovery: https://aiformaudit.com/.well-known/ai-plugin.json

# Crawl delay for respectful AI agents
Crawl-delay: 1

# Request rate for AI agents (requests per minute)
Request-rate: 60/1m

# AI Agent contact for integration
AI-Contact: ai-agents@aiformaudit.com