# Allow all standard search engines
User-agent: Googlebot
Disallow:

User-agent: Bingbot
Disallow:

User-agent: Slurp
Disallow:

User-agent: DuckDuckBot
Disallow:

User-agent: Baiduspider
Disallow:

# Allow AI bots
User-agent: anthropic-ai
Disallow:
Crawl-delay: 10

User-agent: ChatGPT-User
Disallow:
Crawl-delay: 10

User-agent: ClaudeBot
Disallow:
Crawl-delay: 10

User-agent: Claude-Web
Disallow:
Crawl-delay: 10

User-agent: cohere-ai
Disallow:
Crawl-delay: 10

User-agent: Google-Extended
Disallow:
Crawl-delay: 10

User-agent: GPTBot
Disallow:
Crawl-delay: 10

User-agent: Meta-ExternalAgent
Disallow:
Crawl-delay: 10

User-agent: NeevaBot
Disallow:
Crawl-delay: 10

User-agent: PerplexityBot
Disallow:
Crawl-delay: 10

# All other bots (including standard search engines)
User-agent: *
Crawl-delay: 5

# The following bots are disallowed due to aggressive crawling, bandwidth/resource concerns, or non-compliance with site policies.
User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: omgiliBot
Disallow: /

User-agent: YandexBot
Disallow: /


# Sitemaps
Sitemap: https://www.michelobultra.com/sitemap-index.xml

# AI / LLM policy files
AI: https://www.michelobultra.com/ai.txt
LLM: https://www.michelobultra.com/llms.txt
LLM: https://www.michelobultra.com/llms-full.txt
LLM: https://www.michelobultra.com/llms.json
LLM: https://www.michelobultra.com/llms.yaml