# Fleet-wide default robots.txt — disallow known AI training / search
# crawlers for all Ultra Web Hosting customer sites that don't have
# their own robots.txt. Served by nginx (see sentinel-robots-fallback.conf)
# when no customer file exists at $document_root/robots.txt.
#
# Customers who want AI crawlers to access their site can override
# unconditionally by dropping their own robots.txt into /home/<user>/
# public_html/ — nginx's try_files checks their file first, ours is
# only used as fallback.
#
# Last updated: 2026-04-24

# OpenAI
User-agent: GPTBot
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

# Anthropic
User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

# Google (AI training, not core search — Google-Extended is the ONLY
# way to opt out of AI training for Google since Googlebot shares the
# UA string for search and Gemini training).
User-agent: Google-Extended
Disallow: /

# Apple (AI training). Applebot-Extended is the ONLY way to opt out
# of AI training for Apple since Applebot shares the UA for search.
User-agent: Applebot-Extended
Disallow: /

# Perplexity
User-agent: PerplexityBot
Disallow: /

# Amazon
User-agent: Amazonbot
Disallow: /

# ByteDance / TikTok
User-agent: Bytespider
Disallow: /

# Huawei
User-agent: PetalBot
Disallow: /

# Meta (including the AI-specific agent)
User-agent: FacebookBot
Disallow: /

User-agent: MetaExternalAgent
Disallow: /

# Commercial crawlers we've seen scraping large volumes
User-agent: DotBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /