# Search engines and inspection tools MUST be able to fetch faceted hub URLs; otherwise they can never
# read the `noindex, nofollow` those URLs serve (src/lib/seo/query-identity.ts) and the URLs stay stuck
# as "Indexed, though blocked by robots.txt". GTA Online and GTA 6 vehicle hubs use KV list snapshots, so
# search engines stay open to read noindex. Story vehicles have no KV list snapshot yet (uncached D1), so that
# facet space stays closed for everyone until the snapshot is enabled.
User-agent: *
Allow: /
Disallow: /gta-5/vehicles?

# AI-training / SEO-tool crawlers plus the July 2026 facet enumerators: the hub query space is closed;
# the bare hubs and detail pages carry the content. Google-Extended and Applebot-Extended are training
# control tokens, not crawlers (facet Disallows under them are no-ops; kept so training opt-out for the
# facet space stays explicit). Crawl-delay is advisory (ignored by Google, honored by some).
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Applebot-Extended
User-agent: Google-Extended
User-agent: meta-externalagent
User-agent: meta-webindexer
User-agent: meta-externalads
User-agent: Amazonbot
User-agent: Bytespider
User-agent: PetalBot
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: DataForSeoBot
User-agent: SleepBot
Allow: /
Disallow: /gta-5-online/vehicles?
Disallow: /gta-5/vehicles?
Disallow: /gta-6/vehicles?
Disallow: /gta-5-online/properties?
Disallow: /gta-5-online/weapons?
Crawl-delay: 10

Sitemap: https://www.gtaboss.gg/sitemap.xml