User-agent: *
Allow: /
Disallow: /admin
Disallow: /api/

# AI crawlers — explicitly welcomed for AI search / Overview citation surface.
# Each bot gets its own block so future per-bot tuning (e.g. crawl-delay) is trivial.

# OpenAI training crawler (used to improve future ChatGPT models)
User-agent: GPTBot
Allow: /
Disallow: /admin
Disallow: /api/

# OpenAI search crawler (powers ChatGPT Search live results)
User-agent: OAI-SearchBot
Allow: /
Disallow: /admin
Disallow: /api/

# Anthropic crawler (Claude training + Claude.ai citations)
User-agent: ClaudeBot
Allow: /
Disallow: /admin
Disallow: /api/

# Anthropic user-triggered fetch (when a Claude user pastes a URL)
User-agent: Claude-User
Allow: /
Disallow: /admin
Disallow: /api/

# Legacy Anthropic crawler (older user-agent string, predates ClaudeBot)
User-agent: anthropic-ai
Allow: /
Disallow: /admin
Disallow: /api/

# Perplexity crawler (powers Perplexity AI answers + citations)
User-agent: PerplexityBot
Allow: /
Disallow: /admin
Disallow: /api/

# Perplexity user-triggered fetch
User-agent: Perplexity-User
Allow: /
Disallow: /admin
Disallow: /api/

# Google's AI / Gemini training crawler (separate from Googlebot, opt-in)
User-agent: Google-Extended
Allow: /
Disallow: /admin
Disallow: /api/

# Common Crawl (broadly used as training data + cited by some AI tools)
User-agent: CCBot
Allow: /
Disallow: /admin
Disallow: /api/

Sitemap: https://skatesetups.com/sitemap-index.xml