# Standard crawlers
User-agent: *
Allow: /
Disallow: /api/

# OpenAI GPTBot
User-agent: GPTBot
Allow: /
Allow: /about
Allow: /product
Allow: /solutions
Disallow: /api/

# Google AI bot
User-agent: Google-Extended
Allow: /
Allow: /about
Allow: /product
Allow: /solutions
Disallow: /api/

# Anthropic Claude bot
User-agent: AnthropicBot
Allow: /
Allow: /about
Allow: /product
Allow: /solutions
Disallow: /api/

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Allow: /about
Allow: /product
Allow: /solutions
Disallow: /api/

# Cohere AI
User-agent: cohere-ai
Allow: /
Allow: /about
Allow: /product
Allow: /solutions
Disallow: /api/

# Crawl delay for AI bots to prevent overloading server
User-agent: GPTBot
Crawl-delay: 2

User-agent: AnthropicBot
Crawl-delay: 2

User-agent: PerplexityBot
Crawl-delay: 2

User-agent: Google-Extended
Crawl-delay: 1

# Sitemaps
Sitemap: https://www.bloq.in/sitemap.xml