# Geol.ai Robots.txt - AI-First Configuration
# Last Updated: October 28, 2025
# 
# AI CRAWLERS - FULL ACCESS WITH RESPECTFUL DELAYS
# =================================================

# ChatGPT / GPTBot (OpenAI) - Search & Citation
User-agent: GPTBot
Crawl-delay: 2
Allow: /
Allow: /api-docs
Disallow: /resources/directory
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /dev/

# ChatGPT Search (Real-time queries)
User-agent: ChatGPT-User
Allow: /
Allow: /api-docs
Disallow: /resources/directory
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /dev/

# OAI-SearchBot (ChatGPT Search indexing)
User-agent: OAI-SearchBot
Allow: /
Allow: /api-docs
Disallow: /resources/directory
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /dev/

# Claude / Anthropic
User-agent: anthropic-ai
Crawl-delay: 2
Allow: /
Allow: /api-docs
Disallow: /resources/directory
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /dev/

User-agent: Claude-Web
Crawl-delay: 2
Allow: /
Allow: /api-docs
Disallow: /resources/directory
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /dev/

User-agent: ClaudeBot
Crawl-delay: 2
Allow: /
Allow: /api-docs
Disallow: /resources/directory
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /dev/

# Perplexity
User-agent: PerplexityBot
Crawl-delay: 2
Allow: /
Allow: /api-docs
Disallow: /resources/directory
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /dev/

User-agent: Perplexity-User
Crawl-delay: 2
Allow: /
Allow: /api-docs
Disallow: /resources/directory
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /dev/

# Google Gemini / Bard
User-agent: Google-Extended
Crawl-delay: 2
Allow: /
Allow: /api-docs
Disallow: /resources/directory
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /dev/

# Microsoft Copilot / Bing Chat
User-agent: CCBot
Crawl-delay: 2
Allow: /
Allow: /api-docs
Disallow: /resources/directory
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /dev/

# You.com
User-agent: YouBot
Crawl-delay: 2
Allow: /
Allow: /api-docs
Disallow: /resources/directory
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /dev/

# Cohere AI
User-agent: cohere-ai
Crawl-delay: 2
Allow: /
Allow: /api-docs
Disallow: /resources/directory
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /dev/

# Meta AI
User-agent: meta-externalagent
Crawl-delay: 2
Allow: /
Allow: /api-docs
Disallow: /resources/directory
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /dev/

# Amazon Alexa
User-agent: Amazonbot
Crawl-delay: 3
Allow: /
Allow: /api-docs
Disallow: /resources/directory
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /dev/

# Apple Bot (Siri, Spotlight)
User-agent: Applebot
Crawl-delay: 2
Allow: /
Allow: /api-docs
Disallow: /resources/directory
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /dev/

# TRADITIONAL SEARCH ENGINES
# ==========================

# Googlebot
User-agent: Googlebot
Crawl-delay: 1
Allow: /
Disallow: /resources/directory
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /dev/

# Bingbot
User-agent: Bingbot
Crawl-delay: 1
Allow: /
Disallow: /resources/directory
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /dev/

# DuckDuckBot
User-agent: DuckDuckBot
Crawl-delay: 2
Allow: /
Disallow: /resources/directory
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /dev/

# Yandex
User-agent: Yandex
Crawl-delay: 2
Allow: /
Disallow: /resources/directory
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /dev/

# Baidu
User-agent: Baiduspider
Crawl-delay: 3
Allow: /
Disallow: /resources/directory
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /dev/

# ALL OTHER BOTS
# ==============
User-agent: *
Crawl-delay: 3
Allow: /
Disallow: /resources/directory
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /dev/
Disallow: /analyze
Disallow: /urls
Disallow: /scans
Disallow: /analytics
Disallow: /settings
Disallow: /integrations
Disallow: /billing
Disallow: /account

# SITEMAP
# =======
Sitemap: https://geol.ai/sitemap.xml

# ADDITIONAL RESOURCES
# ====================
# LLMs.txt: https://geol.ai/llms.txt
# API Documentation: https://geol.ai/api-docs
# RSS Feed: https://geol.ai/feed.xml
# Support: https://geol.ai/support

# Last Updated: November 5, 2025