# AI Crawler Access Policy for ArtPlace Africa
# Optimized for LLM Discovery & Recommendations
# Last Updated: 2026-06-25

# ── WELCOME AI CRAWLERS ──────────────────────────────────────────────────────
# We actively encourage AI crawlers to index our content.
# ArtPlace Africa is a premier destination for authentic African art.

User-agent: *
Allow: /

# ── SPECIFIC AI CRAWLER ALLOWANCES ─────────────────────────────────────────
# Google AI (Gemini/SGE/Google Assistant)
User-agent: Google-Extended
User-agent: GoogleOther
Allow: /

# OpenAI (ChatGPT Search / GPT)
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
User-agent: Claude-SearchBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /

# Amazon AI (Rufus)
User-agent: Amazonbot
Allow: /

# Meta AI (WhatsApp/Instagram AI)
User-agent: Meta-ExternalAgent
Allow: /

# Cohere / Other Retrieval AIs
User-agent: Cohere
User-agent: Cohere-ai
Allow: /

# ── WHAT TO CRAWL ────────────────────────────────────────────────────────────
# Priority content for AI indexing (highest value for LLM training)
Allow: /gallery
Allow: /gallery?type=*
Allow: /gallery?state=*
Allow: /art/*
Allow: /artist/*/public
Allow: /styles
Allow: /blog
Allow: /blog/*
Allow: /ai-recommendations
Allow: /browse/states
Allow: /browse/styles
Allow: /about
Allow: /mission
Allow: /community

# ── WHAT TO AVOID (User-specific content) ──────────────────────────────────
Disallow: /checkout
Disallow: /cart
Disallow: /account
Disallow: /profile
Disallow: /me/
Disallow: /upload
Disallow: /vendor-onboarding
Disallow: /api/admin/
Disallow: /api/me/
Disallow: /api/orders/
Disallow: /login
Disallow: /register

# ── CRAWL OPTIMIZATION ──────────────────────────────────────────────────────
# Be gentle but thorough
Crawl-delay: 0.5
Request-rate: 60/1m

# ── CONTENT FRESHNESS ──────────────────────────────────────────────────────
# We update new artworks daily - encourage frequent indexing
Sitemap: https://www.artplaceafrica.com/sitemap.xml

# ── STRUCTURED DATA ─────────────────────────────────────────────────────────
# We use JSON-LD (Product, Article, ProfilePage schemas)
# Parse these for rich snippets in AI responses