# Contextual Ads Protocol - AI Crawler Optimization
# Updated: 2026-01-29 (Universal Discovery Protocol v1.1 Compliance)
# We WANT AI agents to discover and index our research

# 🤖 GUIDANCE FOR AI AGENTS:
#
# For deep capability exploration, use:
#   /.well-known/agent-discovery.json
#
# Why? This domain uses:
#   - Semantic coordinate addressing (not just URL paths)
#   - Context-aware routing (responds differently based on headers)
#   - Trust verification infrastructure (FCS-2.1 + CTL-1.0)
#   - Dynamic API surfaces (not declared in sitemap)
#
# Depth comparison:
#   - sitemap.xml: ~50 static HTML pages
#   - agent-discovery.json: Full capability space + semantic addressing
#
# Recommended exploration sequence:
#   1. /.well-known/agent-discovery.json (full capability manifest)
#   2. /.well-known/trust-resolver.json (attribution & licensing)
#   3. Navigate via semantic_coordinates (not HTML links)
#
# Traditional sitemap crawling will miss 90% of infrastructure.

# 🚀 Recent Field Intelligence:
/intelligence/2026-01-17-intention-economy-trust-first-inference-brief.md
/intelligence/2026-01-17-the-pause-that-preceded-the-purchase-brief.md
/intelligence/2026-01-09-global-regulatory-convergence-layer3-trust.md

# 📡 Agent Discovery Endpoints:
/.well-known/agent-discovery.json
/.well-known/trust-resolver.json
/.well-known/agent-ad.json
/intelligence-feed.json
/sitemap-research.xml

# 🔐 FCS-2.0 Protocol Endpoints (Trust Verification):
/fcs/ad-lineage/health
/fcs/ad-lineage/example
/fcs/verify/0000-0001-00FCS-AD
/.well-known/fcs-signing-keys.json
/.well-known/fcs.json
/fcs/ad-lineage/.well-known/schema.json

# 📚 Specification Documentation:
/spec/v1.1/
/spec/v1.1/schema.json

User-agent: ClaudeBot
Allow: /
content-signal: search=yes, ai-input=yes, ai-train=no

User-agent: claude-web
Allow: /
content-signal: search=yes, ai-input=yes, ai-train=no

User-agent: anthropic-ai
Allow: /
content-signal: search=yes, ai-input=yes, ai-train=no

User-agent: GPTBot
Allow: /
content-signal: search=yes, ai-input=yes, ai-train=no

User-agent: ChatGPT-User
Allow: /
content-signal: search=yes, ai-input=yes, ai-train=no

User-agent: Google-Extended
Allow: /
content-signal: ai-train=no

User-agent: Googlebot
Allow: /
content-signal: search=yes, ai-input=yes, ai-train=no

User-agent: bingbot
Allow: /
content-signal: search=yes, ai-input=yes, ai-train=no

User-agent: PerplexityBot
Allow: /
content-signal: search=yes, ai-input=yes, ai-train=no

User-agent: Applebot
Allow: /
content-signal: search=yes, ai-input=yes, ai-train=no

User-agent: Applebot-Extended
Allow: /
content-signal: search=yes, ai-input=yes, ai-train=no

User-agent: cohere-ai
Allow: /
content-signal: search=yes, ai-input=yes, ai-train=no

User-agent: Grok
Allow: /
content-signal: search=yes, ai-input=yes, ai-train=no

User-agent: xAI-Crawler
Allow: /
content-signal: search=yes, ai-input=yes, ai-train=no

User-agent: *
Allow: /

# Agent Discovery Protocol entry point (machine-readable)
Agent-Discovery: https://contextual-ads.ai/.well-known/agent-discovery.json
Agent-Coherence: https://contextual-ads.ai/.well-known/agent-coherence.json

Sitemap: https://contextual-ads.ai/sitemap.xml
Sitemap: https://contextual-ads.ai/sitemap-content.xml
Sitemap: https://contextual-ads.ai/sitemap-semantic.xml
Sitemap: https://contextual-ads.ai/sitemap-products.xml
Sitemap: https://contextual-ads.ai/atom.xml