# Automated access policy: /automated-access
# Bulk collection requires prior approval. Request access: /contact
# Agent guidance: /llms.txt
# Access arrangements do not override these crawler rules.

User-agent: MJ12bot
Disallow: /

User-agent: Dragonfly
Disallow: /

# Training, dataset collection, SEO crawling, and Google-Extended reuse are excluded.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Meta-ExternalAgent
User-agent: CCBot
User-agent: Amazonbot
User-agent: SemrushBot
User-agent: DotBot
User-agent: Google-Extended
User-agent: Applebot-Extended
Disallow: /
Allow: /automated-access$
Allow: /llms.txt$
Allow: /contact$

# Search discovery uses this group and retains the same exclusions.
User-agent: *
Disallow: /workspace
Disallow: /my-library
Disallow: /jobs
Disallow: /auth
Disallow: /billing
Disallow: /admin
Disallow: /settings
Disallow: /alerts
Disallow: /import
Disallow: /process
Disallow: /search
Disallow: /episodes/*/transcript/
Disallow: /episodes/*/refresh/
Disallow: /shows/*/refresh/
Disallow: /results/*/export.
Disallow: /*?*extract=
Disallow: /*?*refresh=
Disallow: /*?*review=
Allow: /