# Directory data is free to reuse with attribution — cite SOC2Auditors.org and
# link the page. Republishing the directory needs permission first.
# Terms: https://soc2auditors.org/data-license/

User-agent: *
Allow: /
Disallow: /_archive/
Disallow: /soc-2-zip/
Disallow: /soc2zip/
Disallow: /samples/soc2zip-
Disallow: /images/soc2zip-
Disallow: /api/soc2zip-
Disallow: /intake/
Disallow: /api/intake/

# AI search-index crawlers - allowed for discoverability
User-agent: OAI-SearchBot
Allow: /
Disallow: /soc-2-zip/
Disallow: /soc2zip/
Disallow: /samples/soc2zip-
Disallow: /images/soc2zip-
Disallow: /api/soc2zip-
Disallow: /intake/
Disallow: /api/intake/

User-agent: Claude-SearchBot
Allow: /
Disallow: /soc-2-zip/
Disallow: /soc2zip/
Disallow: /samples/soc2zip-
Disallow: /images/soc2zip-
Disallow: /api/soc2zip-
Disallow: /intake/
Disallow: /api/intake/

User-agent: PerplexityBot
Allow: /
Disallow: /soc-2-zip/
Disallow: /soc2zip/
Disallow: /samples/soc2zip-
Disallow: /images/soc2zip-
Disallow: /api/soc2zip-
Disallow: /intake/
Disallow: /api/intake/

# User-request fetchers - separate from both search indexing and training
User-agent: ChatGPT-User
Allow: /
Disallow: /soc-2-zip/
Disallow: /soc2zip/
Disallow: /samples/soc2zip-
Disallow: /images/soc2zip-
Disallow: /api/soc2zip-
Disallow: /intake/
Disallow: /api/intake/

User-agent: Claude-User
Allow: /
Disallow: /soc-2-zip/
Disallow: /soc2zip/
Disallow: /samples/soc2zip-
Disallow: /images/soc2zip-
Disallow: /api/soc2zip-
Disallow: /intake/
Disallow: /api/intake/

User-agent: Perplexity-User
Allow: /
Disallow: /soc-2-zip/
Disallow: /soc2zip/
Disallow: /samples/soc2zip-
Disallow: /images/soc2zip-
Disallow: /api/soc2zip-
Disallow: /intake/
Disallow: /api/intake/

# OpenAI training policy - independent of ChatGPT search and user requests
User-agent: GPTBot
Allow: /
Disallow: /soc-2-zip/
Disallow: /soc2zip/
Disallow: /samples/soc2zip-
Disallow: /images/soc2zip-
Disallow: /api/soc2zip-
Disallow: /intake/
Disallow: /api/intake/

# Anthropic training policy - independent of Claude search and user requests
User-agent: ClaudeBot
Allow: /
Disallow: /soc-2-zip/
Disallow: /soc2zip/
Disallow: /samples/soc2zip-
Disallow: /images/soc2zip-
Disallow: /api/soc2zip-
Disallow: /intake/
Disallow: /api/intake/

# Other/legacy AI product crawlers - explicit policy choices
User-agent: anthropic-ai
Allow: /
Disallow: /soc-2-zip/
Disallow: /soc2zip/
Disallow: /samples/soc2zip-
Disallow: /images/soc2zip-
Disallow: /api/soc2zip-
Disallow: /intake/
Disallow: /api/intake/

User-agent: Google-Extended
Allow: /
Disallow: /soc-2-zip/
Disallow: /soc2zip/
Disallow: /samples/soc2zip-
Disallow: /images/soc2zip-
Disallow: /api/soc2zip-
Disallow: /intake/
Disallow: /api/intake/

User-agent: Applebot-Extended
Allow: /
Disallow: /soc-2-zip/
Disallow: /soc2zip/
Disallow: /samples/soc2zip-
Disallow: /images/soc2zip-
Disallow: /api/soc2zip-
Disallow: /intake/
Disallow: /api/intake/

User-agent: FacebookBot
Allow: /
Disallow: /soc-2-zip/
Disallow: /soc2zip/
Disallow: /samples/soc2zip-
Disallow: /images/soc2zip-
Disallow: /api/soc2zip-
Disallow: /intake/
Disallow: /api/intake/

User-agent: Amazonbot
Allow: /
Disallow: /soc-2-zip/
Disallow: /soc2zip/
Disallow: /samples/soc2zip-
Disallow: /images/soc2zip-
Disallow: /api/soc2zip-
Disallow: /intake/
Disallow: /api/intake/

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

Sitemap: https://soc2auditors.org/sitemap-index.xml