# KeepMD robots.txt
# Purpose:
# - Keep normal public website crawlable.
# - Welcome major AI crawlers (search, retrieval and training) to public content.
# - Allow /llms.txt and /llms-full.txt discovery.
# - Preserve private/admin disallows inside each specific crawler group.
# - Do not block Googlebot from LLM context files because Google must crawl them to see X-Robots-Tag: googlebot: noindex.

User-agent: OAI-SearchBot
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin/
Disallow: /cdn-cgi/
Allow: /

User-agent: ChatGPT-User
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin/
Disallow: /cdn-cgi/
Allow: /

User-agent: GPTBot
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin/
Disallow: /cdn-cgi/
Allow: /

User-agent: Claude-SearchBot
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin/
Disallow: /cdn-cgi/
Allow: /

User-agent: Claude-User
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin/
Disallow: /cdn-cgi/
Allow: /

User-agent: ClaudeBot
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin/
Disallow: /cdn-cgi/
Allow: /

User-agent: PerplexityBot
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin/
Disallow: /cdn-cgi/
Allow: /

User-agent: Perplexity-User
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin/
Disallow: /cdn-cgi/
Allow: /

User-agent: Googlebot
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin/
Disallow: /cdn-cgi/
Allow: /

User-agent: Google-Extended
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin/
Disallow: /cdn-cgi/
Allow: /

User-agent: bingbot
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin/
Disallow: /cdn-cgi/
Allow: /

User-agent: Applebot
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin/
Disallow: /cdn-cgi/
Allow: /

User-agent: Applebot-Extended
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin/
Disallow: /cdn-cgi/
Allow: /

User-agent: Amazonbot
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin/
Disallow: /cdn-cgi/
Allow: /

User-agent: Meta-ExternalAgent
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin/
Disallow: /cdn-cgi/
Allow: /

# Default public crawl rules.
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /cdn-cgi/

# Sitemap location.
Sitemap: https://keepmd.org/sitemap.xml