# https://digitised.net/robots.txt
User-agent: *
Allow: /
Disallow: /auth/
Disallow: /profile/
Disallow: /UserPanel
Disallow: /archive-owner

# AI crawlers - search input allowed, training not granted
User-agent: GPTBot
Allow: /
Disallow: /auth/
Disallow: /profile/

User-agent: OAI-SearchBot
Allow: /
Disallow: /auth/
Disallow: /profile/

User-agent: ClaudeBot
Allow: /
Disallow: /auth/
Disallow: /profile/

User-agent: Google-Extended
Allow: /
Disallow: /auth/
Disallow: /profile/

User-agent: PerplexityBot
Allow: /
Disallow: /auth/
Disallow: /profile/

User-agent: Bytespider
Disallow: /

Content-Signal: search=yes, ai-train=no, ai-input=yes

Sitemap: https://digitised.net/sitemap.xml
Sitemap: https://digitised.net/sitemaps/sitemap-arabic.xml
Sitemap: https://digitised.net/sitemaps/sitemap-english.xml

LLMs: https://digitised.net/llms.txt
LLMs-Full: https://digitised.net/llms-full.txt