# Disallow for all bots (general rules)

User-agent: *
Disallow: /en-GB/
Disallow: /web/
Disallow: /meinhays/
Disallow: /tinyurl/
Disallow: /c/portal/login
Disallow: /personaldienstleistung/kontakt-formular-vakanz-melden?*
Disallow: /*?doAsUserId=
Disallow: /login/
 
# === Allow known, legitimate AI crawlers ===
User-agent: GPTBot
Allow: /
# Source: https://openai.com/gptbot
 
User-agent: Google-Extended
Allow: /
# Source: https://developers.google.com/search/docs/crawling-indexing/googlebot
 
User-agent: AnthropicBot
Allow: /
# Source: https://www.anthropic.com/anthropicbot
 
User-agent: ClaudeBot
Allow: /
 
User-agent: CCBot
Allow: /
# Common Crawl bot
 
User-agent: PerplexityBot
Allow: /
# Source: https://www.perplexity.ai/blog/perplexitybot
 
User-agent: ChatGPT-User
Allow: /
 
User-agent: Applebot
Allow: /
 
User-agent: Bingbot
Allow: /
 
# Optional: Crawl delay for all bots
User-agent: *
Crawl-delay: 10
 
Sitemap: https://www.hays.de/sitemap.xml