# ============================================
# General Rules for All Crawlers
# ============================================
User-agent: *

# Block access to sensitive directories
Disallow: /admin/
Disallow: /api/
Disallow: /config/
Disallow: /database/
Disallow: /logs/
Disallow: /temp/
Disallow: /include/
Disallow: /cgi-bin/
Disallow: /static/backup/
Disallow: /install/

# Block low-value content
Disallow: /search/
Disallow: /tag/
Disallow: /*?utm_*
Disallow: /*?sessionid*
Disallow: /*?PHPSESSID*

# Allow static resources for rendering
Allow: /*.css$
Allow: /*.js$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.webp$
Allow: /*.svg$
Allow: /*.pdf$

# Request delay for all crawlers
Crawl-delay: 1

# Website sitemap location
Sitemap: https://www.ahacyb.com/sitemap.xml


# ============================================
# Rules for AI Crawlers (Welcome to Products)
# ============================================
User-agent: GPTBot
User-agent: Google-Extended
User-agent: ClaudeBot
User-agent: DeepSeekBot
User-agent: Baidu-Extended
User-agent: Applebot
User-agent: Amazonbot
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: CohereBot
User-agent: MistralBot
User-agent: YouBot
User-agent: FacebookBot
User-agent: Twitterbot
User-agent: Bytespider

# Allow AI crawlers to access product content, but not admin or API
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /config/
Disallow: /search/
Crawl-delay: 1


# ============================================
# Rules for Search Engine Crawlers
# ============================================
User-agent: Baiduspider
User-agent: Googlebot
User-agent: Bingbot

# Allow full access for search engines, but block admin and API
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /config/
Disallow: /search/
Crawl-delay: 1