# ============================================================
# CACD — robots.txt
# Generated: 2026-08-30
# Website: https://china-asean-digital.academy
# ============================================================

# Allow all major search engines
User-agent: *
Allow: /

# Disallow sensitive / utility paths
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /draft/
Disallow: /preview/
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /trackback/
Disallow: /*?s=
Disallow: /*&print=
Disallow: /*.pdf$
Disallow: /*.doc$
Disallow: /*.docx$

# Allow specific asset directories for image search
Allow: /static/images/
Allow: /static/svg/

# Crawl-delay (politeness, seconds)
Crawl-delay: 1

# ============================================================
# Specific search engine directives
# ============================================================

# Googlebot
User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 1

# Googlebot-Image
User-agent: Googlebot-Image
Allow: /static/images/
Allow: /static/svg/

# Bingbot
User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 1

# Baidu Spider
User-agent: Baiduspider
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 2

# Yandex
User-agent: Yandex
Allow: /
Disallow: /admin/
Disallow: /api/

# ============================================================
# Block known bad / aggressive crawlers
# ============================================================

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: curl
Disallow: /admin/
Disallow: /api/

# ============================================================
# Sitemap location
# ============================================================

Sitemap: https://china-asean-digital.academy/sitemap.xml
Sitemap: https://china-asean-digital.academy/sitemap.html

# ============================================================
# Host directive (Yandex)
# ============================================================

Host: https://china-asean-digital.academy