# robots.txt for DexAPK

User-agent: *
Allow: /*

# Disallow admin pages
Disallow: /admin/
Disallow: /admin/*

# Normalize URLs - prevent crawling of uppercase URLs
Disallow: /*?*
Disallow: /*.php$
Disallow: /*.html$
Disallow: /*index
Disallow: /*?
Disallow: /*&
Disallow: /*%

# Image optimization rules
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.webp$
Allow: /*.svg$

# Sitemap location
Sitemap: https://dexapk.com/sitemap.xml
Sitemap: https://dexapk.com/image-sitemap.xml

# Crawl delay to prevent server overload
Crawl-delay: 1

# Allow specific bots with custom rules
User-agent: Googlebot
Allow: /
Crawl-delay: 0.5

User-agent: Bingbot
Allow: /
Crawl-delay: 0.5

User-agent: Slurp
Allow: /
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

User-agent: Baiduspider
Allow: /
Crawl-delay: 2

User-agent: Yandex
Allow: /
Crawl-delay: 2

# Disallow certain bots
User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /