# =====================
# ✅ Allow Good Bots
# =====================

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Yandex
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Exabot
Allow: /

User-agent: Applebot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: ChatGPT-User
Allow: /

# Default allow for all others
User-agent: *
Disallow:

# Sitemap
Sitemap: https://mg-nrega.com/sitemap_index.xml

# =====================
# 🚫 Block Bad Bots / Crawlers
# =====================

User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Yeti
Disallow: /

User-agent: omgili
Disallow: /

User-agent: meta-external-agent
Disallow: /

# =====================
# ⏳ Crawl Delay for Aggressive Bots
# =====================

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: DotBot
Crawl-delay: 10

User-agent: PetalBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 15