# Comprehensive robots.txt to block unwanted bots
# Last updated: March 2025

User-agent: *
Crawl-delay: 10

# Block SEMrush bot
User-agent: SemrushBot
User-agent: SemrushBot-SA
User-agent: SemrushBot-BA
User-agent: SemrushBot-SI
User-agent: SemrushBot-SWA
User-agent: SemrushBot-CT
User-agent: SemrushBot-BM
Disallow: /

# Block TikTok bots
User-agent: TikTok-Bot
User-agent: Bytespider
Disallow: /

# Block Ahrefs bot
User-agent: AhrefsBot
Disallow: /

# Block Moz bot
User-agent: rogerbot
User-agent: dotbot
Disallow: /

# Block Majestic bot
User-agent: MJ12bot
Disallow: /

# Block DotBot
User-agent: dotbot
Disallow: /

# Block DataForSeoBot 
User-agent: DataForSeoBot
Disallow: /

# Block Screaming Frog
User-agent: Screaming Frog SEO Spider
Disallow: /

# Block BLEXBot
User-agent: BLEXBot
Disallow: /

# Block PetalBot (Huawei)
User-agent: PetalBot
Disallow: /

# Block Yandex
User-agent: Yandex
User-agent: YandexBot
User-agent: YandexImages
User-agent: YandexMobileBot
Disallow: /

# Block Baidu
User-agent: Baiduspider
Disallow: /

# Block Sogou
User-agent: Sogou web spider
User-agent: Sogou inst spider
Disallow: /

# Block Mail.RU Bot
User-agent: Mail.RU_Bot
Disallow: /

# Block UptimeBot
User-agent: UptimeBot
Disallow: /

# Block other common SEO/scraping bots
User-agent: proximic
User-agent: AspiegelBot
User-agent: SpiderLing
User-agent: TurnitinBot
User-agent: netEstate NE Crawler
User-agent: magpie-crawler
User-agent: Nuclei
User-agent: 360Spider
User-agent: spider
User-agent: Semanticbot
User-agent: BDCbot
User-agent: Qwantify
Disallow: /

# Allow common good bots
User-agent: Googlebot
Allow: /

User-agent: bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

# Default rule for all other bots
User-agent: *
Allow: /