# ===== Sitemaps =====
# Sitemap must be an absolute URL (protocol + domain), not a relative path

# Sitemap: https://www.arto.dk/sitemap.xml


# ===== General rules for all bots =====

User-agent: *
Disallow: /site/admin/
Disallow: /libs/


# ===== Slow down aggressive but legitimate crawlers =====

User-agent: Bingbot
Crawl-delay: 10

User-agent: MSNBot
Crawl-delay: 10


# ===== Safe to Block (no major SEO impact) =====

User-agent: AhrefsBot
Disallow: /
# SEO backlink crawler, high request rate, no direct SEO value unless you use Ahrefs.

User-agent: SemrushBot
Disallow: /
# SEO auditing crawler, often hits hard, only useful if you pay for SEMrush.

User-agent: MJ12bot
Disallow: /
# Majestic SEO crawler, mainly for their backlink index, not useful unless you use Majestic.

User-agent: DotBot
Disallow: /
# Moz crawler for link data, can be very aggressive, only useful for Moz subscribers.

User-agent: MegaIndex
Disallow: /
# Russian SEO bot, known for scraping, not a real search engine.

User-agent: MauiBot
Disallow: /
# Hawaiian ISP-based crawler used for scraping, often ignores crawl-delay.

User-agent: PetalBot
Disallow: /
# Huawei's search engine bot, no SEO loss unless targeting Huawei search.

User-agent: SeekportBot
Disallow: /
# Old European search engine crawler, low relevance today.

User-agent: BLEXBot
Disallow: /
# Aggressive link scraper used for SEO databases, no benefit unless you use BLEX.

User-agent: CCBot
Disallow: /
# Common Crawl bot, archives web pages for public datasets, not needed for SEO.

User-agent: Wotbox
Disallow: /
# Obsolete search engine bot, now mostly used for scraping.

User-agent: Exabot
Disallow: /
# French search engine bot, low relevance unless targeting France.


# ===== Maybe Block (depends on audience) =====

User-agent: Baiduspider
Disallow: /
# Chinese search engine, block unless you target mainland China traffic.

User-agent: YandexBot
Disallow: /
# Russian search engine, block unless you target Russian-speaking audiences.

User-agent: ZumBot
Disallow: /
# South Korean search engine bot, block unless targeting South Korea.

User-agent: Sogou
Disallow: /
# Chinese search engine crawler, block unless targeting Chinese market.