#################################
# Google
#################################
User-agent: Googlebot

# Sitecore internals (hard block)
Disallow: /sitecore/
Disallow: /sitecore_modules/
Disallow: /sitecore/api/
Disallow: /sitecore/shell/
Disallow: /sitecore/client/

# System / config
Disallow: /App_Config/
Disallow: /App_Data/
Disallow: /bin/
Disallow: /temp/
Disallow: /xsl/

# APIs
Disallow: /api/sitecore/
Disallow: /api/sitecore/Multilanguage/LanguageSwitcher

# Parameters (biggest load reducer)
Disallow: /*?*

# Allow media (images use parameters)
Allow: /-/media/

Crawl-delay: 5


#################################
# Bing
#################################
User-agent: Bingbot
Disallow: /sitecore/
Disallow: /sitecore_modules/
Disallow: /sitecore/api/
Disallow: /sitecore/shell/
Disallow: /sitecore/client/
Disallow: /App_Config/
Disallow: /App_Data/
Disallow: /bin/
Disallow: /temp/
Disallow: /xsl/
Disallow: /api/sitecore/
Disallow: /api/sitecore/Multilanguage/LanguageSwitcher
Disallow: /*?*
Allow: /-/media/
Crawl-delay: 10


#################################
# Secondary search engines
#################################
User-agent: DuckDuckBot
Allow: /
Disallow: /*?*
Crawl-delay: 20

User-agent: Slurp
Allow: /
Disallow: /*?*
Crawl-delay: 20

User-agent: YandexBot
Allow: /
Disallow: /*?*
Crawl-delay: 30


#################################
# Social media preview bots
#################################
User-agent: facebookexternalhit
Allow: /
Disallow: /*?*
Crawl-delay: 30

User-agent: LinkedInBot
Allow: /
Disallow: /*?*
Crawl-delay: 30

User-agent: Twitterbot
Allow: /
Disallow: /*?*
Crawl-delay: 30


#################################
# Aggressive SEO crawlers (blocked)
#################################
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /


#################################
# Fallback (unknown bots)
#################################
User-agent: *
Disallow: /
Crawl-delay: 60


#################################
# Sitemap
#################################
Sitemap: https://www.nortonrosefulbright.com/sitemap-NRFWeb.xml

Sitemap: https://www.nortonrosefulbright.com/sitemap-NRFWeb.xml