Robots.txt
# For all robots
User-agent: *
Disallow: /*/country*
Allow: /*/country$
# Block access to specific groups of pages
Disallow: /*/cart
Disallow: /*/checkout
Disallow: /*/my-account
Disallow: /*/releaseVersion.txt
Disallow: /*/buildDescription.txt
Disallow: /*/dealers/locator
# PLP filters
Disallow: /*?q=
Disallow: /*&q=
Allow: /*?q=%3Arelevant
Allow: /*&q=%3Arelevant
Disallow: /*?number=
Disallow: /*&number=
Allow: /*?number=24
Allow: /*&number=24
Allow: /*?page=
Allow: /*&page=
# Avoid bad mix country/lang
Allow: /global/en/
### Europe
Allow: /at/de/
Allow: /be/fr/
Allow: /be/en/
Allow: /ch/de/
Allow: /ch/fr/
Allow: /ch/it/
Allow: /cz/cs/
Allow: /de/de/
Allow: /dk/en/
Allow: /es/es/
Allow: /fr/fr/
Allow: /gb/en/
Allow: /it/it/
Allow: /nl/en/
Allow: /pl/pl/
Allow: /pt/pt/
Allow: /se/en/
### Americas
Allow: /ar/es/
Allow: /br/pt/
Allow: /ca/en/
Allow: /ca/fr/
Allow: /cl/es/
Allow: /co/es/
Allow: /mx/es/
Allow: /us/en/
### Oceania
Allow: /au/en/
Allow: /nz/en/
### Asia
Allow: /cn/zh/
Allow: /in/en/
Allow: /jp/ja/
Allow: /kr/ko/
### Africa
Allow: /za/en/
### Disallow
Disallow: /ar/
Disallow: /at/
Disallow: /au/
Disallow: /be/
Disallow: /bg/
Disallow: /br/
Disallow: /ca/
Disallow: /ch/
Disallow: /cl/
Disallow: /cn/
Disallow: /co/
Disallow: /cr/
Disallow: /cy/
Disallow: /cz/
Disallow: /de/
Disallow: /dk/
Disallow: /do/
Disallow: /ec/
Disallow: /ee/
Disallow: /es/
Disallow: /fi/
Disallow: /fr/
Disallow: /gb/
Disallow: /global/
Disallow: /gt/
Disallow: /hk/
Disallow: /hr/
Disallow: /ie/
Disallow: /il/
Disallow: /in/
Disallow: /is/
Disallow: /it/
Disallow: /jp/
Disallow: /kr/
Disallow: /lu/
Disallow: /lv/
Disallow: /mx/
Disallow: /nl/
Disallow: /no/
Disallow: /nz/
Disallow: /pa/
Disallow: /pe/
Disallow: /pl/
Disallow: /pt/
Disallow: /re/
Disallow: /rs/
Disallow: /ru/
Disallow: /se/
Disallow: /sg/
Disallow: /si/
Disallow: /sk/
Disallow: /tw/
Disallow: /ua/
Disallow: /us/
Disallow: /uy/
Disallow: /za/
# Allow search crawlers to discover the sitemap
#europe
Sitemap: https://www.scott-sports.com/at/de/sitemap.xml
Sitemap: https://www.scott-sports.com/be/fr/sitemap.xml
Sitemap: https://www.scott-sports.com/be/en/sitemap.xml
Sitemap: https://www.scott-sports.com/ch/de/sitemap.xml
Sitemap: https://www.scott-sports.com/ch/fr/sitemap.xml
Sitemap: https://www.scott-sports.com/ch/it/sitemap.xml
Sitemap: https://www.scott-sports.com/cz/cs/sitemap.xml
Sitemap: https://www.scott-sports.com/de/de/sitemap.xml
Sitemap: https://www.scott-sports.com/dk/en/sitemap.xml
Sitemap: https://www.scott-sports.com/es/es/sitemap.xml
Sitemap: https://www.scott-sports.com/fr/fr/sitemap.xml
Sitemap: https://www.scott-sports.com/gb/en/sitemap.xml
Sitemap: https://www.scott-sports.com/it/it/sitemap.xml
Sitemap: https://www.scott-sports.com/nl/en/sitemap.xml
Sitemap: https://www.scott-sports.com/pl/pl/sitemap.xml
Sitemap: https://www.scott-sports.com/pt/pt/sitemap.xml
Sitemap: https://www.scott-sports.com/se/en/sitemap.xml
#americas
Sitemap: https://www.scott-sports.com/ar/es/sitemap.xml
Sitemap: https://www.scott-sports.com/br/pt/sitemap.xml
Sitemap: https://www.scott-sports.com/ca/en/sitemap.xml
Sitemap: https://www.scott-sports.com/ca/fr/sitemap.xml
Sitemap: https://www.scott-sports.com/cl/es/sitemap.xml
Sitemap: https://www.scott-sports.com/co/es/sitemap.xml
Sitemap: https://www.scott-sports.com/mx/es/sitemap.xml
Sitemap: https://www.scott-sports.com/us/en/sitemap.xml
#oceania
Sitemap: https://www.scott-sports.com/au/en/sitemap.xml
Sitemap: https://www.scott-sports.com/nz/en/sitemap.xml
#asia
Sitemap: https://www.scott-sports.com/cn/zh/sitemap.xml
Sitemap: https://www.scott-sports.com/in/en/sitemap.xml
Sitemap: https://www.scott-sports.com/jp/ja/sitemap.xml
Sitemap: https://www.scott-sports.com/kr/ko/sitemap.xml
#africa
Sitemap: https://www.scott-sports.com/za/en/sitemap.xml
#global
Sitemap: https://www.scott-sports.com/global/en/sitemap.xml
# Block CazoodleBot as it does not present correct accept content headers
User-agent: CazoodleBot
Disallow: /
# Block MJ12bot as it is just noise
User-agent: MJ12bot
Disallow: /
# Block dotbot as it cannot parse base urls properly
User-agent: dotbot
Disallow: /
# Block Gigabot
User-agent: Gigabot
Disallow: /
# Allow Marketing tool SEMRush bot
User-agent: SemrushBot
Allow: /
# Block AhrefsBot
User-agent: AhrefsBot
Disallow: /
# Block Blexbot
User-agent: Blexbot
Disallow: /
(Nice to have)