Robots.txt
# Madara Cosmetics Crawler directives
User-agent: *
crawl-delay: 10
# Disallow: /$ - Disabled due to crawling issues. As there's a redirect in place to always redirect bots to a store-code version, this can be safely disabled
Disallow: /index.php/
# Disallow query string in the URL, but allow configurable options
Disallow: /*?
Allow: /*opt=
Disallow: /checkout/
Disallow: /app/
Disallow: /lib/
Disallow: /*.php$
Disallow: /pkginfo/
Disallow: /report/
Disallow: /var/
Disallow: /catalog/
Disallow: /customer/
Disallow: /sendfriend/
Disallow: /review/
Disallow: /*SID=
Disallow: /sendfriend/product/send/id/
Disallow: /review/product/listAjax/id/
Disallow: /searchsuiteautocomplete/ajax/index/
Disallow: /*/sendfriend/product/send/id/
Disallow: /*/review/product/listAjax/id/
Disallow: /*/searchsuiteautocomplete/ajax/index/
# Sitemap Locations
# English store Sitemap Location
Sitemap: https://www.madaracosmetics.com/media/sitemaps/sitemap_en.xml
# Estonian store Sitemap Location
Sitemap: https://www.madaracosmetics.com/media/sitemaps/sitemap_ee.xml
# Latvian store Sitemap Location
Sitemap: https://www.madaracosmetics.com/media/sitemaps/sitemap_lv.xml
# Italian store Sitemap Location
Sitemap: https://www.madaracosmetics.com/media/sitemaps/sitemap_it.xml
# German store Sitemap Location
Sitemap: https://www.madaracosmetics.com/media/sitemaps/sitemap_de.xml
# Finnish store Sitemap Location
Sitemap: https://www.madaracosmetics.com/media/sitemaps/sitemap_fi.xml
# French store Sitemap Location
Sitemap: https://www.madaracosmetics.com/media/sitemaps/sitemap_fr.xml
# US store Sitemap Location
Sitemap: https://www.madaracosmetics.com/media/sitemaps/sitemap_us.xml
# UK store Sitemap Location
Sitemap: https://www.madaracosmetics.com/media/sitemaps/sitemap_uk.xml
# Spanish store Sitemap Location
Sitemap: https://www.madaracosmetics.com/media/sitemaps/sitemap_es.xml
# Polish store Sitemap Location
Sitemap: https://www.madaracosmetics.com/media/sitemaps/sitemap_pl.xml
# Lithuanian store Sitemap Location
Sitemap: https://www.madaracosmetics.com/media/sitemaps/sitemap_lt.xml
# Dutch store Sitemap Location
Sitemap: https://www.madaracosmetics.com/media/sitemaps/sitemap_nl.xml
# Swedish store Sitemap Location
Sitemap: https://www.madaracosmetics.com/media/sitemaps/sitemap_se.xml
# Norwegian store Sitemap Location
Sitemap: https://www.madaracosmetics.com/media/sitemaps/sitemap_no.xml
(Nice to have)