Eshop.pma-tools.de - SEO Check

Meta Tags

Medienliste

Überschriftenstruktur

Gefundene Links auf dieser Seite

HTTP-Header

Links von Wikipedia

Robots.txt

# Michael van Grinsven 27.11.2023 08:36
User-agent: *
Disallow: /admin/
Disallow: /Core/
Disallow: /tmp/
Disallow: /views/
Disallow: /Setup/
Disallow: /log/
# TEST 20201106
Disallow: /newsletter/
Disallow: /en/newsletter/
Disallow: /index.php?cl=newsletter
#
Disallow: /AGB/
Disallow: /en/Terms-and-Conditions/
#
Disallow: /warenkorb/
Disallow: /en/cart/
Disallow: /index.php?cl=basket
#
Disallow: /mein-konto/
Disallow: /en/my-account/
Disallow: /index.php?cl=account
#
Disallow: /mein-merkzettel/
Disallow: /en/my-wishlist/
Disallow: /index.php?cl=account_noticelist
#
Disallow: /mein-wunschzettel/
Disallow: /en/my-gift-registry/
Disallow: /index.php?cl=account_wishlist
#
Disallow: /konto-eroeffnen/
Disallow: /en/open-account/
Disallow: /index.php?cl=register
#
Disallow: /passwort-vergessen/
Disallow: /en/forgot-password/
Disallow: /index.php?cl=forgotpwd
#
Disallow: /index.php?cl=moredetails
#
Disallow: /index.php?cl=review
#
Disallow: /index.php?cl=search
#
Disallow: /EXCEPTION_LOG.txt
#
# wildcards at the end, because of some crawlers see it as errors
Disallow: /*?cl=newsletter
Disallow: /*&cl=newsletter
#
Disallow: /*?cl=basket
Disallow: /*&cl=basket
#
Disallow: /*?cl=account
Disallow: /*&cl=account
#
Disallow: /*?cl=account_noticelist
Disallow: /*&cl=account_noticelist
#
Disallow: /*?cl=account_wishlist
Disallow: /*&cl=account_wishlist
#
Disallow: /*?cl=register
Disallow: /*&cl=register
#
Disallow: /*?cl=forgotpwd
Disallow: /*&cl=forgotpwd
#
Disallow: /*?cl=moredetails
Disallow: /*&cl=moredetails
#
Disallow: /*?cl=review
Disallow: /*&cl=review
#
Disallow: /*?cl=search
Disallow: /*&cl=search
#
Disallow: /*&fnc=tobasket
Disallow: /*&fnc=tocomparelist
Disallow: /*&addcompare=
#
Disallow: /*/sid/
Disallow: /*?sid=
Disallow: /*&sid=
#
Disallow: /*?cur=
Disallow: /*&cur

# bingbot and its numerous siblings (which are, in part, not documented :-((  ) 
# require specific (and slower) crawl rate settings because they simply ignore
# the crawl delay settings. so if you actually want a crawl delay of, say, 2 then there
# must be a crawl delay setting of 20 to 30 - according to bing support.
User-agent: bingbot
User-agent: msnbot
User-agent: msnbot-NewsBlogs
User-agent: msnbot-media
User-agent: msnbot-UDiscovery
Crawl-delay: 20

# Bot Operators: If you are listed here, please contact us.
# And yes, we know that multiple User-Agent-Lines with
# identical rules could be folded. But honestly: who would trust
# a rogue bot operator to implement parsing correctly?

User-agent: Yandex
Disallow: /
User-agent: MJ12bot
# Test 202208-202209: will Mj12bot accept Allow Syntax and will it behave
Allow: /branchen
Disallow: /
User-agent: proximic
Disallow: /
User-agent: UnisterBot
Disallow: /

# Next ones in line for getting blocked:  nachtschatten, neofonie
# And now the final fallback instructions for the rest of the bots

User-agent: *
Crawl-delay: 2
# at least some bots might unterstand wildcards and crawl delay ...
Disallow: /*?

User-agent: MJ12bot
Disallow: /

User-agent: dotbot
Disallow: /

Suchvorschau