Built.de - SEO Check

Meta Tags

Medienliste

Überschriftenstruktur

Gefundene Links auf dieser Seite

HTTP-Header

Links von Wikipedia

Robots.txt

#
# ACHTUNG!!!
#
# An dieser Datei KEINE Aenderungen vornehmen!!!
# Dies ist eine automatisch erstellte Datei!
# Aenderungen werden ohne Sicherung automatisch 
# ueberschrieben!
#
#-------------------------------------------------
#      Version: 2018-06-04 20:47
#    Generiert: 04.06.2018 20:47
# durch Script: esRemoteAccessSettings.class.php
#-------------------------------------------------


#  80legs (http://www.80legs.com/webcrawler.html)
User-agent: 008
Disallow: /

#  Abonti (http://www.abonti.com)
User-agent: Abonti
Disallow: /

#  AcoonBot (http://www.acoon.de/robot.asp)
User-agent: Acoon
Disallow: /

User-agent: AcoonBot
Disallow: /

User-agent: OpenAcoon
Disallow: /

#  Ahrefs.com (http://ahrefs.com/robot/)
User-agent: AhrefsBot
Disallow: / 

#  Baiduspider (http://www.baidu.com/search/spider.htm)
User-agent: baiduspider
Disallow: / 

#  Birubot [Bot]
User-agent: Birubot
Disallow: /

#  Bixo Labs (http://bixolabs.com)
User-agent: bixolabs
Disallow: /

#  BLEXBot (http://webmeup.com/crawler.html)
User-agent: BLEXBot
Disallow: /

#  BotOnParade (www.bots-on-para.de/bot.html)
User-agent: BotOnParade
Disallow: / 

#  BUbiNG  (http://law.di.unimi.it/BUbiNG.html)
User-agent: BUbiNG
Disallow: /

#  CareerBot (http://www.career-x.de/bot.html)
User-agent: careerbot
Disallow: /

#  CCBot/2 (commoncrawl.org)
User-agent: CCBot
Disallow: /

#  Cliqzbot (http://cliqz.com/company/cliqzbot)
User-agent: Cliqzbot
Disallow: *

#  discobot (http://discoveryengine.com/discobot.html)
User-agent: discobot
Disallow: / 

#  DotBot/1.1 (http://www.opensiteexplorer.org/dotbot)
User-agent: dotbot
Disallow: /

#  Exabot/3.0 (http://www.exabot.com/go/robot)
User-agent: Exabot
Disallow: /

#  Gigabot (http://www.gigablast.com/spider.html)
User-agent: gigabot
Disallow: /

#  Gnip [Bot] (http://www.gnip.com)
User-agent: UnwindFetchor
Disallow: /

#  Huawei Symantec (http://www.huaweisymantec.com/en/IRL/spider/)
User-agent: HuaweiSymantecSpider
Disallow: / 

#  iCjobs Stellenangebote Jobs (www.icjobs.de/bot.htm)
User-agent: ICCrawler - iCjobs 
Disallow: / 

#  LCC (http://corpora.informatik.uni-leipzig.de/crawler_faq.html)
User-agent: LCC
Disallow: /

#  Linguee Bot (http://www.linguee.com/bot)
User-agent: Linguee
Disallow: /

#  Majestic-12 (http://www.majestic12.co.uk/bot.php)
User-agent: MJ12bot
Disallow: /

#  meanpathbot/ (http://www.meanpath.com/meanpathbot.html)
User-agent: meanpathbot
Disallow: /

#  MojeekBot/0.2 (http://www.mojeek.com/bot.html)
User-agent: MojeekBot
Disallow: / 

#  NerdByNature (http://www.nerdbynature.net/bot)
User-agent: NerdByNature.Bot
Disallow: / 

#  oBot/2.3.1 (http://filterdb.iss.net/crawler/)
User-agent: oBot
Disallow: / 

#  OneRiot [Bot] (http://www.oneriot.com)
User-agent: OneRiot
Disallow: /

#  OpenLinkProfiler (http://OpenLinkProfiler.org/bot)
User-agent: spbot
Disallow: /

#  Plukkie (http://www.botje.com/plukkie.htm)
User-agent: plukkie
Disallow: / 

#  Ruby
User-agent: Ruby
Disallow: /

#  ScoutJet (http://www.scoutjet.com/)
User-agent: ScoutJet
Disallow: /

#  Search17Bot (http://www.search17.com/bot.php)
User-agent: search17
Disallow: /

#  SearchmetricsBot (http://www.searchmetrics.com/en/searchmetrics-bot/)
User-agent: SearchmetricsBot
Disallow: /

#  SemrushBot (http://www.semrush.com/bot.html)
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-SA
Disallow: /

#  Sosospider (http://help.soso.com/webspider.htm)
User-agent: Sosospider
Disallow: / 

#  Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)
User-agent: Speedy
Disallow: / 

#  Spinn3r (http://spinn3r.com/robot)
User-agent: Spinn3r
Disallow: / 

#  TurnitinBot (http://www.turnitin.com/robot/crawlerinfo.html)
User-agent: TurnitinBot
Disallow: /

#  TwengaBot (http://www.twenga.com/bot.html)
User-agent: twenga2.com
Disallow: /

User-agent: twenga.com
Disallow: /

#  WBSearchBot (http://www.warebay.com/bot.html)
User-agent: WBSearchBot
Disallow: /

#  XoviBot/2 (http://www.xovibot.net/)
User-agent: XoviBot
Disallow: /

#  Yandex (http://yandex.com/bots)
User-agent: Yandex
Disallow: / 

#  Zookabot (http://zookabot.com)
User-agent: Zookabot
Disallow: /

#
# Erweiterungen des Shopbetreibers
#
#User-agent: *
#Disallow: /

User-agent: Mediapartners-Google
Disallow:

User-agent: Googlebot
Disallow: /eshop.php?action=article_detail
Disallow: /eshop.php?action=article_list
Disallow: /eshop.php?action=catalog_list
Crawl-delay: 120

User-agent: Adsbot-Google
Disallow:

User-agent: Googlebot-Image
Disallow:

User-agent: Googlebot-Mobile
Disallow:

Suchvorschau