Barracudamsp.com - SEO Checker

Meta tags

Media list

Heading structure

Found links on this page

HTTP Response Header

Links from Wikipedia

Robots.txt

User-agent: *

# Prevent Crawling for WordPress Pages
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: */wp-admin/
Disallow: /wp-includes/
Disallow: */wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /trackback*
Disallow: */trackback/
Disallow: /feed*
Disallow: */feed/
Disallow: /comments*
Disallow: */comments/
Disallow: /tag/

# Prevent Crawling Search Results
Disallow: /search/?searchTerm*


# Prevent Crawling for Thank You Pages
Disallow: */*LP-OnDemand*
Disallow: */*LP-ThankYou*
Disallow: */*thankyou*
Disallow: */*thank-you*
Disallow: */thankyou/*
Disallow: */thankyou*
Disallow: */thankYou*
Disallow: /thankyou-download*
Disallow: /thankyou-contact*
Disallow: /thankyou-quote*
Disallow: /thankyou-demo*

# Prevent Crawling for Login & Register
Disallow: */login/*
Disallow: */login*
Disallow: */Login*
Disallow: */register/*
Disallow: */customers/s/login/*
Disallow: */auth/login/*

# Prevent Error Crawling for Templates
Disallow: /error/

# Prevent Crawling for Captcha
Disallow: */captcha/*

# Prevent Crawling for Admin
Disallow: */admin/*
Disallow: */administrator/*

# Prevent Crawling for User
Disallow: */user/*
Disallow: */users/*

# Prevent Crawling for CGI
Disallow: /cgi-bin/

# Prevent Crawling for cache and log
Disallow: */cache/*
Disallow: */log/*
Disallow: */no-cache*
Disallow: */nocache*


########## SITEMAP FILES ##########
Sitemap: https://www.barracudamsp.com/sitemap.xml



########## PREVENT BAD BOTS TO CRAWL ##########

# http://www.opensiteexplorer.org/dotbot
User-agent: DotBot
User-agent: dotbot
Disallow: /

# http://help.sogou.com/homepage/search-engine/add-crawl.html
User-agent: Sogou Spider
Disallow: /

# http://www.exalead.com
User-agent: Exabot
Disallow: /

# https://www.youdao.com/help/webmaster/spider/
User-agent: YoudaoBot
Disallow: /

# https://megaindex.com/crawler
User-agent: MegaIndex.ru
Disallow: /

User-agent: megaindex.com
Disallow: /

# https://napoveda.seznam.cz/en/fulltext-search/crawling/
User-agent: SeznamBot
Disallow: /

# https://cliqz.com/company/cliqz-bot
User-agent: Cliqzbot
Disallow: /

# https://www.gigablast.com/spider.html
User-agent: Gigabot
Disallow: /

# http://openlinkprofiler.org/bot
User-agent: spbot
Disallow: /

# https://developers.ezooms.com/ezooms/ezooms-bot
User-agent: Ezooms
Disallow: /

# https://scoutjet.com/
User-agent: ScoutJet
Disallow: /

# http://webmeup-crawler.com/
User-agent: BLEXBot
Disallow: /

# Disallow: Sistrix
User-agent: sistrix
User-agent: SISTRIX Crawler
User-agent: SISTRIX
Disallow: /

# Disallow: SEOkicks-Robot
User-agent: SEOkicks
Disallow:/

User-agent: SEOkicks-Robot
Disallow:/

# http://www.searchmetrics.com
User-agent: SearchmetricsBot
Disallow: /

# http://www.domaintools.com/webmasters/surveybot.php
User-agent: SurveyBot
Disallow: /

# http://commoncrawl.org/faq/
User-agent: CCBot
Disallow: /

# http://cliqz.com/company/cliqzbot
User-agent: Cliqzbot
Disallow: /

# https://domainstats.com/pages/our-bot
User-agent: DomainStatsBot
Disallow: /

# http://www.soso.com/help/webmasters.htm
User-agent : Sosospider
Disallow: /

# https://teleport.dev/
User-agent: Teleport
Disallow: /

# http://www.tenmax.com/teleportpro/
User-agent: TeleportPro
Disallow: /

# https://www.httrack.com/
User-agent: HTTrack
Disallow: /

########## END ##########

Search preview