Dslrpros.com - SEO Checker

Meta tags

Media list

Heading structure

Found links on this page

HTTP Response Header

Links from Wikipedia

Robots.txt

# GENERAL SETTINGS

## Enable robots.txt rules for all crawlers
User-agent: *
Allow: /

## Disallow pslogin
Disallow: /pslogin/*
Disallow: /page_cache/*

## Magento sitemap: 
Sitemap: https://www.dslrpros.com/media/dslrpros_sitemap.xml
Sitemap: https://www.dslrpros.com/media/blog_dslrpros_sitemap.xml

## DEVELOPMENT RELATED SETTINGS

## CVS, svn directories and dump files
Disallow: /*.csv$
Disallow: /*.xls$
Disallow: /*.svn$
Disallow: /*.idea$
Disallow: /*.sql$
Disallow: /*.php$

## GENERAL MAGENTO SETTINGS

## MAGENTO SEO IMPROVEMENTS

## Do not crawl sub category pages that are sorted or filtered.
Disallow: *?dir*
Disallow: *?dir=desc
Disallow: *?dir=asc
Disallow: *?limit=
Disallow: *?limit=all
Disallow: *?__store=
Disallow: *?mode

## Do not crawl links with session IDs
Disallow: /*?SID=

## Do not crawl checkout and user account pages
Disallow: /checkout/
Disallow: /onestepcheckout/
Disallow: /customer/
Disallow: /customer/account/
Disallow: /checkout/cart/add/uenc/
Disallow: /customer/account/login/
Disallow: /sales/

## Do not crawl search pages and not-SEO optimized catalog links
Disallow: /catalogsearch/
Disallow: /catalog/product_compare/add/product/*
Disallow: /catalog/product_compare/
Disallow: /catalog/category/view/
Disallow: /catalog/product/view/
Disallow: /catalogsearch/*
Disallow: /ajaxcart/*
Disallow: /tag/
Disallow: /wishlist/
Disallow: /clickheat/
Disallow: /catalog/
Disallow: /sendfriend/
Disallow: /index.php/

## Do not crawl URLs with parameters.
Disallow: *?PageSpeed=noscript
Disallow: *?dronemodel=
Disallow: *?producttype=
Disallow: *?p=
Disallow: *?yoReviewsPage=
Disallow: *?mode=list
Disallow: *?mode=grid
Disallow: *?variant=
Disallow: *?order=price
Disallow: *?order=position
Disallow: *?price=amshopby_slider_from-amshopby_slider_to
Disallow: *?manufacturer=
Disallow: *?options=cart

## SERVER SETTINGS

## Do not crawl common server technical folders and files
Disallow: /cgi-bin/
Disallow: /cleanup.php
Disallow: /apc.php
Disallow: /memcache.php
Disallow: /phpinfo.php

## Common Magento Technical Folders
Disallow: /app/
Disallow: /downloader/
Disallow: /errors/
Disallow: /includes/
Disallow: /lib/
Disallow: /pkginfo/
Disallow: /shell/
Disallow: /var/
Disallow: /bin/
Disallow: /dev/
Disallow: /phpserver/
Disallow: /pub/
Disallow: /report/

## Common Magento Files
Disallow: /api.php
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /get.php
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /README.txt
Disallow: /RELEASE_NOTES.txt

## DISALLOW DUPLICATE CONTENT
Disallow: /tag/
Disallow: /review/

Search preview