Wiwi.uni-muenster.de - SEO Checker

Meta tags

Media list

Heading structure

Found links on this page

HTTP Response Header

Links from Wikipedia

• Andreas Löschel

Robots.txt

#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.

User-agent: *
Crawl-delay: 10
# CSS, JS, Images
Allow: /*/misc/*.css$
Allow: /*/misc/*.css?
Allow: /*/misc/*.js$
Allow: /*/misc/*.js?
Allow: /*/misc/*.gif
Allow: /*/misc/*.jpg
Allow: /*/misc/*.jpeg
Allow: /*/misc/*.png
Allow: /*/modules/*.css$
Allow: /*/modules/*.css?
Allow: /*/modules/*.js$
Allow: /*/modules/*.js?
Allow: /*/modules/*.gif
Allow: /*/modules/*.jpg
Allow: /*/modules/*.jpeg
Allow: /*/modules/*.png
# Drupal-Directories
Disallow: /*/includes/
Disallow: /*/misc/
Disallow: /*/modules/
Disallow: /*/profiles/
Disallow: /*/scripts/
Disallow: /*/themes/
# Drupal-Files
Disallow: /*/cron.php
Disallow: /*/install.php
Disallow: /*/update.php
Disallow: /*/xmlrpc.php
# Drupal-Paths (clean URLs)
Disallow: /*/admin/
Disallow: /*/comment/reply/
Disallow: /*/filter/tips/
Disallow: /*/node/add/
Disallow: /*/search/
Disallow: /*/user
Disallow: /*/user/
Disallow: /*/user/login?destination=*
Disallow: /*/user/register?destination=*
Disallow: /*/user?destination=*
Disallow: /*/de/admin/
Disallow: /*/de/comment/reply/
Disallow: /*/de/filter/tips/
Disallow: /*/de/node/add/
Disallow: /*/de/search/
Disallow: /*/de/user
Disallow: /*/de/user/
Disallow: /*/de/user/login?destination=*
Disallow: /*/de/user/register?destination=*
Disallow: /*/de/user?destination=*
Disallow: /*/en/admin/
Disallow: /*/en/comment/reply/
Disallow: /*/en/filter/tips/
Disallow: /*/en/node/add/
Disallow: /*/en/search/
Disallow: /*/en/user
Disallow: /*/en/user/
Disallow: /*/en/user/login?destination=*
Disallow: /*/en/user/register?destination=*
Disallow: /*/en/user?destination=*
# Drupal-Paths (no clean URLs)
Disallow: /*/?q=admin/
Disallow: /*/?q=comment/reply/
Disallow: /*/?q=filter/tips/
Disallow: /*/?q=node/add/
Disallow: /*/?q=user
Disallow: /*/?q=user/
Disallow: /*/?q=search/
Disallow: /wiwi/
Disallow: /intranet/
Disallow: /at/
# Specific Drupal-Site-Paths
Disallow: /fakultaet/de/anmeldung-edw
Disallow: /fakultaet/de/teilnahme-edw
Disallow: /fakultaet/en/node/
Disallow: /fakultaet/en/news
Disallow: /fakultaet/en/news/
Disallow: /fakultaet/en/termine
Disallow: /fakultaet/en/termine/
# Other paths
Disallow: /phpmyadmin/
Disallow: /orsee/
Disallow: /exam_time/

Search preview