# # robots.txt # # This file is to prevent the crawling and indexing of certain parts # of your site by web crawlers and spiders run by sites like Yahoo! # and Google. By telling these "robots" where not to go on your site, # you save bandwidth and server resources. # # This file will be ignored unless it is at the root of your host: # Used: http://example.com/robots.txt # Ignored: http://example.com/site/robots.txt # # For more information about the robots.txt standard, see: # http://www.robotstxt.org/robotstxt.html User-agent: Twiceler Disallow: / User-agent: oBot Disallow: / User-agent: * Disallow: */edit Disallow: /ogm Disallow: /hr/senior_promotions/ Disallow: /UCL-Info Disallow: /search-webglimps Disallow: /Registry Disallow: /ceacb Disallow: /drupal/ Disallow: /dpl/ Disallow: /tobedeleted-sites/ Disallow: /silva/ Disallow: /www-mail/ Disallow: /digital-presence-services/chat/ #Drupal default # CSS, JS, Images Allow: /misc/*.css$ Allow: /misc/*.css? Allow: /misc/*.js$ Allow: /misc/*.js? Allow: /misc/*.gif Allow: /misc/*.jpg Allow: /misc/*.jpeg Allow: /misc/*.png Allow: /modules/*.css$ Allow: /modules/*.css? Allow: /modules/*.js$ Allow: /modules/*.js? Allow: /modules/*.gif Allow: /modules/*.jpg Allow: /modules/*.jpeg Allow: /modules/*.png Allow: /profiles/*.css$ Allow: /profiles/*.css? Allow: /profiles/*.js$ Allow: /profiles/*.js? Allow: /profiles/*.gif Allow: /profiles/*.jpg Allow: /profiles/*.jpeg Allow: /profiles/*.png Allow: /themes/*.css$ Allow: /themes/*.css? Allow: /themes/*.js$ Allow: /themes/*.js? Allow: /themes/*.gif Allow: /themes/*.jpg Allow: /themes/*.jpeg Allow: /themes/*.png Allow: */misc/*.css$ Allow: */misc/*.css? Allow: */misc/*.js$ Allow: */misc/*.js? Allow: */misc/*.gif Allow: */misc/*.jpg Allow: */misc/*.jpeg Allow: */misc/*.png Allow: */modules/*.css$ Allow: */modules/*.css? Allow: */modules/*.js$ Allow: */modules/*.js? Allow: */modules/*.gif Allow: */modules/*.jpg Allow: */modules/*.jpeg Allow: */modules/*.png Allow: */profiles/*.css$ Allow: */profiles/*.css? Allow: */profiles/*.js$ Allow: */profiles/*.js? Allow: */profiles/*.gif Allow: */profiles/*.jpg Allow: */profiles/*.jpeg Allow: */profiles/*.png Allow: */themes/*.css$ Allow: */themes/*.css? Allow: */themes/*.js$ Allow: */themes/*.js? Allow: */themes/*.gif Allow: */themes/*.jpg Allow: */themes/*.jpeg Allow: */themes/*.png # Directories Disallow: /includes/ Disallow: /misc/ Disallow: /modules/ Disallow: /profiles/ Disallow: /scripts/ Disallow: /themes/ Disallow: /node/ Disallow: */includes/ Disallow: */misc/ Disallow: */modules/ Disallow: */profiles/ Disallow: */scripts/ Disallow: */themes/ Disallow: */node/ # Files Disallow: /CHANGELOG.txt Disallow: /cron.php Disallow: /INSTALL.mysql.txt Disallow: /INSTALL.pgsql.txt Disallow: /INSTALL.sqlite.txt Disallow: /install.php Disallow: /INSTALL.txt Disallow: /LICENSE.txt Disallow: /MAINTAINERS.txt Disallow: /update.php Disallow: /UPGRADE.txt Disallow: /xmlrpc.php Disallow: */CHANGELOG.txt Disallow: */cron.php Disallow: */INSTALL.mysql.txt Disallow: */INSTALL.pgsql.txt Disallow: */INSTALL.sqlite.txt Disallow: */install.php Disallow: */INSTALL.txt Disallow: */LICENSE.txt Disallow: */MAINTAINERS.txt Disallow: */update.php Disallow: */UPGRADE.txt Disallow: */xmlrpc.php # Paths (clean URLs) Disallow: /admin/ Disallow: /comment/reply/ Disallow: /filter/tips/ Disallow: /node/add/ Disallow: /search/ Disallow: /user/register/ Disallow: /user/password/ Disallow: /user/login/ Disallow: /user/logout/ Disallow: */admin/ Disallow: */comment/reply/ Disallow: */filter/tips/ Disallow: */node/add/ Disallow: */search/ Disallow: */user/register/ Disallow: */user/password/ Disallow: */user/login/ Disallow: */user/logout/ # Paths (no clean URLs) Disallow: /?q=admin/ Disallow: /?q=comment/reply/ Disallow: /?q=filter/tips/ Disallow: /?q=node/add/ Disallow: /?q=search/ Disallow: /?q=user/password/ Disallow: /?q=user/register/ Disallow: /?q=user/login/ Disallow: /?q=user/logout/ Disallow: */?q=admin/ Disallow: */?q=comment/reply/ Disallow: */?q=filter/tips/ Disallow: */?q=node/add/ Disallow: */?q=search/ Disallow: */?q=user/password/ Disallow: */?q=user/register/ Disallow: */?q=user/login/ Disallow: */?q=user/logout/ # Paths (clean URLs) - fixed Disallow: /admin Disallow: /comment/reply Disallow: /filter/tips Disallow: /node/add Disallow: /search Disallow: /user/register Disallow: /user/password Disallow: /user/login Disallow: /user/logout Disallow: */admin Disallow: */comment/reply Disallow: */filter/tips Disallow: */node/add Disallow: */user/register Disallow: */user/password Disallow: */user/login Disallow: */user/logout # Paths (no clean URLs) - fixed Disallow: /?q=admin Disallow: /?q=comment/reply Disallow: /?q=filter/tips Disallow: /?q=node/add Disallow: /?q=search Disallow: /?q=user/password Disallow: /?q=user/register Disallow: /?q=user/login Disallow: /?q=user/logout Disallow: */?q=admin Disallow: */?q=comment/reply Disallow: */?q=filter/tips Disallow: */?q=node/add Disallow: */?q=search Disallow: */?q=user/password Disallow: */?q=user/register Disallow: */?q=user/login Disallow: */?q=user/logout # Sites Disallow: /tobedeleted-sites Disallow: /culture/staff/staff Disallow: /culture/projects/projects Disallow: /drupal-training Disallow: /prospective-students/undergraduate-redesign Disallow: /work-at-ucl/redeployment # Sites - fixed Disallow: /culture/staff/staff/ Disallow: /culture/projects/projects/ Disallow: /drupal-training/ Disallow: /prospective-students/undergraduate-redesign/ Disallow: /work-at-ucl/redeployment/ User-Agent: Funnelback Allow: /drupal/ Allow: /module-catalogue/ Allow: /prospective-students/undergraduate-redesign/
Folgende Keywords wurden erkannt. Überprüfe die Optimierung dieser Keywords für Deine Seite.
(Nice to have)