# robots.txt for http://palimpsest.stanford.edu/ # $Id: robots.txt,v 1.39 2016/07/21 18:18:32 waiscool Exp $ # # # # note: as far as I can tell, you must put the general disallows *before* the specific exceptions ## Is this true? Why? ## also you can only have one block of rules per user-agent pattern, so if you want to distinguish temporary ## from permanent rules, all you can do is include both in the same block and use comments # # # Validator (see below) says to put User-agent: * LAST # # NB Validate this with http://tool.motoricerca.info/robots-checker.phtml # Valid on date: Nov 21, 2006 User-agent: * Disallow: /wp Disallow: /xyz Disallow: /defgh Disallow: /xyz_jcms Disallow: /bnbn Disallow: /mnopq # temporarily bar everyone, while we set up cool.conservation-us.org #User-agent: * #Disallow: / ############ Specific agents # I want the Internet Archiver to index the mailing-lists and some # special dirs User-agent: ia_archiver # # Permanent disallows Disallow: /Architext/ Disallow: /cgi-bin/ Disallow: /waac/admin/ Disallow: /waac/board-business/ Disallow: /testarea/ Disallow: /admin/ Disallow: /RCS/ # not sure whether I want the archives to have these because of confidentiality issues # but I'll depend on my server/file-system perms for now #Disallow: /byform/mailing-lists/cdl/instances/ # the following no longer exists in the web space #Disallow: /byform/mailing-lists/cdl/tagged/ Disallow: /_admin/ Disallow: /private/ Disallow: /sg/osg/private/ Disallow: /sg/bpg/private/ Disallow: /sg/cipp/private/ Disallow: /scripts/ #Disallow: /img/ #Disallow: /gfx/ Disallow: /scripts/ #Disallow: /css/ Disallow: /coolaic/testdir Disallow: /coolaic/css Disallow: /coolaic/sg/obs Disallow: /coolaic/sg/pmg.bak Disallow: /coolaic/sg/wag.bak Disallow: /coolaic/sg/asg/test Disallow: /coolaic/sg/bpg/RCS Disallow: /coolaic/sg/bpg/obs Disallow: /coolaic/sg/bpg/private Disallow: /coolaic/sg/cipp/private Disallow: /coolaic/sg/emg/RCS Disallow: /coolaic/sg/emg/obs Disallow: /coolaic/sg/emg/private Disallow: /coolaic/sg/osg/RCS Disallow: /coolaic/sg/osg/private Disallow: /coolaic/sg/pmg/private Disallow: /coolaic/sg/pmg/RCS Disallow: /coolaic/sg/psg/private Disallow: /coolaic/sg/psg/RCS Disallow: /coolaic/sg/rats/private Disallow: /coolaic/sg/rats/RCS Disallow: /coolaic/sg/tsg/RCS Disallow: /coolaic/sg/tsg/obs Disallow: /coolaic/sg/tsg/private Disallow: /coolaic/sg/html/private Disallow: /coolaic/sg/assets/private Disallow: /coolaic/jaic/RCS Disallow: /coolaic/jaic/test # We're not currently using ht//dig but we don't object to remote sites indexing us # but let's be careful User-agent: htdig # # temporary disallows during htdig setup # # Permanent disallows during setup Disallow: /help/ Disallow: /jcms/current/ Disallow: /byform/mailing-lists/osg-l/ Disallow: /byform/mailing-lists/aic-prog/ Disallow: /byform/mailing-lists/emg-board/ Disallow: /byform/mailing-lists/emg-education/ Disallow: /byform/mailing-lists/bpg-board/ Disallow: /byform/mailing-lists/bpg-edu/ Disallow: /byform/mailing-lists/bpg-pubc/ Disallow: /byform/mailing-lists/cippnews-l/ Disallow: /byform/mailing-lists/aic-prog/ Disallow: /byform/mailing-lists/asglist/ Disallow: /byform/mailing-lists/aic-paintings/ Disallow: /byform/mailing-lists/aic-photographic/ Disallow: /Architext/ Disallow: /cgi-bin/ Disallow: /waac/admin/ Disallow: /waac/board-business/ Disallow: /testarea/ Disallow: /admin/ Disallow: /RCS/ Disallow: /misc/people/chunked/ Disallow: /misc/people/nations/ Disallow: /byform/mailing-lists/cdl/instances/ Disallow: /byform/mailing-lists/cdl/tagged/ Disallow: /wcg/Old/ Disallow: /icom/Old/ Disallow: /_admin/ Disallow: /sg/osg/private/ Disallow: /sg/bpg/private/ Disallow: /sg/cipp/private/ Disallow: /scripts/ Disallow: /img/ Disallow: /gfx/ Disallow: /scripts/ Disallow: /css/ #################################### THIS SHOULD GO LAST ################################# ###### Global -- NB Keep block together User-agent: * # # temporary disallows during setup # # Permanent disallows Disallow: /bugger Disallow: */bugger* Disallow: /crap Disallow: *crap* Disallow: /inc Disallow: /icom.bak Disallow: /cool_files Disallow: /Templates Disallow: /help/ Disallow: /mirrors/jcms/current/ Disallow: /byform/mailing-lists/osg-l/ Disallow: /byform/mailing-lists/aic-prog/ Disallow: /byform/mailing-lists/emg-board/ Disallow: /byform/mailing-lists/emg-education/ Disallow: /byform/mailing-lists/bpg-board/ Disallow: /byform/mailing-lists/bpg-edu/ Disallow: /byform/mailing-lists/bpg-pubc/ Disallow: /byform/mailing-lists/cippnews-l/ Disallow: /byform/mailing-lists/aic-prog/ Disallow: /byform/mailing-lists/asglist/ Disallow: /byform/mailing-lists/osg-l/ Disallow: /byform/mailing-lists/aic-paintings/ Disallow: /byform/mailing-lists/aic-photographic/ Disallow: /Architext/ Disallow: /cgi-bin/ Disallow: /waac/admin/ Disallow: /waac/board-business/ Disallow: /byorg/baman/inc/ Disallow: /testarea/ Disallow: /admin/ Disallow: /RCS/ Disallow: /misc/people/chunked/ Disallow: /misc/people/nations/ Disallow: /byform/mailing-lists/cdl/instances/ Disallow: /byform/mailing-lists/cdl/tagged/ Disallow: /wcg/Old/ Disallow: /icom/Old/ Disallow: /private/ Disallow: /_admin/ Disallow: /scripts/ Disallow: /img/ Disallow: /scripts/ Disallow: /coolaic/gfx Disallow: /coolaic/img Disallow: /coolaic/images Disallow: /coolaic/_sitelib Disallow: /coolaic/sg/_sitelib Disallow: /coolaic/sg_sitelib Disallow: /coolaic/testdir Disallow: /coolaic/css Disallow: /coolaic/sg/obs Disallow: /coolaic/sg/_baks Disallow: /coolaic/sg/_notes Disallow: /coolaic/sg/pmg.bak Disallow: /coolaic/sg/wag.bak Disallow: /coolaic/sg/asg/test Disallow: /coolaic/sg/asg/_notes Disallow: /coolaic/sg/bpg/RCS Disallow: /coolaic/sg/bpg/Templates Disallow: /coolaic/sg/bpg/img Disallow: /coolaic/sg/bpg/obs Disallow: /coolaic/sg/bpg/private Disallow: /coolaic/sg/cipp/RCS Disallow: /coolaic/sg/cipp/private Disallow: /coolaic/sg/emg/RCS Disallow: /coolaic/sg/emg/obs Disallow: /coolaic/sg/emg/private Disallow: /coolaic/sg/osg/RCS Disallow: /coolaic/sg/osg/private Disallow: /coolaic/sg/pmg/private Disallow: /coolaic/sg/pmg/RCS Disallow: /coolaic/sg/psg/private Disallow: /coolaic/sg/psg/RCS Disallow: /coolaic/sg/rats/private Disallow: /coolaic/sg/rats/RCS Disallow: /coolaic/sg/tsg/RCS Disallow: /coolaic/sg/tsg/obs Disallow: /coolaic/sg/tsg/private Disallow: /coolaic/sg/html/private Disallow: /coolaic/sg/assets/private Disallow: /coolaic/sg/wag/private Disallow: /coolaic/jaic/RCS Disallow: /coolaic/jaic/test # Google User-agent: GoogleBot Disallow: /anagpic/about.htm Disallow: /anagpic/bylaws.htm Disallow: /anagpic/member_programs.htm Disallow: /anagpic/submissions.htm #################################### NOTHING below here -- globals should be the last #################################
Following keywords were found. You can check the keyword optimization of this page for each keyword.
(Nice to have)