Robots.txt
User-agent: AdsBot-Google
disallow: /account/
disallow: /admin/
disallow: /players/
disallow: /registrations/
disallow: /financial-aid-applications/
disallow: /overdue-payments/
disallow: /waivers/
disallow: /styleguide/
User-agent: Applebot
disallow: /account/
disallow: /admin/
disallow: /players/
disallow: /registrations/
disallow: /financial-aid-applications/
disallow: /overdue-payments/
disallow: /waivers/
disallow: /styleguide/
User-agent: Bingbot
disallow: /account/
disallow: /admin/
disallow: /players/
disallow: /registrations/
disallow: /financial-aid-applications/
disallow: /overdue-payments/
disallow: /waivers/
disallow: /styleguide/
User-agent: Discordbot
disallow: /account/
disallow: /admin/
disallow: /players/
disallow: /registrations/
disallow: /financial-aid-applications/
disallow: /overdue-payments/
disallow: /waivers/
disallow: /styleguide/
User-agent: DuckDuckBot
disallow: /account/
disallow: /admin/
disallow: /players/
disallow: /registrations/
disallow: /financial-aid-applications/
disallow: /overdue-payments/
disallow: /waivers/
disallow: /styleguide/
User-agent: facebookexternalhit
disallow: /account/
disallow: /admin/
disallow: /players/
disallow: /registrations/
disallow: /financial-aid-applications/
disallow: /overdue-payments/
disallow: /waivers/
disallow: /styleguide/
User-agent: Googlebot
disallow: /account/
disallow: /admin/
disallow: /players/
disallow: /registrations/
disallow: /financial-aid-applications/
disallow: /overdue-payments/
disallow: /waivers/
disallow: /styleguide/
User-agent: Googlebot-Image
disallow: /account/
disallow: /admin/
disallow: /players/
disallow: /registrations/
disallow: /financial-aid-applications/
disallow: /overdue-payments/
disallow: /waivers/
disallow: /styleguide/
User-agent: ia_archiver
disallow: /account/
disallow: /admin/
disallow: /players/
disallow: /registrations/
disallow: /financial-aid-applications/
disallow: /overdue-payments/
disallow: /waivers/
disallow: /styleguide/
User-agent: LinkedInBot
disallow: /account/
disallow: /admin/
disallow: /players/
disallow: /registrations/
disallow: /financial-aid-applications/
disallow: /overdue-payments/
disallow: /waivers/
disallow: /styleguide/
User-agent: msnbot
disallow: /account/
disallow: /admin/
disallow: /players/
disallow: /registrations/
disallow: /financial-aid-applications/
disallow: /overdue-payments/
disallow: /waivers/
disallow: /styleguide/
User-agent: Naverbot
disallow: /account/
disallow: /admin/
disallow: /players/
disallow: /registrations/
disallow: /financial-aid-applications/
disallow: /overdue-payments/
disallow: /waivers/
disallow: /styleguide/
User-agent: Pinterestbot
disallow: /account/
disallow: /admin/
disallow: /players/
disallow: /registrations/
disallow: /financial-aid-applications/
disallow: /overdue-payments/
disallow: /waivers/
disallow: /styleguide/
User-agent: Slurp
disallow: /account/
disallow: /admin/
disallow: /players/
disallow: /registrations/
disallow: /financial-aid-applications/
disallow: /overdue-payments/
disallow: /waivers/
disallow: /styleguide/
User-agent: teoma
disallow: /account/
disallow: /admin/
disallow: /players/
disallow: /registrations/
disallow: /financial-aid-applications/
disallow: /overdue-payments/
disallow: /waivers/
disallow: /styleguide/
User-agent: TelegramBot
disallow: /account/
disallow: /admin/
disallow: /players/
disallow: /registrations/
disallow: /financial-aid-applications/
disallow: /overdue-payments/
disallow: /waivers/
disallow: /styleguide/
User-agent: Twitterbot
disallow: /account/
disallow: /admin/
disallow: /players/
disallow: /registrations/
disallow: /financial-aid-applications/
disallow: /overdue-payments/
disallow: /waivers/
disallow: /styleguide/
User-agent: Yeti
disallow: /account/
disallow: /admin/
disallow: /players/
disallow: /registrations/
disallow: /financial-aid-applications/
disallow: /overdue-payments/
disallow: /waivers/
disallow: /styleguide/
User-agent: *
disallow: /account/
disallow: /admin/
disallow: /players/
disallow: /registrations/
disallow: /financial-aid-applications/
disallow: /overdue-payments/
disallow: /waivers/
disallow: /styleguide/
# Non-american bots
# China
User-agent: baiduspider
disallow: /
# Russia
User-agent: Yandex
disallow: /
# Czech
User-agent: seznambot
disallow: /
# Bad bots according to
# https://simtechdev.com/blog/good-and-bad-bots-to-control-to-save-server-resources-and-improve-performance/
# UK
User-agent: MJ12Bot
disallow: /
User-agent: PetalBot
disallow: /
User-agent: AspiegelBot
disallow: /
User-agent: AhrefsBot
disallow: /
User-agent: SEMrushBot
disallow: /
User-agent: DotBot
disallow: /
User-agent: MauiBot
disallow: /
# ##################################################################### #
# Robot permissions picked up from wikipedia.org #
# ##################################################################### #
# Crawlers that are kind enough to obey, but which we'd rather not have
# unless they're feeding search engines.
User-agent: UbiCrawler
Disallow: /
User-agent: DOC
Disallow: /
User-agent: Zao
Disallow: /
# Some bots are known to be trouble, particularly those designed to copy
# entire sites. Please obey robots.txt.
User-agent: sitecheck.internetseer.com
Disallow: /
User-agent: Zealbot
Disallow: /
User-agent: MSIECrawler
Disallow: /
User-agent: SiteSnagger
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: Fetch
Disallow: /
User-agent: Offline Explorer
Disallow: /
User-agent: Teleport
Disallow: /
User-agent: TeleportPro
Disallow: /
User-agent: WebZIP
Disallow: /
User-agent: linko
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: Microsoft.URL.Control
Disallow: /
User-agent: Xenu
Disallow: /
User-agent: larbin
Disallow: /
User-agent: libwww
Disallow: /
User-agent: ZyBORG
Disallow: /
User-agent: Download Ninja
Disallow: /
# Misbehaving: requests much too fast:
User-agent: fast
Disallow: /
#
# Sorry, wget in its recursive mode is a frequent problem.
# Please read the man page and use it properly; there is a
# --wait option you can use to set the delay between hits,
# for instance.
#
User-agent: wget
Disallow: /
#
# The 'grub' distributed client has been *very* poorly behaved.
#
User-agent: grub-client
Disallow: /
#
# Doesn't follow robots.txt anyway, but...
#
User-agent: k2spider
Disallow: /
#
# Hits many times per second, not acceptable
# http://www.nameprotect.com/botinfo.html
User-agent: NPBot
Disallow: /
# A capture bot, downloads gazillions of pages with no public benefit
# http://www.webreaper.net/
User-agent: WebReaper
Disallow: /
(Nice to have)