Robots.txt
# User-agent: bingbot
# Disallow: /
User-agent: AhrefsBot
User-agent: AdnormCrawler
User-agent: Baiduspider
User-agent: betaBot
User-agent: bingbot
User-agent: BLEXBot
User-agent: CCBot
User-agent: Cliqzbot
User-agent: cms+crawler
User-agent: coccoc
User-agent: DomainCrawler
User-agent: Exabot
User-agent: genico
User-agent: HaosouSpider
User-agent: ia_archiver
User-agent: istellabot
User-agent: JobboerseBot
User-agent: Linguee
User-agent: Linguee Bot
User-agent: linkapediabot
User-agent: linkdexbot
User-agent: Lipperhey
User-agent: ltx71
User-agent: Lipperhey-Kaus-Australis
User-agent: Mail.RU_Bot
User-agent: meanpathbot
User-agent: megaindex
User-agent: MetaJobBot
User-agent: MJ12bot
User-agent: netEstate NE Crawler
User-agent: Plunkkie
User-agent: psbot
User-agent: org_bot
User-agent: Qwantify
User-agent: rv
User-agent: safedns
User-agent: SEOkicks
User-agent: seoscanners.net
User-agent: SemrushBot
User-agent: SemrushBot-SA
User-agent: SeznamBot
User-agent: SMTBot
User-agent: Slurp
User-agent: Sogou
User-agent: Uptimebot
User-agent: vebidoobot
User-agent: Wget
User-agent: YandexImages
User-agent: YandexBot
User-agent: yoozBot
Disallow: /
User-agent: *
Disallow: /images/
Disallow: /pdf/
(Deseable)