Robots.txt
Sitemap: https://www.trekaroo.com/sitemap.xml
User-agent: Mediapartners-Google
User-agent: AdsBot-Google-Mobile
User-agent: AdsBot-Google
Disallow:
User-agent: Googlebot
User-agent: Applebot
User-agent: TechnoratiBot
User-agent: AdsBot-Google
User-agent: Feedfetcher-Google
User-agent: baiduspider
User-agent: msnbot
User-agent: MJ12bot
User-agent: naverbot
User-agent: seznambot
User-agent: Slurp
User-agent: teoma
User-agent: twiceler
User-agent: Yandex
User-agent: pingdom.com_bot
User-agent: ELB-HealthChecker
User-agent: LinkedInBot
User-agent: Sogou web spider
User-agent: Sosospider
User-agent: facebookexternalhit
User-agent: snap.com
User-agent: bingbot
User-agent: Facebook share follower
User-agent: PostPost
User-agent: Summify
User-agent: FlipboardProxy
User-agent: BaiduMobaider
User-agent: Amazon CloudFront
User-agent: Twitterbot
User-Agent: archive.org_bot
User-Agent: ia_archiver
User-Agent: ScoutJet
Disallow: */edit$
Disallow: /people/*/guides
Disallow: /reviews/digg_it/*
Disallow: /people/*/compliments
Disallow: /people/*/been_theres
Disallow: /guides/*-wishlist
Disallow: /mobile/reviews/*
Disallow: /map$
Disallow: /photos$
Disallow: /*BUS_CENSUS_NAME*
User-agent: dotbot
User-agent: rogerbot
Disallow: */edit$
Disallow: /people/*/guides
Disallow: /reviews/digg_it/*
Disallow: /people/*/compliments
Disallow: /people/*/been_theres
Disallow: /guides/*-wishlist
Disallow: /mobile/reviews/*
Disallow: /map$
Disallow: /photos$
Disallow: /nearby/
User-agent: Fasterfox
User-agent: Nutch
User-agent: spock
User-agent: OmniExplorer_Bot
User-agent: TurnitinBot
User-agent: BecomeBot
User-agent: genieBot
User-agent: MLBot
User-agent: 80bot
User-agent: NextGenSearchBot
User-agent: WinkBot
User-agent: Spock
User-agent: SpokeSpider
User-agent: disco
User-agent: IAB
User-agent: HouxouCrawler
User-agent: zibber
User-agent: NetinfoBot
User-agent: 008
Disallow: /
(Nice to have)