Robots.txt
Sitemap: https://www.rubylane.com/sitemaps/rubylaneSitemap.xml
Sitemap: https://www.rubylane.com/sitemaps/rubylaneSitemapindex_listing.xml
User-agent: AhrefsBot
User-agent: 008
User-agent: psbot
User-agent: spbot
User-agent: linksmanager
User-agent: RedCarpet
User-agent: OmniExplorer_Bot
User-agent: TurnitinBot
User-agent: Twiceler
User-agent: grub-client
User-agent: NaverBot
User-agent: e-SocietyRobot
User-agent: PCNBot
User-agent: aipbot
User-agent: PGBot
User-agent: MJ12bot
User-agent: VoilaBot
User-agent: DealOzBot
User-agent: pimeyes.com crawler
User-agent: PetalBot
User-agent: SemrushBot
User-agent: SEOkicks
User-agent: The Knowledge AI
User-agent: SiteAuditBot
User-agent: SemrushBot-BA
User-agent: SemrushBot-SI
User-agent: SemrushBot-SWA
User-agent: SemrushBot-CT
User-agent: SemrushBot-BM
User-agent: SplitSignalBot
User-agent: SemrushBot-COUB
User-agent: DataForSeoBot
User-agent: dotbot
User-agent: Yandex
User-agent: YandexBot
User-agent: Sogou
User-agent: ImagesiftBot
User-agent: ClaudeBot
Disallow: /
User-Agent: Pinterestbot
Crawl-delay: 2
Disallow: /cgi-bin
Disallow: /clickv1
Disallow: /too
Disallow: /en/
Disallow: /ni/
Disallow: /_hp
User-agent: bingbot
User-agent: Applebot
Crawl-delay: 2
Disallow: /_pod/
Disallow: /_podl/
Disallow: /cgi-bin
Disallow: /clickv1
Disallow: /too
Disallow: /en/
Disallow: /ni/
Disallow: /_hp
User-agent: *
Disallow: /cgi-bin
Disallow: /clickv1
Disallow: /too
Disallow: /en/
Disallow: /ni/
Disallow: /_hp
# specification: https://www.robotstxt.org/norobots-rfc.txt
(Deseable)