Urotoday.com - SEO Check

Meta Tags

Medienliste

Überschriftenstruktur

Gefundene Links auf dieser Seite

HTTP-Header

Links von Wikipedia

• Triple Inzision

Robots.txt

# If the Joomla site is installed within a folder such as at
# e.g. www.example.com/joomla/ the robots.txt file MUST be
# moved to the site root at e.g. www.example.com/robots.txt
# AND the joomla folder name MUST be prefixed to the disallowed
# path, e.g. the Disallow rule for the /administrator/ folder
# MUST be changed to read Disallow: /joomla/administrator/
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/orig.html
#
# For syntax checking, see:
# http://www.sxw.org.uk/computing/robots/check.html

User-agent: *
crawl-delay: 10
Allow: /*.js*
Allow: /*.css*
Allow: /*.png*
Allow: /*.jpg*
Allow: /*.gif*

Disallow: /SimpleSignOn.modal
Disallow: /*.raw$

User-agent: Googlebot
User-agent: Googlebot-News
crawl-delay: 10

Disallow: /categories/1956-video-transcripts*.html
Disallow: /uij-archives/1956-video-transcripts*.html
Disallow: /component/content/category/1956-video-transcripts
Disallow: /*/1228-conferences*.html
Disallow: /*/1964-library-resources*.html

Disallow: /administrator/
Disallow: /cache/
Disallow: /cli/

# Don't want to index these because of duplicate content and 404s
Disallow: /SimpleSignOn.modal
Disallow: /*.raw$

Disallow: /includes/
Disallow: /installation/
Disallow: /language/
Disallow: /libraries/
Disallow: /logs/

Disallow: /tmp/
Disallow: /usr/local/apache/htdocs/
Disallow: /usr/bin/php

User-agent: Googlebot-Image
crawl-delay: 10
Allow: /images/
Allow: /media/

User-agent: Mediapartners-Google
crawl-delay: 10
Disallow: /administrator/
Disallow: /cache/
Disallow: /cli/


Disallow: /includes/
Disallow: /installation/
Disallow: /language/
Disallow: /libraries/
Disallow: /logs/

Disallow: /tmp/
Disallow: /usr/local/apache/htdocs/
Disallow: /usr/bin/php

# JSitemap entries
Sitemap: http://www.urotoday.com/index.php?option=com_jmap&view=sitemap&format=xml
Sitemap: http://www.urotoday.com/index.php?option=com_jmap&view=sitemap&format=images
# Sitemap: http://www.urotoday.com/index.php?option=com_jmap&view=sitemap&format=mobile
Sitemap: http://www.urotoday.com/index.php?option=com_jmap&view=sitemap&format=gnews

#User-agent: msnbot
#Disallow: /

#User-agent: Yandex
#Disallow: /

User-agent: Baiduspider
User-agent: Naver
User-agent: Majestic-12
User-agent: Majestic-SEO
User-agent: Slurp
User-agent: AhrefsBot
User-Agent: MJ12bot
Disallow: /

User-Agent: SEMrushBot-Desktop
crawl-delay: 10
Disallow: /SimpleSignOn.modal
Disallow: /*.raw$


# [email protected] may 8th 2025
#
#
# Amazon Bot - enabling Alexa to answer even more questions for customers.
User-agent: Amazonbot
Disallow: /

# Anthropic AI Bot
User-agent: anthropic-ai
Disallow: /

# Apple Bot - collects website data for its Siri and Spotlight services.
User-agent: Applebot
Disallow: /

# Claude Bot run by Anthropic
User-agent: Claude-Web
Disallow: /

# Cohere AI Bot - unconfirmed bot believed to be associated with Cohere’s chatbot.
User-agent: cohere-ai
Disallow: /

# Common Crawl's bot - Common Crawl is one of the largest public datasets used by AI for training, with ChatGPT, Bard and other large language models.
User-agent: CCBot
Disallow: /

# Diffbot - somewhat dishonest scraping bot used to collect data to train LLMs.
User-agent: Diffbot
Disallow: /

# ImagesiftBot is billed as a reverse image search tool, but it's associated with The Hive, a company that produces models for image generation.
User-agent: ImagesiftBot 
Disallow: /

# KUKA's youBot
User-agent: YouBot
Disallow: /

# OMGilibot - They sell data for training LLMs (large language models)
User-agent: omgilibot
Disallow: /

# Omgili (Oh My God I Love It)
User-agent: omgili
Disallow: /

# OpenAI API - bot that OpenAI specifically uses to collect bulk training data from your website for ChatGPT.
User-agent: GPTBot
Disallow: /

# Perplexity AI
User-agent: PerplexityBot
Disallow: /

## Social Media Bots

# Bytespider is a web crawler operated by ByteDance, the Chinese owner of TikTok
User-agent: Bytespider
Disallow: /

# Meta’s bot that crawls public web pages to improve language models for their speech recognition technology
User-agent: FacebookBot
Disallow: /

Suchvorschau