Namepros.com - SEO Check

Meta Tags

Medienliste

Überschriftenstruktur

Gefundene Links auf dieser Seite

HTTP-Header

Links von Wikipedia

• Jeffrey Beall

Robots.txt

# vim: ft=robots

# Mediapartners-Google and AdsBot-Google ignore wildcard groups.
User-agent: *


##
# Inaccessible by crawlers because of authentication requirements
##

# Require login
Disallow: /account/
Disallow: /blog/*/metrics$
Disallow: /blog/*/reply$
Disallow: /blog/*/reply?
Disallow: /blog/*/unread$
Disallow: /blog/*/watch$
Disallow: /blog/create-thread
Disallow: /blog/post-thread
Disallow: /chat/
Disallow: /conversations/
Disallow: /credits/
Disallow: /forums/*/create-thread
Disallow: /forums/*/post-thread
Disallow: /forums/*/watch$
Disallow: /parking/*
Allow: /parking/$
Disallow: /points/*/add$
Disallow: /points/*/remove$
Disallow: /pq/points/*/*/add$
Disallow: /pq/points/*/*/remove$
Disallow: /pq/quick-replies/*/*/mode?
Disallow: /pq/quick-replies/*/*/reply?
Disallow: /quick-reply/*/reply?
Disallow: /quick-reply/*/toggle-mode?
Disallow: /tags/*/watch$
Disallow: /threads/*/metrics$
Disallow: /threads/*/reply$
Disallow: /threads/*/reply?
Disallow: /threads/*/unread$
Disallow: /threads/*/watch$

# Require POST
Disallow: /login/login
Disallow: /register/register

# Require login often enough to cause problems
Disallow: /members/list/
Disallow: /members/list$
Disallow: /members/list?
Disallow: /members/*/followers
Disallow: /members/*/following
Disallow: /members/*/post-areas
Disallow: /members/*/recent-activity
Disallow: /members/*/recent-content
Disallow: /members/*/trophies

# Transient pages
Disallow: /search/1*/
Disallow: /search/2*/
Disallow: /search/3*/
Disallow: /search/4*/
Disallow: /search/5*/
Disallow: /search/6*/
Disallow: /search/7*/
Disallow: /search/8*/
Disallow: /search/9*/
Disallow: /search/member$
Disallow: /search/member?

##
# Only intended for real, authentic, genuine homo sapiens
##

Disallow: /bclk/
Disallow: /bevt/


##
# Internal stuff.
##

# If you're a hacker, go straight to /internal/ because it's obviously the most vulnerable.
# That's where we hide our ion cannon.  (We have cake!)

Disallow: /admin.php
Disallow: /internal/


##
# Bots that ignore *
##

User-agent: AdsBot-Google
User-agent: Mediapartners-Google
Disallow: /bclk/
Disallow: /bevt/


##
# Less-than-intelligent bots that can't properly parse constructs such as rel="nofollow", X-Robots-Tag, or base tags.
# They also tend not to deduplicate URLs (so if a link to /misc/style appears on every page, they'll crawl it once for each page).
# If your crawler appears here, it's probably snowballing, resulting in a ridiculous number of requests.
##

User-agent: AhrefsBot
User-agent: Linguee
Disallow: /account/
Disallow: /conversations/
Disallow: /credits/
Disallow: /posts/
Disallow: /find-new/
Disallow: /search/
Disallow: /tradereviews/
Disallow: /members/
Disallow: /misc/
Disallow: /goto/
Disallow: /data/
Disallow: /thanks/
Disallow: /bclk/
Disallow: /bevt/


# THE CAKE IS A LIE
# THE CAKE IS A LIE
# THE CAKE IS^D

Suchvorschau