Nintendoreporters.com - SEO Checker

Meta tags

Media list

Heading structure

Found links on this page

HTTP Response Header

Links from Wikipedia

Robots.txt

# robots.txt - NintendoReporters.com - all ur data belongs to us ;)
# This robots.txt is optimized for both SEO and AEO.
# We ♥ humans, robots, Nintendo gamers & helpful AI… but not indiscriminate scrapers/crawlers.
# Created in the distant future (2XXX) after a robotic uprising.
# Collection of any data through automated means is prohibited unless you have express (written) permission.

##################################################
# AI / LLM CRAWLERS – ALLOW EVERYTHING HERE      #
##################################################

# — OpenAI

User-agent: OAI-SearchBot
# Model-training: NO - Indexes pages so SearchGPT / ChatGPT “Browse” can cite & link your content in real time.
User-agent: ChatGPT-User
# Model-training: NO - v1.* string - One-off fetcher when a ChatGPT user pastes or clicks a URL.
User-agent: ChatGPT-User/2.0
# Model-training: NO - v2.* string - Model-training: NO - One-off fetcher when a ChatGPT user pastes or clicks a URL.
User-agent: GPTBot
# Model-training: YES - Bulk crawler for GPT-4/5 and future model training and safety tuning.
User-agent: Operator
# Model-training: NO - It’s interactive, not a web-wide scraper. Loads pages in a headless browser only when a user asks the Operator agent to perform a task (e.g. book tickets). It behaves like a human session, not a bulk spider.

# — Anthropic

User-agent: Claude-Web
# Model-training: Probably YES - Collects web data “to enhance the utility and safety of our generative AI models".
User-agent: ClaudeBot
# Model-training: YES - Collects web data “to enhance the utility and safety of our generative AI models.”
User-agent: anthropic-ai
# Model-training: Probably YES - Alias seen in logs; likewise tagged as an undocumented AI agent believed to gather training data.

# — Google

User-agent: Google-Extended
# Model-training: YES - If you allow it. Policy flag, not a crawler: tells Google whether its Bard/Gemini & Vertex AI models may use already-crawled content for training.

# — Perplexity

User-agent: PerplexityBot
# Model-training: NO - Builds the Perplexity.ai search index; returns citations with links.

User-agent: Perplexity-User/1.0
# Model-training: NO - "Not used to crawl content for AI foundation models". User-triggered fetcher that grabs a page so Perplexity can quote & link it in an answer.

# — Apple

User-agent: Applebot-Extended
# Model-training: YES - If allowed. - Opt-in flag that lets Apple reuse Applebot-crawled pages to train its forthcoming “Apple Intelligence” models.

# — Microsoft / Bing / Copilot
User-agent: BingBot
# Model-training: NO – direct model-training not claimed. Content is fed into Bing’s index and may be quoted via retrieval-augmented generation, but the text is not bulk-shoveled into foundation-model checkpoints.Copilot uses Bing’s indexb

# — Amazon

User-agent: Amazonbot
# Model-training: Likely YES - Amazon confirms the data is reused for question-answering AI; most observers treat it as a model-training + retrieval crawler.

# — Meta / Facebook & Instagram

User-agent: FacebookBot
# Model-training: NO - It’s event-triggered, not continuous, and Meta lists it only for preview caching, not AI.
User-agent: meta-externalagent
# Model-training: YES - Explicitly a model-training bot. Large-scale crawler “for use cases such as training AI models or improving products by indexing content directly.” Runs even when no user has shared the link.

# — ByteDance

User-agent: Bytespider
# Model-training: YES - Mass-scale scraper for Doubao / TikTok LLM training.

# — Cohere

User-agent: cohere-ai
# Model-training: ASSUMED YES - Undocumented bot; thought to fetch pages for Cohere’s chat / RAG demos and model work.

# — Common Crawl & open data

User-agent: Diffbot
# Model-training: Potentially YES — Diffbot’s data is marketed for “AI model training.” Crawls sites and turns pages into structured JSON that clients (some being AI labs) can license.

# — Others

User-agent: DuckAssistBot
# Model-training: NO – DuckDuckGo states the data “is not used to train AI models.”
User-agent: YouBot
# Model-training: Probably NO - Framed as an “AI search crawler”; no mention of using text in model training.
User-agent: MistralAI-User
# Model-training: NO - Docs say it “is not used… for generative AI training.” On-demand fetcher for user questions in LeChat; includes source links in answers.

Allow: /
# Allow everything for the agents listed above

##################################################
# SEARCH-ENGINE & SEO EXCEPTIONS                 #
##################################################

User-agent: Googlebot-Image
Allow: /storage/files/

User-agent: Adsbot-Google
User-agent: Googlebot-Mobile
User-agent: AhrefsBot
User-agent: AhrefsSiteAudit
Allow: /

##################################################
#  BLOCKED MODEL-TRACKING SCRAPERS               #
##################################################

User-agent: CCBot
# Model-training: YES - Purpose is to publish data sets that power LLM training. Mass crawler that builds the openly-licensed Common Crawl corpus used by many research labs and companies.
User-agent: AI2Bot
# Model-training: YES - Described as “downloads data to train open-source AI models.” Collects scholarly and general-web content for AI2 research and open-source model work.
User-agent: TimpiBot
# Model-training: YES - By design. The index “can be used to train LLMs.” Distributed crawler whose index can be licensed for LLM training.
User-agent: KangarooBot# Model-training: YES - Explicitly an “AI data scraper” for model training. Scrapes web pages to train open-source models focused on Australian language & culture.
Disallow: /

##################################################
# BLOCKED SCRAPERS & “BAD” BOTS                  #
##################################################

# Generic frameworks / high-volume scrapers
User-agent: Scrapy
User-agent: magpie-crawler
User-agent: Node/simplecrawler
User-agent: omgili
User-agent: omgilibot
Disallow: /

# Misc legacy bandwidth hogs
User-agent: Fasterfox
User-agent: 008
User-agent: MJ12bot
User-agent: Exabot
User-agent: proximic
User-agent: YoudaoBot
User-agent: discobot
User-agent: dotbot
User-agent: yacybot
Disallow: /

##################################################
# UNIVERSAL RULES (“*”)                          #
##################################################

###############################
# Highlight content           #
###############################

User-agent: *

# — Highlighted content
Allow: /nieuws/
Allow: /releases/
Allow: /en/news/
Allow: /en/releases/

###############################
# Standard WordPress hygiene  #
###############################

Disallow: /cgi-bin/
Disallow: /wp-admin/
Allow:    /wp-admin/admin-ajax.php

###############################
# WordPress duplicate pages   #
###############################

Disallow: */feed/*
# posts, cats & tags in /feed/ form - Feeds are near-duplicate content and waste crawl budget.
Disallow: */page/*
# paginated archives (?page/2/, /page/3/) - Prevents Google from indexing dozens of thin “page/2/” URLs.
Disallow: */trackback/
# legacy pingback endpoint - Removes legacy trackback endpoints. Almost never needed today.
Disallow: */embed/
# oEmbed versions of posts & pages - Stops WordPress’s oEmbed copies (/embed/).They add no SEO value.

###############################
# WordPress REST & XML-RPC    #
###############################

Disallow: /wp-json/
# REST API root - Stops crawlers hitting the REST API - Saves server resources unless you surface content via REST. Yoast’s “Crawl Optimisation” toggle does the same.
Disallow: /?rest_route=*
# REST endpoints via query string - Stops crawlers hitting the REST API. - Saves server resources unless you surface content via REST. Yoast’s “Crawl Optimisation” toggle does the same.
Disallow: /xmlrpc.php
# old remote-publishing API - Blocks the legacy XML-RPC interface. Reduces brute-force login attempts; no SEO loss.

###############################
# WordPress attachment dupes  #
###############################

Disallow: /?attachment_id=*
# thin “media attachment” pages - Hides thin “attachment” pages generated for each image.	Avoids useless index entries; the media file itself remains crawlable.

###############################
# WordPress internal search   #
###############################

Disallow: /?s=*
# Prevents search queries from being indexed.
Disallow: /search/*
# Prevents Custom search queries from being indexed.

###############################
# Allow image / asset folders #
###############################

Allow: /storage/files/
# Explicitly lets images, PDFs, videos be crawled. Makes sure Google can render posts correctly even though other wp- paths may be disallowed.
# (Leave CSS/JS open – blocking them hurts rendering.)

###############################
# House-keeping               #
###############################

Disallow: /*.pdf$
# Block PDF Files from Indexing.
Crawl-delay: 3
# Make sure crawlers who respect this value give some room. Googlebot ignores this.

##################################################
# SITEMAPS                                       #
##################################################

Sitemap: https://www.nintendoreporters.com/sitemap_index.xml
Sitemap: https://www.nintendoreporters.com/newsarticle-sitemap.xml
Sitemap: https://www.nintendoreporters.com/video-sitemap.xml

Search preview