# robots.txt — xtoolkit.live
# Last updated: 2026-05-16

# ── General crawlers ──────────────────────────────────────────────────
User-agent: *
Allow: /
Allow: /chrome-extension
Disallow: /api/
Disallow: /admin

# ── Googlebot — explicit priority hints ───────────────────────────────
# The /chrome-extension page is a key product page with rich structured
# data (SoftwareApplication, WebPage, BreadcrumbList, HowTo, FAQPage).
# Explicit Allow ensures it is never shadowed by the /api/ Disallow rule.
User-agent: Googlebot
Allow: /chrome-extension
Allow: /
Disallow: /api/
Disallow: /admin

# ── AI training crawlers — block all ─────────────────────────────────
# These bots scrape content for LLM training without providing traffic.
# We block them to protect content value and reduce server load.

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: Meltwater
Disallow: /

# ── Sitemap index ─────────────────────────────────────────────────────
# Sitemap is split into sub-sitemaps for better crawl organisation:
#   sitemap-pages.xml  — homepage, category landings, static pages
#   sitemap-tools.xml  — all tool pages (55+)
#   sitemap-blog.xml   — blog articles
#   sitemap-images.xml — image sitemap for OG/preview images

Sitemap: https://xtoolkit.live/sitemap.xml
Sitemap: https://xtoolkit.live/sitemap-images.xml