# Local Removals  — robots.txt (optimized)
# Last-updated: 2025-10-28

# Sitemap(s)
Sitemap: https://local-removals.co.uk/sitemap_index.xml

########################
# Global rules (all bots)
########################
User-agent: *
# Block sensitive/admin and non-HTML endpoints
Disallow: /cgi-bin/
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /wp-config.php
Disallow: /xmlrpc.php
Disallow: /*.php$
# Keep feeds, search results, and trackbacks out of indexes
Disallow: /trackback/
Disallow: /feed/
Disallow: /*/feed/
Disallow: /*?s=
# Allow assets for proper rendering (do not block CSS/JS/images)
Allow: /*.js$
Allow: /*.css$
Allow: /wp-includes/js/
Allow: /wp-includes/css/
Allow: /wp-content/uploads/
# (Intentionally NOT blocking /wp-content/plugins/ or /wp-content/themes/ to avoid asset issues)

# Note: No crawl-delay — major search engines ignore it and it slows discovery.
# If you must throttle, do it in server/CDN rate controls.

########################
# AI/LLM crawlers (explicit)
########################
# OpenAI
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /
User-agent: Claude-Web
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Apple
User-agent: Applebot-Extended
Allow: /

# Google AI extension (training use). Allow = opt-in. Switch to Disallow to opt-out.
User-agent: Google-Extended
Allow: /

# Common Crawl (used by multiple research/AI efforts). Allow = opt-in.
User-agent: CCBot
Allow: /