# robots.txt — software-supplies.co.uk # Updated May 2026 ############################################### # BLOCKED — aggressive scrapers, no index value ############################################### User-agent: Bytespider Disallow: / User-agent: PetalBot Disallow: / User-agent: DataForSeoBot Disallow: / ############################################### # SEO CRAWLERS — full access # Google, Bing, and major search engines ############################################### User-agent: Googlebot Allow: / User-agent: Googlebot-Image Allow: / User-agent: Googlebot-Video Allow: / User-agent: AdsBot-Google Allow: / User-agent: Bingbot Allow: / User-agent: Slurp Allow: / User-agent: DuckDuckBot Allow: / User-agent: Baiduspider Allow: / User-agent: YandexBot Allow: / ############################################### # AI CRAWLERS — full access # Covers indexing, citation, AI search answers ############################################### User-agent: GPTBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: ClaudeBot Allow: / User-agent: Claude-Web Allow: / User-agent: anthropic-ai Allow: / User-agent: Google-Extended Allow: / User-agent: PerplexityBot Allow: / User-agent: CCBot Allow: / User-agent: Applebot-Extended Allow: / User-agent: meta-externalagent Allow: / User-agent: Amazonbot Allow: / User-agent: Diffbot Allow: / User-agent: YouBot Allow: / User-agent: cohere-ai Allow: / User-agent: AI2Bot Allow: / ############################################### # DEFAULT — all other crawlers ############################################### User-agent: * # WordPress admin Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php # WooCommerce thin and transactional pages Disallow: /cart/ Disallow: /checkout/ Disallow: /my-account/ Disallow: /?s= Disallow: /search/ Disallow: /add-to-cart/ # WooCommerce internal directories Disallow: /wp-content/uploads/wc-logs/ Disallow: /wp-content/uploads/woocommerce_transient_files/ Disallow: /wp-content/uploads/woocommerce_uploads/ # Complianz GDPR — dynamically versioned CSS, no index value Disallow: /wp-content/uploads/complianz/ # WooCommerce REST API and analytics endpoints Disallow: /wp-json/wc/ Disallow: /wp-json/woocommerce-analytics/ # Crawlable assets Allow: /wp-content/uploads/ Allow: /wp-content/plugins/ Allow: /wp-content/themes/ Allow: /wp-includes/js/ ############################################### # SITEMAP ############################################### Sitemap: https://software-supplies.co.uk/sitemap_index.xml