# robots.txt for McGovern Reid Court Lawyers
# Generated by VisibilityKit — A1 SEO Glasgow — 2026-06-18
# Goal: Preserve all existing crawler rules, add AI bot directives.
# Existing rules: PRESERVED unchanged below.
# AI bot rules: ADDED at top (explicit) — do not conflict with existing.

# AI Answer Engines: explicitly allowed 
# These bots provide AI-generated answers and citations.
# Blocking them reduces your visibility in AI search results.

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

# AI Training Scrapers: blocked 
# These bots bulk-download content for AI training datasets.

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: peer39_crawler
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: Wget
Disallow: /

User-agent: curl
Disallow: /

User-agent: python-requests
Disallow: /

# Existing rules: preserved from live robots.txt 
# Scraped 2026-06-18. Edit directly if the site's rules have changed.

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Sitemaps 
Sitemap: http://mcgoverncourtlawyers.co.uk/sitemap.xml
Sitemap: https://mcgoverncourtlawyers.co.uk/sitemap_index.xml