{
 "study": "Do dental websites block AI crawlers? 2026",
 "publisher": "Tepexa",
 "recorded": "2026-10-03",
 "license": "CC BY 4.0",
 "source_url": "https://tepexa.com/blog/do-dental-websites-block-ai-crawlers",
 "method": {
  "frame": "OpenStreetMap dentist listings in the contiguous US that publish a website",
  "domains_attempted": 3742,
  "robots_served": 2503,
  "states": 49,
  "requests": "one HTTPS request for /robots.txt per domain, nothing else fetched",
  "parsing": "RFC 9309 grouping; a bot counts as blocked when its own group disallows the site root"
 },
 "metrics": {
  "domains_attempted": 3742,
  "robots_served": 2503,
  "states": 49,
  "served_%": 66.9,
  "server_refused_unknown_crawler_%": 10.4,
  "no_robots_file_%": 3.3,
  "unreachable_%": 16.2,
  "other_response_%": 3.1,
  "no_rules_%": 6.2,
  "star_blocks_whole_site_%": 3.6,
  "names_any_ai_bot_%": 10.9,
  "blocks_any_ai_bot_%": 1.5,
  "allows_any_ai_bot_%": 10.3,
  "sitemap_declared_%": 80.1,
  "median_bytes": 177
 },
 "per_bot": {
  "GPTBot": {
   "named_%": 7.4,
   "blocked_%": 0.4
  },
  "OAI-SearchBot": {
   "named_%": 5.2,
   "blocked_%": 0.0
  },
  "ChatGPT-User": {
   "named_%": 4.6,
   "blocked_%": 0.1
  },
  "ClaudeBot": {
   "named_%": 7.5,
   "blocked_%": 0.5
  },
  "Claude-User": {
   "named_%": 1.0,
   "blocked_%": 0.1
  },
  "PerplexityBot": {
   "named_%": 2.7,
   "blocked_%": 0.1
  },
  "Perplexity-User": {
   "named_%": 4.1,
   "blocked_%": 0.0
  },
  "Google-Extended": {
   "named_%": 10.1,
   "blocked_%": 0.3
  },
  "Applebot-Extended": {
   "named_%": 5.8,
   "blocked_%": 0.4
  },
  "CCBot": {
   "named_%": 6.7,
   "blocked_%": 1.1
  },
  "Bytespider": {
   "named_%": 5.9,
   "blocked_%": 1.2
  },
  "Meta-ExternalAgent": {
   "named_%": 5.8,
   "blocked_%": 0.5
  },
  "Amazonbot": {
   "named_%": 8.8,
   "blocked_%": 0.3
  },
  "Googlebot": {
   "named_%": 6.1,
   "blocked_%": 0.0
  },
  "Bingbot": {
   "named_%": 4.8,
   "blocked_%": 0.0
  }
 },
 "limits": [
  "sites that refuse unknown crawlers at the server are absent from the 2,503 parsed files",
  "robots.txt is a request, not an access control — a crawler may ignore it",
  "server-level bot protection can block a crawler that robots.txt allows, and the reverse",
  "the OpenStreetMap frame skews toward practices someone mapped"
 ]
}