{
  "id": 11367,
  "url": "https://arxiv.org/abs/2607.14447v1",
  "title": "Do Generative AI Assistants Respect robots.txt? Tracing Web Access Beyond Visible Answers",
  "summary": "AI assistants increasingly retrieve web content at inference time to provide fresh and grounded answers, yet it remains unclear whether these search-augmented capabilities respect website-owner restrictions expressed through robots.txt. We present a controlled empirical study of ten widely used AI assistants with advertised web-search capabilities. For each assistant, we first identify a configuration that actually produces observable web-browsing behavior and record the user-agent exposed durin",
  "authors": "Gabriel Lopez-Fonseca, David Rodriguez, Stefan Bechtold, Jose M. Del Alamo",
  "category": "research",
  "topics": "agents-autonomy",
  "orgs": null,
  "regions": null,
  "published_at": "2026-07-16T00:45:06.000Z",
  "fetched_at": "2026-07-18T05:10:55.931Z",
  "source_slug": "arxiv-ethics",
  "source_name": "arXiv",
  "source_homepage": "https://arxiv.org",
  "ethics_ai_record_url": "https://ethics.ai/record/11367",
  "original_url": "https://arxiv.org/abs/2607.14447v1",
  "evidence_status": "source-only",
  "attribution": "via ethics.ai"
}