{
  "id": 1040,
  "url": "https://arxiv.org/abs/2606.15029v1",
  "title": "Metric Match: A Subset Selection Approach to Evaluating LLM Judge Reliability",
  "summary": "LLM judges are used to reduce the need for costly human labor in evaluating open-ended text generation. However, the reliability of these judges depends critically on their alignment with human raters -- a property that itself depends on costly human annotations. In this work, we develop a method (Metric Match) for estimating correlation-based reliability metrics of LLM judges from limited annotations. Metric Match selects a subset of samples for human annotation such that the subset matches the",
  "authors": "Alyssa Unell, Natalie Dullerud, Naomi Boneh, Meena Jagadeesan, Tatsu Hashimoto, Nigam Shah et al.",
  "category": "research",
  "topics": "safety-alignment,jobs-economy",
  "orgs": null,
  "regions": null,
  "published_at": "2026-06-12T23:54:16.000Z",
  "fetched_at": "2026-07-14T14:14:59.014Z",
  "source_slug": "arxiv-ethics",
  "source_name": "arXiv",
  "source_homepage": "https://arxiv.org",
  "ethics_ai_record_url": "https://ethics.ai/record/1040",
  "original_url": "https://arxiv.org/abs/2606.15029v1",
  "evidence_status": "source-only",
  "attribution": "via ethics.ai"
}