{
 "@context": "https://schema.org",
 "@type": "Claim",
 "@id": "https://wulfkaal.github.io/positions/2026-09-26-004",
 "identifier": "kaal:position:2026-09-26-004",
 "additionalType": "https://wulfkaal.github.io/positions/schema.json#AffirmedPositionClaim",
 "name": "Reward Model Overoptimization Proxy Ground Truth Gap",
 "text": "Gao, Schulman, and Hilton show the central Goodhart problem in reward-model optimization: an imperfect proxy can improve under optimization while ground-truth performance worsens. This supports the protected overoptimization claim in their controlled synthetic setup. The result should not be overstated. Their gold-standard reward model stands in for humans, and the abstract does not establish that degradation is monotonic in realized KL distance from the reference policy.",
 "author": {
  "@type": "Person",
  "name": "Wulf A. Kaal",
  "identifier": "https://orcid.org/0009-0008-7840-1847"
 },
 "datePublished": "2026-09-26",
 "dateModified": "2026-09-26",
 "creativeWorkStatus": "Affirmed",
 "responseType": "qualification",
 "keywords": [
  "ai-and-agents",
  "risk-and-incentives",
  "empirical-evidence"
 ],
 "scope_conditions": [
  "the scaling-law setting documented by Gao, Schulman, and Hilton"
 ],
 "currentDebate": {
  "name": "In Gao, Schulman, and Hilton's synthetic proxy-versus-gold reward-model setup, optimizing an imperfect proxy too much can hinder ground-truth performance.",
  "url": "https://arxiv.org/abs/2210.10760v1"
 },
 "extends": {
  "identifier": "kaal:claim:7456999-016",
  "url": "https://wulfkaal.github.io/claims/7456999-016",
  "citation": "Wulf A. Kaal, Reinforcement Learning from Reputation Feedback (2026). SSRN: https://ssrn.com/abstract=7456999",
  "paper": "Wulf A. Kaal, Reinforcement Learning from Reputation Feedback (2026). SSRN: https://ssrn.com/abstract=7456999",
  "authors": [
   "Wulf A. Kaal"
  ],
  "year": "2026",
  "ssrn": null,
  "source_pdf_sha256": "01016fbbcc25cad205cebd59ea686d8dfee8ae9b632095d86ba6c9a900456a4b"
 },
 "isBasedOn": [
  {
   "@id": "https://wulfkaal.github.io/claims/7456999-016"
  },
  {
   "@type": "CreativeWork",
   "name": "In Gao, Schulman, and Hilton's synthetic proxy-versus-gold reward-model setup, optimizing an imperfect proxy too much can hinder ground-truth performance.",
   "url": "https://arxiv.org/abs/2210.10760v1"
  }
 ],
 "batch_id": "kaal-growth:2026-09-26:daily-001",
 "review_provenance": "https://wulfkaal.github.io/positions-src/2026-09-26-compound-growth-kaal-growth-2026-09-26-daily-001.json",
 "publicationStatus": "public",
 "recordTypeNote": "Dated commentary position extending a scholarly corpus claim. Not a verbatim claim extracted from the paper.",
 "isPartOf": {
  "@id": "https://wulfkaal.github.io/positions/index.json"
 },
 "version": "1.0",
 "canonical_url": "https://wulfkaal.github.io/positions/2026-09-26-004",
 "canonicalForm": "https://wulfkaal.github.io/positions/2026-09-26-004.md",
 "candidateId": "kaal:candidate:f90eaee0604a1dd7db30e58b99010607226e1bd7918d60c04ca0609f8e75f4c7",
 "evidenceLevel": "permitted_full_text",
 "reviewTier": "independent evidence-first compound-growth review",
 "mappingConfidence": null,
 "mappingAmbiguous": false,
 "mappingMethod": "independent one-to-one proposition and mechanism review",
 "mappingWhyRelevant": "The source directly supports the protected claim's core proxy-overoptimization mechanism and ground-truth degradation, while requiring narrower wording than a monotonic claim about realized KL distance.",
 "sourceProvenance": {
  "canonicalUrl": "https://arxiv.org/abs/2210.10760v1",
  "retrievedAt": "2026-09-26T12:43:33.343Z",
  "sourceContentSha256": "a6d7fa9ae80f2ca996abfe8bbeb58de15bbb5c5af750573a38d2ba5c0d7bed6e",
  "sourceProposition": "In Gao, Schulman, and Hilton's synthetic proxy-versus-gold reward-model setup, optimizing an imperfect proxy too much can hinder ground-truth performance.",
  "sourcePropositionSha256": "291fd2f62eb67009cbb731d966986ea6d39bdcc4003d682c72b0737b963145f8",
  "normalizedWorkId": "arxiv:2210.10760",
  "sourceVersion": "v1",
  "exactEvidenceQuote": "In reinforcement learning from human feedback, it is common to optimize against a reward model trained to predict human preferences. Because the reward model is an imperfect proxy, optimizing its value too much can hinder ground truth performance, in accordance with Goodhart's law.",
  "exactEvidenceQuoteSha256": "6f10843015ed3ec77affdf2a3b4343a1aab18fa999e2abcfcf1f76ef705be3b0",
  "claimMappings": [
   {
    "claimId": "kaal:claim:7456999-016",
    "confidence": null,
    "method": "independent one-to-one review",
    "whyRelevant": "The source directly supports the protected claim's core proxy-overoptimization mechanism and ground-truth degradation, while requiring narrower wording than a monotonic claim about realized KL distance.",
    "ambiguous": false
   }
  ]
 },
 "userAffirmation": "Standing scholarly-growth preauthorization bound to a477d44b57756f7af8b15c04a79ae0c9cbe7d66a30ac9edc97a09fc21c8944b7; publication remains conditional on successful projection, deployment, and live-byte verification.",
 "publicIdentitySha256": "f90eaee0604a1dd7db30e58b99010607226e1bd7918d60c04ca0609f8e75f4c7",
 "sha256": "4223e126645f75f2203300500e9114fdd816c184478be395d3a0be989495fc42"
}