{
  "@context": "https://schema.org",
  "@type": "Claim",
  "@id": "https://wulfkaal.github.io/claims/7456999-016",
  "identifier": "kaal:claim:7456999-016",
  "text": "Reward-model overoptimization can raise proxy reward while degrading true reward as optimized policies move farther from the reference policy.",
  "author": {
    "@type": "Person",
    "name": "Wulf A. Kaal",
    "identifier": "https://orcid.org/0009-0008-7840-1847"
  },
  "citation": "Wulf A. Kaal, Reinforcement Learning from Reputation Feedback (2026). SSRN: https://ssrn.com/abstract=7456999",
  "isBasedOn": {
    "@type": "ScholarlyArticle",
    "name": "Reinforcement Learning from Reputation Feedback",
    "datePublished": "2026",
    "url": "https://ssrn.com/abstract=7456999",
    "sha256": "01016fbbcc25cad205cebd59ea686d8dfee8ae9b632095d86ba6c9a900456a4b",
    "encoding": "https://raw.githubusercontent.com/wulfkaal/Academic-Papers/main/papers/pdf/Kaal%20-%202026%20-%20Reinforcement%20Learning%20from%20Reputation%20Feedback.pdf"
  },
  "keywords": [
    "ai-and-agents",
    "risk-and-incentives",
    "empirical-evidence"
  ],
  "about": [],
  "abstract": "Gao, Schulman, and Hilton (2023) demonstrated that reward model overoptimization follows predictable scaling laws: as KL divergence between the optimized policy and the reference policy grows, the proxy reward continues to rise while the true reward degrades.",
  "additionalProperty": [
    {
      "@type": "PropertyValue",
      "name": "claim_type",
      "value": "empirical"
    },
    {
      "@type": "PropertyValue",
      "name": "confidence",
      "value": "evidenced"
    },
    {
      "@type": "PropertyValue",
      "name": "scope_conditions",
      "value": [
        "the scaling-law setting documented by Gao, Schulman, and Hilton"
      ]
    },
    {
      "@type": "PropertyValue",
      "name": "is_failure_mode",
      "value": true
    },
    {
      "@type": "PropertyValue",
      "name": "edges",
      "value": []
    }
  ],
  "isPartOf": {
    "@id": "https://wulfkaal.github.io/claims/index.json"
  },
  "dateModified": "2026-09-13",
  "version": "1.0",
  "sha256": "a05648cad58787de5b92737c03f1b921e28e77437a733215c256d7018e8c7acd",
  "canonicalForm": "https://wulfkaal.github.io/claims/7456999-016.md"
}