{
  "batch_id": "kaal-growth:2026-09-26:daily-001",
  "date": "2026-09-26",
  "status": "standing-preauthorized",
  "repository_mode": "public",
  "private_compilation": false,
  "review_provenance": "https://wulfkaal.github.io/positions-src/2026-09-26-compound-growth-kaal-growth-2026-09-26-daily-001.json",
  "source_snapshot_sha256": "a477d44b57756f7af8b15c04a79ae0c9cbe7d66a30ac9edc97a09fc21c8944b7",
  "review_ledger_sha256": "4d27457a7b10a77f7be55d7d7056e3a27a4e98e292748377aa18673471185206",
  "exact_authorization": "Publish frozen scholarly-growth packets automatically only after every evidence, identity, uniqueness, authority, protected-layer, deterministic-build, deployment, and live-byte verification gate passes.",
  "positions": [
    {
      "sequence": 3,
      "response_type": "agreement",
      "slug": "instructgpt-three-stage-rlhf-pipeline",
      "topics": [
        "ai-and-agents",
        "research-methods"
      ],
      "text": "Ouyang and coauthors operationalize RLHF as three linked stages: supervised fine-tuning, reward-model training from labeler-ranked outputs, and PPO optimization against that reward model. This directly supports the protected pipeline description within the InstructGPT design. The correspondence should remain bounded. It does not establish that every later preference-optimization method uses an explicit reward model or PPO, and it does not by itself validate the broader RLRF proposal.",
      "scope_conditions": [
        "the canonical RLHF pipeline described by the cited literature"
      ],
      "current_debate": {
        "name": "Ouyang et al.'s InstructGPT method uses supervised fine-tuning, reward-model training on labeler-ranked outputs, and PPO optimization against that reward model.",
        "url": "https://arxiv.org/abs/2203.02155v1"
      },
      "extends": {
        "identifier": "kaal:claim:7456999-015",
        "url": "https://wulfkaal.github.io/claims/7456999-015",
        "citation": "Wulf A. Kaal, Reinforcement Learning from Reputation Feedback (2026). SSRN: https://ssrn.com/abstract=7456999",
        "paper": "Wulf A. Kaal, Reinforcement Learning from Reputation Feedback (2026). SSRN: https://ssrn.com/abstract=7456999",
        "authors": [
          "Wulf A. Kaal"
        ],
        "year": "2026",
        "ssrn": null,
        "source_pdf_sha256": "01016fbbcc25cad205cebd59ea686d8dfee8ae9b632095d86ba6c9a900456a4b"
      },
      "candidate_id": "kaal:candidate:0a89bee805f767babdf32d8db7b45fa8c2f8760590d1e7dfb83f587747fda1b2",
      "evidence_level": "permitted_full_text",
      "review_tier": "independent evidence-first compound-growth review",
      "mapping_confidence": null,
      "mapping_ambiguous": false,
      "mapping_method": "independent one-to-one proposition and mechanism review",
      "mapping_why_relevant": "This is a direct mechanism-level match to the protected three-stage RLHF description, but it supports Ouyang et al.'s InstructGPT pipeline rather than a universal claim about every modern preference-optimization method.",
      "source_provenance": {
        "canonicalUrl": "https://arxiv.org/abs/2203.02155v1",
        "retrievedAt": "2026-09-26T12:43:33.343Z",
        "sourceContentSha256": "6d8e81f561c9b6f3b9b125197b23ce59f3b1f70c198be30fb9f45f3c0cb579bc",
        "sourceProposition": "Ouyang et al.'s InstructGPT method uses supervised fine-tuning, reward-model training on labeler-ranked outputs, and PPO optimization against that reward model.",
        "sourcePropositionSha256": "fdb607c838f76d5e543460fa65d762b4cce630b44972748894dc274fafe8967d",
        "normalizedWorkId": "arxiv:2203.02155",
        "sourceVersion": "v1",
        "exactEvidenceQuote": "A diagram illustrating the three steps of our method: (1) supervised fine-tuning (SFT), (2) reward model (RM) training, and (3) reinforcement learning via proximal policy optimization (PPO) on this reward model. Blue arrows indicate that this data is used to train one of our models. In Step 2, boxes A-D are samples from our models that get ranked by labelers.",
        "exactEvidenceQuoteSha256": "f0369bc97a14e5dd23ce5c53ebd6c4577068143c668bb843084b03974a1fed98",
        "claimMappings": [
          {
            "claimId": "kaal:claim:7456999-015",
            "confidence": null,
            "method": "independent one-to-one review",
            "whyRelevant": "This is a direct mechanism-level match to the protected three-stage RLHF description, but it supports Ouyang et al.'s InstructGPT pipeline rather than a universal claim about every modern preference-optimization method.",
            "ambiguous": false
          }
        ]
      },
      "user_affirmation": "Standing scholarly-growth preauthorization bound to a477d44b57756f7af8b15c04a79ae0c9cbe7d66a30ac9edc97a09fc21c8944b7; publication remains conditional on successful projection, deployment, and live-byte verification."
    },
    {
      "sequence": 4,
      "response_type": "qualification",
      "slug": "reward-model-overoptimization-proxy-ground-truth-gap",
      "topics": [
        "ai-and-agents",
        "risk-and-incentives",
        "empirical-evidence"
      ],
      "text": "Gao, Schulman, and Hilton show the central Goodhart problem in reward-model optimization: an imperfect proxy can improve under optimization while ground-truth performance worsens. This supports the protected overoptimization claim in their controlled synthetic setup. The result should not be overstated. Their gold-standard reward model stands in for humans, and the abstract does not establish that degradation is monotonic in realized KL distance from the reference policy.",
      "scope_conditions": [
        "the scaling-law setting documented by Gao, Schulman, and Hilton"
      ],
      "current_debate": {
        "name": "In Gao, Schulman, and Hilton's synthetic proxy-versus-gold reward-model setup, optimizing an imperfect proxy too much can hinder ground-truth performance.",
        "url": "https://arxiv.org/abs/2210.10760v1"
      },
      "extends": {
        "identifier": "kaal:claim:7456999-016",
        "url": "https://wulfkaal.github.io/claims/7456999-016",
        "citation": "Wulf A. Kaal, Reinforcement Learning from Reputation Feedback (2026). SSRN: https://ssrn.com/abstract=7456999",
        "paper": "Wulf A. Kaal, Reinforcement Learning from Reputation Feedback (2026). SSRN: https://ssrn.com/abstract=7456999",
        "authors": [
          "Wulf A. Kaal"
        ],
        "year": "2026",
        "ssrn": null,
        "source_pdf_sha256": "01016fbbcc25cad205cebd59ea686d8dfee8ae9b632095d86ba6c9a900456a4b"
      },
      "candidate_id": "kaal:candidate:f90eaee0604a1dd7db30e58b99010607226e1bd7918d60c04ca0609f8e75f4c7",
      "evidence_level": "permitted_full_text",
      "review_tier": "independent evidence-first compound-growth review",
      "mapping_confidence": null,
      "mapping_ambiguous": false,
      "mapping_method": "independent one-to-one proposition and mechanism review",
      "mapping_why_relevant": "The source directly supports the protected claim's core proxy-overoptimization mechanism and ground-truth degradation, while requiring narrower wording than a monotonic claim about realized KL distance.",
      "source_provenance": {
        "canonicalUrl": "https://arxiv.org/abs/2210.10760v1",
        "retrievedAt": "2026-09-26T12:43:33.343Z",
        "sourceContentSha256": "a6d7fa9ae80f2ca996abfe8bbeb58de15bbb5c5af750573a38d2ba5c0d7bed6e",
        "sourceProposition": "In Gao, Schulman, and Hilton's synthetic proxy-versus-gold reward-model setup, optimizing an imperfect proxy too much can hinder ground-truth performance.",
        "sourcePropositionSha256": "291fd2f62eb67009cbb731d966986ea6d39bdcc4003d682c72b0737b963145f8",
        "normalizedWorkId": "arxiv:2210.10760",
        "sourceVersion": "v1",
        "exactEvidenceQuote": "In reinforcement learning from human feedback, it is common to optimize against a reward model trained to predict human preferences. Because the reward model is an imperfect proxy, optimizing its value too much can hinder ground truth performance, in accordance with Goodhart's law.",
        "exactEvidenceQuoteSha256": "6f10843015ed3ec77affdf2a3b4343a1aab18fa999e2abcfcf1f76ef705be3b0",
        "claimMappings": [
          {
            "claimId": "kaal:claim:7456999-016",
            "confidence": null,
            "method": "independent one-to-one review",
            "whyRelevant": "The source directly supports the protected claim's core proxy-overoptimization mechanism and ground-truth degradation, while requiring narrower wording than a monotonic claim about realized KL distance.",
            "ambiguous": false
          }
        ]
      },
      "user_affirmation": "Standing scholarly-growth preauthorization bound to a477d44b57756f7af8b15c04a79ae0c9cbe7d66a30ac9edc97a09fc21c8944b7; publication remains conditional on successful projection, deployment, and live-byte verification."
    }
  ],
  "public_authority": {
    "authorityReceipt": "kaal-authority:2026-08-27:compound-growth-standing-preauthorization",
    "lane": "scholarly-growth",
    "standingPreauthorization": true,
    "batchManifestSha256": "a477d44b57756f7af8b15c04a79ae0c9cbe7d66a30ac9edc97a09fc21c8944b7"
  }
}