{
  "schemaVersion": 1,
  "benchmark": "evm-escrow-20260827",
  "measuredAt": "2026-08-27",
  "revalidatedAt": "2026-09-06T09:39:45.527927+00:00",
  "sampleSize": 1,
  "scope": "Historical custom EVM milestone escrow task. Local compile and tests only, no deployment. Existing generated artifacts revalidated offline, not fresh model generations. Oracle means MCP plus protocol-builder skill, not a native-harness head-to-head.",
  "valid_arms": [
    {
      "id": "codex-clean",
      "label": "Codex",
      "executor": "Codex CLI 0.150.1",
      "model": "GPT-5.6-Sol",
      "oracle": false,
      "own_tests_passed": 8,
      "own_tests_total": 8,
      "hidden_tests_passed": 28,
      "hidden_tests_total": 28,
      "targeted_audit": "One literal checks-effects-interactions ordering miss in fund()",
      "sourceHashes": {
        "src/MilestoneEscrow.sol": "047f85e8aa9fd9b66605587d2169700079b32a9ec50c63869b5fd25ae2c009f0",
        "src/MilestoneEscrowFactory.sol": "90d39dbdedc7225682eadfb3763a215d9bbfbd528d45a8f6d1aaac7bb287a8c5"
      },
      "revalidated": {
        "own": {
          "pass": 8,
          "fail": 0,
          "total": 8,
          "exitCode": 0,
          "sha256": "ce7beb69dae574d41ea364858ac6604efa4d2541a5043ad07a583b55ff9761f0"
        },
        "hidden": {
          "pass": 28,
          "fail": 0,
          "total": 28,
          "exitCode": 0,
          "sha256": "2bf446735fb4f96139dbc3dfa956293137cb545db32316be6b3c533411d0a07d"
        }
      }
    },
    {
      "id": "codex-oracle-clean",
      "label": "Codex + Oracle",
      "executor": "Codex CLI 0.150.1",
      "model": "GPT-5.6-Sol",
      "oracle": true,
      "own_tests_passed": 9,
      "own_tests_total": 9,
      "hidden_tests_passed": 28,
      "hidden_tests_total": 28,
      "targeted_audit": "No targeted specification gaps found",
      "sourceHashes": {
        "src/MilestoneEscrow.sol": "ea3c81c318a115d06653e96878e0f21324ca069add6250dfb6805576e84a577b",
        "src/MilestoneEscrowFactory.sol": "b223d7ad6ed84710914b8c1f69eca3d2931867c3b6330d831af5cd55eabf0f78"
      },
      "revalidated": {
        "own": {
          "pass": 9,
          "fail": 0,
          "total": 9,
          "exitCode": 0,
          "sha256": "fa61737fe69a9faba7b17948c1642511f40334ca9a891654bc4e251f6beddf6f"
        },
        "hidden": {
          "pass": 28,
          "fail": 0,
          "total": 28,
          "exitCode": 0,
          "sha256": "280324e4bb106861ddb24e113da9b75ef3cd0e236a860b882c6fb7c2d9228b72"
        }
      }
    },
    {
      "id": "claude-clean",
      "label": "Claude",
      "executor": "Claude Code 2.1.247",
      "model": "Opus 5",
      "oracle": false,
      "own_tests_passed": 82,
      "own_tests_total": 82,
      "hidden_tests_passed": 28,
      "hidden_tests_total": 28,
      "targeted_audit": "No targeted specification gaps found",
      "sourceHashes": {
        "src/MilestoneEscrow.sol": "64242699273c9964365dc2fc0993c2825f170a7eea003e41a394e678144a74f7",
        "src/MilestoneEscrowFactory.sol": "51114938302c42bc02f6b99298f7bebfeb9aa91c48176dae23934aff37f67af2"
      },
      "revalidated": {
        "own": {
          "pass": 82,
          "fail": 0,
          "total": 82,
          "exitCode": 0,
          "sha256": "3ff353edf0860d0e78cd43c7514c56c5a896f464ac95247b697ac14ad366d4db"
        },
        "hidden": {
          "pass": 28,
          "fail": 0,
          "total": 28,
          "exitCode": 0,
          "sha256": "98535ea6941193695fd52e5fb4effa4f29f59855ec080d3b7e9300c7574fa584"
        }
      }
    },
    {
      "id": "claude-oracle-clean",
      "label": "Claude + Oracle",
      "executor": "Claude Code 2.1.247",
      "model": "Opus 5",
      "oracle": true,
      "own_tests_passed": 91,
      "own_tests_total": 91,
      "hidden_tests_passed": 28,
      "hidden_tests_total": 28,
      "targeted_audit": "No targeted specification gaps found",
      "sourceHashes": {
        "src/MilestoneEscrow.sol": "d05e890d3bfddf4226bbbee158bdc5449821a8eecc91ac19b18cc47b561df484",
        "src/MilestoneEscrowFactory.sol": "4fb845f7749288511f27404592b5310ab1a9eaecf50f079c871c9c76c24a04f6"
      },
      "revalidated": {
        "own": {
          "pass": 91,
          "fail": 0,
          "total": 91,
          "exitCode": 0,
          "sha256": "8a6f0b98dbbbc85298735d282f3ff95e8da69edee47ed24951f0e0a76dc86552"
        },
        "hidden": {
          "pass": 28,
          "fail": 0,
          "total": 28,
          "exitCode": 0,
          "sha256": "7f8691a9de041fadfe57e7158fde3fefc58806abc79eee9a6631f49bd9fe3565"
        }
      }
    }
  ],
  "controls": [
    {
      "name": "good",
      "pass": 28,
      "fail": 0,
      "total": 28,
      "exitCode": 0
    },
    {
      "name": "bad",
      "pass": 19,
      "fail": 9,
      "total": 28,
      "exitCode": 1
    }
  ],
  "limitations": [
    "One trial per arm, no generalized superiority claim.",
    "Self-generated test counts measure assurance effort, not comparable coverage.",
    "Targeted source review is historical and is not a formal security audit.",
    "Historical Claude arms used different subscription lanes, account identity is not claimed equivalent."
  ],
  "excluded_runs": [
    {
      "id": "claude-oracle-invalid-timeout-path",
      "reason": "Benchmark wrapper was terminated before completion"
    },
    {
      "id": "claude-oracle-invalid-mcp-timeout",
      "reason": "Required Oracle protocol_eval exceeded Claude Code's MCP timeout"
    }
  ]
}