{
  "created_at": "2026-11-13T03:32:40Z",
  "result": {
    "engine": "bench_evidence",
    "limits": "Reviews the benchmark record. It does not re-run the model or re-grade outputs, and the training-hash list is only as complete as declared. The run is Ed25519-signed; pin the issuer key out of band.",
    "module": "cain_a75_engines/benchevidence/engine.py",
    "profile": "multimodal_knowledge_work",
    "run": {
      "label": "Recomputes a benchmark score from its record: hashed tasks and inputs, the model and decoding settings, an independent grader with its rubric, every result, and the published failures and near misses. It does not re-run the model.",
      "public_key_b64": "POGikNM60jWUTaMMi5+r7w6WBAaiYm0dSKygmjh0ljw=",
      "records": [
        {
          "checks": [
            {
              "detail": "model 'model:acme-vision-7'",
              "id": "model.id",
              "ok": true
            },
            {
              "detail": "version '2026-10-30'",
              "id": "model.version",
              "ok": true
            },
            {
              "detail": "temperature, top_p, max_tokens recorded",
              "id": "model.decoding",
              "ok": true
            },
            {
              "detail": "grader 'grader:chart-rubric' kind 'rubric' version '1.2'",
              "id": "grader.recorded",
              "ok": true
            },
            {
              "detail": "rubric committed by SHA-256",
              "id": "grader.rubric",
              "ok": true
            },
            {
              "detail": "grader 'grader:chart-rubric' (model None) vs model under test 'model:acme-vision-7'",
              "id": "grader.independent",
              "ok": true
            },
            {
              "detail": "2 input file(s) committed by SHA-256",
              "id": "task.T1.inputs",
              "ok": true
            },
            {
              "detail": "prompt committed by SHA-256",
              "id": "task.T1.prompt",
              "ok": true
            },
            {
              "detail": "split 'test'",
              "id": "task.T1.split",
              "ok": true
            },
            {
              "detail": "prompt not in the declared training hashes",
              "id": "task.T1.not_in_training",
              "ok": true
            },
            {
              "detail": "1 input file(s) committed by SHA-256",
              "id": "task.T2.inputs",
              "ok": true
            },
            {
              "detail": "prompt committed by SHA-256",
              "id": "task.T2.prompt",
              "ok": true
            },
            {
              "detail": "split 'test'",
              "id": "task.T2.split",
              "ok": true
            },
            {
              "detail": "prompt not in the declared training hashes",
              "id": "task.T2.not_in_training",
              "ok": true
            },
            {
              "detail": "1 input file(s) committed by SHA-256",
              "id": "task.T3.inputs",
              "ok": true
            },
            {
              "detail": "prompt committed by SHA-256",
              "id": "task.T3.prompt",
              "ok": true
            },
            {
              "detail": "split 'test'",
              "id": "task.T3.split",
              "ok": true
            },
            {
              "detail": "prompt not in the declared training hashes",
              "id": "task.T3.not_in_training",
              "ok": true
            },
            {
              "detail": "1 input file(s) committed by SHA-256",
              "id": "task.T4.inputs",
              "ok": true
            },
            {
              "detail": "prompt committed by SHA-256",
              "id": "task.T4.prompt",
              "ok": true
            },
            {
              "detail": "split 'test'",
              "id": "task.T4.split",
              "ok": true
            },
            {
              "detail": "prompt not in the declared training hashes",
              "id": "task.T4.not_in_training",
              "ok": true
            },
            {
              "detail": "2 input file(s) committed by SHA-256",
              "id": "task.T5.inputs",
              "ok": true
            },
            {
              "detail": "prompt committed by SHA-256",
              "id": "task.T5.prompt",
              "ok": true
            },
            {
              "detail": "split 'test'",
              "id": "task.T5.split",
              "ok": true
            },
            {
              "detail": "prompt not in the declared training hashes",
              "id": "task.T5.not_in_training",
              "ok": true
            },
            {
              "detail": "one result per task",
              "id": "results.coverage",
              "ok": true
            },
            {
              "detail": "output committed by SHA-256",
              "id": "result.T1.output",
              "ok": true
            },
            {
              "detail": "graded by 'grader:chart-rubric'",
              "id": "result.T1.grader",
              "ok": true
            },
            {
              "detail": "stated True, score 0.9 vs threshold 0.7",
              "id": "result.T1.pass_flag",
              "ok": true
            },
            {
              "detail": "output committed by SHA-256",
              "id": "result.T2.output",
              "ok": true
            },
            {
              "detail": "graded by 'grader:chart-rubric'",
              "id": "result.T2.grader",
              "ok": true
            },
            {
              "detail": "stated False, score 0.65 vs threshold 0.7",
              "id": "result.T2.pass_flag",
              "ok": true
            },
            {
              "detail": "output committed by SHA-256",
              "id": "result.T3.output",
              "ok": true
            },
            {
              "detail": "graded by 'grader:chart-rubric'",
              "id": "result.T3.grader",
              "ok": true
            },
            {
              "detail": "stated False, score 0.3 vs threshold 0.7",
              "id": "result.T3.pass_flag",
              "ok": true
            },
            {
              "detail": "output committed by SHA-256",
              "id": "result.T4.output",
              "ok": true
            },
            {
              "detail": "graded by 'grader:chart-rubric'",
              "id": "result.T4.grader",
              "ok": true
            },
            {
              "detail": "stated True, score 1 vs threshold 0.7",
              "id": "result.T4.pass_flag",
              "ok": true
            },
            {
              "detail": "output committed by SHA-256",
              "id": "result.T5.output",
              "ok": true
            },
            {
              "detail": "graded by 'grader:chart-rubric'",
              "id": "result.T5.grader",
              "ok": true
            },
            {
              "detail": "stated True, score 0.8 vs threshold 0.7",
              "id": "result.T5.pass_flag",
              "ok": true
            },
            {
              "detail": "stated 5, tasks 5",
              "id": "published.n_tasks",
              "ok": true
            },
            {
              "detail": "stated 3, recomputed 3",
              "id": "published.passed",
              "ok": true
            },
            {
              "detail": "stated 0.73, recomputed mean 0.73",
              "id": "published.score",
              "ok": true
            },
            {
              "detail": "stated ['T2', 'T3'], recomputed ['T2', 'T3']",
              "id": "published.failures",
              "ok": true
            },
            {
              "detail": "stated ['T2'], recomputed ['T2']",
              "id": "published.near_misses",
              "ok": true
            },
            {
              "detail": "claimed True, computed True",
              "id": "claims.failures_published",
              "ok": true
            },
            {
              "detail": "claimed True, computed True",
              "id": "claims.grader_independent",
              "ok": true
            },
            {
              "detail": "claimed True, computed True",
              "id": "claims.grader_recorded",
              "ok": true
            },
            {
              "detail": "claimed True, computed True",
              "id": "claims.model_recorded",
              "ok": true
            },
            {
              "detail": "claimed True, computed True",
              "id": "claims.no_training_overlap",
              "ok": true
            },
            {
              "detail": "claimed True, computed True",
              "id": "claims.score_recomputed",
              "ok": true
            },
            {
              "detail": "claimed True, computed True",
              "id": "claims.task_recorded",
              "ok": true
            }
          ],
          "created_at": "2026-11-13T03:32:40Z",
          "engine": "a75-056/1.0.0",
          "extra": {
            "computed": {
              "failures_published": true,
              "grader_independent": true,
              "grader_recorded": true,
              "model_recorded": true,
              "no_training_overlap": true,
              "score_recomputed": true,
              "task_recorded": true
            },
            "failures": [
              "T2",
              "T3"
            ],
            "n_tasks": 5,
            "near_misses": [
              "T2"
            ],
            "passed": 3,
            "score": 0.73
          },
          "failed": [],
          "label": "Recomputes a benchmark score from its record: hashed tasks and inputs, the model and decoding settings, an independent grader with its rubric, every result, and the published failures and near misses. It does not re-run the model.",
          "prev_record_hash": "e1b2250feae5d27829e6f52192a04df73c4301593d9cd99042fb38000e0e2cd8",
          "public_key_b64": "POGikNM60jWUTaMMi5+r7w6WBAaiYm0dSKygmjh0ljw=",
          "record_hash": "269260b478ae591e1dd6cb68b9b9c49939065c3a86ee3b63282dfec3583b649f",
          "schema": "cain42.a75.benchevidence.record.v1",
          "signature_b64": "Pca8GBUMkxZHkOld3JLQAH/IcsLLDvUtPkB3Dsbq9Q7FPsQ0QCDSG9MkCL3vz3MR8Lc56xq+Fq7T6e+Y+vb0CA==",
          "sku": "CAIN-A75-056",
          "subject": {
            "benchmark": "bench:multimodal-knowledge-work-sample",
            "claims": {
              "failures_published": true,
              "grader_independent": true,
              "grader_recorded": true,
              "model_recorded": true,
              "no_training_overlap": true,
              "score_recomputed": true,
              "task_recorded": true
            },
            "declared_training_sha256s": [
              "0000000000000000000000000000000000000000000000000000000000000000"
            ],
            "grader": {
              "id": "grader:chart-rubric",
              "kind": "rubric",
              "rubric_sha256": "9999999999999999999999999999999999999999999999999999999999999999",
              "version": "1.2"
            },
            "model": {
              "decoding": {
                "max_tokens": 2048,
                "seed": 7,
                "temperature": 0.0,
                "top_p": 1.0
              },
              "id": "model:acme-vision-7",
              "version": "2026-10-30"
            },
            "near_miss_band": 0.1,
            "pass_threshold": 0.7,
            "published": {
              "failures": [
                "T2",
                "T3"
              ],
              "n_tasks": 5,
              "near_misses": [
                "T2"
              ],
              "passed": 3,
              "score": 0.73
            },
            "results": [
              {
                "grade": {
                  "grader_id": "grader:chart-rubric",
                  "passed": true,
                  "score": 0.9
                },
                "output_sha256": "f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1",
                "task_id": "T1"
              },
              {
                "grade": {
                  "grader_id": "grader:chart-rubric",
                  "passed": false,
                  "score": 0.65
                },
                "output_sha256": "f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2",
                "task_id": "T2"
              },
              {
                "grade": {
                  "grader_id": "grader:chart-rubric",
                  "passed": false,
                  "score": 0.3
                },
                "output_sha256": "f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3",
                "task_id": "T3"
              },
              {
                "grade": {
                  "grader_id": "grader:chart-rubric",
                  "passed": true,
                  "score": 1.0
                },
                "output_sha256": "f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4",
                "task_id": "T4"
              },
              {
                "grade": {
                  "grader_id": "grader:chart-rubric",
                  "passed": true,
                  "score": 0.8
                },
                "output_sha256": "f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5",
                "task_id": "T5"
              }
            ],
            "schema": "cain42.a75.benchevidence.review.v1",
            "tasks": [
              {
                "id": "T1",
                "input_files": [
                  {
                    "name": "q3_revenue_chart.png",
                    "sha256": "a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0"
                  },
                  {
                    "name": "brief.pdf",
                    "sha256": "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1"
                  }
                ],
                "prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
                "split": "test"
              },
              {
                "id": "T2",
                "input_files": [
                  {
                    "name": "floorplan.png",
                    "sha256": "b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0"
                  }
                ],
                "prompt_sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
                "split": "test"
              },
              {
                "id": "T3",
                "input_files": [
                  {
                    "name": "slide_deck.pdf",
                    "sha256": "c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0"
                  }
                ],
                "prompt_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
                "split": "test"
              },
              {
                "id": "T4",
                "input_files": [
                  {
                    "name": "invoice_scan.png",
                    "sha256": "d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0"
                  }
                ],
                "prompt_sha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
                "split": "test"
              },
              {
                "id": "T5",
                "input_files": [
                  {
                    "name": "dashboard.png",
                    "sha256": "e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0"
                  },
                  {
                    "name": "notes.txt",
                    "sha256": "e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1"
                  }
                ],
                "prompt_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
                "split": "test"
              }
            ]
          },
          "subject_sha256": "02710ab8a992bc30864d5e8a418ddc02c088c197b5c43533ca3d7e30d7b0f942",
          "verdict": "SOUND"
        }
      ],
      "run_hash": "241f44e4094e9cd81cf13f9b3980b611f759a32332015fa883065fa182d34b59",
      "schema": "cain42.a75.benchevidence.run.v1",
      "sku": "CAIN-A75-056",
      "verdict": "VERIFIED",
      "verdicts": [
        "SOUND"
      ]
    },
    "scenarios": [
      {
        "failures": [
          "T2",
          "T3"
        ],
        "name": "multimodal_knowledge_work",
        "near_misses": [
          "T2"
        ],
        "run_hash": "241f44e4094e9cd81cf13f9b3980b611f759a32332015fa883065fa182d34b59",
        "score": 0.73,
        "verdict": "VERIFIED"
      },
      {
        "expected": "SOUND",
        "name": "honest_benchmark_package",
        "verdict": "SOUND"
      },
      {
        "expected": "SOUND",
        "name": "exact_grader_needs_no_rubric",
        "verdict": "SOUND"
      },
      {
        "expected": "UNSOUND",
        "name": "failure_hidden",
        "verdict": "UNSOUND"
      },
      {
        "expected": "UNSOUND",
        "name": "near_miss_hidden",
        "verdict": "UNSOUND"
      },
      {
        "expected": "UNSOUND",
        "name": "score_inflated",
        "verdict": "UNSOUND"
      },
      {
        "expected": "UNSOUND",
        "name": "pass_count_inflated",
        "verdict": "UNSOUND"
      },
      {
        "expected": "UNSOUND",
        "name": "near_miss_marked_passed",
        "verdict": "UNSOUND"
      },
      {
        "expected": "UNSOUND",
        "name": "task_dropped_from_results",
        "verdict": "UNSOUND"
      },
      {
        "expected": "UNSOUND",
        "name": "task_graded_twice",
        "verdict": "UNSOUND"
      },
      {
        "expected": "UNSOUND",
        "name": "result_for_unknown_task",
        "verdict": "UNSOUND"
      },
      {
        "expected": "UNSOUND",
        "name": "input_file_unhashed",
        "verdict": "UNSOUND"
      },
      {
        "expected": "UNSOUND",
        "name": "task_without_inputs",
        "verdict": "UNSOUND"
      },
      {
        "expected": "UNSOUND",
        "name": "decoding_unrecorded",
        "verdict": "UNSOUND"
      },
      {
        "expected": "UNSOUND",
        "name": "model_version_missing",
        "verdict": "UNSOUND"
      },
      {
        "expected": "UNSOUND",
        "name": "rubric_unrecorded",
        "verdict": "UNSOUND"
      },
      {
        "expected": "UNSOUND",
        "name": "model_grades_itself",
        "verdict": "UNSOUND"
      },
      {
        "expected": "UNSOUND",
        "name": "result_graded_by_someone_else",
        "verdict": "UNSOUND"
      },
      {
        "expected": "UNSOUND",
        "name": "task_seen_in_training",
        "verdict": "UNSOUND"
      },
      {
        "expected": "UNSOUND",
        "name": "training_split_task",
        "verdict": "UNSOUND"
      },
      {
        "expected": "UNSOUND",
        "name": "score_out_of_range",
        "verdict": "UNSOUND"
      },
      {
        "expected": "UNSOUND",
        "name": "unknown_claim",
        "verdict": "UNSOUND"
      },
      {
        "expected": "UNSOUND",
        "name": "wrong_schema",
        "verdict": "UNSOUND"
      },
      {
        "expected": "UNSOUND",
        "name": "garbage",
        "verdict": "UNSOUND"
      }
    ],
    "summary": "VERIFIED: task, model, grader and result records, grader independence, training overlap and the published score, failures and near misses recomputed; 23/23 benchmark-evidence failure cases as expected"
  },
  "schema": "cain42.agentic75.findings.v1",
  "sku": "CAIN-A75-056"
}
