{
 "schema": "cain.e89.clbench.preregistration.v1",
 "registered_at": "2026-11-17T23:01:56Z",
 "benchmark": {"name": "Continual Learning Bench", "repo": "https://github.com/pgasawa/continual-learning-bench",
   "commit": "5f8c50eb1e84b2eda2ef4faff757dfc812a0ea26", "paper": "arXiv:2606.05661",
   "task": "blind_spectrum_monitoring", "schedule": "default (90 scans, 3 stages x 30)",
   "corpus_id": "mixed_grid_lifecycle", "task_modified": false, "scorer_modified": false},
 "model": {"id": "Qwen/Qwen2.5-1.5B-Instruct", "serving": "scripts/evolution89/local_openai_server.py (greedy, CPU, fp32)",
   "max_new_tokens": 512, "paid_usd": 0},
 "arms": {
   "A_stateless": "the harness's reset-each-instance baseline, produced inside every arm's run",
   "B_icl": "upstream icl system, context capped at 4,000 tokens (CPU latency)",
   "B2_icl_notepad": "upstream icl_notepad system, same cap (ordinary in-context memory baseline)",
   "C_cain_retrieval": "cain_memory mode=retrieval: E88 MemoryStore episodic records, top-3 lexical hits",
   "D_cain_structured": "cain_memory mode=structured: E88 episodic + semantic consolidation of own past actions"},
 "not_run": {
   "E_skill_promotion": "NOT EVALUABLE on BSM: per-scan reward is hidden from the system (feedback is an acknowledgement), so no outcome can verify a skill",
   "F_causal_retrieval": "NOT EVALUABLE on BSM for the same reason: no observed outcome to link a cause to",
   "G_full_stack": "equal to D on this task once E/F are not evaluable; not run as a separate arm"},
 "primary_metric": "harness cumulative_gain (rollout reward minus the stateless baseline reward, per instance, summed) and total_reward",
 "hypotheses": {
   "H1": "D_cain_structured has positive cumulative gain over its own stateless baseline.",
   "H2": "D_cain_structured beats B_icl on total reward on the same schedule.",
   "H3": "C_cain_retrieval does not beat B_icl (recency matters more than similarity on BSM)."},
 "analysis": "per-scan paired differences (rollout vs baseline, and arm vs arm) with a paired sign test and bootstrap 95% CI; 1 run per arm, so run-to-run variance is not measured",
 "limits": ["1 run per arm instead of the default 5 (CPU time)", "1.5B CPU model, not a frontier model",
            "ICL context capped at 4,000 tokens; frontier systems on the leaderboard used up to 256k",
            "parameters k=3, rel_tol=0.02, max_rows=24 fixed before any BSM run with a real model"]
}
