From 3e8fd73f680efe522c8fb0c757186c59a192760d Mon Sep 17 00:00:00 2001 From: Brummel Date: Fri, 26 Jun 2026 21:35:10 +0200 Subject: [PATCH] tidy: clear the milestone fieldtest findings (refusal reasons, --help, ledger + glossary) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves the four non-bug findings from the inferential-validation milestone fieldtest (the bug — the broken-pipe panic — is fixed in 1088320): - [friction] generalize refusals now carry a reason instead of a bare usage dump: <2 instruments, a multi-value candidate flag, a missing knob, a duplicate symbol, and a non-stage1-r strategy each get a one-line why (mirroring the R-only metric refusal). Two parser tests strengthened to pin the message. - [friction] --select added to the walkforward branch of the global USAGE (aura --help) — the plateau objective was undiscoverable from the only help the CLI prints (the per-subcommand usage already carried it). - [spec_gap] C1 gains a scope note: bit-identity is per-run; a derived metric recomputed by two different command paths may differ by floating-point reassociation (<=1 ULP), which is not a C1 violation — ratifies the fieldtester's reading of the sweep-vs-generalize sqn difference. - [spec_gap] the milestone vocabulary is added to docs/glossary.md: cross-instrument generalization, deflated score, overfit probability, neighbourhood score, plateau selection, sign-agreement, worst-case floor. The fieldtest triage spec (docs/specs/fieldtest-*) is removed now its findings are resolved; the tracked fieldtests/ scenario corpus stays. Verified: cargo test --workspace green (0 failed), clippy --workspace --all-targets -D warnings clean; `aura --help` shows --select; the refusals print their reason. refs #146 --- crates/aura-cli/src/main.rs | 60 +++-- docs/design/INDEX.md | 8 + docs/glossary.md | 28 +++ ...ldtest-milestone-inferential-validation.md | 216 ------------------ 4 files changed, 74 insertions(+), 238 deletions(-) delete mode 100644 docs/specs/fieldtest-milestone-inferential-validation.md diff --git a/crates/aura-cli/src/main.rs b/crates/aura-cli/src/main.rs index 5867f6b..bd7ca3c 100644 --- a/crates/aura-cli/src/main.rs +++ b/crates/aura-cli/src/main.rs @@ -1683,44 +1683,56 @@ fn parse_generalize_args( let mut stop_k: Option = None; let mut metric = "expectancy_r".to_string(); let mut name = "generalize".to_string(); - // a single grid knob, required and single-valued (csv with exactly one item) - let one = |value: &str, usage: &dyn Fn() -> String| -> Result<(), String> { + // generalize validates ONE candidate cell, so each grid knob takes a single value; + // refuse a multi-value flag with a reason, not a bare usage dump (fieldtest friction). + let one = |value: &str| -> Result<(), String> { let items: Vec<&str> = value.split(',').collect(); - if items.len() != 1 || items[0].is_empty() { return Err(usage()); } + if items.len() != 1 || items[0].is_empty() { + return Err("generalize validates a single candidate: --fast / --slow / --stop-length / --stop-k each take exactly one value".to_string()); + } Ok(()) }; + let knobs = || "generalize requires all four candidate knobs: --fast --slow --stop-length --stop-k, each a single value".to_string(); let mut tail = rest; while let Some((flag, t)) = tail.split_first() { let (value, t) = t.split_first().ok_or_else(usage)?; match *flag { - "--strategy" => { if *value != "stage1-r" { return Err(usage()); } } + "--strategy" => { if *value != "stage1-r" { return Err("generalize requires --strategy stage1-r (the candidate must produce R)".to_string()); } } "--real" => { let parts: Vec = value.split(',').map(|s| s.to_string()).collect(); - if parts.iter().any(|s| s.is_empty()) { return Err(usage()); } + if parts.iter().any(|s| s.is_empty()) { return Err("generalize: --real takes a comma list of non-empty symbols (e.g. GER40,USDJPY)".to_string()); } symbols = Some(parts); } "--from" => from_ms = Some(value.parse().map_err(|_| usage())?), "--to" => to_ms = Some(value.parse().map_err(|_| usage())?), - "--fast" => { one(value, &usage)?; fast = Some(value.parse().map_err(|_| usage())?); } - "--slow" => { one(value, &usage)?; slow = Some(value.parse().map_err(|_| usage())?); } - "--stop-length" => { one(value, &usage)?; stop_length = Some(value.parse().map_err(|_| usage())?); } - "--stop-k" => { one(value, &usage)?; stop_k = Some(value.parse().map_err(|_| usage())?); } + "--fast" => { one(value)?; fast = Some(value.parse().map_err(|_| usage())?); } + "--slow" => { one(value)?; slow = Some(value.parse().map_err(|_| usage())?); } + "--stop-length" => { one(value)?; stop_length = Some(value.parse().map_err(|_| usage())?); } + "--stop-k" => { one(value)?; stop_k = Some(value.parse().map_err(|_| usage())?); } "--metric" => metric = (*value).to_string(), "--name" => name = (*value).to_string(), _ => return Err(usage()), } tail = t; } - let symbols = symbols.ok_or_else(usage)?; - if symbols.len() < 2 { return Err(usage()); } + let symbols = symbols + .ok_or_else(|| "generalize requires --real — a comma list of two or more instruments".to_string())?; + if symbols.len() < 2 { + return Err(format!( + "generalize needs at least two instruments to compare across; got {} (--real takes a comma list of >=2 symbols)", + symbols.len() + )); + } // distinct: a duplicate would double-count one instrument in the floor / sign count let mut seen = std::collections::HashSet::new(); - if !symbols.iter().all(|s| seen.insert(s.clone())) { return Err(usage()); } + if !symbols.iter().all(|s| seen.insert(s.clone())) { + return Err("generalize: each instrument may appear once; --real has a duplicate symbol".to_string()); + } let grid = Stage1RGrid { - fast: vec![fast.ok_or_else(usage)?], - slow: vec![slow.ok_or_else(usage)?], - stop_length: vec![stop_length.ok_or_else(usage)?], - stop_k: vec![stop_k.ok_or_else(usage)?], + fast: vec![fast.ok_or_else(knobs)?], + slow: vec![slow.ok_or_else(knobs)?], + stop_length: vec![stop_length.ok_or_else(knobs)?], + stop_k: vec![stop_k.ok_or_else(knobs)?], ..Stage1RGrid::default() }; Ok((name, symbols, grid, metric, from_ms, to_ms)) @@ -3079,7 +3091,7 @@ fn run_dispatch(args: RunArgs) -> Result { } const USAGE: &str = - "usage: aura run [--harness ] [--real [--from ] [--to ]] [--trace ] | aura chart [--tap ] [--panels] | aura graph | aura sweep [--strategy ] [--real [--from ] [--to ]] [--name |--trace ] | aura mc [--name |--trace ] | aura mc --strategy stage1-r [--real [--from ] [--to ]] [--fast ] [--slow ] [--stop-length ] [--stop-k ] [--block-len ] [--resamples ] [--seed ] | aura walkforward [--strategy ] [--real [--from ] [--to ]] [--name |--trace ] [--fast ] [--slow ] [--stop-length ] [--stop-k ] | aura generalize [--strategy stage1-r] --real --fast --slow --stop-length --stop-k [--from ] [--to ] [--metric ] [--name ] | aura runs families | aura runs family [rank ]"; + "usage: aura run [--harness ] [--real [--from ] [--to ]] [--trace ] | aura chart [--tap ] [--panels] | aura graph | aura sweep [--strategy ] [--real [--from ] [--to ]] [--name |--trace ] | aura mc [--name |--trace ] | aura mc --strategy stage1-r [--real [--from ] [--to ]] [--fast ] [--slow ] [--stop-length ] [--stop-k ] [--block-len ] [--resamples ] [--seed ] | aura walkforward [--strategy ] [--real [--from ] [--to ]] [--name |--trace ] [--fast ] [--slow ] [--stop-length ] [--stop-k ] [--select ] | aura generalize [--strategy stage1-r] --real --fast --slow --stop-length --stop-k [--from ] [--to ] [--metric ] [--name ] | aura runs families | aura runs family [rank ]"; fn main() { // Restore the default SIGPIPE disposition. Rust's runtime sets SIGPIPE to SIG_IGN @@ -4097,11 +4109,13 @@ mod tests { #[test] fn parse_generalize_requires_a_single_value_candidate() { - // a candidate is one cell: a multi-value grid flag is refused - assert!(parse_generalize_args(&[ + // a candidate is one cell: a multi-value grid flag is refused WITH A REASON, + // not a bare usage dump (fieldtest friction). + let err = parse_generalize_args(&[ "--real", "GER40,USDJPY", "--fast", "2,3", "--slow", "12", "--stop-length", "14", "--stop-k", "2.0", - ]).is_err()); + ]).unwrap_err(); + assert!(err.contains("single candidate"), "refusal must say why: {err}"); } #[test] @@ -4115,10 +4129,12 @@ mod tests { #[test] fn parse_generalize_refuses_fewer_than_two_instruments() { - assert!(parse_generalize_args(&[ + // refused WITH A REASON naming the >=2 requirement (fieldtest friction). + let err = parse_generalize_args(&[ "--real", "GER40", "--fast", "3", "--slow", "12", "--stop-length", "14", "--stop-k", "2.0", - ]).is_err()); + ]).unwrap_err(); + assert!(err.contains("at least two instruments"), "refusal must say why: {err}"); } #[test] diff --git a/docs/design/INDEX.md b/docs/design/INDEX.md index ea99733..d999c07 100644 --- a/docs/design/INDEX.md +++ b/docs/design/INDEX.md @@ -113,6 +113,14 @@ source of truth — not session memory. loop that reaches a unique state after each input tick. Same input (incl. seed) → bit-identical run. Two backtests are fully disjoint and run concurrently without locking. +**Scope (ratified, fieldtest 0078).** The bit-identity is *per run* — one backtest +of given (inputs, seed) reproduces byte-for-byte. A *derived metric* recomputed for +the same params by two *different command paths* (e.g. a swept member's `sqn` vs the +same cell re-run under `aura generalize`) may differ by floating-point reassociation +(≤1 ULP), since the two paths accumulate the same logical reduction in a different +operation order (IEEE-754 non-associativity). This is not a C1 violation: C1 governs +the determinism of a single run, not the cross-command bit-identity of a re-derived +statistic. **Forbids.** Concurrency *within* a single sim; any nondeterministic input that is not captured as an explicit input (see C11, C12). **Why.** Real money rides on backtest results; reproducibility and an audit diff --git a/docs/glossary.md b/docs/glossary.md index d12d153..ffdc911 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -55,10 +55,18 @@ The type-erased 64-bit word holding one scalar-base-type value with its kind str **Avoid:** — A node that wires a sub-graph and exposes one output (a combined signal, or a strategy) — composition is fractal and acyclic. Also names the multi-column stream a node emits: the **record** a producer's `eval` returns, bundling 1..K base scalar columns (e.g. OHLCV), each bound field-wise by a consumer (C7/C8). +### cross-instrument generalization +**Avoid:** cross-symbol pooling, pooled generalization +The validation read that grades how consistently one *brought* candidate holds across a set of instruments, scored on its weakest one — the across-instrument axis of the anti-false-discovery discipline. Realised by `aura generalize`; an aggregator (a recomputable family score), never a selector that picks a winner. + ### cycle **Avoid:** — One data-driven clock step: one input record = one cycle, advanced in global timestamp order with a monotonic `cycle_id`. (In the pipeline-process sense "cycle" also names one milestone round; the engine sense is this clock step.) +### deflated score +**Avoid:** trials-adjusted score +A sweep winner's metric penalised for the number of configurations tried (trials-deflation), recorded additively on the winning member's `selection` without changing which member won. Paired with an **overfit probability**; the across-trials defence against a winner that is luck at sweep scale. + ### edge **Avoid:** — A directed wiring link in a harness that forwards **one field** (`from_field`) of a producer node's `eval` output record into a consumer node's input slot (the engine's `Edge`); the source-side variant binding the source value into an input slot is a **source target** (`Target`). Edges define the DAG the bootstrap topologically orders; a per-field scalar-kind mismatch across an edge is rejected at bootstrap. @@ -115,10 +123,22 @@ The reproducible metadata record of a run (node-commit + params + data-window + **Avoid:** MC An orchestration axis running N seeded realizations that perturb the input; each realization is itself deterministic given its seed (Monte-Carlo = sweep over seeds). +### neighbourhood score +**Avoid:** smoothed score +The metric of a sweep cell averaged (or worst-cased) over its closed grid neighbourhood — the surface a **plateau selection** argmaxes instead of the bare peak. Recorded on a plateau winner's `selection` alongside its neighbour count. + ### node **Avoid:** block The universal composable dataflow unit, implementing `lookbacks()` + `eval(ctx)` — a producer, a pure consumer (sink), or both — with at most one output port; a producer's output is a **record of 1..K base-scalar columns** (a scalar is the degenerate K=1 case). Everything that plugs into the engine is fractally a node. +### overfit probability +**Avoid:** P(overfit), luck probability +The recorded chance that a deflated sweep winner is noise rather than edge — a rank of the winner's metric against a centred no-edge null over the family. Paired with the **deflated score** on the winner's `selection`. + +### plateau selection +**Avoid:** plateau-over-peak (as a noun) +A selection objective that argmaxes the neighbourhood-smoothed metric surface (mean or worst-case) rather than the bare in-sample peak, preferring a robust parameter plateau to a lucky spike. Opt-in via `--select plateau:mean|plateau:worst`; the default selection stays a bare argmax. + ### playground **Avoid:** — The egui-native visual face (`aura play`) that plays any harness — program structure before a run, live sink streams during, recorded traces and meta-views after. A trace explorer / execution viewer, never a scene editor. @@ -171,6 +191,10 @@ The four streamed scalar kinds — `i64`, `f64`, `bool`, `timestamp` (a newtype **Avoid:** — A node that exposes session context as scalar streams (`bars_since_open`, `in_session`, `session_open_ts`) so session logic stays inside the stream model. Calendars and instrument specs remain metadata beside the hot path. +### sign-agreement +**Avoid:** sign consistency +In a cross-instrument generalization, the count of instruments on which the candidate's R-metric is net positive. Reported beside the **worst-case floor** to express consistency of direction across instruments. + ### signal **Avoid:** — A node whose output is a score, feeding the `signals (scores) → decision node → bias stream` chain. A specific node role — distinct from a general node and from a strategy. @@ -230,3 +254,7 @@ An orchestration axis: rolling in-sample optimize + out-of-sample test across mo ### World **Avoid:** — The project's meta-level program that dynamically constructs and orchestrates *families* of harnesses (walk-forward / sweep / optimize / Monte-Carlo / comparison). aura's differentiator and product, as opposed to the single-backtest substrate. + +### worst-case floor +**Avoid:** min-over-instruments +The headline cross-instrument generalization score: the minimum of a candidate's R-metric across the instruments, so a single strong market cannot flatter it (non-dominable by construction). The cross-instrument sibling of a worst-case **plateau selection**. diff --git a/docs/specs/fieldtest-milestone-inferential-validation.md b/docs/specs/fieldtest-milestone-inferential-validation.md deleted file mode 100644 index 3b14854..0000000 --- a/docs/specs/fieldtest-milestone-inferential-validation.md +++ /dev/null @@ -1,216 +0,0 @@ -# Fieldtest — milestone "Inferential validation" — 2026-06-26 - -**Status:** Draft — awaiting orchestrator triage -**Author:** fieldtester (dispatched by fieldtest skill, milestone-close gate) - -## Scope - -The milestone **"Inferential validation (defend against false discovery at sweep -scale)"** makes aura's massively-parallel sweep defensible rather than a -false-discovery engine, via three recorded-provenance checks now all shipped: -(a) **trials-deflation** — deflate the selected winner's metric for the number of -configurations tried (#144); (b) **plateau-over-peak** — an opt-in selection -objective preferring a robust parameter plateau over the in-sample peak (#145); -(c) **cross-instrument generalization** — the new `aura generalize` subcommand -grading how consistently a brought candidate holds across several instruments -(#146). The promise field-tested: *"given a parameter sweep, tell me whether an -apparent edge is REAL or an artefact of luck / overfitting."* - -This fieldtest works only from the public surface: the `aura` CLI (`--help`), the -design ledger (`docs/design/INDEX.md`), the glossary, and the `aura-ingest` -example corpus. The implementation source was not read. Exact CLI syntax was -discovered from the single usage string `aura --help` prints. Build exercised: -`cargo run -q --bin aura --` (debug, built from HEAD `15f5c0b` at the start of -the run; the workspace built clean before any scenario). - -Real Pepperstone M1 bars from `/mnt/tickdata/Pepperstone` (GER40, USDJPY, EURUSD -all carry Sept-2024; GER40 carries the full 2024 calendar year). - -## Examples - -### fieldtests/milestone-inferential-validation/scenario_1_trials_deflation.sh — trials-deflation provenance -- A walk-forward over a 2x2 stage1-r grid on real GER40 (full 2024, 9 IS+OOS - windows). Asserts each window's recorded winner carries a `selection` block - with `deflated_score`, `overfit_probability`, `n_trials`, and the resampling - provenance (`n_resamples`, `block_len`, `seed`), default mode `Argmax`, and - that the deflated score sits below the raw winner metric. -- Fits the (a) trials-deflation piece: the recorded selection is exactly the - anti-false-discovery deflation #144 promised. -- Outcome: built, ran, matched expected. PASS. (`raw_winner_metric` 0.0618 → - `deflated_score` -0.153, `overfit_probability` 0.685 for the family of 4.) - -### fieldtests/milestone-inferential-validation/scenario_2_plateau_over_peak.sh — plateau is opt-in, default unchanged -- The same walk-forward run with no flag, with `--select plateau:worst`, with - `--select plateau:mean`, and with a bad token. Asserts default = `Argmax` + - deflation annotation (plateau keys absent); `plateau:worst` = `PlateauWorst` + - `neighbourhood_score`/`n_neighbours` (deflation keys absent); `plateau:mean` = - `PlateauMean`; bad token rejected non-zero. -- Fits the (b) plateau-over-peak piece: opt-in, default-preserving, with the - plateau annotation reaching the recorded winner (orthogonal rule × annotation). -- Outcome: built, ran, matched expected. PASS. - -### fieldtests/milestone-inferential-validation/scenario_3_generalize.sh — cross-instrument generalization -- Happy path: `aura generalize` over GER40,USDJPY,EURUSD (Sept-2024). Asserts the - per-instrument breakdown, that `worst_case` equals the min over instruments, - a `sign_agreement` count, and that the run persists as a `CrossInstrument` - family whose 3 members each carry their `instrument` lineage field. Plus the - three refusals (single `--real` symbol, non-R `--metric`, multi-value candidate - flag), each exit-2. -- Fits the (c) cross-instrument piece — the newest surface. -- Outcome: built, ran, matched expected. PASS. (`worst_case` = -0.0055 = the - EURUSD `expectancy_r`, `sign_agreement` 2/3; all refusals exit 2.) - -### fieldtests/milestone-inferential-validation/scenario_4_composition.sh — the end-to-end "real or overfit?" story -- The chain a researcher runs: sweep a stage1-r grid on GER40 → rank by sqn to - read the in-sample winner → generalize that winner across GER40,USDJPY,EURUSD. - Asserts the chain runs coherently, renders a verdict from `worst_case` + - `sign_agreement`, and that the GER40 leg of generalize reproduces the swept - winner's sqn (within 1e-9). -- Fits the milestone promise as a whole: it composes all three pieces into the - single "is this edge real?" verdict. -- Outcome: built, ran, matched expected (within tolerance). PASS, with the - verdict the toolkit is meant to give: the GER40 in-sample winner (sqn 0.994) - does NOT generalize (worst_case -0.149 on EURUSD, sign_agreement 2/3) → - "instrument-specific / likely overfit." The honest answer is the win. - -## Findings - -### [working] All three milestone pieces reach the output and read sensibly -- All four scenarios. -- Trials-deflation: the recorded `selection` block carries `deflated_score`, - `overfit_probability`, `n_trials`, `n_resamples`, `block_len`, `seed`, - `selection_metric`, `raw_winner_metric`, `mode` — a fully self-describing - provenance record on the winning member's manifest, exactly the #144 contract. - Default mode is `Argmax`; the deflated score sits below the raw metric. -- Plateau: `--select plateau:worst`/`plateau:mean` flips `mode` to - `PlateauWorst`/`PlateauMean` and swaps the deflation annotation for - `neighbourhood_score` + `n_neighbours`; the no-flag default stays - `Argmax`+deflation byte-for-byte (C23). The orthogonal rule × annotation - reshape is visible and correct from the JSON alone. -- Generalize: the per-instrument breakdown + worst-case floor + sign-agreement - print live; the M members persist as a `CrossInstrument` family each stamped - with its `instrument` lineage and the correct per-instrument pip geometry - (`pip_size=1`/`0.01`/`0.0001` for GER40/USDJPY/EURUSD). The non-R metric refusal - carries a precise message naming the four allowed R-metrics. -- Why working: the new surface was reached for the way a researcher would, and - the recorded provenance is legible and correct on the first try. These are the - wins that protect the milestone from drift. -- Recommended downstream action: carry-on. - -### [bug] `aura | head` panics with "Broken pipe (os error 32)" on stdout -- Surfaced in scenario 2 (`walkforward … | head -1`) and scenario 4 - (`runs family … rank sqn | head -1`). -- What happened (verbatim): - ``` - thread 'main' (…) panicked at …/library/std/src/io/stdio.rs:1165:9: - failed printing to stdout: Broken pipe (os error 32) - note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace - ``` - Under `set -o pipefail` this propagated exit code 101 and aborted the script. -- Why a bug: piping a multi-line family/sweep listing into `head` (or any reader - that closes early — `head`, `less`, a closed UI pane) is the single most natural - thing a CLI consumer does. The family-listing subcommands stream many JSONL - lines and do not install a SIGPIPE handler / tolerate `EPIPE`, so the writer - panics instead of exiting quietly. The wrong-output is a Rust panic + backtrace - note where a clean truncated stream is expected. `aura run` (single line) does - not hit it; the family-emitting commands (`walkforward`, `sweep`, `mc`, - `runs family … rank`, `runs families`) do whenever they emit more lines than the - reader consumes. -- One-line repro: - `cargo run -q --bin aura -- walkforward --strategy stage1-r --real GER40 --from 1704067200000 --to 1735689599999 --fast 2,3 --slow 8,12 --stop-length 14 --stop-k 2.0 | head -1` -- Recommended downstream action: debug (RED test: a family-emitting command piped - into a head-like early-closing reader exits 0/SIGPIPE with no panic — restore - the default SIGPIPE disposition or swallow `ErrorKind::BrokenPipe` at the print - loop). - -### [friction] Two of the three `generalize` refusals dump bare usage with no reason -- Scenario 3 refusals (b) single `--real` symbol and (d) multi-value candidate - flag (`--fast 3,5`). -- What happened: both print only the usage string and exit 2: - `aura: generalize [--strategy stage1-r] --real --fast …` - whereas the (c) non-R metric refusal gives a precise, actionable message: - `aura: metric 'total_pips' is not comparable across instruments; - cross-instrument scoring is R-only (sqn, sqn_normalized, expectancy_r, - net_expectancy_r)`. -- Why friction: the task completes (correct exit code), but the consumer is not - told *why*. `--real GER40` (one symbol) is refused because generalization needs - ≥2 instruments — the whole point of the worst-case-across-instruments framing — - yet the message never says "needs at least two instruments." `--fast 3,5` is - refused because the candidate must be a single grid cell (generalize validates - *one* brought candidate, not a sweep) — but the usage string does not convey - "single value per candidate flag." The (c) message proves the project knows how - to write a good refusal; (b) and (d) just fall through to usage. -- Recommended downstream action: plan (tidy iteration) — give the arity/min-symbol - refusals a one-line reason mirroring the R-only message ("needs ≥2 instruments", - "candidate flags take a single value; generalize validates one cell"). - -### [friction] The `--select` plateau objective is invisible in `aura --help` -- Scenario 2 (the milestone's plateau-over-peak feature). -- What happened: `aura --help` (and every per-subcommand `--help`, which all - print the same single usage string) shows the walkforward form as - `aura walkforward [--strategy …] [--real …] [--name |--trace ] [--fast …] - [--slow …] [--stop-length …] [--stop-k …]` — with **no** `--select - `. The flag works (scenario 2 proves it) but - a consumer reading the help would never discover the plateau objective exists. -- Why friction: a shipped, opt-in milestone feature is undiscoverable from the - only help the CLI offers. The feature is reachable only if you already know the - flag name (from the ledger / commit log, which a downstream consumer does not - read). The mc subcommand's `--block-len`/`--resamples`/`--seed` and the - generalize `--metric` are documented in the usage; `--select` is the gap. -- Recommended downstream action: plan (tidy iteration) — add - `[--select ]` to the walkforward branch of - the usage string. - -### [spec_gap] Two orchestration commands compute the same member's R metric with a 1-ULP difference -- Scenario 4 (and confirmed by a direct repeat probe). -- What happened: for the identical (params fast=3 slow=6 stop_length=3 stop_k=2.0, - Sept-2024 window, GER40, metric sqn), the **sweep** path records - `sqn = 0.9941661927806448` while the **generalize** path records - `0.9941661927806447` — a last-ULP difference, stable and reproducible across - re-runs (each path is internally bit-deterministic; the two paths differ). -- Why spec_gap: the ledger's C1 guarantees "same input (incl. seed) → - bit-identical run," and the cross-instrument realization note calls the - generalization aggregate "recomputable from the members." But the ledger is - silent on whether two *different commands* computing the same member's metric - must agree to the bit. A naive consumer asking "did my swept winner's GER40 - score survive into generalize?" by exact string-equality would conclude they - differ. I chose the reading that this is a benign floating-point reassociation - between two reduction paths (within 1e-9), not a determinism violation — but the - equally-plausible reading is that "recomputable" should mean bit-identical, in - which case the two paths should share one reduction kernel. The contract does - not say which. -- Recommended downstream action: ratify (record in the ledger that cross-*command* - recomputation of a member metric is bit-equal-up-to-reassociation, not - bit-identical — or tighten it to share the kernel). Either resolution is a - one-line ledger note, not code the fieldtest should write. - -### [spec_gap] The milestone's core vocabulary is absent from the glossary -- All scenarios (a documentation observation across the milestone surface). -- What happened: `docs/glossary.md` defines bias, R, R metrics, sweep, - walk-forward, Monte-Carlo, etc., but has **no** entry for "trials-deflation," - "deflated score," "overfit probability," "plateau (over peak)," "cross-instrument - generalization," "worst-case floor," or "sign-agreement" — the entire vocabulary - the milestone's JSON output uses (`deflated_score`, `overfit_probability`, - `neighbourhood_score`, `worst_case`, `sign_agreement`). -- Why spec_gap: a downstream researcher reading `overfit_probability` off a winner - manifest, or `worst_case` off a generalize run, has no public definition to look - up — the terms are load-bearing for interpreting the output but undocumented on - the public surface. The ledger's open-thread prose explains them, but the - glossary (the consumer-facing term index) is silent, so the reading of each term - is left to the consumer. I read the numbers as "deflated = luck-penalised, - overfit_probability = P(this winner is noise), worst_case = the floor over - instruments" — plausible, but the public surface does not pin it. -- Recommended downstream action: tighten the design ledger / glossary — add the - five-to-seven milestone terms to `docs/glossary.md` (a docwriter task), so the - output is self-explanatory from the public interface alone. - -## Recommendation summary - -| Finding | Class | Action | -|---|---|---| -| All three pieces reach output, read sensibly | working | carry-on | -| `… \| head` panics with Broken pipe on stdout | bug | debug | -| Two generalize refusals dump bare usage, no reason | friction | plan | -| `--select` plateau objective absent from `--help` | friction | plan | -| Sweep vs generalize 1-ULP metric mismatch | spec_gap | ratify | -| Milestone vocabulary absent from the glossary | spec_gap | tighten the design ledger |