From 7e1dd25a7644428da9a99a48df2c1576f515cdfc Mon Sep 17 00:00:00 2001 From: claude Date: Fri, 24 Jul 2026 00:04:53 +0200 Subject: [PATCH] =?UTF-8?q?fieldtest:=20milestone-stderr-honesty=20?= =?UTF-8?q?=E2=80=94=203=20examples,=201=20bug=20/=202=20friction=20/=202?= =?UTF-8?q?=20spec-gap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Milestone fieldtest for "Surface honesty on stderr", run source-blind against the release binary only (binary-only knowledge floor, user direction 2026-07-23): op-lists authored via the binary's own graph build/introspect surfaces, project + real archive reached through help alone. Markers grep-separable, zero-trade notice exact, abort paths correctly unmarked (4x working). Findings routed: synthetic member-fault panic (exit 101, source-path leak) -> debug; null-result and partial-failure widening -> tracker; op-script discoverability -> #323 evidence; silent unknown-op-field drop -> tracker; plural nit -> tidy. refs #278, refs #313 --- .../captured/example1_combined.log | 6 +++ .../captured/example1_runB.err | 1 + .../captured/example2_flat.err | 1 + .../captured/example2_momentum.err | 0 .../captured/example2_singular.err | 1 + .../captured/example3_survey.log | 43 +++++++++++++++ .../drivers/example1_marker_separability.sh | 38 ++++++++++++++ .../example2_zero_trade_walkforward.sh | 32 ++++++++++++ .../example3_diagnostic_completeness.sh | 52 +++++++++++++++++++ .../ops/flat.ops.json | 13 +++++ .../ops/flat_closed.ops.json | 13 +++++ .../ops/momentum.ops.json | 11 ++++ 12 files changed, 211 insertions(+) create mode 100644 fieldtests/milestone-stderr-honesty/captured/example1_combined.log create mode 100644 fieldtests/milestone-stderr-honesty/captured/example1_runB.err create mode 100644 fieldtests/milestone-stderr-honesty/captured/example2_flat.err create mode 100644 fieldtests/milestone-stderr-honesty/captured/example2_momentum.err create mode 100644 fieldtests/milestone-stderr-honesty/captured/example2_singular.err create mode 100644 fieldtests/milestone-stderr-honesty/captured/example3_survey.log create mode 100755 fieldtests/milestone-stderr-honesty/drivers/example1_marker_separability.sh create mode 100755 fieldtests/milestone-stderr-honesty/drivers/example2_zero_trade_walkforward.sh create mode 100755 fieldtests/milestone-stderr-honesty/drivers/example3_diagnostic_completeness.sh create mode 100644 fieldtests/milestone-stderr-honesty/ops/flat.ops.json create mode 100644 fieldtests/milestone-stderr-honesty/ops/flat_closed.ops.json create mode 100644 fieldtests/milestone-stderr-honesty/ops/momentum.ops.json diff --git a/fieldtests/milestone-stderr-honesty/captured/example1_combined.log b/fieldtests/milestone-stderr-honesty/captured/example1_combined.log new file mode 100644 index 0000000..ebeb16a --- /dev/null +++ b/fieldtests/milestone-stderr-honesty/captured/example1_combined.log @@ -0,0 +1,6 @@ +### run A: zero-trade walkforward (synthetic) — expect note, exit 0 +aura: note: all 3 walk-forward windows recorded zero trades +# exitA=0 +### run B: real sweep with a poison axis member — expect warning, exit 3 +# exitB=3 +aura: warning: cell (df772d885a1b9c6b46a6ea51ba1339f0f3ee5c22e47ac4a3c95867ebc9b3b9d2, EURUSD, [1577923200000, 1609451819999]) failed at stage 0: a member panicked: SMA length must be >= 1 — recorded, campaign continues diff --git a/fieldtests/milestone-stderr-honesty/captured/example1_runB.err b/fieldtests/milestone-stderr-honesty/captured/example1_runB.err new file mode 100644 index 0000000..3229b14 --- /dev/null +++ b/fieldtests/milestone-stderr-honesty/captured/example1_runB.err @@ -0,0 +1 @@ +aura: warning: cell (df772d885a1b9c6b46a6ea51ba1339f0f3ee5c22e47ac4a3c95867ebc9b3b9d2, EURUSD, [1577923200000, 1609451819999]) failed at stage 0: a member panicked: SMA length must be >= 1 — recorded, campaign continues diff --git a/fieldtests/milestone-stderr-honesty/captured/example2_flat.err b/fieldtests/milestone-stderr-honesty/captured/example2_flat.err new file mode 100644 index 0000000..0b9d7cc --- /dev/null +++ b/fieldtests/milestone-stderr-honesty/captured/example2_flat.err @@ -0,0 +1 @@ +aura: note: all 3 walk-forward windows recorded zero trades diff --git a/fieldtests/milestone-stderr-honesty/captured/example2_momentum.err b/fieldtests/milestone-stderr-honesty/captured/example2_momentum.err new file mode 100644 index 0000000..e69de29 diff --git a/fieldtests/milestone-stderr-honesty/captured/example2_singular.err b/fieldtests/milestone-stderr-honesty/captured/example2_singular.err new file mode 100644 index 0000000..6d62099 --- /dev/null +++ b/fieldtests/milestone-stderr-honesty/captured/example2_singular.err @@ -0,0 +1 @@ +aura: note: all 1 walk-forward windows recorded zero trades diff --git a/fieldtests/milestone-stderr-honesty/captured/example3_survey.log b/fieldtests/milestone-stderr-honesty/captured/example3_survey.log new file mode 100644 index 0000000..55bfce9 --- /dev/null +++ b/fieldtests/milestone-stderr-honesty/captured/example3_survey.log @@ -0,0 +1,43 @@ +### real sweep, poison member (CAUGHT -> warning?, survives?) +# exit=3 + aura: warning: cell (df772d885a1b9c6b46a6ea51ba1339f0f3ee5c22e47ac4a3c95867ebc9b3b9d2, EURUSD, [1577923200000, 1609451819999]) failed at stage 0: a member panicked: SMA length must be >= 1 — recorded, campaign continues + +### synthetic walkforward, SAME poison (uncaught panic?) +# exit=101 + + thread '' (2537074) panicked at crates/aura-std/src/sma.rs:46:9: + SMA length must be >= 1 + note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + + thread '' (2537076) panicked at crates/aura-std/src/sma.rs:46:9: + SMA length must be >= 1 + + thread '' (2537094) panicked at crates/aura-std/src/sma.rs:46:9: + SMA length must be >= 1 + +### sweep, every member zero trades (any notice?) +# exit=0 + +### mc, vacuous / all seeds identical (marker? exit?) +# exit=2 + aura: mc is vacuous: every seed produced an identical result — the strategy never warmed over the synthetic walk, so no seed reached a distinguishable realization; use a shallower-lookback blueprint or a longer walk + +### generalize, every instrument zero trades (any notice?) +# exit=0 + +### generalize, one instrument has no data in window (marker? exit?) +# exit=1 + aura: generalize produced no cross-instrument grade + +### walkforward yielding a single OOS window (plural grammar) +# exit=0 + aura: note: all 1 walk-forward windows recorded zero trades + +### walkforward without a project (real mode) +# exit=1 + aura: walkforward needs a project: strategies resolve against the project store and vocabulary (no Aura.toml found up from /tmp/tmp.lVNHxAPupn) + +### degenerate axis on a NON-campaign call (graph build finalize) +# exit=1 + aura: finalize: slot const.clock is unconnected + diff --git a/fieldtests/milestone-stderr-honesty/drivers/example1_marker_separability.sh b/fieldtests/milestone-stderr-honesty/drivers/example1_marker_separability.sh new file mode 100755 index 0000000..d7d0d47 --- /dev/null +++ b/fieldtests/milestone-stderr-honesty/drivers/example1_marker_separability.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Example 1 — Mission (1): a consumer separates benign notices from real faults +# by FORM alone (grep / CI log scan), and checks exit codes against the binary's +# own conventions. +# +# Finding proven here: the two markers cannot co-occur in ONE invocation (the +# only note-emitting verb, walkforward, is single-cell all-or-nothing: the note +# fires only when every window SUCCEEDS with zero trades, i.e. exit 0; any +# warning implies a cell/fold failure, i.e. exit 3). So separability is shown +# the way the promise's "CI log scan" actually consumes it: over the accumulated +# stderr of several runs. +set -u +AURA="${AURA:?set AURA to the release binary path}" +HERE="$(cd "$(dirname "$0")/.." && pwd)" +WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT +LOG="$HERE/captured/example1_combined.log" + +# Build the synthetic zero-trade strategy from its canonical op-list. +"$AURA" graph build < "$HERE/ops/flat.ops.json" > "$WORK/flat.json" + +# A real project is required to reach the `aura: warning:` class at all +# (synthetic member panics are uncaught — see example 3). Scaffold one. +( cd "$WORK" && "$AURA" new proj >/dev/null ) + +: > "$LOG" +echo "### run A: zero-trade walkforward (synthetic) — expect note, exit 0" >> "$LOG" +"$AURA" walkforward "$WORK/flat.json" --axis graph.sma.length=2,3,5 >/dev/null 2>>"$LOG" +echo "# exitA=$?" >> "$LOG" +echo "### run B: real sweep with a poison axis member — expect warning, exit 3" >> "$LOG" +( cd "$WORK/proj" && "$AURA" sweep blueprints/signal.json --real EURUSD \ + --from 1577836800000 --to 1609459200000 --axis proj_signal.fast.length=0,2 \ + >/dev/null 2>>"$HERE/captured/example1_runB.err" ) +echo "# exitB=$?" >> "$LOG" +cat "$HERE/captured/example1_runB.err" >> "$LOG" + +echo "=== combined log ==="; cat "$LOG" +echo "=== benign notices (grep '^aura: note: ') ==="; grep '^aura: note: ' "$LOG" || true +echo "=== real faults (grep '^aura: warning: ') ==="; grep '^aura: warning: ' "$LOG" || true diff --git a/fieldtests/milestone-stderr-honesty/drivers/example2_zero_trade_walkforward.sh b/fieldtests/milestone-stderr-honesty/drivers/example2_zero_trade_walkforward.sh new file mode 100755 index 0000000..d997b34 --- /dev/null +++ b/fieldtests/milestone-stderr-honesty/drivers/example2_zero_trade_walkforward.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Example 2 — Mission (2): a walk-forward whose every window records zero trades +# must announce itself with an explicit notice; one that trades must not; exit +# code stays success either way. Blueprints authored via the binary's own +# op-script surface (`aura graph build`). +set -u +AURA="${AURA:?set AURA to the release binary path}" +HERE="$(cd "$(dirname "$0")/.." && pwd)" +WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT + +"$AURA" graph build < "$HERE/ops/flat.ops.json" > "$WORK/flat.json" +"$AURA" graph build < "$HERE/ops/momentum.ops.json" > "$WORK/momentum.json" +cd "$WORK" # synthetic families record to ./runs — keep them in the temp dir + +echo "=== A: flat (zero-trade) walkforward — expect note + exit 0 ===" +"$AURA" walkforward "$WORK/flat.json" --axis graph.sma.length=2,3,5 \ + >/dev/null 2>"$HERE/captured/example2_flat.err" +echo "# exit=$?"; cat "$HERE/captured/example2_flat.err" + +echo "=== B: momentum (trades) walkforward — expect NO note + exit 0 ===" +"$AURA" walkforward "$WORK/momentum.json" --axis graph.sma.length=2,3,5 \ + >/dev/null 2>"$HERE/captured/example2_momentum.err" +echo "# exit=$?"; cat "$HERE/captured/example2_momentum.err"; echo "(stderr empty above = no note)" + +echo "=== C: single-window edge — expect the 'all 1 windows' plural nit ===" +# The synthetic roller always yields 3 windows; a sub-120-day REAL window yields +# exactly one OOS window, which exposes the ungrammatical "all 1 ... windows". +( cd "$WORK" && "$AURA" new proj >/dev/null ) +( cd "$WORK/proj" && "$AURA" walkforward blueprints/signal.json --real EURUSD \ + --from 1577836800000 --to 1583020800000 --axis proj_signal.fast.length=4 \ + >/dev/null 2>"$HERE/captured/example2_singular.err" ) +echo "# exit=$?"; cat "$HERE/captured/example2_singular.err" diff --git a/fieldtests/milestone-stderr-honesty/drivers/example3_diagnostic_completeness.sh b/fieldtests/milestone-stderr-honesty/drivers/example3_diagnostic_completeness.sh new file mode 100755 index 0000000..d97dc4e --- /dev/null +++ b/fieldtests/milestone-stderr-honesty/drivers/example3_diagnostic_completeness.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Example 3 — Mission (3): from the consumer's seat, do any OTHER diagnostics +# encountered en route fall OUTSIDE both stderr classes (note/warning) when they +# plausibly belong to one? Survey the diagnostics a driver actually hits and tag +# each with (marker?, exit code, verdict). Reads only the binary's own output. +set -u +AURA="${AURA:?set AURA to the release binary path}" +HERE="$(cd "$(dirname "$0")/.." && pwd)" +WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT +OUT="$HERE/captured/example3_survey.log" +: > "$OUT" + +run() { # label ; then the command via "$@" + local label="$1"; shift + "$@" >/dev/null 2>"$WORK/e"; local e=$? + { echo "### $label"; echo "# exit=$e"; sed 's/^/ /' "$WORK/e"; echo; } >> "$OUT" +} + +"$AURA" graph build < "$HERE/ops/flat.ops.json" > "$WORK/flat.json" +"$AURA" graph build < "$HERE/ops/momentum.ops.json" > "$WORK/momentum.json" +"$AURA" graph build < "$HERE/ops/flat_closed.ops.json" > "$WORK/flat_closed.json" +( cd "$WORK" && "$AURA" new proj >/dev/null ) + +# (A) SAME fault class, two surfaces: real campaign vs synthetic. +run "real sweep, poison member (CAUGHT -> warning?, survives?)" \ + bash -c "cd '$WORK/proj' && '$AURA' sweep blueprints/signal.json --real EURUSD --from 1577836800000 --to 1609459200000 --axis proj_signal.fast.length=0,2" +run "synthetic walkforward, SAME poison (uncaught panic?)" \ + "$AURA" walkforward "$WORK/momentum.json" --axis graph.sma.length=0,3 + +# (B) null-result siblings of the zero-trade note that get NO note. +run "sweep, every member zero trades (any notice?)" \ + "$AURA" sweep "$WORK/flat.json" --axis graph.sma.length=2,3,5 +run "mc, vacuous / all seeds identical (marker? exit?)" \ + "$AURA" mc "$WORK/flat_closed.json" --seeds 5 +run "generalize, every instrument zero trades (any notice?)" \ + bash -c "cd '$WORK/proj' && '$AURA' generalize blueprints/signal.json --real EURUSD,GBPUSD --from 1577836800000 --to 1609459200000 --axis proj_signal.fast.length=4" + +# (C) partial-failure surfaced without a marker. +run "generalize, one instrument has no data in window (marker? exit?)" \ + bash -c "cd '$WORK/proj' && '$AURA' generalize blueprints/signal.json --real EURUSD,Copper --from 1514764800000 --to 1530403200000 --axis proj_signal.fast.length=4" + +# (D) plural-agreement nit on the note itself. +run "walkforward yielding a single OOS window (plural grammar)" \ + bash -c "cd '$WORK/proj' && '$AURA' walkforward blueprints/signal.json --real EURUSD --from 1577836800000 --to 1583020800000 --axis proj_signal.fast.length=4" + +# (E) hard errors that (correctly) sit outside both classes. +run "walkforward without a project (real mode)" \ + bash -c "cd '$WORK' && '$AURA' walkforward proj/blueprints/signal.json --real EURUSD --axis proj_signal.fast.length=2" +run "degenerate axis on a NON-campaign call (graph build finalize)" \ + bash -c "echo '[{\"op\":\"add\",\"type\":\"Const\"}]' | '$AURA' graph build" + +cat "$OUT" diff --git a/fieldtests/milestone-stderr-honesty/ops/flat.ops.json b/fieldtests/milestone-stderr-honesty/ops/flat.ops.json new file mode 100644 index 0000000..d68795b --- /dev/null +++ b/fieldtests/milestone-stderr-honesty/ops/flat.ops.json @@ -0,0 +1,13 @@ +[ + {"op":"doc","text":"flat: bias = 0 * sign(close - SMA(close,length)); Scale factor bound to 0 so no window ever trades, while sma.length stays an open axis"}, + {"op":"source","role":"close","kind":"F64"}, + {"op":"add","type":"SMA","name":"sma"}, + {"op":"add","type":"Sub","name":"d"}, + {"op":"add","type":"Sign","name":"s"}, + {"op":"add","type":"Scale","name":"z","bind":{"factor":{"F64":0.0}}}, + {"op":"feed","role":"close","into":["sma.series","d.lhs"]}, + {"op":"connect","from":"sma.value","to":"d.rhs"}, + {"op":"connect","from":"d.value","to":"s.value"}, + {"op":"connect","from":"s.value","to":"z.signal"}, + {"op":"expose","from":"z.value","as":"bias"} +] diff --git a/fieldtests/milestone-stderr-honesty/ops/flat_closed.ops.json b/fieldtests/milestone-stderr-honesty/ops/flat_closed.ops.json new file mode 100644 index 0000000..27bd845 --- /dev/null +++ b/fieldtests/milestone-stderr-honesty/ops/flat_closed.ops.json @@ -0,0 +1,13 @@ +[ + {"op":"doc","text":"flat-closed: bias = 0, all params bound (mc requires a closed blueprint)"}, + {"op":"source","role":"close","kind":"F64"}, + {"op":"add","type":"SMA","name":"sma","bind":{"length":{"I64":3}}}, + {"op":"add","type":"Sub","name":"d"}, + {"op":"add","type":"Sign","name":"s"}, + {"op":"add","type":"Scale","name":"z","bind":{"factor":{"F64":0.0}}}, + {"op":"feed","role":"close","into":["sma.series","d.lhs"]}, + {"op":"connect","from":"sma.value","to":"d.rhs"}, + {"op":"connect","from":"d.value","to":"s.value"}, + {"op":"connect","from":"s.value","to":"z.signal"}, + {"op":"expose","from":"z.value","as":"bias"} +] diff --git a/fieldtests/milestone-stderr-honesty/ops/momentum.ops.json b/fieldtests/milestone-stderr-honesty/ops/momentum.ops.json new file mode 100644 index 0000000..1358faf --- /dev/null +++ b/fieldtests/milestone-stderr-honesty/ops/momentum.ops.json @@ -0,0 +1,11 @@ +[ + {"op":"doc","text":"momentum: bias = sign(close - SMA(close,length)); length is the open IS-refit axis"}, + {"op":"source","role":"close","kind":"F64"}, + {"op":"add","type":"SMA","name":"sma"}, + {"op":"add","type":"Sub","name":"d"}, + {"op":"add","type":"Sign","name":"s"}, + {"op":"feed","role":"close","into":["sma.series","d.lhs"]}, + {"op":"connect","from":"sma.value","to":"d.rhs"}, + {"op":"connect","from":"d.value","to":"s.value"}, + {"op":"expose","from":"s.value","as":"bias"} +]