claude
7e1dd25a76
Milestone fieldtest for "Surface honesty on stderr", run source-blind against the release binary only (binary-only knowledge floor, user direction 2026-07-23): op-lists authored via the binary's own graph build/introspect surfaces, project + real archive reached through help alone. Markers grep-separable, zero-trade notice exact, abort paths correctly unmarked (4x working). Findings routed: synthetic member-fault panic (exit 101, source-path leak) -> debug; null-result and partial-failure widening -> tracker; op-script discoverability -> #323 evidence; silent unknown-op-field drop -> tracker; plural nit -> tidy. refs #278, refs #313
53 lines
2.9 KiB
Bash
Executable File
53 lines
2.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Example 3 — Mission (3): from the consumer's seat, do any OTHER diagnostics
|
|
# encountered en route fall OUTSIDE both stderr classes (note/warning) when they
|
|
# plausibly belong to one? Survey the diagnostics a driver actually hits and tag
|
|
# each with (marker?, exit code, verdict). Reads only the binary's own output.
|
|
set -u
|
|
AURA="${AURA:?set AURA to the release binary path}"
|
|
HERE="$(cd "$(dirname "$0")/.." && pwd)"
|
|
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT
|
|
OUT="$HERE/captured/example3_survey.log"
|
|
: > "$OUT"
|
|
|
|
run() { # label ; then the command via "$@"
|
|
local label="$1"; shift
|
|
"$@" >/dev/null 2>"$WORK/e"; local e=$?
|
|
{ echo "### $label"; echo "# exit=$e"; sed 's/^/ /' "$WORK/e"; echo; } >> "$OUT"
|
|
}
|
|
|
|
"$AURA" graph build < "$HERE/ops/flat.ops.json" > "$WORK/flat.json"
|
|
"$AURA" graph build < "$HERE/ops/momentum.ops.json" > "$WORK/momentum.json"
|
|
"$AURA" graph build < "$HERE/ops/flat_closed.ops.json" > "$WORK/flat_closed.json"
|
|
( cd "$WORK" && "$AURA" new proj >/dev/null )
|
|
|
|
# (A) SAME fault class, two surfaces: real campaign vs synthetic.
|
|
run "real sweep, poison member (CAUGHT -> warning?, survives?)" \
|
|
bash -c "cd '$WORK/proj' && '$AURA' sweep blueprints/signal.json --real EURUSD --from 1577836800000 --to 1609459200000 --axis proj_signal.fast.length=0,2"
|
|
run "synthetic walkforward, SAME poison (uncaught panic?)" \
|
|
"$AURA" walkforward "$WORK/momentum.json" --axis graph.sma.length=0,3
|
|
|
|
# (B) null-result siblings of the zero-trade note that get NO note.
|
|
run "sweep, every member zero trades (any notice?)" \
|
|
"$AURA" sweep "$WORK/flat.json" --axis graph.sma.length=2,3,5
|
|
run "mc, vacuous / all seeds identical (marker? exit?)" \
|
|
"$AURA" mc "$WORK/flat_closed.json" --seeds 5
|
|
run "generalize, every instrument zero trades (any notice?)" \
|
|
bash -c "cd '$WORK/proj' && '$AURA' generalize blueprints/signal.json --real EURUSD,GBPUSD --from 1577836800000 --to 1609459200000 --axis proj_signal.fast.length=4"
|
|
|
|
# (C) partial-failure surfaced without a marker.
|
|
run "generalize, one instrument has no data in window (marker? exit?)" \
|
|
bash -c "cd '$WORK/proj' && '$AURA' generalize blueprints/signal.json --real EURUSD,Copper --from 1514764800000 --to 1530403200000 --axis proj_signal.fast.length=4"
|
|
|
|
# (D) plural-agreement nit on the note itself.
|
|
run "walkforward yielding a single OOS window (plural grammar)" \
|
|
bash -c "cd '$WORK/proj' && '$AURA' walkforward blueprints/signal.json --real EURUSD --from 1577836800000 --to 1583020800000 --axis proj_signal.fast.length=4"
|
|
|
|
# (E) hard errors that (correctly) sit outside both classes.
|
|
run "walkforward without a project (real mode)" \
|
|
bash -c "cd '$WORK' && '$AURA' walkforward proj/blueprints/signal.json --real EURUSD --axis proj_signal.fast.length=2"
|
|
run "degenerate axis on a NON-campaign call (graph build finalize)" \
|
|
bash -c "echo '[{\"op\":\"add\",\"type\":\"Const\"}]' | '$AURA' graph build"
|
|
|
|
cat "$OUT"
|