fieldtest: trace-identity cycle — 4 examples, 1 bug / 1 spec-gap / 7 friction / 4 working
A source-blind consumer drove the shipped binary across recording and re-recording, addressing a run after the fact, the description-edit authoring loop, and a cold read of the documentation. Both decisions this cycle made deliberately hold in every shape tested. A parameter pinned on the command line to the value it already has gives the same handle as not passing it, across one-at-a-time, both scalar kinds, and all at once; the order of several `--override` flags never matters; a description-only edit converges on the same directory. Eight invocations, three directories, and the right three. The field test also falsified two sentences this cycle wrote, both fixed here. The contract clause claimed that two blueprints differing only in debug symbols "compute bit-identically, so one address is the correct answer". They do compute bit-identically — and are addressed separately, because `manifest.params` records node-qualified names and the merged parameter vector is hashed. The audit's substitution blanks those names inside the topology hash's slot; the parameter vector re-admits them through a field it does not touch. Whether the handle should follow the record or #171's name-blind projection is a genuine fork between two established positions here, so the behaviour stands and the clause now states what actually holds, naming #354 for the fork. The authoring guide predicted two directories for a no-op `--override`, since it listed "an `--override` value" among the splitting causes without mentioning the merge. It now states the rule the cycle actually shipped — effective parameterisation, not how it was supplied — and names the provenance exclusions in the one place an author looks. Routed: #354 (the rename fork), #355 (five friction items on the path from "a run recorded something" to "I am looking at it" — handle discoverability, `measure ic` refusal parity, asking for an identity without running, empty-state answers, the README's command map), and the consumer-visible face of the non-pruning class onto #352. refs #311
This commit is contained in:
+150
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env bash
|
||||
# Field test for cycle #311 — identity-keyed single-run trace directories.
|
||||
#
|
||||
# Replays the whole test from scratch: scaffolds a lab project, builds the
|
||||
# fixtures, runs them, and captures every step's stdout/stderr under out/.
|
||||
# Source-blind: every command here is one a downstream consumer can type.
|
||||
#
|
||||
# ./c311_0_run_all.sh # uses ../../target/release/aura
|
||||
# AURA=/path/to/aura ./c311_0_run_all.sh
|
||||
set -u
|
||||
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
AURA="${AURA:-$HERE/../../target/release/aura}"
|
||||
OUT="$HERE/out"
|
||||
LAB="$HERE/lab"
|
||||
|
||||
rm -rf "$LAB" "$OUT"
|
||||
mkdir -p "$OUT"
|
||||
|
||||
step() { printf '\n===== %s =====\n' "$1" | tee -a "$OUT/transcript.txt"; }
|
||||
|
||||
# run a command, echoing it and capturing both streams into the transcript
|
||||
run() {
|
||||
{ printf '$ %s\n' "$*"; "$@" 2>&1; printf '[exit %d]\n' "$?"; } \
|
||||
| tee -a "$OUT/transcript.txt"
|
||||
}
|
||||
|
||||
# same, but keep chart's 70 KB of HTML out of the transcript
|
||||
run_chart() {
|
||||
{ printf '$ %s\n' "$*"; "$@" 2>&1 >/dev/null; printf '[exit %d]\n' "$?"; } \
|
||||
| tee -a "$OUT/transcript.txt"
|
||||
}
|
||||
|
||||
# print just the handle a run reported
|
||||
handle() {
|
||||
"$@" 2>/dev/null | python3 -c 'import sys,json; print(json.load(sys.stdin).get("trace_name"))'
|
||||
}
|
||||
|
||||
cd "$HERE"
|
||||
step "scaffold"
|
||||
run "$AURA" new lab
|
||||
cd "$LAB"
|
||||
|
||||
step "build the fixtures"
|
||||
for f in 1_spread 2_ic_measure 3a_spread_redoc 3b_spread_rename_nodes \
|
||||
3c_spread_rename_graph 3d_spread_rename_tap; do
|
||||
"$AURA" graph build < "$HERE/c311_$f.ops.json" > "blueprints/$f.bp.json" \
|
||||
|| echo "BUILD FAILED: $f" | tee -a "$OUT/transcript.txt"
|
||||
done
|
||||
run "$AURA" graph introspect --params blueprints/1_spread.bp.json
|
||||
run "$AURA" graph introspect --taps blueprints/1_spread.bp.json
|
||||
|
||||
# ---------------------------------------------------------------- example 1
|
||||
# Axis 1: recording and re-recording. A change that matters must split the
|
||||
# directory; a change that does not must not.
|
||||
step "example 1 — recording and re-recording"
|
||||
run "$AURA" exec blueprints/1_spread.bp.json
|
||||
{
|
||||
printf '\n-- handle per invocation --\n'
|
||||
printf '%-42s %s\n' 'plain' \
|
||||
"$(handle "$AURA" exec blueprints/1_spread.bp.json)"
|
||||
printf '%-42s %s\n' '--override fast.length=3 (matters)' \
|
||||
"$(handle "$AURA" exec blueprints/1_spread.bp.json --override fast.length=3)"
|
||||
printf '%-42s %s\n' '--override fast.length=2 (no-op)' \
|
||||
"$(handle "$AURA" exec blueprints/1_spread.bp.json --override fast.length=2)"
|
||||
printf '%-42s %s\n' '--override slow.length=4 (no-op)' \
|
||||
"$(handle "$AURA" exec blueprints/1_spread.bp.json --override slow.length=4)"
|
||||
printf '%-42s %s\n' '--override bias.scale=1.0 (no-op)' \
|
||||
"$(handle "$AURA" exec blueprints/1_spread.bp.json --override bias.scale=1.0)"
|
||||
printf '%-42s %s\n' 'all three pinned at their defaults' \
|
||||
"$(handle "$AURA" exec blueprints/1_spread.bp.json --override fast.length=2 --override slow.length=4 --override bias.scale=1.0)"
|
||||
printf '%-42s %s\n' 'two real overrides, order X' \
|
||||
"$(handle "$AURA" exec blueprints/1_spread.bp.json --override fast.length=3 --override slow.length=5)"
|
||||
printf '%-42s %s\n' 'two real overrides, order Y' \
|
||||
"$(handle "$AURA" exec blueprints/1_spread.bp.json --override slow.length=5 --override fast.length=3)"
|
||||
} | tee -a "$OUT/transcript.txt"
|
||||
run "$AURA" exec blueprints/1_spread.bp.json --override bias.scale=1
|
||||
run find runs/traces -maxdepth 1 -mindepth 1 -type d
|
||||
|
||||
# ---------------------------------------------------------------- example 2
|
||||
# Axis 2: addressing a run after the fact — every verb that takes a handle,
|
||||
# then the blueprint's plain name instead.
|
||||
step "example 2 — addressing a run after the fact"
|
||||
"$AURA" exec blueprints/2_ic_measure.bp.json | tee -a "$OUT/transcript.txt"
|
||||
IC=$(handle "$AURA" exec blueprints/2_ic_measure.bp.json)
|
||||
run_chart "$AURA" chart spreadwatch-7958ba7c
|
||||
run_chart "$AURA" chart spreadwatch-7958ba7c --tap spread
|
||||
run_chart "$AURA" chart spreadwatch-7958ba7c --tap bogus
|
||||
run "$AURA" measure ic "$IC" --signal signal --price price
|
||||
run "$AURA" runs families
|
||||
run "$AURA" runs family spreadwatch-7958ba7c
|
||||
run "$AURA" reproduce spreadwatch-7958ba7c
|
||||
# ... and now the bare blueprint name instead of a handle
|
||||
run_chart "$AURA" chart spreadwatch
|
||||
run_chart "$AURA" chart nosuchthing
|
||||
run "$AURA" measure ic graph --signal signal --price price
|
||||
|
||||
# ---------------------------------------------------------------- example 3
|
||||
# Axis 3: the authoring loop. Edits that change no run semantics must not
|
||||
# mint a fresh directory.
|
||||
step "example 3 — the authoring loop (names and descriptions)"
|
||||
# the obvious way to ask "did my edit change the identity?" first:
|
||||
run "$AURA" graph introspect --identity-id blueprints/1_spread.bp.json
|
||||
{
|
||||
printf '%-24s %-66s %s\n' fixture 'identity id (--content-id FILE --identity-id)' handle
|
||||
for f in 1_spread 3a_spread_redoc 3b_spread_rename_nodes \
|
||||
3c_spread_rename_graph 3d_spread_rename_tap; do
|
||||
id=$("$AURA" graph introspect --content-id "blueprints/$f.bp.json" --identity-id | tail -1)
|
||||
printf '%-24s %-66s %s\n' "$f" "$id" \
|
||||
"$(handle "$AURA" exec "blueprints/$f.bp.json")"
|
||||
done
|
||||
} | tee -a "$OUT/transcript.txt"
|
||||
run find runs/traces -type f
|
||||
# one identity id, three directories — do the three actually hold the same data?
|
||||
{
|
||||
printf '$ cmp the recorded series across the three addresses\n'
|
||||
for d in spreadwatch-6e269cb5 otherwatch-7958ba7c; do
|
||||
if cmp -s runs/traces/spreadwatch-7958ba7c/spread.json "runs/traces/$d/spread.json"
|
||||
then printf ' %-22s byte-identical to spreadwatch-7958ba7c\n' "$d"
|
||||
else printf ' %-22s DIFFERS from spreadwatch-7958ba7c\n' "$d"; fi
|
||||
done
|
||||
printf '$ the manifest key that splits them (defaults vector):\n'
|
||||
for d in spreadwatch-7958ba7c spreadwatch-6e269cb5; do
|
||||
printf ' %-22s ' "$d"
|
||||
python3 -c "import json;print(json.load(open('runs/traces/$d/index.json'))['manifest']['defaults'])"
|
||||
done
|
||||
} | tee -a "$OUT/transcript.txt"
|
||||
run cat runs/traces/spreadwatch-7958ba7c/index.json
|
||||
run_chart "$AURA" chart spreadwatch-7958ba7c --tap spread
|
||||
|
||||
# ---------------------------------------------------------------- example 4
|
||||
# Axis 4: does the documentation let you predict where the output lands?
|
||||
# The prediction below was written from docs/authoring-guide.md §1 + §3 and
|
||||
# docs/glossary.md alone, before this blueprint was ever run.
|
||||
step "example 4 — cold prediction, unnamed blueprint"
|
||||
{
|
||||
printf 'PREDICTED (from the guide + glossary, before running):\n'
|
||||
printf ' runs/traces/graph-<8 hex>/{index.json,price.json,signal.json}\n'
|
||||
printf ' stdout carries "trace_name":"graph-<8 hex>"\n'
|
||||
printf ' the 8 hex digits themselves are NOT predictable from any public doc\n'
|
||||
} | tee -a "$OUT/transcript.txt"
|
||||
run "$AURA" exec blueprints/2_ic_measure.bp.json
|
||||
run ls runs/traces/graph-cb78c78d
|
||||
# a narrower re-run of the same identity: what happens to the tap already there
|
||||
run "$AURA" exec blueprints/2_ic_measure.bp.json --tap signal=record
|
||||
run ls runs/traces/graph-cb78c78d
|
||||
run cat runs/traces/graph-cb78c78d/index.json
|
||||
run "$AURA" measure ic graph-cb78c78d --signal signal --price price
|
||||
|
||||
printf '\ntranscript: %s\n' "$OUT/transcript.txt"
|
||||
@@ -0,0 +1,15 @@
|
||||
[
|
||||
{"op": "name", "name": "spreadwatch"},
|
||||
{"op": "doc", "text": "SMA crossover bias, taps the raw fast-slow spread"},
|
||||
{"op": "source", "role": "price", "kind": "F64"},
|
||||
{"op": "add", "type": "SMA", "name": "fast", "bind": {"length": {"I64": 2}}},
|
||||
{"op": "add", "type": "SMA", "name": "slow", "bind": {"length": {"I64": 4}}},
|
||||
{"op": "feed", "role": "price", "into": ["fast.series", "slow.series"]},
|
||||
{"op": "add", "type": "Sub", "name": "sub"},
|
||||
{"op": "connect", "from": "fast.value", "to": "sub.lhs"},
|
||||
{"op": "connect", "from": "slow.value", "to": "sub.rhs"},
|
||||
{"op": "add", "type": "Bias", "name": "bias", "bind": {"scale": {"F64": 1.0}}},
|
||||
{"op": "connect", "from": "sub.value", "to": "bias.signal"},
|
||||
{"op": "tap", "from": "sub.value", "as": "spread"},
|
||||
{"op": "expose", "from": "bias.bias", "as": "bias"}
|
||||
]
|
||||
@@ -0,0 +1,13 @@
|
||||
[
|
||||
{"op": "doc", "text": "Measurement-only graph: records the price passthrough and an EMA-SMA signal"},
|
||||
{"op": "source", "role": "price", "kind": "F64"},
|
||||
{"op": "add", "type": "Scale", "name": "pass", "bind": {"factor": {"F64": 1.0}}},
|
||||
{"op": "add", "type": "EMA", "name": "ema", "bind": {"length": {"I64": 3}}},
|
||||
{"op": "add", "type": "SMA", "name": "sma", "bind": {"length": {"I64": 6}}},
|
||||
{"op": "feed", "role": "price", "into": ["pass.signal", "ema.series", "sma.series"]},
|
||||
{"op": "add", "type": "Sub", "name": "sig"},
|
||||
{"op": "connect", "from": "ema.value", "to": "sig.lhs"},
|
||||
{"op": "connect", "from": "sma.value", "to": "sig.rhs"},
|
||||
{"op": "tap", "from": "pass.value", "as": "price"},
|
||||
{"op": "tap", "from": "sig.value", "as": "signal"}
|
||||
]
|
||||
@@ -0,0 +1,83 @@
|
||||
[
|
||||
{
|
||||
"op": "name",
|
||||
"name": "spreadwatch"
|
||||
},
|
||||
{
|
||||
"op": "doc",
|
||||
"text": "Crossover bias with a spread tap; rewritten description, same graph"
|
||||
},
|
||||
{
|
||||
"op": "source",
|
||||
"role": "price",
|
||||
"kind": "F64"
|
||||
},
|
||||
{
|
||||
"op": "add",
|
||||
"type": "SMA",
|
||||
"name": "fast",
|
||||
"bind": {
|
||||
"length": {
|
||||
"I64": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"op": "add",
|
||||
"type": "SMA",
|
||||
"name": "slow",
|
||||
"bind": {
|
||||
"length": {
|
||||
"I64": 4
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"op": "feed",
|
||||
"role": "price",
|
||||
"into": [
|
||||
"fast.series",
|
||||
"slow.series"
|
||||
]
|
||||
},
|
||||
{
|
||||
"op": "add",
|
||||
"type": "Sub",
|
||||
"name": "sub"
|
||||
},
|
||||
{
|
||||
"op": "connect",
|
||||
"from": "fast.value",
|
||||
"to": "sub.lhs"
|
||||
},
|
||||
{
|
||||
"op": "connect",
|
||||
"from": "slow.value",
|
||||
"to": "sub.rhs"
|
||||
},
|
||||
{
|
||||
"op": "add",
|
||||
"type": "Bias",
|
||||
"name": "bias",
|
||||
"bind": {
|
||||
"scale": {
|
||||
"F64": 1.0
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"op": "connect",
|
||||
"from": "sub.value",
|
||||
"to": "bias.signal"
|
||||
},
|
||||
{
|
||||
"op": "tap",
|
||||
"from": "sub.value",
|
||||
"as": "spread"
|
||||
},
|
||||
{
|
||||
"op": "expose",
|
||||
"from": "bias.bias",
|
||||
"as": "bias"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,83 @@
|
||||
[
|
||||
{
|
||||
"op": "name",
|
||||
"name": "spreadwatch"
|
||||
},
|
||||
{
|
||||
"op": "doc",
|
||||
"text": "SMA crossover bias, taps the raw fast-slow spread"
|
||||
},
|
||||
{
|
||||
"op": "source",
|
||||
"role": "price",
|
||||
"kind": "F64"
|
||||
},
|
||||
{
|
||||
"op": "add",
|
||||
"type": "SMA",
|
||||
"name": "quick",
|
||||
"bind": {
|
||||
"length": {
|
||||
"I64": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"op": "add",
|
||||
"type": "SMA",
|
||||
"name": "lazy",
|
||||
"bind": {
|
||||
"length": {
|
||||
"I64": 4
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"op": "feed",
|
||||
"role": "price",
|
||||
"into": [
|
||||
"quick.series",
|
||||
"lazy.series"
|
||||
]
|
||||
},
|
||||
{
|
||||
"op": "add",
|
||||
"type": "Sub",
|
||||
"name": "delta"
|
||||
},
|
||||
{
|
||||
"op": "connect",
|
||||
"from": "quick.value",
|
||||
"to": "delta.lhs"
|
||||
},
|
||||
{
|
||||
"op": "connect",
|
||||
"from": "lazy.value",
|
||||
"to": "delta.rhs"
|
||||
},
|
||||
{
|
||||
"op": "add",
|
||||
"type": "Bias",
|
||||
"name": "b",
|
||||
"bind": {
|
||||
"scale": {
|
||||
"F64": 1.0
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"op": "connect",
|
||||
"from": "delta.value",
|
||||
"to": "b.signal"
|
||||
},
|
||||
{
|
||||
"op": "tap",
|
||||
"from": "delta.value",
|
||||
"as": "spread"
|
||||
},
|
||||
{
|
||||
"op": "expose",
|
||||
"from": "b.bias",
|
||||
"as": "bias"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,83 @@
|
||||
[
|
||||
{
|
||||
"op": "name",
|
||||
"name": "otherwatch"
|
||||
},
|
||||
{
|
||||
"op": "doc",
|
||||
"text": "SMA crossover bias, taps the raw fast-slow spread"
|
||||
},
|
||||
{
|
||||
"op": "source",
|
||||
"role": "price",
|
||||
"kind": "F64"
|
||||
},
|
||||
{
|
||||
"op": "add",
|
||||
"type": "SMA",
|
||||
"name": "fast",
|
||||
"bind": {
|
||||
"length": {
|
||||
"I64": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"op": "add",
|
||||
"type": "SMA",
|
||||
"name": "slow",
|
||||
"bind": {
|
||||
"length": {
|
||||
"I64": 4
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"op": "feed",
|
||||
"role": "price",
|
||||
"into": [
|
||||
"fast.series",
|
||||
"slow.series"
|
||||
]
|
||||
},
|
||||
{
|
||||
"op": "add",
|
||||
"type": "Sub",
|
||||
"name": "sub"
|
||||
},
|
||||
{
|
||||
"op": "connect",
|
||||
"from": "fast.value",
|
||||
"to": "sub.lhs"
|
||||
},
|
||||
{
|
||||
"op": "connect",
|
||||
"from": "slow.value",
|
||||
"to": "sub.rhs"
|
||||
},
|
||||
{
|
||||
"op": "add",
|
||||
"type": "Bias",
|
||||
"name": "bias",
|
||||
"bind": {
|
||||
"scale": {
|
||||
"F64": 1.0
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"op": "connect",
|
||||
"from": "sub.value",
|
||||
"to": "bias.signal"
|
||||
},
|
||||
{
|
||||
"op": "tap",
|
||||
"from": "sub.value",
|
||||
"as": "spread"
|
||||
},
|
||||
{
|
||||
"op": "expose",
|
||||
"from": "bias.bias",
|
||||
"as": "bias"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,83 @@
|
||||
[
|
||||
{
|
||||
"op": "name",
|
||||
"name": "spreadwatch"
|
||||
},
|
||||
{
|
||||
"op": "doc",
|
||||
"text": "SMA crossover bias, taps the raw fast-slow spread"
|
||||
},
|
||||
{
|
||||
"op": "source",
|
||||
"role": "price",
|
||||
"kind": "F64"
|
||||
},
|
||||
{
|
||||
"op": "add",
|
||||
"type": "SMA",
|
||||
"name": "fast",
|
||||
"bind": {
|
||||
"length": {
|
||||
"I64": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"op": "add",
|
||||
"type": "SMA",
|
||||
"name": "slow",
|
||||
"bind": {
|
||||
"length": {
|
||||
"I64": 4
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"op": "feed",
|
||||
"role": "price",
|
||||
"into": [
|
||||
"fast.series",
|
||||
"slow.series"
|
||||
]
|
||||
},
|
||||
{
|
||||
"op": "add",
|
||||
"type": "Sub",
|
||||
"name": "sub"
|
||||
},
|
||||
{
|
||||
"op": "connect",
|
||||
"from": "fast.value",
|
||||
"to": "sub.lhs"
|
||||
},
|
||||
{
|
||||
"op": "connect",
|
||||
"from": "slow.value",
|
||||
"to": "sub.rhs"
|
||||
},
|
||||
{
|
||||
"op": "add",
|
||||
"type": "Bias",
|
||||
"name": "bias",
|
||||
"bind": {
|
||||
"scale": {
|
||||
"F64": 1.0
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"op": "connect",
|
||||
"from": "sub.value",
|
||||
"to": "bias.signal"
|
||||
},
|
||||
{
|
||||
"op": "tap",
|
||||
"from": "sub.value",
|
||||
"as": "gap"
|
||||
},
|
||||
{
|
||||
"op": "expose",
|
||||
"from": "bias.bias",
|
||||
"as": "bias"
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user