claude
9636b00314
Source-blind milestone fieldtest of the embedding promise (range
024e865..32f6be8): a path-dep World program driving refusals as values
and a live in-process tap, reproduce-as-data with both guard classes and
a tampered-metric divergence probe, 22 CLI exit-class probes derived
from the docs beforehand, and the stale-dylib handshake via the nodes-new
scaffold plus a rev-pinned external crate. Replayable via m4_0_run_all.sh.
Findings: 2 code bugs fixed in-cycle RED-first (99e9300 — the library
exposes-neither panic was the one real promise breach), 2 doc bugs fixed
in the same commit, 4 friction items bundled as #349, 2 spec gaps
ratified into C30 (rev+lockfile clause; the scaffold's absolute path also
in #349 — the committed lab-nodes fixture is relativized), 6 working.
refs #296
90 lines
4.8 KiB
Bash
90 lines
4.8 KiB
Bash
#!/usr/bin/env bash
|
|
# Fieldtest, milestone 39, axis 2 — the CLI exit-class partition.
|
|
#
|
|
# The downstream task: I am writing a CI wrapper around `aura` and I want to
|
|
# branch on the failure CLASS without parsing stderr, exactly as C14 promises
|
|
# (2 = the content of what argv named, 1 = the environment/recorded state,
|
|
# 3 = campaign completed with failed cells, 0 = success). Every probe below
|
|
# is a refusal a real author trips over; `want` is the class I derived from
|
|
# docs/authoring-guide.md's exit-code passage + C14 BEFORE running it.
|
|
#
|
|
# Run from the `lab/` project directory.
|
|
set -u
|
|
AURA=${AURA:-../../../target/release/aura}
|
|
BP=blueprints/embed-probe.json
|
|
tmp=$(mktemp -d)
|
|
|
|
pass=0; fail=0
|
|
probe() { # probe <want> <label> -- <cmd...>
|
|
local want=$1 label=$2; shift 3
|
|
local out; out=$("$@" 2>&1 >/dev/null); local got=$?
|
|
local verdict="OK "; if [ "$got" != "$want" ]; then verdict="DIFF"; fi
|
|
if [ "$got" = "$want" ]; then pass=$((pass+1)); else fail=$((fail+1)); fi
|
|
printf '%s want=%s got=%s %-46s | %s\n' "$verdict" "$want" "$got" "$label" "$(echo "$out" | head -1)"
|
|
}
|
|
|
|
# --- 0: success -------------------------------------------------------------
|
|
probe 0 "closed blueprint, one synthetic run" -- $AURA exec $BP
|
|
probe 0 "blueprint + declared tap folded" -- $AURA exec $BP --tap spread=mean
|
|
|
|
# --- 2: the content of what argv named --------------------------------------
|
|
printf 'not json at all\n' > "$tmp/notjson.json"
|
|
probe 2 "target file is not JSON" -- $AURA exec "$tmp/notjson.json"
|
|
cp ../m4_1_tapped_signal.ops.json "$tmp/oplist.json"
|
|
probe 2 "op-script array handed to exec" -- $AURA exec "$tmp/oplist.json"
|
|
probe 2 "process document handed to exec" -- $AURA exec ../m4_2_process_sweep_only.json
|
|
printf '{"format_version":1,"blueprint":{"nodes":[],"edges":[]}}\n' > "$tmp/broken.json"
|
|
probe 2 "blueprint envelope that will not build" -- $AURA exec "$tmp/broken.json"
|
|
python3 - <<PY
|
|
import json
|
|
ops = json.load(open("../m4_1_tapped_signal.ops.json"))
|
|
op = [dict(o) for o in ops]
|
|
for o in op:
|
|
o.pop("bind", None)
|
|
if o.get("op") == "name": o["name"] = "open-probe"
|
|
json.dump(op, open("$tmp/open.ops.json", "w"))
|
|
bl = [o for o in ops if o.get("op") not in ("expose", "tap")]
|
|
for o in bl:
|
|
if o.get("op") == "name": o["name"] = "blind-probe"
|
|
json.dump(bl, open("$tmp/blind.ops.json", "w"))
|
|
PY
|
|
$AURA graph build < "$tmp/open.ops.json" > "$tmp/open.bp.json"
|
|
$AURA graph build < "$tmp/blind.ops.json" > "$tmp/blind.bp.json"
|
|
probe 2 "open blueprint (free knobs) run directly" -- $AURA exec "$tmp/open.bp.json"
|
|
probe 2 "blueprint exposing neither bias nor tap" -- $AURA exec "$tmp/blind.bp.json"
|
|
probe 2 "unknown fold label on a declared tap" -- $AURA exec $BP --tap spread=median
|
|
probe 2 "tap name the blueprint does not declare" -- $AURA exec $BP --tap spred=mean
|
|
probe 2 "same tap subscribed twice" -- $AURA exec $BP --tap spread=mean --tap spread=count
|
|
probe 2 "override token with no '='" -- $AURA exec $BP --override fast.length
|
|
probe 2 "override value that is not a literal" -- $AURA exec $BP --override fast.length=abc
|
|
probe 2 "override names no param of the graph" -- $AURA exec $BP --override nope.length=3
|
|
probe 2 "override value of the wrong kind" -- $AURA exec $BP --override fast.length=1.5
|
|
|
|
# --- 1: the environment / recorded state ------------------------------------
|
|
BPID=00fd2bc044fe4c9b947aac97da740e6d08bf4103ec376058f0cf52590f7863aa
|
|
PRID=e4a97c684686c2982f034a27dabb7f26e54c76d7578dd985eaa41ef67c77f275
|
|
# same campaign as axis 1, but a window the archive cannot serve (year 2124)
|
|
sed -e 's/"from_ms": 1725148800000/"from_ms": 4870281600000/' \
|
|
-e 's/"to_ms": 1725753600000/"to_ms": 4870886400000/' \
|
|
-e 's/"name": "m4-embed-reproduce-probe"/"name": "m4-window-with-no-data"/' \
|
|
../m4_2_campaign_ger40.json > "$tmp/nodata.json"
|
|
# ... one over an instrument the archive has never heard of
|
|
sed -e 's/"GER40"/"NOSUCHSYM"/' \
|
|
-e 's/"name": "m4-embed-reproduce-probe"/"name": "m4-absent-symbol"/' \
|
|
../m4_2_campaign_ger40.json > "$tmp/absent.json"
|
|
# ... and one whose axis carries no values at all
|
|
sed -e 's/"values": \[2, 3\]/"values": []/' \
|
|
-e 's/"name": "m4-embed-reproduce-probe"/"name": "m4-empty-axis"/' \
|
|
../m4_2_campaign_ger40.json > "$tmp/emptyaxis.json"
|
|
|
|
probe 1 "override value out of the node's domain" -- $AURA exec $BP --override fast.length=-1
|
|
probe 1 "target is neither file nor content id" -- $AURA exec no-such-target
|
|
probe 1 "well-formed but unregistered content id" -- $AURA exec 0000000000000000000000000000000000000000000000000000000000000000
|
|
probe 1 "reproduce an unknown family id" -- $AURA reproduce no-such-family-0
|
|
probe 1 "campaign over a window the archive lacks" -- $AURA exec "$tmp/nodata.json"
|
|
probe 1 "campaign over a symbol the archive lacks" -- $AURA exec "$tmp/absent.json"
|
|
probe 1 "campaign document content fails its gate" -- $AURA exec "$tmp/emptyaxis.json"
|
|
|
|
echo "--- $pass matched the class I derived from the docs, $fail did not"
|
|
rm -rf "$tmp"
|