fieldtest: shell-boundary — 4 examples, 6 findings
Downstream-consumer corpus for the #295 library surface: a standalone workspace (path-deps, no aura-cli in the link graph — cargo tree verified) exercising the four cycle axes as a World program would, public interface only: a DefaultMemberRunner member run over real GER40, a hand-authored campaign document through aura_campaign::execute, the IC + registry deflation dispatch as library calls, and load_family/reproduce round-trips. Run transcripts committed beside the sources. Findings routed separately: the family-id mismatch between the registry enumeration and the reproduce keying (list-then-reproduce dead-ends) to debug; the reproduce-API friction (no report on the simple path; DataSource/pip re-specification the manifest already holds, wrong pip silently DIVERGED) to a follow-up. Four working confirmations, chief among them the cycle's acceptance claim: all four axes compiled first-try from the public docs alone. refs #295
This commit is contained in:
+1294
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,51 @@
|
||||
# Standalone downstream-consumer crate for the shell-boundary (#295) fieldtest.
|
||||
#
|
||||
# The cycle's claim to substantiate empirically: a downstream World program can
|
||||
# run the canonical member backtest and drive the family/validation machinery
|
||||
# through the LIBRARY ALONE — no `aura` binary, no `aura-cli` in the link graph.
|
||||
#
|
||||
# Like the prior fixtures, this is NOT a member of the aura workspace (empty
|
||||
# [workspace] table below = own workspace root); it path-deps the library crates
|
||||
# exactly as a real C16 research project would, and drives the public surface
|
||||
# discovered from the design ledger + glossary + authoring guide + `cargo doc`
|
||||
# rustdoc + the shipped worked example (no crates/*/src was read).
|
||||
#
|
||||
# NB: aura-cli is deliberately ABSENT from [dependencies] — the link graph is the
|
||||
# acceptance evidence. Run e.g.:
|
||||
# cargo run --manifest-path fieldtests/cycle-295-shell-boundary/Cargo.toml --bin sb_1_member_run
|
||||
# so HEAD source is always what runs.
|
||||
[workspace]
|
||||
|
||||
[package]
|
||||
name = "sb295-fieldtest"
|
||||
version = "0.0.0"
|
||||
edition = "2024"
|
||||
publish = false
|
||||
|
||||
[dependencies]
|
||||
aura-core = { path = "../../crates/aura-core" }
|
||||
aura-runner = { path = "../../crates/aura-runner" }
|
||||
aura-campaign = { path = "../../crates/aura-campaign" }
|
||||
aura-research = { path = "../../crates/aura-research" }
|
||||
aura-registry = { path = "../../crates/aura-registry" }
|
||||
aura-ingest = { path = "../../crates/aura-ingest" }
|
||||
aura-measurement = { path = "../../crates/aura-measurement" }
|
||||
# Reached for directly: DefaultMemberRunner::new / DataSource::Real want an
|
||||
# Arc<DataServer>, and aura-ingest re-exports only the constructed default_data_server().
|
||||
data-server = { git = "http://192.168.178.103:3000/Brummel/data-server.git", branch = "main" }
|
||||
|
||||
[[bin]]
|
||||
name = "sb_1_member_run"
|
||||
path = "sb_1_member_run.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "sb_2_campaign"
|
||||
path = "sb_2_campaign.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "sb_3_ic"
|
||||
path = "sb_3_ic.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "sb_4_reproduce"
|
||||
path = "sb_4_reproduce.rs"
|
||||
@@ -0,0 +1,4 @@
|
||||
running member: strategy_id=884b1ba0640b2f5bb661c8d47e169db85ac2bb729ff5aa1ff3f9c930085da6d9 instrument=GER40
|
||||
member ran OK. metrics:
|
||||
total_pips=79.35 max_drawdown=862.4750 bias_sign_flips=7253
|
||||
R: n_trades=6604 expectancy_r=0.0042 win_rate=0.363 sqn=0.255 sqn_normalized=0.031 net_expectancy_r=0.0042
|
||||
@@ -0,0 +1,6 @@
|
||||
runs root: runs
|
||||
campaign_id=7c892fcdced6a808bf036564a350e1fd1fd2d47324abbadb5dcd39e02f13d1aa
|
||||
executing campaign (1 cell, sweep over fast.length in [2,3])…
|
||||
campaign ran OK. run#=0 cells=1
|
||||
cell 0: 1 stage-families
|
||||
stage 0 block std::sweep family_id=7c892fcd-0-GER40-w0-r0-s0-0 members=2
|
||||
@@ -0,0 +1,9 @@
|
||||
informed signal:
|
||||
information_coefficient = 0.8909
|
||||
overfit_probability = 0.0020
|
||||
n_pairs = 399
|
||||
noise signal:
|
||||
information_coefficient = -0.0191
|
||||
overfit_probability = 0.6507
|
||||
n_pairs = 399
|
||||
deterministic re-run identical: true
|
||||
@@ -0,0 +1,22 @@
|
||||
### Case A — auto-discover the family id via registry.load_family_members() (the natural 'list then reproduce' path)
|
||||
runs root: runs
|
||||
reproducing family: 7c892fcd-0-GER40-w0-r0-s0
|
||||
load_family failed: exit=1 msg=no such family '7c892fcd-0-GER40-w0-r0-s0'
|
||||
--- reproduce_family(id, env) ---
|
||||
reproduce_family failed: exit=1 msg=no such family '7c892fcd-0-GER40-w0-r0-s0'
|
||||
--- reproduce_family_in(reg, id, data, env) -> ReproduceReport ---
|
||||
reproduce_family_in failed: exit=1 msg=no such family '7c892fcd-0-GER40-w0-r0-s0'
|
||||
|
||||
### Case B — the family id as returned by execute()'s StageFamily.family_id (with the stage '-0' suffix)
|
||||
runs root: runs
|
||||
reproducing family: 7c892fcd-0-GER40-w0-r0-s0-0
|
||||
load_family OK: kind=Sweep members=2
|
||||
--- reproduce_family(id, env) ---
|
||||
7c892fcd-0-GER40-w0-r0-s0-0 member sma_signal.fast.length=2, stop_length=3, stop_k=2 reproduced: bit-identical
|
||||
7c892fcd-0-GER40-w0-r0-s0-0 member sma_signal.fast.length=3, stop_length=3, stop_k=2 reproduced: bit-identical
|
||||
reproduced 2/2 members bit-identically
|
||||
reproduce_family returned Ok(()) — bit-identical (no per-member detail returned).
|
||||
--- reproduce_family_in(reg, id, data, env) -> ReproduceReport ---
|
||||
ReproduceReport: 2/2 members bit-identical
|
||||
IDENTICAL sma_signal.fast.length=2, stop_length=3, stop_k=2
|
||||
IDENTICAL sma_signal.fast.length=3, stop_length=3, stop_k=2
|
||||
@@ -0,0 +1,58 @@
|
||||
//! Axis 1 — a single member backtest driven through `DefaultMemberRunner`, the
|
||||
//! shipped `aura_campaign::MemberRunner` implementor, over a shipped example
|
||||
//! blueprint (r_sma) and real GER40 archive data. No `aura` binary involved;
|
||||
//! this program links the library crates only (see Cargo.toml).
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use aura_campaign::{CellSpec, MemberRunner};
|
||||
use aura_ingest::default_data_server;
|
||||
use aura_runner::{DefaultMemberRunner, Env};
|
||||
|
||||
const SYMBOL: &str = "GER40";
|
||||
// September 2024, inclusive epoch-ms — the window the shipped ger40 examples use.
|
||||
const WINDOW_MS: (i64, i64) = (1_725_148_800_000, 1_727_740_799_999);
|
||||
|
||||
fn main() {
|
||||
let server = default_data_server();
|
||||
if !server.has_symbol(SYMBOL) {
|
||||
println!("skip: no local {SYMBOL} archive — nothing to demonstrate.");
|
||||
return;
|
||||
}
|
||||
|
||||
let blueprint_json = std::fs::read_to_string(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/../../crates/aura-cli/examples/r_sma.json"
|
||||
))
|
||||
.expect("shipped example blueprint r_sma.json");
|
||||
|
||||
let env = Env::std();
|
||||
let runner = DefaultMemberRunner::new(&env, server, BTreeMap::new(), Vec::new());
|
||||
|
||||
let cell = CellSpec {
|
||||
strategy_ordinal: 0,
|
||||
strategy_id: aura_research::content_id_of(&blueprint_json),
|
||||
blueprint_json,
|
||||
axes: BTreeMap::new(),
|
||||
instrument: SYMBOL.to_string(),
|
||||
window_ms: WINDOW_MS,
|
||||
regime: None,
|
||||
regime_ordinal: 0,
|
||||
};
|
||||
|
||||
println!("running member: strategy_id={} instrument={SYMBOL}", cell.strategy_id);
|
||||
match runner.run_member(&cell, &[], WINDOW_MS) {
|
||||
Ok(report) => {
|
||||
let m = &report.metrics;
|
||||
println!("member ran OK. metrics:");
|
||||
println!(" total_pips={:.2} max_drawdown={:.4} bias_sign_flips={}", m.total_pips, m.max_drawdown, m.bias_sign_flips);
|
||||
match &m.r {
|
||||
Some(r) => println!(
|
||||
" R: n_trades={} expectancy_r={:.4} win_rate={:.3} sqn={:.3} sqn_normalized={:.3} net_expectancy_r={:.4}",
|
||||
r.n_trades, r.expectancy_r, r.win_rate, r.sqn, r.sqn_normalized, r.net_expectancy_r
|
||||
),
|
||||
None => println!(" R: (none)"),
|
||||
}
|
||||
}
|
||||
Err(fault) => println!("member fault (data-dependent): {fault:?}"),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
//! Axis 2 — campaign execution driven through `aura_campaign::execute` with the
|
||||
//! default runner over a tiny hand-authored campaign + process document. Author
|
||||
//! the two closed-vocabulary documents as JSON, canonicalize + content-address
|
||||
//! them via aura-research, register the artifacts, then run the whole family/
|
||||
//! validation machinery. Library crates only; no `aura` binary.
|
||||
use std::collections::BTreeMap;
|
||||
use std::num::NonZeroUsize;
|
||||
|
||||
use aura_campaign::{execute, MemberRunner};
|
||||
use aura_ingest::default_data_server;
|
||||
use aura_research::{
|
||||
campaign_to_json, content_id_of, parse_campaign, parse_process, process_to_json,
|
||||
};
|
||||
use aura_runner::{DefaultMemberRunner, Env};
|
||||
|
||||
const SYMBOL: &str = "GER40";
|
||||
const FROM_MS: i64 = 1_725_148_800_000; // 2024-09-01
|
||||
const TO_MS: i64 = 1_727_740_800_000; // 2024-10-01
|
||||
|
||||
fn main() {
|
||||
let server = default_data_server();
|
||||
if !server.has_symbol(SYMBOL) {
|
||||
println!("skip: no local {SYMBOL} archive — nothing to demonstrate.");
|
||||
return;
|
||||
}
|
||||
|
||||
// --- 1. the strategy blueprint (shipped example) + its content id ------------
|
||||
let blueprint_json = std::fs::read_to_string(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/../../crates/aura-cli/examples/r_sma.json"
|
||||
))
|
||||
.expect("shipped example blueprint r_sma.json");
|
||||
let bp_id = content_id_of(&blueprint_json);
|
||||
|
||||
// --- 2. a minimal process document: one selection-free sweep -----------------
|
||||
let process_src = r#"{
|
||||
"format_version": 1,
|
||||
"kind": "process",
|
||||
"name": "sb-explore-only-sweep",
|
||||
"pipeline": [ { "block": "std::sweep" } ]
|
||||
}"#;
|
||||
let process_doc = parse_process(process_src).expect("process parses");
|
||||
let process_canon = process_to_json(&process_doc);
|
||||
let process_id = content_id_of(&process_canon);
|
||||
|
||||
// --- 3. a minimal campaign document referencing both by content id -----------
|
||||
let campaign_src = format!(
|
||||
r#"{{
|
||||
"format_version": 1,
|
||||
"kind": "campaign",
|
||||
"name": "sb-ger40-sma-min",
|
||||
"seed": 7,
|
||||
"data": {{
|
||||
"instruments": ["{SYMBOL}"],
|
||||
"windows": [ {{ "from_ms": {FROM_MS}, "to_ms": {TO_MS} }} ]
|
||||
}},
|
||||
"strategies": [
|
||||
{{
|
||||
"ref": {{ "content_id": "{bp_id}" }},
|
||||
"axes": {{ "fast.length": {{ "kind": "I64", "values": [2, 3] }} }}
|
||||
}}
|
||||
],
|
||||
"process": {{ "ref": {{ "content_id": "{process_id}" }} }},
|
||||
"presentation": {{ "persist_taps": [], "emit": ["family_table"] }}
|
||||
}}"#
|
||||
);
|
||||
let campaign_doc = parse_campaign(&campaign_src).expect("campaign parses");
|
||||
let campaign_canon = campaign_to_json(&campaign_doc);
|
||||
let campaign_id = content_id_of(&campaign_canon);
|
||||
|
||||
// --- 4. wire the runner + registry, register the artifacts -------------------
|
||||
let env = Env::std();
|
||||
println!("runs root: {}", env.runs_root().display());
|
||||
let reg = env.registry();
|
||||
reg.put_blueprint(&bp_id, &blueprint_json).expect("store blueprint");
|
||||
reg.put_process(&process_canon).expect("store process");
|
||||
reg.put_campaign(&campaign_canon).expect("store campaign");
|
||||
|
||||
let runner = DefaultMemberRunner::new(&env, server, BTreeMap::new(), Vec::new());
|
||||
let strategies = vec![(bp_id.clone(), blueprint_json.clone())];
|
||||
|
||||
println!("campaign_id={campaign_id}");
|
||||
println!("executing campaign (1 cell, sweep over fast.length in [2,3])…");
|
||||
match execute(
|
||||
&campaign_id,
|
||||
&campaign_doc,
|
||||
&process_doc,
|
||||
&strategies,
|
||||
&runner as &dyn MemberRunner,
|
||||
®,
|
||||
NonZeroUsize::new(1).unwrap(),
|
||||
) {
|
||||
Ok(outcome) => {
|
||||
println!("campaign ran OK. run#={} cells={}", outcome.run, outcome.cells.len());
|
||||
for (ci, cell) in outcome.cells.iter().enumerate() {
|
||||
println!(" cell {ci}: {} stage-families", cell.families.len());
|
||||
for fam in &cell.families {
|
||||
println!(
|
||||
" stage {} block {} family_id={} members={}",
|
||||
fam.stage,
|
||||
fam.block,
|
||||
fam.family_id,
|
||||
fam.reports.len()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(fault) => println!("campaign fault: {fault:?}"),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
//! Axis 3 — the Information Coefficient as a pure library call
|
||||
//! (`aura_measurement::information_coefficient`), including its permutation-null
|
||||
//! deflation (the returned `overfit_probability`). No data archive, no `aura`
|
||||
//! binary: just two in-memory (timestamp, value) series a downstream measurement
|
||||
//! World program would hold. Runs an *informed* signal (correlated with the
|
||||
//! forward return) against a *noise* signal to show the deflation discriminates.
|
||||
use aura_core::Timestamp;
|
||||
use aura_measurement::information_coefficient;
|
||||
|
||||
const N: usize = 400;
|
||||
const HORIZON: usize = 1;
|
||||
const PERMUTATIONS: usize = 500;
|
||||
const SEED: u64 = 12_345;
|
||||
|
||||
// A tiny deterministic LCG so the fixture is reproducible without an rng dep.
|
||||
struct Lcg(u64);
|
||||
impl Lcg {
|
||||
fn next_unit(&mut self) -> f64 {
|
||||
self.0 = self.0.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407);
|
||||
// top 53 bits -> [0,1)
|
||||
((self.0 >> 11) as f64) / ((1u64 << 53) as f64)
|
||||
}
|
||||
fn next_signed(&mut self) -> f64 {
|
||||
self.next_unit() * 2.0 - 1.0
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let mut rng = Lcg(0x1234_5678_9abc_def0);
|
||||
|
||||
// A random-walk price and its forward one-step return.
|
||||
let mut price = Vec::with_capacity(N);
|
||||
let mut level = 100.0f64;
|
||||
for _ in 0..N {
|
||||
level += rng.next_signed();
|
||||
price.push(level);
|
||||
}
|
||||
let price_series: Vec<(Timestamp, f64)> =
|
||||
price.iter().enumerate().map(|(i, &p)| (Timestamp(i as i64 * 60_000), p)).collect();
|
||||
|
||||
// Informed signal: the forward return plus noise (so it genuinely forecasts).
|
||||
let mut informed = Vec::with_capacity(N);
|
||||
for i in 0..N {
|
||||
let fwd = if i + HORIZON < N { price[i + HORIZON] - price[i] } else { 0.0 };
|
||||
informed.push(fwd + 0.5 * rng.next_signed());
|
||||
}
|
||||
let informed_series: Vec<(Timestamp, f64)> =
|
||||
informed.iter().enumerate().map(|(i, &s)| (Timestamp(i as i64 * 60_000), s)).collect();
|
||||
|
||||
// Noise signal: uncorrelated with the forward return.
|
||||
let noise_series: Vec<(Timestamp, f64)> =
|
||||
(0..N).map(|i| (Timestamp(i as i64 * 60_000), rng.next_signed())).collect();
|
||||
|
||||
let informed_ic = information_coefficient(&informed_series, &price_series, HORIZON, PERMUTATIONS, SEED);
|
||||
let noise_ic = information_coefficient(&noise_series, &price_series, HORIZON, PERMUTATIONS, SEED);
|
||||
|
||||
println!("informed signal:");
|
||||
println!(" information_coefficient = {:.4}", informed_ic.information_coefficient);
|
||||
println!(" overfit_probability = {:.4}", informed_ic.overfit_probability);
|
||||
println!(" n_pairs = {}", informed_ic.n_pairs);
|
||||
println!("noise signal:");
|
||||
println!(" information_coefficient = {:.4}", noise_ic.information_coefficient);
|
||||
println!(" overfit_probability = {:.4}", noise_ic.overfit_probability);
|
||||
println!(" n_pairs = {}", noise_ic.n_pairs);
|
||||
|
||||
// Determinism check (C1/C18 for a measurement): same inputs -> same bytes.
|
||||
let again = information_coefficient(&informed_series, &price_series, HORIZON, PERMUTATIONS, SEED);
|
||||
let deterministic = again.information_coefficient == informed_ic.information_coefficient
|
||||
&& again.overfit_probability == informed_ic.overfit_probability;
|
||||
println!("deterministic re-run identical: {deterministic}");
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
//! Axis 4 — project loading + bit-identical reproduction (C18/C1) as a library
|
||||
//! consumer. Opens the default `Env`, discovers a persisted family in the run
|
||||
//! registry (written by the sb_2_campaign fixture), inspects it via `load_family`,
|
||||
//! and re-derives it via `reproduce_family`. Also probes `reproduce_family_in`,
|
||||
//! the only variant that returns a programmatic `ReproduceReport`. No `aura`
|
||||
//! binary; library crates only.
|
||||
//!
|
||||
//! Usage: `… --bin sb_4_reproduce [<family-id>]`
|
||||
//! (omit the id to auto-pick the first family the registry holds).
|
||||
use aura_ingest::default_data_server;
|
||||
use aura_runner::family::DataSource;
|
||||
use aura_runner::reproduce::{load_family, reproduce_family, reproduce_family_in};
|
||||
use aura_runner::Env;
|
||||
|
||||
const SYMBOL: &str = "GER40";
|
||||
const FROM_MS: i64 = 1_725_148_800_000;
|
||||
const TO_MS: i64 = 1_727_740_800_000;
|
||||
|
||||
fn main() {
|
||||
let env = Env::std();
|
||||
let reg = env.registry();
|
||||
println!("runs root: {}", env.runs_root().display());
|
||||
|
||||
// Discover a persisted family id (or take one from argv).
|
||||
let arg_id = std::env::args().nth(1);
|
||||
let family_id = match arg_id {
|
||||
Some(id) => id,
|
||||
None => {
|
||||
let members = match reg.load_family_members() {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
println!("could not read registry: {e:?}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
match members.first() {
|
||||
Some(rec) => rec.family.clone(),
|
||||
None => {
|
||||
println!("no persisted families — run the sb_2_campaign fixture first.");
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
println!("reproducing family: {family_id}");
|
||||
|
||||
// 1. Inspect the persisted family programmatically.
|
||||
match load_family(®, &family_id) {
|
||||
Ok(fam) => println!(" load_family OK: kind={:?} members={}", fam.kind, fam.members.len()),
|
||||
Err(e) => println!(" load_family failed: exit={} msg={}", e.exit_code, e.message),
|
||||
}
|
||||
|
||||
// 2. The ergonomic path: reproduce_family(id, env) -> Result<(), RunnerError>.
|
||||
// Auto-derives the data source from the stored manifest.
|
||||
println!(" --- reproduce_family(id, env) ---");
|
||||
match reproduce_family(&family_id, &env) {
|
||||
Ok(()) => println!(" reproduce_family returned Ok(()) — bit-identical (no per-member detail returned)."),
|
||||
Err(e) => println!(" reproduce_family failed: exit={} msg={}", e.exit_code, e.message),
|
||||
}
|
||||
|
||||
// 3. The only variant that returns a ReproduceReport: reproduce_family_in,
|
||||
// which forces the consumer to re-specify the DataSource (symbol/window/pip)
|
||||
// that the family manifest already records. pip re-specified by hand here.
|
||||
println!(" --- reproduce_family_in(reg, id, data, env) -> ReproduceReport ---");
|
||||
let data = DataSource::Real {
|
||||
server: default_data_server(),
|
||||
symbol: SYMBOL.to_string(),
|
||||
from_ms: Some(FROM_MS),
|
||||
to_ms: Some(TO_MS),
|
||||
pip: 1.0,
|
||||
};
|
||||
match reproduce_family_in(®, &family_id, &data, &env) {
|
||||
Ok(report) => {
|
||||
let n = report.outcomes.len();
|
||||
let identical = report.outcomes.iter().filter(|(_, ok)| *ok).count();
|
||||
println!(" ReproduceReport: {identical}/{n} members bit-identical");
|
||||
for (member, ok) in &report.outcomes {
|
||||
println!(" {} {member}", if *ok { "IDENTICAL" } else { "DIVERGED " });
|
||||
}
|
||||
}
|
||||
Err(e) => println!(" reproduce_family_in failed: exit={} msg={}", e.exit_code, e.message),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user