diff --git a/ail-embed/Cargo.toml b/ail-embed/Cargo.toml index 4c5f5bb..7bc408a 100644 --- a/ail-embed/Cargo.toml +++ b/ail-embed/Cargo.toml @@ -14,13 +14,23 @@ license = "MIT" publish = false description = "Lean embedding of an AILang M3-frozen staticlib kernel into a Rust host. Not an AILang [workspace] member (Invariant 1)." -# Iter m5.1: the library itself has ZERO dependencies — the embedding -# core only touches the frozen C ABI. `data-server` is dev-only, -# exercised solely by the hermetic smoke test, so Invariant 1 holds -# in the dependency graph, not just on paper. +# The embedding core (`src/lib.rs`) touches only the frozen C ABI and +# carries zero finance knowledge; `data-server` enters solely through +# the additive `adapter` module + the `swarm_runner` bin. `ail-embed` +# is the sole sanctioned data-server↔AILang meeting point — Invariant 1 +# governs the *compiler* crates (`ailang-*`, `crates/ail`, `runtime/`), +# which remain data-server-free, not this workspace-excluded crate. [dependencies] +# The adapter + swarm_runner bin link data-server (ail-embed is the +# sole data-server↔AILang meeting point — Invariant 1 governs the +# *compiler* crates, not this crate). Path is manifest-relative +# (data-server is at /home/brummel/dev/libs/data-server). +data-server = { path = "../../libs/data-server" } [dev-dependencies] -data-server = { path = "../../libs/data-server" } zip = "2" tempfile = "3" + +[[bin]] +name = "swarm_runner" +path = "src/bin/swarm_runner.rs" diff --git a/ail-embed/src/adapter.rs b/ail-embed/src/adapter.rs new file mode 100644 index 0000000..4539359 --- /dev/null +++ b/ail-embed/src/adapter.rs @@ -0,0 +1,82 @@ +//! The sole `data-server`-knowing layer of `ail-embed`: maps +//! `data_server::records::TickParsed` to the single scalar the M3 +//! `Tick` record carries (mid price), and lazily streams +//! `SymbolChunkIter` chunks into the data-server-free `Kernel` +//! (host-side chunk unroll — the adapter owns chunk iteration). + +use std::sync::Arc; + +use data_server::records::TickParsed; +use data_server::{DataServer, SymbolChunkIter}; + +use crate::Kernel; + +/// Mid price — the single scalar the M3 `Tick` record carries. +#[inline] +pub fn tick_to_px(t: &TickParsed) -> f64 { + (t.ask + t.bid) / 2.0 +} + +/// Lazy `Iterator` over a `data-server` tick stream, +/// yielding the per-tick mid. Pulls `next_chunk` on demand (the +/// adapter owns chunk iteration; chunk boundaries are invisible to +/// the fold) so a whole symbol is never materialised. +pub struct MidPriceStream { + it: SymbolChunkIter, + chunk: Option>, + idx: usize, +} + +impl MidPriceStream { + pub fn new(it: SymbolChunkIter) -> Self { + Self { it, chunk: None, idx: 0 } + } +} + +impl Iterator for MidPriceStream { + type Item = f64; + fn next(&mut self) -> Option { + loop { + if let Some(c) = &self.chunk { + if self.idx < c.len() { + let px = tick_to_px(&c[self.idx]); + self.idx += 1; + return Some(px); + } + } + match self.it.next_chunk() { + Some(c) => { + self.chunk = Some(c); + self.idx = 0; + } + None => return None, + } + } + } +} + +/// Fold up to `max_ticks` of `symbol`'s mid-price stream through the +/// M3 kernel. `None` if the symbol has no tick stream. Returns +/// `(Σ mid, count)` — the kernel's `acc += px; n += 1` over the +/// bounded prefix, identical-order to a host reference fold. +pub fn fold_symbol( + server: &Arc, + symbol: &str, + max_ticks: usize, +) -> Option<(f64, i64)> { + let it = server.stream_tick(symbol)?; + Some(Kernel::new().run(MidPriceStream::new(it).take(max_ticks))) +} + +#[cfg(test)] +mod tests { + use super::tick_to_px; + use data_server::records::TickParsed; + + /// mid = (ask + bid) / 2. Exact in f64 for these inputs. + #[test] + fn tick_to_px_is_mid() { + let t = TickParsed { time_ms: 0, ask: 2.0, bid: 4.0 }; + assert_eq!(tick_to_px(&t), 3.0); + } +} diff --git a/ail-embed/src/bin/swarm_runner.rs b/ail-embed/src/bin/swarm_runner.rs new file mode 100644 index 0000000..2cacc40 --- /dev/null +++ b/ail-embed/src/bin/swarm_runner.rs @@ -0,0 +1,73 @@ +//! Symbol-fan swarm harness (spec §3 symbol-fan / Testing §2). Run +//! as a subprocess by `tests/swarm.rs` under `AILANG_RC_STATS=1` so +//! the C-runtime atexit `ailang_rc_stats:` line is observable. +//! +//! argv[1] = data dir (defaults to `data_server::DEFAULT_DATA_PATH`). +//! Enumerates up to `N_SYMBOLS` symbols that have tick files +//! (requires >= 2 for a meaningful swarm), spawns one thread per +//! symbol — each thread owns its `Kernel` (`Ctx: !Send` forces +//! one ctx per thread) — folds at most `MAX_TICKS` mids, and prints +//! one `RESULT ` line per symbol to +//! stdout. Exits 0 on success, non-zero (with a message) otherwise. + +use std::sync::Arc; + +use ail_embed::adapter::fold_symbol; +use data_server::records::DataFormat; +use data_server::{DataServer, DEFAULT_DATA_PATH}; + +/// Deterministic bounded prefix per symbol (chunk-aligned via +/// `Iterator::take`): enough real volume to be a genuine proof, +/// bounded so the E2E stays ~seconds. m5.3 owns the friction +/// timing; here it is only a runtime bound. +const MAX_TICKS: usize = 2_000_000; +/// Max threads/symbols fanned. Real data has >= 2 tick symbols +/// (EURUSD, XAUUSD); take min(N_SYMBOLS, available). +const N_SYMBOLS: usize = 4; + +fn main() { + let data_dir = std::env::args() + .nth(1) + .unwrap_or_else(|| DEFAULT_DATA_PATH.to_string()); + + let server = Arc::new(DataServer::new(&data_dir)); + + let mut tick_syms: Vec = server + .symbols() + .into_iter() + .filter(|s| { + server.file_count(s, DataFormat::Tick).unwrap_or(0) > 0 + }) + .map(|s| s.to_string()) + .collect(); + tick_syms.truncate(N_SYMBOLS); + + if tick_syms.len() < 2 { + eprintln!( + "swarm_runner: need >= 2 tick symbols under {data_dir}, found {}", + tick_syms.len() + ); + std::process::exit(2); + } + + let handles: Vec<_> = tick_syms + .into_iter() + .map(|sym| { + let srv = Arc::clone(&server); + // Each thread builds its own Kernel inside the closure. + // `Kernel`/`Ctx` are `!Send`, so this is the ONLY way + // this compiles — one ctx per thread, enforced by the + // type system, not by convention. + std::thread::spawn(move || { + let (acc, n) = fold_symbol(&srv, &sym, MAX_TICKS) + .expect("symbol has a tick stream"); + (sym, acc, n) + }) + }) + .collect(); + + for h in handles { + let (sym, acc, n) = h.join().expect("worker thread panicked"); + println!("RESULT {sym} {:016x} {n}", acc.to_bits()); + } +} diff --git a/ail-embed/src/lib.rs b/ail-embed/src/lib.rs index 05b1c9c..6977c41 100644 --- a/ail-embed/src/lib.rs +++ b/ail-embed/src/lib.rs @@ -6,6 +6,10 @@ use std::ffi::c_void; +/// The sole `data-server`-knowing layer (spec §2). Additive; the +/// embedding core below has zero finance knowledge. +pub mod adapter; + // The M3-frozen C ABI. Symbol names + signatures mirror // `crates/ail/tests/embed/tick_roundtrip.c:31-36`. `backtest_step_tick` // is the author-chosen export of `examples/embed_backtest_step_tick.ail`. diff --git a/ail-embed/tests/swarm.rs b/ail-embed/tests/swarm.rs new file mode 100644 index 0000000..4cdf41e --- /dev/null +++ b/ail-embed/tests/swarm.rs @@ -0,0 +1,164 @@ +//! Symbol-fan swarm E2E (spec Testing §2). Runs the `swarm_runner` +//! bin as a subprocess under `AILANG_RC_STATS=1` (the C-runtime +//! atexit stat line, runtime/rc.c:124-138, is only observable from a +//! separate process that exits). Skips when /mnt tick data is absent +//! — mirroring data-server's own tests/data_server.rs::skip_if_no_data(). +//! +//! Two assertions, deliberately SPLIT into two tests: +//! +//! - `symbol_fan_swarm_bit_exact` — the existence proof. Per symbol, +//! the kernel `(acc,n)` is BIT-EXACT vs an independent same-order +//! host reference fold. This is GREEN and live: it proves the +//! real-data-server → adapter → M3-kernel swarm computes correctly. +//! +//! - `symbol_fan_swarm_leak_free` — `#[ignore]`d pending an escalated +//! design decision. The global `Σallocs==Σfrees` measurement +//! (ported from crates/ail/tests/embed_tick_e2e.rs:94-104) is +//! *invalid under a multi-threaded host*: host-side box allocs run +//! with `__ail_tls_ctx == NULL` and fall through to the +//! **non-atomic** global `g_rc_*` counters (runtime/rc.c:90-91, +//! 161,212), which rc.c's own header explicitly documents as +//! single-threaded/non-atomic ("when it acquires [concurrency], +//! atomic-vs-non-atomic becomes a separate decision"). The swarm is +//! *actually* leak-free (no box crosses a thread — `Ctx: !Send`; +//! bit-exact is GREEN every run); only the global *counter* races, +//! losing increments. The assertion body is preserved verbatim and +//! will be re-armed (un-`#[ignore]`d) by the runtime iteration that +//! makes the global fallback counters atomic. See +//! docs/journals/2026-05-19-iter-embedding-abi-m5.2.md and the M5 +//! bounce-back. Quarantined, NOT weakened — what it asserts is +//! unchanged; it is gated on a decision the Boss escalated. + +use std::process::Command; +use std::sync::Arc; + +use data_server::records::DataFormat; +use data_server::{DataServer, DEFAULT_DATA_PATH}; + +// Must match swarm_runner.rs. +const MAX_TICKS: usize = 2_000_000; +const N_SYMBOLS: usize = 4; + +fn skip_if_no_data() -> bool { + !std::path::Path::new(DEFAULT_DATA_PATH).exists() +} + +/// Independent host reference: same symbols, same order, same cap, +/// pure-Rust fold (`acc += mid; n += 1`). Bit-exact with the kernel +/// fold because the addition order is identical. +fn reference(symbol: &str) -> (f64, i64) { + let server = Arc::new(DataServer::new(DEFAULT_DATA_PATH)); + let mut it = server.stream_tick(symbol).expect("tick stream"); + let (mut acc, mut n) = (0.0_f64, 0_i64); + 'outer: while let Some(chunk) = it.next_chunk() { + for r in chunk.iter() { + if n as usize >= MAX_TICKS { + break 'outer; + } + acc += (r.ask + r.bid) / 2.0; + n += 1; + } + } + (acc, n) +} + +/// Spawn `swarm_runner` under `AILANG_RC_STATS=1`. `None` on the +/// skip-if-absent path (no /mnt data); otherwise `(stdout, stderr)` +/// after asserting a clean child exit. +fn run_swarm_or_skip() -> Option<(String, String)> { + if skip_if_no_data() { + eprintln!("skipping: {DEFAULT_DATA_PATH} absent (mirrors data-server)"); + return None; + } + let out = Command::new(env!("CARGO_BIN_EXE_swarm_runner")) + .arg(DEFAULT_DATA_PATH) + .env("AILANG_RC_STATS", "1") + .output() + .expect("spawn swarm_runner"); + let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); + let stderr = String::from_utf8_lossy(&out.stderr).into_owned(); + assert!( + out.status.success(), + "swarm_runner exited {:?}\nstdout:\n{stdout}\nstderr:\n{stderr}", + out.status.code() + ); + Some((stdout, stderr)) +} + +/// Existence proof (GREEN, live): each symbol's kernel `(acc,n)` is +/// bit-exact vs an independent same-order host reference fold. Real +/// data-server → adapter → M3-frozen kernel, one symbol per thread. +#[test] +fn symbol_fan_swarm_bit_exact() { + let Some((stdout, _stderr)) = run_swarm_or_skip() else { return }; + + let mut symbols_checked = 0usize; + for line in stdout.lines().filter(|l| l.starts_with("RESULT ")) { + let mut t = line.split_whitespace(); + let _ = t.next(); // "RESULT" + let sym = t.next().expect("symbol field"); + let acc_bits = u64::from_str_radix( + t.next().expect("acc-bits field"), 16, + ).expect("acc-bits hex"); + let n: i64 = t.next().expect("n field").parse().expect("n int"); + + let kernel_acc = f64::from_bits(acc_bits); + let (ref_acc, ref_n) = reference(sym); + assert_eq!( + kernel_acc.to_bits(), ref_acc.to_bits(), + "{sym}: kernel acc bit-exact vs same-order host reference" + ); + assert_eq!(n, ref_n, "{sym}: tick count matches reference"); + assert!(ref_n > 0, "{sym}: non-empty stream"); + symbols_checked += 1; + } + assert!( + (2..=N_SYMBOLS).contains(&symbols_checked), + "expected 2..={N_SYMBOLS} symbols fanned, got {symbols_checked}" + ); +} + +/// Global leak-freedom: Σallocs == Σfrees across ALL `ailang_rc_stats:` +/// lines (the M2 dual-stat-line model, ported from +/// embed_tick_e2e.rs:94-104). The invariant is the global Σ, not +/// per-line balance (M2 TLS-ctx cross-attribution). +/// +/// `#[ignore]`d: this measurement is invalid under a multi-threaded +/// host. Host-side box allocs run with `__ail_tls_ctx == NULL` and +/// hit the non-atomic global `g_rc_*` counters (runtime/rc.c:90-91, +/// 161,212) — single-threaded/non-atomic *by rc.c's own documented +/// design*. 3 worker threads race the counter, losing increments; +/// the *counter* is unreliable even though the swarm is actually +/// leak-free (no box crosses a thread; `symbol_fan_swarm_bit_exact` +/// is GREEN every run). Un-`#[ignore]` this — body unchanged — once +/// the runtime iteration makes the global fallback counters atomic +/// (the deferred "concurrency arrived" decision rc.c:44-49 names). +/// See docs/journals/2026-05-19-iter-embedding-abi-m5.2.md. +#[test] +#[ignore = "blocked: global g_rc_* counter is non-atomic by rc.c design; \ + invalid under the multi-thread host. Swarm is actually \ + leak-free (bit-exact is green; no box crosses a thread). \ + Un-ignore when the runtime makes the global fallback \ + counters atomic — see the m5.2 journal + M5 bounce-back."] +fn symbol_fan_swarm_leak_free() { + let Some((_stdout, stderr)) = run_swarm_or_skip() else { return }; + + let mut allocs: u64 = 0; + let mut frees: u64 = 0; + let mut seen = 0usize; + for line in stderr.lines().filter(|l| l.starts_with("ailang_rc_stats:")) { + seen += 1; + for tok in line.split_whitespace() { + if let Some(v) = tok.strip_prefix("allocs=") { + allocs += v.parse::().expect("allocs= u64"); + } else if let Some(v) = tok.strip_prefix("frees=") { + frees += v.parse::().expect("frees= u64"); + } + } + } + assert!(seen > 0, "no ailang_rc_stats line; stderr:\n{stderr}"); + assert_eq!( + allocs, frees, + "globally leak-free: Σallocs==Σfrees across {seen} stat line(s)" + ); +} diff --git a/bench/orchestrator-stats/2026-05-19-iter-embedding-abi-m5.2.json b/bench/orchestrator-stats/2026-05-19-iter-embedding-abi-m5.2.json new file mode 100644 index 0000000..838da0e --- /dev/null +++ b/bench/orchestrator-stats/2026-05-19-iter-embedding-abi-m5.2.json @@ -0,0 +1,13 @@ +{ + "iter_id": "embedding-abi-m5.2", + "date": "2026-05-19", + "mode": "standard", + "outcome": "PARTIAL", + "tasks_total": 3, + "tasks_completed": 2, + "reloops_per_task": { "1": 0, "2": 0, "3": 0 }, + "review_loops_spec": 0, + "review_loops_quality": 1, + "blocked_reason": "worker-blocked", + "notes": "Tasks 1+2 clean (Task 1 DONE_WITH_CONCERNS: plan Step-6 command ordering defect + quality-phase stale-comment honesty fix, 1 quality re-loop). Task 3 BLOCKED: the plan's verbatim leak-free assertion correctly surfaced a non-atomic global g_rc counter race (host-side ail-embed allocs run with __ail_tls_ctx==NULL, falling through to runtime/rc.c:161,212 raced by 3 worker threads). Not re-loopable (verbatim plan code), not in-scope-fixable (zero runtime/ diff mandated; runtime/ M3-frozen). Bit-exact half of the E2E sound. Invariant 1 clean independently (Step4=0, Step5 empty)." +} diff --git a/docs/journals/2026-05-19-iter-embedding-abi-m5.2.md b/docs/journals/2026-05-19-iter-embedding-abi-m5.2.md new file mode 100644 index 0000000..3bd4fe0 --- /dev/null +++ b/docs/journals/2026-05-19-iter-embedding-abi-m5.2.md @@ -0,0 +1,234 @@ +# iter embedding-abi-m5.2 — data-server adapter + symbol-fan swarm; leak-free gate surfaced a runtime race + +**Date:** 2026-05-19 +**Started from:** 9cc9d9c5170eb8d52152dcef071726c7a6ffd05d +**Status:** PARTIAL +**Tasks completed:** 2 of 3 + +## Summary + +Tasks 1 and 2 landed clean and verbatim-to-plan: `data-server` was +promoted dev-dep → real dep, the additive `adapter` module +(`tick_to_px` / `MidPriceStream` / `fold_symbol`) was added with a +genuine compile-error RED → GREEN cycle, and the `swarm_runner` bin +compiled — that compile *is* the compile-time per-thread-ctx proof +(`Ctx: !Send`; a shared-ctx design is `error[E0277]` and could not +have built). Invariant 1 holds independently: the AILang workspace +graph has zero `data-server` (Step 4 = `0`) and there is zero +compiler-surface diff (Step 5 path filter empty — only `ail-embed/**` +touched, not even the root `Cargo.toml`). + +Task 3 is BLOCKED. The plan's verbatim leak-free assertion +(`Σallocs == Σfrees` across all `ailang_rc_stats:` lines) is correct +and is doing its job: it surfaced a genuine concurrency defect. The +swarm E2E passed once in isolation by luck (the racing counter +happened to land on the balanced value) and then FAILED in the +full-suite run and on 4 of 5 direct re-runs. The host-side box +allocations in `ail-embed/src/lib.rs` (`make_state` / `make_tick` / +the final `ailang_rc_dec`) run on the Rust worker thread where +`__ail_tls_ctx == NULL` — the M3 forwarder only sets the TLS ctx for +the synchronous duration of the kernel call, not for the host's own +allocs. With `__ail_tls_ctx` null those allocs fall through to the +**non-atomic global statics** `g_rc_alloc_count` / `g_rc_free_count` +(`runtime/rc.c:161,212`), incremented concurrently by 3 worker +threads with no synchronisation. The atexit `g_rc` line's `allocs` +is consequently non-deterministic across runs +(5,993,321 … 5,999,318 over 5 runs; should be a stable 6,000,000) +while `frees` stays at a stable 3. The bit-exactness half of the +test is sound and passed every run; only the global leak-accounting +is racy. + +This is exactly the class of finding the symbol-fan swarm milestone +exists to expose — and it is out of m5.2 scope to fix (see Blocked +detail). The Boss decides: extend the spec / m5.3 to cover the +host-side-alloc TLS-ctx routing (or atomic global counters), or +re-scope the leak-free acceptance. + +## Per-task notes + +- iter embedding-abi-m5.2.1: promote `data-server` dev-dep → real + dep + additive `adapter` module. RED (`E0432 unresolved import + super::tick_to_px`) → impl `tick_to_px`/`MidPriceStream`/ + `fold_symbol` → GREEN (`tick_to_px_is_mid`, + + `kernel_run_sums_prices` unregressed under `--lib`). Cargo.toml + + `lib.rs:7` verbatim-to-plan. Stale m5.1 "ZERO dependencies" + manifest comment corrected for present-tense honesty (quality + phase). +- iter embedding-abi-m5.2.2: `src/bin/swarm_runner.rs` written + verbatim; `cargo build --bin swarm_runner` → exit 0. The clean + compile IS the compile-time per-thread-ctx proof (`Ctx: !Send`; + no `E0277`; per-thread-owned-Kernel structure intact, not + collapsed to a shared ctx). +- iter embedding-abi-m5.2.3: `tests/swarm.rs` written verbatim. The + E2E genuinely executed (NOT the skip path — 3 tick symbols + present: EURUSD, GER40, XAUUSD; ~4 s, 3×2M FFI folds). Per-symbol + bit-exactness vs independent host reference: sound, passed every + run. Global `Σallocs==Σfrees` leak-free assertion: BLOCKED — + correctly detects a non-atomic global-counter race (see Summary + + Blocked detail). Not re-loopable (the plan's verbatim code is + exactly as specified; no re-attempt makes a racy C counter + deterministic) and not fixable in-scope (Step 5 mandates zero + `runtime/` diff; `runtime/` is M3-frozen). + +## Concerns + +- Plan Task 1 Step 6's literal command + (`cargo test --manifest-path ail-embed/Cargo.toml`, unfiltered) + is unsatisfiable in Task 1's isolation: the `[[bin]] swarm_runner` + declaration added by Step 2's verbatim Cargo.toml replacement + references `src/bin/swarm_runner.rs`, which Task 2 Step 1 creates. + The plan Self-review point 7 ("No later task is required to make + an earlier compile gate pass") is incorrect for this command. + Task 1's no-core-regression intent was fully proven via `--lib` + (`kernel_run_sums_prices` + `tick_to_px_is_mid` GREEN); the + smoke + full-suite gate is covered by Task 3 Step 3 once the bin + exists. The verbatim Cargo.toml was NOT split to dodge this + (splitting it would be the spec-compliance violation; the + `[[bin]]` + dep-promotion is one cohesive manifest change). No + code defect — purely a plan command-ordering defect. +- Quality phase corrected the pre-existing m5.1 Cargo.toml comment + ("the library itself has ZERO dependencies … data-server is + dev-only") which became false the moment `data-server` moved into + `[dependencies]`. The plan's verbatim replacement boundary started + *at* `[dependencies]` and did not script the preceding comment's + removal; leaving a now-false manifest comment violates the + DESIGN.md honesty rule (comments mirror current state). The + replacement comment states the present-tense m5.2 framing, + consistent with the plan's own new block + Boss decision. + +## Known debt + +- The host-side embedding allocation path (`ail-embed/src/lib.rs` + `make_state`/`make_tick`/final `ailang_rc_dec`) executes with + `__ail_tls_ctx == NULL` (the M3 `@` forwarder sets the TLS + ctx only for the synchronous kernel-call duration). Under the + multi-thread swarm these allocs/frees land on the non-atomic + global `g_rc_alloc_count`/`g_rc_free_count` fallback + (`runtime/rc.c:161,212`), which races. Not touched here: + `runtime/` is M3-frozen and Step 5 mandates zero `runtime/` diff; + the host-side-routing alternative is a design change beyond + "additive adapter + swarm bin". Reserved for Boss/m5.3 design. + +## Blocked detail + +- **Task:** 3 (Step 2 / Step 3 — the leak-free `Σallocs==Σfrees` + gate). +- **Reason:** `worker-blocked` — the plan's own verbatim acceptance + gate correctly detects a runtime concurrency defect whose fix is + out of m5.2 scope and would violate this iter's Invariant-1 + Step-5 zero-`runtime/`-diff constraint. +- **Worker says (verbatim):** Task 3's `tests/swarm.rs` is verbatim + to plan and genuinely executes (bit-exact half sound, GREEN every + run). The `Σallocs==Σfrees` half is non-deterministic: full-suite + run FAILED `left: 11997743, right: 12000003`; 5 direct child runs + of the atexit `g_rc` line gave `allocs=` 5993699/5996029/5993321/ + 5999318/5996019 with `frees=3` stable. Root cause: host-side + `ailang_rc_alloc`/`ailang_rc_dec` from `lib.rs` run with + `__ail_tls_ctx==NULL`, falling through to the non-atomic global + `g_rc_*` counters (`rc.c:161,212`) raced by 3 worker threads. Not + re-loopable (verbatim plan code; a racy C counter is not made + deterministic by re-attempting the same code) and not + in-scope-fixable (zero `runtime/` diff mandated; `runtime/` is + M3-frozen). +- **Suggested next step:** Boss decision — either (a) extend the + spec/m5.3 to route host-side `ail-embed` allocs through a + per-thread `ailang_ctx_t` so the swarm's leak accounting is + per-thread-attributed (no global fallback), or make + `g_rc_alloc_count`/`g_rc_free_count` atomic in a *separate* + runtime iteration outside m5.2's zero-`runtime/`-diff constraint; + or (b) re-scope the m5.2 leak-free acceptance. Tasks 1 + 2 are + clean and committable as a PARTIAL; the architect should see the + TSan-discharge reasoning below at milestone close. + +## Boss-decision rationale carried forward (mirrored from the plan, for the architect at milestone close) + +- **TSan discharge (compile-time + cited green precedent, no new + harness).** The spec's "sanitiser-clean (no data race across + per-thread ctxs)" acceptance is discharged by (1) `Ctx`/`Kernel` + being `!Send` (raw `*mut c_void`) — the swarm is *structurally + forced* one-ctx-per-thread; a shared-ctx design is `error[E0277]` + and cannot compile, so Task 2 Step 2's clean build is a + compile-time race-freedom proof for the *per-ctx kernel calls*; + (2) the C-runtime race-freedom of that exact per-thread-ctx + pattern is ratified green by `crates/ail/tests/embed_swarm_tsan.rs` + (C host, 8 pthreads, own ctx each, `clang -fsanitize=thread`, + exit-clean), cited not re-run. NOTE FOR THE ARCHITECT: this + discharge covers the *per-thread-ctx kernel path*. The Task 3 + finding is a *different* surface — the host-side `ail-embed` + allocs that bypass the TLS ctx and hit the non-atomic global + `g_rc_*` fallback. The TSan precedent does not cover that path + (the C host in `embed_swarm_tsan.rs` does not exercise a Rust + host allocating boxes with a null TLS ctx). The compile-time + `!Send` proof and the cited green TSan remain valid for what they + claim; they simply do not reach the global-fallback race m5.2 + newly exposed. +- **`data-server` dev-dep → real dep is sanctioned.** Invariant 1 + governs the *compiler* crates (`ailang-*`, `crates/ail`, + `runtime/`), which gain nothing — verified clean this iter + (Step 4 = `0`, Step 5 empty). `ail-embed` is explicitly the + *sole* data-server↔AILang meeting point. +- **Deferred to m5.3 (not this plan):** time-shard + boundary-invisibility, friction-harvest timing, milestone + close-out. Not implemented here (scope discipline held — no + time-shard, no perf timing, no close-out crept in). + +## Files touched + +- `ail-embed/Cargo.toml` (modified — dep promotion + `[[bin]]` + + stale-comment honesty fix) +- `ail-embed/src/lib.rs` (modified — `pub mod adapter;`, additive) +- `ail-embed/src/adapter.rs` (new — the sole data-server-knowing layer) +- `ail-embed/src/bin/swarm_runner.rs` (new — symbol-fan swarm bin) +- `ail-embed/tests/swarm.rs` (new — symbol-fan leak-free + bit-exact + E2E; bit-exact half sound, leak-free half surfaces the race) + +## Stats + +bench/orchestrator-stats/2026-05-19-iter-embedding-abi-m5.2.json + +## Boss disposition (2026-05-19) + +Boss independently read `runtime/rc.c` and confirms the diagnosis is +**precise and the finding is an instrumentation-measurement race, NOT +a memory bug**: the per-object refcount header ops (`*hdr +=/-= 1`, +rc.c:180/208) are non-atomic but **no box crosses a thread** in the +swarm (`Ctx: !Send` structurally forces alloc/use/free within one +worker), so there is no real refcount race, no double-free, no leak — +the bit-exact half is GREEN every run, confirming the memory +management is actually correct. The *only* raced object is the global +`g_rc_*` statistics counter (rc.c:90-91,161,212), **non-atomic by +rc.c's own explicit design** (header comment rc.c:44-49,85: "AILang +has no concurrency primitives yet; when it acquires them, +atomic-vs-non-atomic becomes a separate decision"). M5 is AILang's +first concurrent consumer — that deferred decision is now due. + +Committed as a PARTIAL (Tasks 1+2 + the GREEN existence proof) so the +sound work + the finding land on the record, main stays green: +`tests/swarm.rs` was Boss-split into `symbol_fan_swarm_bit_exact` +(live, GREEN — the real-data existence proof) and +`symbol_fan_swarm_leak_free` (`#[ignore]`, **body preserved +verbatim** — quarantined, not weakened; un-`#[ignore]` is the +acceptance criterion for the runtime fix). Earlier plan/journal claim +that `embed_swarm_tsan.rs` covers this path was wrong and is +corrected here: that test uses the *scalar* kernel (zero box allocs) +so it never exercised the host-side global-counter path; the +compile-time `!Send` proof + the cited green TSan remain valid only +for the per-thread-ctx *kernel* path, which they always claimed. + +Escalated to the user as an M5 **bounce-back** (touches M3-frozen +`runtime/`; re-frames M5's "zero runtime change" commitment; +multiple substantive options — not unilaterally Boss's to pick in +frozen-runtime territory). Boss recommendation on the record: +**Option A** — make *only the global fallback* `g_rc_*` counters +atomic (leave the per-thread ctx path plain — no contention there), +as its own RED-first runtime micro-iteration (`debug` skill), outside +M5's "zero runtime change" framing; then resume the M5 leak-proof on +the now-valid measurement (un-`#[ignore]` `symbol_fan_swarm_leak_free` +as that iter's GREEN). Rationale: it discharges exactly the deferred +decision rc.c's own comment names, at the defect's actual location, +without touching the frozen value layout / ABI offsets / host-free +rule (the literal content of the M3 freeze); Option B (host-side +TLS-ctx routing) adds a new runtime API surface; Option C (per-symbol +single-thread subprocess leak-proof) is sound but weaker and leaves +the runtime's documented single-thread-stats footgun for every future +concurrent consumer. User picks the direction. diff --git a/docs/journals/INDEX.md b/docs/journals/INDEX.md index 4a2c589..a637888 100644 --- a/docs/journals/INDEX.md +++ b/docs/journals/INDEX.md @@ -107,3 +107,4 @@ - 2026-05-18 — brainstorm embedding-abi-m4 → RETIRED, never speced (premise collapsed under its own feature-acceptance gate during Step-2/3 Q&A; no spec, no grounding-check, no planner handoff — the "problem mis-framed → don't ratify a known-unneeded shape" brainstorm path): `/boss` picked top-P0 "Embedding ABI — M4: sequence crossing via `List`"; user green-lit a continue-here brainstorm; recon (`ailang-plan-recon`) returned a full fact sheet; two user forks resolved in Q&A (own-only `List` param; `List Record`-only element) and Approach A (structural list-shaped `is_c_abi_type` arm, no name-match; B name-anchored / C `std_list`-SSOT-first rejected on language/scope grounds) recommended — all now moot. Struck on **feature-acceptance clause 2**: the shipped M3 gate `is_c_abi_type` (`crates/ailang-check/src/lib.rs:1934-1953`) is a per-parameter loop accepting a C scalar OR a single-ctor all-scalar record *independently per param*, and the forwarder's `llvm_scalar` maps every non-scalar `Type::Con`→`ptr` (M3-frozen), so `(State, Tick) -> State` (both single-ctor all-scalar records) is **already gate-accepted + forwarder-supported today** — the minimal data-server binding is M3 (shipped) + a host-side per-tick loop; cons-list crossing would *add* a 2N+1-box-per-chunk host builder + the deferred flat-array perf debt and removes no redundancy, with no named consumer (M5's adapter unrolls each chunk host-side — a clean adapter, the sole data-server↔AILang meeting point per Invariant 1; whole-chunk in-kernel visibility is semantically void since `State` threads across calls regardless of chunk boundaries). Honest mid-Q&A correction recorded: I had asserted "M5 cannot wire data-server without M4" — false (M5 wires it on M3 + `for tick in chunk`); re-deriving against the code rather than defending the roadmap I wrote is what surfaced the clause-2 failure ("user suggestions ≠ directives, form own judgment"). Outcome: M4 retired in `docs/roadmap.md` (struck entry kept one cycle, never `[x]`); M5 reconciled (`depends on:` M4→M3 + Tick-coverage todo; adapter unrolls host-side; friction feeds the host-per-tick-FFI-vs-batch P2 perf decision); residual = a new `[todo]` "Tick-coverage on M3" (E2E+fixture pinning the two-record-param per-tick `(State, Tick) -> State` shape — capability present today but only E2E-proven for a single record param `State`; every shipped M3 fixture pushes a scalar `Float` sample, none a record `Tick`; test backfill, no brainstorm — the actual "minimal data-server binding"). Forward note: the P2 flat-array item's "1024 cons-cells/chunk" framing is now partly stale (cons-list path dropped) — reconcile when picked up, not now. → 2026-05-18-brainstorm-embedding-abi-m4-retired.md - 2026-05-18 — iter bugfix-over-strict-mode-ctor-rebuild-consume (RED→GREEN, debug→implement mini, DONE 1/1): fixed a conservative `[over-strict-mode]` false-positive surfaced by the Tick-coverage fixtures. The lint's consume-detection (`any_sub_binder_consumed_for`/`pattern_has_consumed_heap_binder`, `crates/ailang-check/src/linearity.rs`) only recognised a consume of an `(own (con T))` param when a *heap-typed* pattern-binder was moved out of `match p`; when `p` was destructured into purely *primitive* fields fed into a `Term::Ctor` rebuilding `p`'s own ctor, that genuine dismantle+rebuild consume was invisible, so the lint spuriously advised `(borrow ...)`. Real harm: an LLM author "fixing" the spurious warning by flipping an export's declared mode `own`→`borrow` would silently invert the ABI ownership contract. RED-first: debugger disproved the carrier's initial nested-`match` hypothesis (the M3 `embed_backtest_step_record.ail` is silent only because its implicit-mode scalar `Float` param disables the lint via the activation gate, linearity.rs:327 — NOT because it handles the rebuild; the defect reproduces single-param, no nesting), wrote the synthetic RED unit `over_strict_mode_silent_when_ctor_rebuilt_from_primitive_fields` committed as its own audit-trail commit `a11cb7c`. GREEN (implement mini): added a 2nd recognition path to `any_sub_binder_consumed_for` — a `match p` arm that destructures binders out of `p`'s ctor and references any of them (primitive or not) inside a `Term::Ctor`'s args in the arm body genuinely consumes `p`; two pure helpers (`ctor_uses_any_binder` + deep `term_mentions_any_binder`, so `(+ acc px)`-mediated flow counts), conservative toward NOT suppressing (a ctor ignoring `p`'s payload still warns — negative-control proven), over-strict-only (by-name shadowing imprecision is extra-silence, never under-strict; recorded as Known debt). Both stale doc comments that mis-attributed the FP to nested `match` corrected for doc-honesty (debugger concern #2, same code region — in-scope, not opportunistic). +166/−20 in linearity.rs only; check-only, zero codegen/runtime/ABI/schema/DESIGN.md change. Boss-verified independently: RED→GREEN, full `cargo test -p ailang-check` 108/0 lib + every binary 0-failed with NO existing test modified, `ail check embed_backtest_step_tick.ail` no longer over-strict on `st`/`tick` (exit 0), `_tick_borrow.ail` + M3 `embed_backtest_step_record.ail` still clean, the already-green `embed_tick_e2e` + bench posture untouched. No audit/fieldtest gate (lint-precision bugfix; the RED test + the green check-suite ARE the regression coverage). RED `a11cb7c` → GREEN this commit. → 2026-05-18-iter-bugfix-over-strict-mode-ctor-rebuild-consume.md - 2026-05-19 — iter embedding-abi-m5.1 (DONE 3/3): M5 iter 1 — stood up the workspace-excluded `ail-embed` crate. A zero-dependency embedding core (Rust port of the audited `crates/ail/tests/embed/tick_roundtrip.c`: `extern "C"` to the M3-frozen ABI + frozen-layout `State`/`Tick` box helpers + a `Kernel` price fold, raw pointers never escaping the type) links the M3 staticlib via a new `build.rs` (no in-repo precedent — `AIL_BIN` env override else nested `cargo build -p ail` against the parent workspace, separate target dir so no cargo-lock deadlock). Plus a hermetic `data-server` smoke: a synthetic Pepperstone-format ZIP fixture written via the crate's own public `RawTickRecord` type (correct-by-construction, mirrors `data-server/src/loader.rs:110-124`) → real `DataServer` → `Kernel`, asserting bit-exact vs a same-order host reference fold and the closed-form `55.0`; runs with no `/mnt`. `ail-embed` is its own cargo workspace root (empty `[workspace]` table) so the AILang compiler workspace owes it nothing; `data-server` is a dev-dependency only ⇒ Invariant 1 holds in the dependency graph, not just on paper. Boss-verified independently: ail-embed suite 2/2 green (`kernel_run_sums_prices` unit RED-first + `hermetic_smoke_data_server_roundtrip` integration), full+no-deps `cargo metadata` on the AILang workspace shows `data-server` count 0, `git status` path filter empty (zero diff to `crates/ailang-*`/`crates/ail/`/`runtime/`/`examples/*.ail`), `src/lib.rs` zero code-level `data_server`, AILang `cargo build --workspace` still clean. Two toolchain-forced corrections to the plan's verbatim `ail-embed/Cargo.toml` (added the empty `[workspace]` table; sibling dev-dep path `../libs`→`../../libs` manifest-relative) — confined to the plan-created manifest, no acceptance gate altered, 0 review re-loops; planner-recon implication recorded in the journal Concerns. Adapter API + thread-swarm explicitly deferred to M5 iter 2+. → 2026-05-19-iter-embedding-abi-m5.1.md +- 2026-05-19 — iter embedding-abi-m5.2 (PARTIAL 2/3; M5 bounce-back): data-server adapter + symbol-fan swarm; the leak-proof did its job and surfaced a real concurrency finding. Tasks 1+2 landed clean: `data-server` promoted dev-dep→real dep (Invariant-1 sanctioned — compiler workspace graph still data-server-count 0, zero compiler-surface diff), additive `adapter` module (`tick_to_px`/`MidPriceStream` lazy `Iterator`/`fold_symbol`, RED-first), and a `swarm_runner` `[[bin]]` whose clean compile *is* the compile-time per-thread-ctx proof (`Ctx: !Send` ⇒ a shared-ctx swarm is `E0277`, cannot build). The real-data symbol-fan swarm runs (EURUSD/GER40/XAUUSD, ~4s) and is **bit-exact vs an independent same-order host reference every run** (`symbol_fan_swarm_bit_exact`, GREEN, live). Task 3's `Σallocs==Σfrees` leak gate is BLOCKED: Boss independently read `runtime/rc.c` and confirms it is an **instrumentation race, not a memory bug** — no box crosses a thread (`!Send`), the real RC is correct (bit-exact green), only the *global* `g_rc_*` stat counters (rc.c:90-91,161,212) are non-atomic *by rc.c's own documented single-threaded design* and lose `++`s when 3 worker threads hit the null-`__ail_tls_ctx` host-side path. M5 is AILang's first concurrent consumer → the deferred "concurrency arrived: atomic-vs-non-atomic" decision rc.c:44-49 names is now due. The leak assertion was Boss-split into `symbol_fan_swarm_leak_free` (`#[ignore]`, **body verbatim** — quarantined not weakened; un-ignore = the runtime fix's acceptance) so main stays green and the finding is pinned. Prior claim that `embed_swarm_tsan.rs` covers this was corrected (it uses the scalar kernel — zero box allocs — never exercised this path). Escalated as a **bounce-back** (touches M3-frozen `runtime/`; re-frames M5's "zero runtime change"; multiple substantive options); Boss recommendation on the record = Option A (atomic *global-fallback* counters only, standalone RED-first runtime micro-iter, M5 framing amended) — user picks the direction. Time-shard + friction-harvest remain m5.3. → 2026-05-19-iter-embedding-abi-m5.2.md