diff --git a/ail-embed/build.rs b/ail-embed/build.rs index 844622f..0e89927 100644 --- a/ail-embed/build.rs +++ b/ail-embed/build.rs @@ -20,6 +20,13 @@ fn main() { println!("cargo:rerun-if-changed={}", kernel.display()); println!("cargo:rerun-if-changed=build.rs"); + // `ail build --emit=staticlib` compiles libailang_rt.a from the + // runtime C sources (rc.c + str.c — see crates/ail build_staticlib). + // Track the whole `runtime/` directory (Cargo recurses mtimes + // under a directory path) so any runtime change relinks the + // archive. A per-file list would silently re-introduce staleness + // for any future runtime source; the directory mechanism cannot. + println!("cargo:rerun-if-changed={}", repo.join("runtime").display()); println!("cargo:rerun-if-env-changed=AIL_BIN"); let ail_bin: PathBuf = match env::var("AIL_BIN") { diff --git a/ail-embed/tests/swarm.rs b/ail-embed/tests/swarm.rs index db21a15..4bed4ba 100644 --- a/ail-embed/tests/swarm.rs +++ b/ail-embed/tests/swarm.rs @@ -11,30 +11,28 @@ //! host reference fold. This is GREEN and live: it proves the //! real-data-server → adapter → M3-kernel swarm computes correctly. //! -//! - `symbol_fan_swarm_leak_free` — STILL `#[ignore]`d. The global +//! - `symbol_fan_swarm_leak_free` — LIVE and GREEN. The global //! `Σallocs==Σfrees` measurement (ported from //! crates/ail/tests/embed_tick_e2e.rs:94-104) was non-deterministic -//! under the multi-threaded host. The runtime fix -//! `bugfix-rc-global-stats-race` (commit `7bfa11e`) made the global -//! `g_rc_*` fallback counters `_Atomic` — that fix is real, -//! committed, and NECESSARY, but it is **not sufficient**: a Boss -//! verification (4 swarm runs) showed the leak Σ is still -//! non-deterministic — `Σfrees` is stable-exact (12000003) every -//! run while `Σallocs` is short by a jittering ~1600–2260 in ~1/3 -//! of runs (e.g. left:11998383 right:12000003 across the 4 stat -//! lines). A SECOND, distinct alloc-undercount remains that the -//! isolated 8×2M pure-global RED did not model. Un-`#[ignore]` is -//! now gated on that residual's own root-cause/debug cycle, NOT on -//! `7bfa11e`. The swarm is still *actually* leak-free (no box -//! crosses a thread — `Ctx: !Send`; `symbol_fan_swarm_bit_exact` -//! GREEN every run); only the alloc *accounting* is still wrong. -//! Body preserved verbatim — quarantined, NOT weakened. See -//! docs/journals/2026-05-19-iter-embedding-abi-m5.2-resume-attempt.md. +//! under the multi-threaded host. Root cause was a build-dependency +//! staleness gap, NOT a residual runtime defect: `ail-embed/build.rs` +//! did not declare the `runtime/` C sources as `rerun-if-changed`, +//! so the atomic-counter runtime fix `bugfix-rc-global-stats-race` +//! (commit `7bfa11e`) never relinked `libailang_rt.a` and the swarm +//! kept linking the stale pre-`7bfa11e` non-atomic +//! `g_rc_alloc_count++` whose race produced the jittering +//! `Σallocs` undercount (`Σfrees` stable-exact). `build.rs` now +//! tracks `runtime/` (this iter, `bugfix-swarm-rc-alloc-undercount`); +//! the fresh atomic archive relinks and the leak Σ balances +//! deterministically (`Σallocs == Σfrees == 12000003`, every run). +//! This test is the integration-level acceptance of both `7bfa11e` +//! and the build-dependency fix. Body preserved verbatim — it was +//! quarantined, never weakened. See +//! docs/journals/2026-05-19-iter-bugfix-swarm-rc-alloc-undercount.md. use std::process::Command; use std::sync::Arc; -use data_server::records::DataFormat; use data_server::{DataServer, DEFAULT_DATA_PATH}; // Must match swarm_runner.rs. @@ -125,27 +123,22 @@ fn symbol_fan_swarm_bit_exact() { /// embed_tick_e2e.rs:94-104). The invariant is the global Σ, not /// per-line balance (M2 TLS-ctx cross-attribution). /// -/// STILL `#[ignore]`d. The runtime fix `bugfix-rc-global-stats-race` -/// (commit `7bfa11e`) made the global `g_rc_*` fallback counters -/// `_Atomic` — necessary and committed, but Boss verification proved -/// it INSUFFICIENT for this swarm: `Σfrees` is stable-exact -/// (12000003) every run, `Σallocs` short by a jittering ~1600–2260 -/// in ~1/3 of runs (e.g. left:11998383 right:12000003, 4 stat -/// lines). A second, distinct alloc-undercount remains — NOT pure -/// global-counter contention (the isolated 8×2M pure-global RED -/// `embed_rc_global_stats_race` is deterministically green). The -/// swarm is still actually leak-free (no box crosses a thread — -/// `Ctx: !Send`; bit-exact GREEN every run); only the alloc -/// accounting is wrong. Un-`#[ignore]` is gated on the residual's -/// own root-cause/debug cycle, not on `7bfa11e`. Body unchanged. -/// See docs/journals/2026-05-19-iter-embedding-abi-m5.2-resume-attempt.md. +/// LIVE and GREEN. The non-determinism was a build-dependency +/// staleness gap, not a residual runtime defect: `ail-embed/build.rs` +/// did not track the `runtime/` C sources as `rerun-if-changed`, so +/// the atomic-counter runtime fix `bugfix-rc-global-stats-race` +/// (commit `7bfa11e`) never relinked `libailang_rt.a` — the swarm +/// kept linking the stale pre-`7bfa11e` non-atomic +/// `g_rc_alloc_count++` whose race undercounted `Σallocs` (`Σfrees` +/// stable-exact). Fixed in `build.rs` this iter +/// (`bugfix-swarm-rc-alloc-undercount`): the fresh atomic archive +/// relinks and the leak Σ now balances deterministically +/// (`Σallocs == Σfrees == 12000003`, every run). This test is the +/// integration-level acceptance of both `7bfa11e` and the +/// build-dependency fix. Body unchanged — it was quarantined, never +/// weakened. See +/// docs/journals/2026-05-19-iter-bugfix-swarm-rc-alloc-undercount.md. #[test] -#[ignore = "still blocked: atomic-global fix 7bfa11e is necessary but \ - INSUFFICIENT — a residual swarm alloc-undercount remains \ - (Σfrees exact, Σallocs short ~1600 jittering, ~1/3 runs; \ - NOT pure global contention — the isolated RED is green). \ - Un-ignore on the residual's own debug cycle, not on \ - 7bfa11e. See the m5.2-resume-attempt journal."] fn symbol_fan_swarm_leak_free() { let Some((_stdout, stderr)) = run_swarm_or_skip() else { return }; diff --git a/bench/orchestrator-stats/2026-05-19-iter-bugfix-swarm-rc-alloc-undercount.json b/bench/orchestrator-stats/2026-05-19-iter-bugfix-swarm-rc-alloc-undercount.json new file mode 100644 index 0000000..a81cc3b --- /dev/null +++ b/bench/orchestrator-stats/2026-05-19-iter-bugfix-swarm-rc-alloc-undercount.json @@ -0,0 +1,12 @@ +{ + "iter_id": "bugfix-swarm-rc-alloc-undercount", + "date": "2026-05-19", + "mode": "mini", + "outcome": "DONE", + "tasks_total": 1, + "tasks_completed": 1, + "reloops_per_task": { "1": 0 }, + "review_loops_spec": 0, + "review_loops_quality": 0, + "blocked_reason": null +} diff --git a/docs/journals/2026-05-19-iter-bugfix-swarm-rc-alloc-undercount.md b/docs/journals/2026-05-19-iter-bugfix-swarm-rc-alloc-undercount.md new file mode 100644 index 0000000..8d36bae --- /dev/null +++ b/docs/journals/2026-05-19-iter-bugfix-swarm-rc-alloc-undercount.md @@ -0,0 +1,97 @@ +# iter bugfix-swarm-rc-alloc-undercount — build.rs missed runtime/ as a rerun-if-changed input + +**Date:** 2026-05-19 +**Started from:** 483117d39be6a38333cb5fae8c4b5b96e108d1c2 +**Status:** DONE +**Tasks completed:** 1 of 1 + +## Summary + +The M5 swarm leak-proof `symbol_fan_swarm_leak_free` was +non-deterministic: `Σfrees` stable-exact (12000003) every run while +`Σallocs` was short by a jittering amount in ~1/3 of runs. The +m5.2-resume-attempt journal framed this as a *residual runtime +defect* distinct from `7bfa11e` (the atomic-global-counter fix). The +`debug` cycle root-caused it instead to a build-dependency +completeness gap, not a runtime bug: `ail-embed/build.rs` declared +`cargo:rerun-if-changed` only for the kernel `.ail` and `build.rs` +itself — NOT the `runtime/` C sources that +`ail build --emit=staticlib` compiles into `libailang_rt.a`. Cargo +therefore never re-ran `build.rs` after `7bfa11e` landed, so +`ail-embed` kept linking a stale pre-`7bfa11e` non-atomic +`libailang_rt.a` whose `g_rc_alloc_count++` raced the M5 swarm's +concurrent TLS-NULL host allocs (the isolated 8×2M pure-global RED +`embed_rc_global_stats_race` structurally could not model this +because it never touches the stale ail-embed archive — it links the +workspace runtime fresh). `runtime/rc.c` at HEAD was already correct; +the entire production fix is one line in the build script. + +The fix adds a directory-level +`println!("cargo:rerun-if-changed={}", repo.join("runtime").display())`. +Confirmed by inspecting `crates/ail/src/main.rs::build_staticlib` +(lines 2519-2534): `libailang_rt.a = ar(rc.o, str.o)` where the +sources are exactly `runtime/rc.c` + `runtime/str.c`, both under +`runtime/`. The directory-level mechanism (Cargo recurses mtimes +under a tracked directory) is preferred over enumerating individual +files precisely because a per-file list would silently re-introduce +the same staleness for any future runtime source (e.g. a new +`runtime/foo.c`). Existing rerun-if-changed lines (kernel `.ail`, +`build.rs`, `AIL_BIN`) were kept. The build.rs edit triggers its own +`rerun-if-changed=build.rs`, which relinks the archive from the +current atomic `runtime/rc.c` — the RED's self-bootstrapping +mechanism. + +The cohesive, in-scope consequence: `symbol_fan_swarm_leak_free` is +un-`#[ignore]`d (the `#[ignore]` attribute and its reason string +removed; the test BODY is byte-for-byte unchanged — it was +quarantined, never weakened). The two surviving doc breadcrumbs +(module-doc bullet, fn-doc) are rewritten to present-tense-resolved +with the accurate build-dependency-staleness rationale and the +corrected journal reference. This test is now the integration-level +acceptance of both `7bfa11e` and the build-dependency fix; the leak Σ +balances deterministically (`Σallocs == Σfrees == 12000003`) across 6 +consecutive runs. + +## Per-task notes + +- iter bugfix-swarm-rc-alloc-undercount.1: added directory-level + `cargo:rerun-if-changed=/runtime` to `ail-embed/build.rs` + (existing kernel/`build.rs`/`AIL_BIN` lines kept); un-`#[ignore]`d + `symbol_fan_swarm_leak_free` in `ail-embed/tests/swarm.rs` (body + verbatim) and corrected its module-doc + fn-doc breadcrumbs to the + resolved build-dep-staleness rationale. RED + `rt_archive_freshness` went 0-lock-insn → GREEN (≥2 lock insns, + matches fresh cross-check). Gate-2 determinism: 6/6 runs, both + `symbol_fan_swarm_leak_free` and `symbol_fan_swarm_bit_exact` + GREEN every run, 0 failed / 0 ignored. + +## Concerns + +(none) + +## Known debt + +- (resolved) `ail-embed/tests/swarm.rs` carried a pre-existing + `warning: unused import: data_server::records::DataFormat` (present + since the m5.2 split at `b724cd1`; `DataFormat` is used only by + `swarm_runner.rs`, never the test). The orchestrator correctly did + not chase it under the minimal-fix constraint; the Boss removed the + dead import inline at commit time (trivial mechanical edit per the + CLAUDE.md carve-out — `swarm.rs` was being committed this iter + regardless, and leaving a warning in the now-final, no-longer- + quarantined M5 leak-proof file is its own debt). `cargo test + --manifest-path ail-embed/Cargo.toml` is warning-clean for this + crate after removal. + +## Blocked detail + +(n/a — DONE) + +## Files touched + +- ail-embed/build.rs +- ail-embed/tests/swarm.rs + +## Stats + +bench/orchestrator-stats/2026-05-19-iter-bugfix-swarm-rc-alloc-undercount.json diff --git a/docs/journals/INDEX.md b/docs/journals/INDEX.md index 751903c..8b6aae0 100644 --- a/docs/journals/INDEX.md +++ b/docs/journals/INDEX.md @@ -110,3 +110,4 @@ - 2026-05-19 — iter embedding-abi-m5.2 (PARTIAL 2/3; M5 bounce-back): data-server adapter + symbol-fan swarm; the leak-proof did its job and surfaced a real concurrency finding. Tasks 1+2 landed clean: `data-server` promoted dev-dep→real dep (Invariant-1 sanctioned — compiler workspace graph still data-server-count 0, zero compiler-surface diff), additive `adapter` module (`tick_to_px`/`MidPriceStream` lazy `Iterator`/`fold_symbol`, RED-first), and a `swarm_runner` `[[bin]]` whose clean compile *is* the compile-time per-thread-ctx proof (`Ctx: !Send` ⇒ a shared-ctx swarm is `E0277`, cannot build). The real-data symbol-fan swarm runs (EURUSD/GER40/XAUUSD, ~4s) and is **bit-exact vs an independent same-order host reference every run** (`symbol_fan_swarm_bit_exact`, GREEN, live). Task 3's `Σallocs==Σfrees` leak gate is BLOCKED: Boss independently read `runtime/rc.c` and confirms it is an **instrumentation race, not a memory bug** — no box crosses a thread (`!Send`), the real RC is correct (bit-exact green), only the *global* `g_rc_*` stat counters (rc.c:90-91,161,212) are non-atomic *by rc.c's own documented single-threaded design* and lose `++`s when 3 worker threads hit the null-`__ail_tls_ctx` host-side path. M5 is AILang's first concurrent consumer → the deferred "concurrency arrived: atomic-vs-non-atomic" decision rc.c:44-49 names is now due. The leak assertion was Boss-split into `symbol_fan_swarm_leak_free` (`#[ignore]`, **body verbatim** — quarantined not weakened; un-ignore = the runtime fix's acceptance) so main stays green and the finding is pinned. Prior claim that `embed_swarm_tsan.rs` covers this was corrected (it uses the scalar kernel — zero box allocs — never exercised this path). Escalated as a **bounce-back** (touches M3-frozen `runtime/`; re-frames M5's "zero runtime change"; multiple substantive options); Boss recommendation on the record = Option A (atomic *global-fallback* counters only, standalone RED-first runtime micro-iter, M5 framing amended) — user picks the direction. Time-shard + friction-harvest remain m5.3. → 2026-05-19-iter-embedding-abi-m5.2.md - 2026-05-19 — iter bugfix-rc-global-stats-race (RED→GREEN, debug→implement mini, DONE 1/1): the M5 iter-2 bounce-back resolution (user-approved Option A). The two null-ctx fallback RC-stats counters `g_rc_alloc_count`/`g_rc_free_count` (`runtime/rc.c:90-91`) were plain `static uint64_t` with a non-atomic `++` on the `__ail_tls_ctx == NULL` arm of `ailang_rc_alloc` (rc.c:161) / the to-zero branch of `ailang_rc_dec` (rc.c:212); a multi-threaded host driving the global fallback (no `ailang_ctx_new`) raced the read-modify-write and lost increments, so the `AILANG_RC_STATS` atexit Σ under-counted non-deterministically — exactly the deferred "concurrency arrived: atomic-vs-non-atomic" decision rc.c's own header (rc.c:44-49) names, M5 being AILang's first concurrent consumer. NOT a memory bug (`Ctx: !Send` keeps every box on one thread; the real refcount/free is correct; programs bit-exact). RED (debugger, audit-trail commit `427b687`): a C host — 8 threads × 2M alloc-then-dec, no ctx so the global path is taken, no box crossing a thread — + integration test asserting the atexit Σ is exact; Boss-verified RED `allocs=2141382 expected 16000000, live=-131242` (deterministic-fail under that contention, not flaky). GREEN (implement mini, this commit): the two globals are now `_Atomic uint64_t`, `atomic_fetch_add_explicit(.., memory_order_relaxed)` at the two fallback `++` sites, `atomic_load_explicit(.., relaxed)` in `ailang_rc_stats_atexit` (its sole reader) — relaxed is correct (pure stats, no happens-before; atexit reader runs post-join). Per-ctx counters + the per-object refcount header left non-atomic BY DESIGN (single-thread-per-ctx; boxes never cross threads) — explicitly out of scope, untouched; frozen value layout / ABI offsets / host-free rule untouched; the genuinely-still-single-threaded drop-worklist doc note correctly left unchanged (a false correction was refused); the two genuinely-stale atomicity doc blocks corrected for honesty. Scope held to `runtime/rc.c` ONLY (37+/14−). Boss-verified independently: RED→GREEN deterministically 3/3 (jitter gone, `allocs==frees==16_000_000`), full `cargo test -p ail` green (every binary 0 failed — the per-ctx tsan harnesses embed_swarm_tsan/embed_rc_accounting_tsan + embed_tick_e2e/embed_record_e2e unaffected since the per-ctx path is untouched; rc.c links into every ail binary so this is the strong regression gate), git-status scope = rc.c + journal + stats only, the RED files unchanged. Unblocks resuming the M5 leak-proof (un-`#[ignore]` `ail-embed`'s `symbol_fan_swarm_leak_free`). RED `427b687` → GREEN this commit. → 2026-05-19-iter-bugfix-rc-global-stats-race.md - 2026-05-19 — iter embedding-abi-m5.2-resume-attempt (RE-QUARANTINED; honesty fix only): attempted to un-`#[ignore]` the M5 swarm leak-proof `symbol_fan_swarm_leak_free` as the integration-level acceptance of the runtime fix `7bfa11e` (bugfix-rc-global-stats-race). Boss verification FALSIFIED the sufficiency premise: across 4–6 swarm runs the leak Σ is still non-deterministic — `Σfrees` stable-exact (12000003) every run, `Σallocs` short by a jittering ~1600–2260 in ~1/3 of runs (e.g. left:11998383 right:12000003 across 4 stat lines). The atomic-global fix is real, committed, NECESSARY (the isolated 8×2M pure-global RED `embed_rc_global_stats_race` is deterministically green) but NOT sufficient — a second, distinct alloc-side undercount remains that the isolated RED did not model (frees never lose ⇒ not a per-ctx race; not pure global contention ⇒ RED green; localisation beyond that is where Boss speculation stops per the debug Iron Law). Un-ignore reverted (never committed); the only shipped diff is the three now-stale `#[ignore]`-rationale breadcrumbs in `ail-embed/tests/swarm.rs` corrected from "un-ignore when the runtime is atomic" to the accurate necessary-but-insufficient state (a future agent seeing `7bfa11e` must NOT un-ignore on that basis — doc-honesty). Test body + the `#[ignore]` itself unchanged (still quarantined, NOT weakened); `symbol_fan_swarm_bit_exact` stays live+GREEN (the swarm IS actually correct and leak-free — only the accounting is wrong). Residual handed to a fresh debug RED-first cycle; two framings flagged for it (genuine residual runtime defect vs Σ-over-heterogeneous-global+per-ctx-lines methodology unsoundness — the latter, if so, is a post-root-cause design decision). main green at the honesty-fix commit; M5 stays open `[~]`; m5.3 remains downstream of a sound leak-proof. → 2026-05-19-iter-embedding-abi-m5.2-resume-attempt.md +- 2026-05-19 — iter bugfix-swarm-rc-alloc-undercount (RED→GREEN, debug→implement mini, DONE 1/1): root-caused and fixed the m5.2-resume-attempt residual. The M5 swarm leak-proof was still non-deterministic after the atomic-counter runtime fix `7bfa11e` — NOT a second runtime bug and NOT test-methodology unsoundness, but a **build-dependency staleness gap**: `ail-embed/build.rs` declared `cargo:rerun-if-changed` only for the kernel `.ail` + `build.rs`, NOT the `runtime/` C sources `ail build --emit=staticlib` compiles into `libailang_rt.a`. So Cargo never re-ran `build.rs` after `7bfa11e`; `ail-embed` kept linking a stale pre-`7bfa11e` non-atomic `libailang_rt.a` whose `g_rc_alloc_count++` raced the swarm's concurrent TLS-NULL host allocs (explaining the frees-stable/allocs-jitter asymmetry, and why the isolated `crates/ail` RED — which rebuilds the staticlib fresh per run — was green while the `ail-embed` swarm was not). `runtime/rc.c` at HEAD was always correct; the entire production fix is one directory-level `cargo:rerun-if-changed=/runtime` line (preferred over a per-file list — the implementer inspected `crates/ail` `build_staticlib` confirming `libailang_rt.a = ar(rc.o,str.o)`, both under `runtime/`; directory tracking is future-proof against any new runtime source). RED (debugger, audit-trail `483117d`): `ail-embed/tests/rt_archive_freshness.rs` pins the *cause* deterministically (objdump the linked archive for `lock`-prefixed atomic increments + a fresh-build cross-check) since the *symptom* is non-deterministic by construction; Boss-verified RED = 0 lock insns ("STALE"). GREEN (implement mini, this commit): the build.rs fix + the cohesive consequence — `symbol_fan_swarm_leak_free` un-`#[ignore]`d (body byte-for-byte verbatim — it was always quarantined, never weakened; the three doc breadcrumbs rewritten to the resolved build-dep-staleness rationale), now the integration-level acceptance of both `7bfa11e` and this fix. Boss removed a pre-existing dead `DataFormat` import in swarm.rs inline (trivial; file committed this iter regardless). Boss-verified independently: RED→GREEN; **swarm determinism 5/5 — `symbol_fan_swarm_leak_free` + `symbol_fan_swarm_bit_exact` GREEN every run, 0 ignored, jitter gone, Σallocs==Σfrees==12000003**; isolated `embed_rc_global_stats_race` still GREEN (untouched); Invariant 1 data-server count 0; scope = `ail-embed/build.rs` + `ail-embed/tests/swarm.rs` + journal + stats only. The M5 swarm leak-proof bounce-back is now fully resolved end-to-end (bounce-back → Option A `7bfa11e` → resume-attempt "necessary-but-insufficient" → this build-dep root-cause+fix). RED `483117d` → GREEN this commit. M5 stays open `[~]`; m5.3 (time-shard + friction-harvest + close-out) remains. → 2026-05-19-iter-bugfix-swarm-rc-alloc-undercount.md