From 65967fb181fce235562a20c63d6e63a9a1f1fadb Mon Sep 17 00:00:00 2001 From: Brummel Date: Wed, 3 Jun 2026 11:37:46 +0200 Subject: [PATCH] plan: cycle 0001 core streaming substrate Five-task plan executing spec 0001: scalar set, Column/Window, KindMismatch, AnyColumn, then a workspace build/test/clippy + surface-purity gate. Each task adds its own module + re-export so the crate compiles green after every task; 14 unit tests pin the substrate contract incl. the wrong-kind-push guard. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/plans/0001-core-streaming-substrate.md | 679 ++++++++++++++++++++ 1 file changed, 679 insertions(+) create mode 100644 docs/plans/0001-core-streaming-substrate.md diff --git a/docs/plans/0001-core-streaming-substrate.md b/docs/plans/0001-core-streaming-substrate.md new file mode 100644 index 0000000..db50fad --- /dev/null +++ b/docs/plans/0001-core-streaming-substrate.md @@ -0,0 +1,679 @@ +# Core Streaming Substrate — Implementation Plan + +> **Parent spec:** `docs/specs/0001-core-streaming-substrate.md` +> +> **For agentic workers:** REQUIRED SUB-SKILL: use the `implement` skill to run +> this plan. Steps use `- [ ]` checkboxes for tracking. + +**Goal:** Deliver the `aura-core` streaming substrate — the four scalar base +types, the bounded-lookback financial-indexed `Column`/`Window`, the type-erased +`AnyColumn` edge, and the `KindMismatch` guard — fully built, tested, and lint-clean. + +**Architecture:** Four new private modules in `aura-core` (`scalar`, `column`, +`any`, `error`), each with its own inline `#[cfg(test)] mod tests`, re-exported +from `lib.rs`. No dependencies added. Interior-mutability-free: a sim is +single-threaded (C1), so columns are owned `&mut` on push, borrowed `&` on read. + +**Tech Stack:** Rust (edition 2024, `aura-core` crate), `cargo build/test/clippy +--workspace`. + +**Module ordering (by dependency):** `scalar` (standalone) → `column` +(generic, standalone) → `error` (uses `scalar::ScalarKind`) → `any` (uses all +three) → final workspace gate. Each task adds its file, its `mod`+`pub use` line +in `lib.rs`, and its tests, so the crate compiles green after every task. + +**Greenfield note:** every type here is new; the tests pin each module's contract +and run green on first build. There is no pre-existing behaviour to RED against — +the must-fail guard (`wrong_kind_push_is_rejected`, Task 4) is the one test whose +*correct behaviour is rejection*. + +**Files this plan creates or modifies:** + +- Create: `crates/aura-core/src/scalar.rs` — `Timestamp`, `Scalar`, `ScalarKind`, `kind()`, four `From` impls. +- Create: `crates/aura-core/src/column.rs` — `Column` ring + zero-copy `Window<'_, T>` with `Index`. +- Create: `crates/aura-core/src/error.rs` — `KindMismatch { expected, got }`. +- Create: `crates/aura-core/src/any.rs` — `AnyColumn` type-erased edge. +- Modify: `crates/aura-core/src/lib.rs:1-19` — module declarations + re-exports (doc prose updated). +- Test: inline `#[cfg(test)] mod tests` in each of the four module files. + +--- + +## Task 1: The scalar set (`scalar.rs`) + +**Files:** +- Create: `crates/aura-core/src/scalar.rs` +- Modify: `crates/aura-core/src/lib.rs` +- Test: inline in `crates/aura-core/src/scalar.rs` + +- [ ] **Step 1: Create `crates/aura-core/src/scalar.rs` with the scalar set** + +```rust +//! The closed four-type scalar set streamed on aura's hot path (C7): +//! `i64`, `f64`, `bool`, and `Timestamp` (epoch-ns UTC). The carrier `Scalar` +//! is a `Copy` POD enum — no type-erased payloads, no heap, no reference counting. + +/// Canonical engine time: epoch-nanoseconds UTC. Newtype over `i64` (C7). +/// `Default` (epoch 0) lets `Column` pre-size its ring. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Timestamp(pub i64); + +/// The kind tag of a scalar / column, used for the edge-time type check (C7). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum ScalarKind { + I64, + F64, + Bool, + Timestamp, +} + +/// The four scalar base types, type-erased into one `Copy` carrier (C7). +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum Scalar { + I64(i64), + F64(f64), + Bool(bool), + Ts(Timestamp), +} + +impl Scalar { + /// The kind tag of this scalar. + pub fn kind(self) -> ScalarKind { + match self { + Scalar::I64(_) => ScalarKind::I64, + Scalar::F64(_) => ScalarKind::F64, + Scalar::Bool(_) => ScalarKind::Bool, + Scalar::Ts(_) => ScalarKind::Timestamp, + } + } +} + +impl From for Scalar { + fn from(v: i64) -> Self { + Scalar::I64(v) + } +} +impl From for Scalar { + fn from(v: f64) -> Self { + Scalar::F64(v) + } +} +impl From for Scalar { + fn from(v: bool) -> Self { + Scalar::Bool(v) + } +} +impl From for Scalar { + fn from(v: Timestamp) -> Self { + Scalar::Ts(v) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn kind_matches_variant() { + assert_eq!(Scalar::I64(1).kind(), ScalarKind::I64); + assert_eq!(Scalar::F64(1.0).kind(), ScalarKind::F64); + assert_eq!(Scalar::Bool(true).kind(), ScalarKind::Bool); + assert_eq!(Scalar::Ts(Timestamp(1)).kind(), ScalarKind::Timestamp); + } + + #[test] + fn from_widenings() { + assert_eq!(Scalar::from(7i64), Scalar::I64(7)); + assert_eq!(Scalar::from(2.5f64), Scalar::F64(2.5)); + assert_eq!(Scalar::from(true), Scalar::Bool(true)); + assert_eq!(Scalar::from(Timestamp(9)), Scalar::Ts(Timestamp(9))); + } + + #[test] + fn timestamp_orders_and_defaults() { + assert!(Timestamp(1) < Timestamp(2)); + assert_eq!(Timestamp::default(), Timestamp(0)); + } + + #[test] + fn scalar_is_copy() { + let s = Scalar::F64(1.0); + let a = s; + let b = s; // Copy: `s` still usable after move-by-value + assert_eq!(a, b); + assert_eq!(s, a); + } +} +``` + +- [ ] **Step 2: Rewrite `lib.rs` doc and declare the `scalar` module only** + +Each task adds only its own `mod` + `pub use` lines so the crate compiles green +after every task (the `mod` lines accumulate; by Task 4 all four are present). +Replace the entire current body of `crates/aura-core/src/lib.rs` (19 lines, +doc-only) with: + +```rust +//! `aura-core` — the shared contract every aura component links against. +//! +//! This crate is the boundary forced by the cdylib hot-reload model: both the +//! engine and every hot-reloadable node cdylib depend on it, so it stays small +//! and stable. +//! +//! Delivered so far (cycle 0001 — the streaming substrate): +//! +//! - the four scalar base types streamed as SoA: [`Scalar`] / [`ScalarKind`] / +//! [`Timestamp`] (C7); +//! - [`Column`] — a fixed-capacity, bounded-lookback ring with financial +//! indexing (index 0 = newest) and a `run_count` freshness primitive (C5/C8), +//! read zero-copy through a [`Window`]; +//! - [`AnyColumn`] — the type-erased edge over the four column kinds, with an +//! edge-time kind check ([`KindMismatch`]) (C7). +//! +//! Still to come (subsequent cycles): the `Node` trait and its `schema`/`eval`, +//! the evaluation context `Ctx`, the firing policies (A: fire-on-any-fresh + +//! hold; B: all-fresh barrier), and the deterministic sim loop. + +mod scalar; + +pub use scalar::{Scalar, ScalarKind, Timestamp}; +``` + +The intra-doc links to `Column`/`Window`/`AnyColumn`/`KindMismatch` resolve once +Tasks 2–4 re-export them; they are not checked by `cargo build`/`test`/`clippy` +(only `cargo doc`), so they break no gate in the interim. + +- [ ] **Step 3: Build and test the scalar module** + +Run: `cargo test -p aura-core scalar::` +Expected: the build succeeds and the four `scalar::tests::*` tests report `ok`; +0 failed. + +--- + +## Task 2: The column and its window (`column.rs`) + +**Files:** +- Create: `crates/aura-core/src/column.rs` +- Test: inline in `crates/aura-core/src/column.rs` + +- [ ] **Step 1: Create `crates/aura-core/src/column.rs`** + +```rust +//! `Column` — a fixed-capacity, bounded-lookback SoA ring with financial +//! indexing (index 0 = newest), and its zero-copy read view `Window` (C5/C8). + +/// A fixed-capacity bounded-lookback column. Pre-sized at construction +/// (C8: no realloc in the hot loop); index 0 = newest; carries a monotonic +/// `run_count` of total pushes — the freshness primitive (C5). +pub struct Column { + buf: Box<[T]>, + head: usize, // next write slot + len: usize, // valid elements, saturates at capacity + run_count: u64, // monotonic total pushes (C5) +} + +impl Column { + /// Create a column sized to `lookback` (must be >= 1). The engine sizes this + /// once at wiring; it is never reallocated (C8). + pub fn with_capacity(lookback: usize) -> Self { + assert!(lookback >= 1, "Column lookback must be >= 1"); + Column { + buf: vec![T::default(); lookback].into_boxed_slice(), + head: 0, + len: 0, + run_count: 0, + } + } + + /// Append the newest value. O(1), allocation-free. On overflow the oldest + /// value is overwritten (true ring). Bumps `run_count`. + pub fn push(&mut self, v: T) { + let cap = self.buf.len(); + self.buf[self.head] = v; + self.head = (self.head + 1) % cap; + if self.len < cap { + self.len += 1; + } + self.run_count += 1; + } + + /// Financial index: 0 = newest, `len()-1` = oldest. `None` if `k >= len` + /// (C8: not-yet-warmed-up / out of range). Zero-copy. + pub fn get(&self, k: usize) -> Option { + if k >= self.len { + return None; + } + let cap = self.buf.len(); + Some(self.buf[(self.head + cap - 1 - k) % cap]) + } + + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + pub fn capacity(&self) -> usize { + self.buf.len() + } + + /// The monotonic push counter — the freshness primitive (C5). Never moved + /// by a read. + pub fn run_count(&self) -> u64 { + self.run_count + } + + /// A zero-copy read view handed to a reading node (C8). + pub fn window(&self) -> Window<'_, T> { + Window { col: self } + } +} + +/// Read-only, zero-copy view into a [`Column`] — what a reading node sees (C8). +/// Carries no storage, only a borrow. +pub struct Window<'a, T> { + col: &'a Column, +} + +impl<'a, T: Copy + Default> Window<'a, T> { + /// Financial index, 0 = newest; `None` if cold / out of range. + pub fn get(&self, k: usize) -> Option { + self.col.get(k) + } + + pub fn len(&self) -> usize { + self.col.len() + } + + pub fn is_empty(&self) -> bool { + self.col.is_empty() + } + + pub fn run_count(&self) -> u64 { + self.col.run_count() + } +} + +/// `window[k]` — newest-at-0 indexed access; panics on out-of-range like a +/// slice. Nodes that may be cold use [`Window::get`] instead. +impl<'a, T: Copy + Default> core::ops::Index for Window<'a, T> { + type Output = T; + + fn index(&self, k: usize) -> &T { + let col = self.col; + assert!(k < col.len, "Window index {k} out of range (len {})", col.len); + let cap = col.buf.len(); + &col.buf[(col.head + cap - 1 - k) % cap] + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn newest_at_zero_oldest_at_len_minus_one() { + let mut c = Column::::with_capacity(4); + c.push(10); + c.push(20); + c.push(30); + assert_eq!(c.len(), 3); + assert_eq!(c.get(0), Some(30)); // newest + assert_eq!(c.get(1), Some(20)); + assert_eq!(c.get(2), Some(10)); // oldest + assert_eq!(c.get(3), None); // beyond len + } + + #[test] + fn wraparound_keeps_newest_capacity_values() { + let mut c = Column::::with_capacity(3); + for v in [1, 2, 3, 4, 5] { + c.push(v); + } + assert_eq!(c.len(), 3); // saturated + assert_eq!(c.capacity(), 3); + assert_eq!(c.get(0), Some(5)); // newest + assert_eq!(c.get(1), Some(4)); + assert_eq!(c.get(2), Some(3)); // oldest retained + assert_eq!(c.get(3), None); // 1 and 2 were dropped + } + + #[test] + fn run_count_counts_pushes_not_len() { + let mut c = Column::::with_capacity(2); + assert_eq!(c.run_count(), 0); + for v in [1, 2, 3, 4, 5] { + c.push(v); + } + assert_eq!(c.run_count(), 5); // counts pushes, past the ring wrap + assert_eq!(c.len(), 2); // while len stays at capacity + let _ = c.get(0); + assert_eq!(c.run_count(), 5); // a read never moves it + } + + #[test] + fn window_is_a_view_matching_the_column() { + let mut c = Column::::with_capacity(4); + c.push(1.0); + c.push(2.0); + let cap_before = c.capacity(); + let w = c.window(); + assert_eq!(w.len(), 2); + assert_eq!(w.get(0), Some(2.0)); + assert_eq!(w[1], 1.0); // Index, newest-at-0 + assert_eq!(w.run_count(), 2); + assert_eq!(c.capacity(), cap_before); // taking a window did not realloc + } + + #[test] + fn empty_column_reads_none() { + let c = Column::::with_capacity(2); + assert!(c.is_empty()); + assert_eq!(c.get(0), None); + assert_eq!(c.run_count(), 0); + } +} +``` + +- [ ] **Step 2: Declare and re-export the `column` module in `lib.rs`** + +Add to `crates/aura-core/src/lib.rs`, after the `mod scalar;` line and after the +`pub use scalar::...;` line respectively: + +```rust +mod column; +``` +```rust +pub use column::{Column, Window}; +``` + +- [ ] **Step 3: Test the column module** + +Run: `cargo test -p aura-core column::` +Expected: the five `column::tests::*` tests report `ok`; 0 failed. + +--- + +## Task 3: The guard error (`error.rs`) + +**Files:** +- Create: `crates/aura-core/src/error.rs` +- Test: inline in `crates/aura-core/src/error.rs` + +- [ ] **Step 1: Create `crates/aura-core/src/error.rs`** + +```rust +//! `KindMismatch` — the wiring-time guard error for a wrong-kind edge push (C7). + +use crate::scalar::ScalarKind; + +/// Returned by [`crate::AnyColumn::push`] when a `Scalar`'s kind does not match +/// the column's kind; the column is left untouched. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct KindMismatch { + /// The kind the column holds. + pub expected: ScalarKind, + /// The kind of the value that was offered. + pub got: ScalarKind, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn kind_mismatch_is_a_plain_value() { + let e = KindMismatch { expected: ScalarKind::I64, got: ScalarKind::F64 }; + assert_eq!(e, KindMismatch { expected: ScalarKind::I64, got: ScalarKind::F64 }); + assert_ne!(e, KindMismatch { expected: ScalarKind::F64, got: ScalarKind::I64 }); + } +} +``` + +- [ ] **Step 2: Declare and re-export the `error` module in `lib.rs`** + +Add to `crates/aura-core/src/lib.rs`: + +```rust +mod error; +``` +```rust +pub use error::KindMismatch; +``` + +- [ ] **Step 3: Test the error module** + +Run: `cargo test -p aura-core error::` +Expected: `error::tests::kind_mismatch_is_a_plain_value` reports `ok`. + +--- + +## Task 4: The type-erased edge (`any.rs`) + +**Files:** +- Create: `crates/aura-core/src/any.rs` +- Test: inline in `crates/aura-core/src/any.rs` (incl. the must-fail guard) + +- [ ] **Step 1: Create `crates/aura-core/src/any.rs`** + +```rust +//! `AnyColumn` — the type-erased edge: the four concrete `Column` behind one +//! enum. The kind check is paid at the edge (wiring time); the hot loop grabs +//! the concrete column once (`as_*_mut`) and pushes monomorphically (C7). + +use crate::column::Column; +use crate::error::KindMismatch; +use crate::scalar::{Scalar, ScalarKind, Timestamp}; + +/// A type-erased edge over the four scalar columns (C7). +pub enum AnyColumn { + I64(Column), + F64(Column), + Bool(Column), + Ts(Column), +} + +impl AnyColumn { + /// Create a type-erased column of `kind`, sized to `lookback` (C8). + pub fn with_capacity(kind: ScalarKind, lookback: usize) -> Self { + match kind { + ScalarKind::I64 => AnyColumn::I64(Column::with_capacity(lookback)), + ScalarKind::F64 => AnyColumn::F64(Column::with_capacity(lookback)), + ScalarKind::Bool => AnyColumn::Bool(Column::with_capacity(lookback)), + ScalarKind::Timestamp => AnyColumn::Ts(Column::with_capacity(lookback)), + } + } + + pub fn kind(&self) -> ScalarKind { + match self { + AnyColumn::I64(_) => ScalarKind::I64, + AnyColumn::F64(_) => ScalarKind::F64, + AnyColumn::Bool(_) => ScalarKind::Bool, + AnyColumn::Ts(_) => ScalarKind::Timestamp, + } + } + + pub fn len(&self) -> usize { + match self { + AnyColumn::I64(c) => c.len(), + AnyColumn::F64(c) => c.len(), + AnyColumn::Bool(c) => c.len(), + AnyColumn::Ts(c) => c.len(), + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn run_count(&self) -> u64 { + match self { + AnyColumn::I64(c) => c.run_count(), + AnyColumn::F64(c) => c.run_count(), + AnyColumn::Bool(c) => c.run_count(), + AnyColumn::Ts(c) => c.run_count(), + } + } + + /// Edge-time typed push. `Err(KindMismatch)` if `v`'s kind != this column's + /// kind; the column is left untouched (C7 guard). The hot path bypasses this + /// by taking the concrete `Column` once via `as_*_mut`. + pub fn push(&mut self, v: Scalar) -> Result<(), KindMismatch> { + match (self, v) { + (AnyColumn::I64(c), Scalar::I64(x)) => { + c.push(x); + Ok(()) + } + (AnyColumn::F64(c), Scalar::F64(x)) => { + c.push(x); + Ok(()) + } + (AnyColumn::Bool(c), Scalar::Bool(x)) => { + c.push(x); + Ok(()) + } + (AnyColumn::Ts(c), Scalar::Ts(x)) => { + c.push(x); + Ok(()) + } + (this, v) => Err(KindMismatch { expected: this.kind(), got: v.kind() }), + } + } + + /// Read one type-erased value (0 = newest). `None` if cold / out of range. + pub fn get(&self, k: usize) -> Option { + match self { + AnyColumn::I64(c) => c.get(k).map(Scalar::I64), + AnyColumn::F64(c) => c.get(k).map(Scalar::F64), + AnyColumn::Bool(c) => c.get(k).map(Scalar::Bool), + AnyColumn::Ts(c) => c.get(k).map(Scalar::Ts), + } + } + + /// Concrete-column accessor for the monomorphic hot path; `Some` only for + /// the matching kind. + pub fn as_i64_mut(&mut self) -> Option<&mut Column> { + match self { + AnyColumn::I64(c) => Some(c), + _ => None, + } + } + + pub fn as_f64_mut(&mut self) -> Option<&mut Column> { + match self { + AnyColumn::F64(c) => Some(c), + _ => None, + } + } + + pub fn as_bool_mut(&mut self) -> Option<&mut Column> { + match self { + AnyColumn::Bool(c) => Some(c), + _ => None, + } + } + + pub fn as_ts_mut(&mut self) -> Option<&mut Column> { + match self { + AnyColumn::Ts(c) => Some(c), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn same_kind_push_round_trips() { + let mut e = AnyColumn::with_capacity(ScalarKind::F64, 4); + e.push(Scalar::F64(1.5)).unwrap(); + e.push(Scalar::F64(2.5)).unwrap(); + assert_eq!(e.kind(), ScalarKind::F64); + assert_eq!(e.len(), 2); + assert_eq!(e.run_count(), 2); + assert_eq!(e.get(0), Some(Scalar::F64(2.5))); // newest + assert_eq!(e.get(1), Some(Scalar::F64(1.5))); + assert_eq!(e.get(2), None); + } + + #[test] + fn wrong_kind_push_is_rejected() { + let mut edge = AnyColumn::with_capacity(ScalarKind::I64, 8); + let err = edge.push(Scalar::F64(1.5)).unwrap_err(); // f64 into an i64 edge + assert_eq!(err, KindMismatch { expected: ScalarKind::I64, got: ScalarKind::F64 }); + assert_eq!(edge.len(), 0); // nothing was stored + assert_eq!(edge.run_count(), 0); // and the counter did not move + } + + #[test] + fn concrete_accessor_matches_only_its_kind() { + let mut e = AnyColumn::with_capacity(ScalarKind::F64, 2); + assert!(e.as_f64_mut().is_some()); + assert!(e.as_i64_mut().is_none()); + assert!(e.as_bool_mut().is_none()); + assert!(e.as_ts_mut().is_none()); + // hot-path push through the concrete column, monomorphic: + e.as_f64_mut().unwrap().push(3.0); + assert_eq!(e.get(0), Some(Scalar::F64(3.0))); + assert_eq!(e.run_count(), 1); + } + + #[test] + fn timestamp_edge_round_trips() { + let mut e = AnyColumn::with_capacity(ScalarKind::Timestamp, 2); + e.push(Scalar::Ts(Timestamp(1_700_000_000_000_000_000))).unwrap(); + assert_eq!(e.get(0), Some(Scalar::Ts(Timestamp(1_700_000_000_000_000_000)))); + } +} +``` + +- [ ] **Step 2: Declare and re-export the `any` module in `lib.rs`** + +Add to `crates/aura-core/src/lib.rs` (this completes the four modules): + +```rust +mod any; +``` +```rust +pub use any::AnyColumn; +``` + +- [ ] **Step 3: Test the any module** + +Run: `cargo test -p aura-core any::` +Expected: the four `any::tests::*` tests report `ok`, including +`wrong_kind_push_is_rejected`. + +--- + +## Task 5: Workspace gate (build + full test + lint) + +**Files:** none (verification only). + +- [ ] **Step 1: Full workspace build** + +Run: `cargo build --workspace` +Expected: `Finished` with 0 errors, 0 warnings. + +- [ ] **Step 2: Full workspace test with a count assertion** + +Run: `cargo test --workspace` +Expected: all tests pass; the `aura-core` unit-test binary reports +`test result: ok. 14 passed; 0 failed` (4 scalar + 5 column + 1 error + 4 any). +If the count is not 14, a test was lost or misnamed — investigate before +proceeding. + +- [ ] **Step 3: Lint clean under `-D warnings`** + +Run: `cargo clippy --workspace --all-targets -- -D warnings` +Expected: `Finished` with no clippy diagnostics (exit 0). + +- [ ] **Step 4: Surface-purity check — no `RefCell`/`Rc`/`dyn Any` leaked** + +Run: `! grep -rnE 'RefCell|Rc<|dyn Any' crates/aura-core/src` +Expected: exit 0 (no matches). The substrate must carry none of these on its +surface (C1/C7). A match is a regression to fix before the cycle closes.