feat(trace-charts): persist recorder taps to disk (iter 1)
Iteration 1 (persist half) of the visual face — spec 0056 / plan 0055:
tap data from a graph and save it to disk so it can later be charted
(the serve/chart half is iteration 2).
- ColumnarTrace (aura-engine): a drained tap <-> columnar SoA form
({tap, kinds, ts, columns}); values coerced to f64 for plotting with
the base type preserved in `kinds` (C7); to_rows rebuilds uniformly-f64
rows for the serve-side join. Pure (C1), beside join_on_ts/summarize.
- TraceStore (aura-registry): a per-run directory under
runs/traces/<name>/ — one <tap>.json per tap + an index.json (manifest
+ tap order), beside the families.jsonl sibling store. Missing run =
NotFound (no panic); corrupt file = Parse{file}.
- CLI: a shared persist_traces helper threaded into run_sample /
run_macd / run_sample_real via an orthogonal `trace: Option<&str>`;
new arms `aura run [--macd] --trace <name>` and `--trace <name>` on
`run --real`. The default (no --trace) run is byte-for-byte unchanged.
Verified by the orchestrator: `cargo test --workspace` green (incl. the
4 columnar_trace, 3 trace_store, 4 cli_run e2e, and the 4-tuple
parse_real_args tests); `cargo clippy --workspace --all-targets -D
warnings` exit 0. Implementer deviations folded in, all
compiler-prescribed: #[derive(Debug)] on RunTraces (the RED tests
panic-format a Result over it), #[allow(clippy::type_complexity)] on
the parse_real_args 4-tuple (mirrors the crate's sample_harness idiom).
Default-run byte-identity stays pinned by run_prints_json_and_exits_zero.
refs #101
This commit is contained in:
@@ -27,6 +27,9 @@ pub use lineage::{
|
||||
FamilyKind, FamilyRunRecord,
|
||||
};
|
||||
|
||||
mod trace_store;
|
||||
pub use trace_store::{RunTraces, TraceStore, TraceStoreError};
|
||||
|
||||
/// An append-only run registry over a JSONL file: one serde_json line per
|
||||
/// `RunReport`.
|
||||
pub struct Registry {
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
//! The per-run trace store: a directory of columnar tap files under
|
||||
//! `<runs_dir>/traces/<name>/`, beside the run registry's `runs.jsonl`. One
|
||||
//! subdirectory per run name; one `<tap>.json` (a `ColumnarTrace`) per tap; plus
|
||||
//! `index.json` carrying the run's `RunManifest` and the tap order. Mirrors the
|
||||
//! registry's directory-co-located sibling-store discipline; a missing run reads as
|
||||
//! `NotFound` (treat-as-absent), never a panic.
|
||||
|
||||
use std::fmt;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use aura_engine::{ColumnarTrace, RunManifest};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// A per-run trace directory rooted at `<runs_dir>/traces`. One subdir per run name.
|
||||
pub struct TraceStore {
|
||||
dir: PathBuf,
|
||||
}
|
||||
|
||||
/// A run's read-back traces: its manifest plus its taps in index (write) order.
|
||||
#[derive(Debug)]
|
||||
pub struct RunTraces {
|
||||
pub manifest: RunManifest,
|
||||
pub taps: Vec<ColumnarTrace>,
|
||||
}
|
||||
|
||||
/// `index.json`: the run manifest + the tap names in write order (= read order).
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct Index {
|
||||
manifest: RunManifest,
|
||||
taps: Vec<String>,
|
||||
}
|
||||
|
||||
impl TraceStore {
|
||||
/// Bind to `<runs_dir>/traces`. No I/O — directories are created on write.
|
||||
pub fn open(runs_dir: impl AsRef<Path>) -> TraceStore {
|
||||
TraceStore { dir: runs_dir.as_ref().join("traces") }
|
||||
}
|
||||
|
||||
/// Write one run's taps + index under `traces/<name>/`, creating directories as
|
||||
/// needed. Each tap lands in `<tap>.json`; the manifest + tap order in
|
||||
/// `index.json`.
|
||||
pub fn write(
|
||||
&self,
|
||||
name: &str,
|
||||
manifest: &RunManifest,
|
||||
taps: &[ColumnarTrace],
|
||||
) -> Result<(), TraceStoreError> {
|
||||
let run_dir = self.dir.join(name);
|
||||
fs::create_dir_all(&run_dir)?;
|
||||
for tap in taps {
|
||||
let path = run_dir.join(format!("{}.json", tap.tap));
|
||||
let body = serde_json::to_string(tap).map_err(|source| TraceStoreError::Parse {
|
||||
file: path.display().to_string(),
|
||||
source,
|
||||
})?;
|
||||
fs::write(&path, body)?;
|
||||
}
|
||||
let index = Index {
|
||||
manifest: manifest.clone(),
|
||||
taps: taps.iter().map(|t| t.tap.clone()).collect(),
|
||||
};
|
||||
let index_path = run_dir.join("index.json");
|
||||
let body = serde_json::to_string(&index).map_err(|source| TraceStoreError::Parse {
|
||||
file: index_path.display().to_string(),
|
||||
source,
|
||||
})?;
|
||||
fs::write(&index_path, body)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read a run's index + taps back, in index order. A missing run directory (no
|
||||
/// `index.json`) is `NotFound`, not a panic; a malformed file is `Parse{file,..}`.
|
||||
pub fn read(&self, name: &str) -> Result<RunTraces, TraceStoreError> {
|
||||
let run_dir = self.dir.join(name);
|
||||
let index_path = run_dir.join("index.json");
|
||||
let index_text = match fs::read_to_string(&index_path) {
|
||||
Ok(t) => t,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
|
||||
return Err(TraceStoreError::NotFound(name.to_string()))
|
||||
}
|
||||
Err(e) => return Err(TraceStoreError::Io(e)),
|
||||
};
|
||||
let index: Index = serde_json::from_str(&index_text).map_err(|source| {
|
||||
TraceStoreError::Parse { file: index_path.display().to_string(), source }
|
||||
})?;
|
||||
let mut taps = Vec::with_capacity(index.taps.len());
|
||||
for tap_name in &index.taps {
|
||||
let path = run_dir.join(format!("{tap_name}.json"));
|
||||
let text = fs::read_to_string(&path)?;
|
||||
let tap: ColumnarTrace = serde_json::from_str(&text).map_err(|source| {
|
||||
TraceStoreError::Parse { file: path.display().to_string(), source }
|
||||
})?;
|
||||
taps.push(tap);
|
||||
}
|
||||
Ok(RunTraces { manifest: index.manifest, taps })
|
||||
}
|
||||
}
|
||||
|
||||
/// What can go wrong reading or writing the trace store.
|
||||
#[derive(Debug)]
|
||||
pub enum TraceStoreError {
|
||||
/// An I/O error reading or writing a trace file.
|
||||
Io(std::io::Error),
|
||||
/// A trace/index file did not parse (the offending file named).
|
||||
Parse { file: String, source: serde_json::Error },
|
||||
/// No recorded run of this name (no `index.json` under `traces/<name>/`).
|
||||
NotFound(String),
|
||||
}
|
||||
|
||||
impl fmt::Display for TraceStoreError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
TraceStoreError::Io(e) => write!(f, "trace store i/o: {e}"),
|
||||
TraceStoreError::Parse { file, source } => {
|
||||
write!(f, "trace store parse error in {file}: {source}")
|
||||
}
|
||||
TraceStoreError::NotFound(name) => {
|
||||
write!(f, "no recorded run '{name}' under runs/traces")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for TraceStoreError {}
|
||||
|
||||
impl From<std::io::Error> for TraceStoreError {
|
||||
fn from(e: std::io::Error) -> Self {
|
||||
TraceStoreError::Io(e)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aura_core::{Scalar, ScalarKind, Timestamp};
|
||||
|
||||
fn temp_traces_root(name: &str) -> PathBuf {
|
||||
let dir =
|
||||
std::env::temp_dir().join(format!("aura-tracestore-{}-{}", std::process::id(), name));
|
||||
let _ = fs::remove_dir_all(&dir);
|
||||
fs::create_dir_all(&dir).expect("create temp traces root");
|
||||
dir
|
||||
}
|
||||
|
||||
fn sample_manifest() -> RunManifest {
|
||||
RunManifest {
|
||||
commit: "c".to_string(),
|
||||
params: vec![("p".to_string(), Scalar::f64(1.0))],
|
||||
window: (Timestamp(1), Timestamp(5)),
|
||||
seed: 0,
|
||||
broker: "sim-optimal(pip_size=0.0001)".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_taps() -> Vec<ColumnarTrace> {
|
||||
vec![
|
||||
ColumnarTrace::from_rows(
|
||||
"equity",
|
||||
&[ScalarKind::F64],
|
||||
&[(Timestamp(1), vec![Scalar::f64(0.0)]), (Timestamp(2), vec![Scalar::f64(0.5)])],
|
||||
),
|
||||
ColumnarTrace::from_rows(
|
||||
"exposure",
|
||||
&[ScalarKind::F64],
|
||||
&[(Timestamp(1), vec![Scalar::f64(1.0)]), (Timestamp(2), vec![Scalar::f64(-1.0)])],
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_then_read_round_trips() {
|
||||
let root = temp_traces_root("roundtrip");
|
||||
let store = TraceStore::open(&root);
|
||||
let manifest = sample_manifest();
|
||||
let taps = sample_taps();
|
||||
store.write("demo", &manifest, &taps).expect("write");
|
||||
let back = store.read("demo").expect("read");
|
||||
assert_eq!(back.manifest, manifest);
|
||||
assert_eq!(back.taps, taps);
|
||||
assert!(root.join("traces/demo/index.json").exists());
|
||||
assert!(root.join("traces/demo/equity.json").exists());
|
||||
assert!(root.join("traces/demo/exposure.json").exists());
|
||||
let _ = fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_missing_run_is_not_found() {
|
||||
let root = temp_traces_root("missing");
|
||||
let store = TraceStore::open(&root);
|
||||
match store.read("nope") {
|
||||
Err(TraceStoreError::NotFound(name)) => assert_eq!(name, "nope"),
|
||||
other => panic!("expected NotFound, got {other:?}"),
|
||||
}
|
||||
let _ = fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupt_tap_file_is_a_parse_error_naming_the_file() {
|
||||
let root = temp_traces_root("corrupt");
|
||||
let store = TraceStore::open(&root);
|
||||
store.write("demo", &sample_manifest(), &sample_taps()).expect("write");
|
||||
fs::write(root.join("traces/demo/equity.json"), "not json").expect("clobber");
|
||||
match store.read("demo") {
|
||||
Err(TraceStoreError::Parse { file, .. }) => assert!(file.ends_with("equity.json")),
|
||||
other => panic!("expected Parse, got {other:?}"),
|
||||
}
|
||||
let _ = fs::remove_dir_all(&root);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user