From a1ce14ccc75f357545882f14c70f2c41eb1809c1 Mon Sep 17 00:00:00 2001 From: Brummel Date: Thu, 25 Jun 2026 14:25:01 +0200 Subject: [PATCH] plan: per-symbol geometry sidecar reader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two-task plan for #3: (1) the meta module — InstrumentGeometry + from_sidecar_bytes parser + serde_json dep + README update + unit tests; (2) DataServer::symbol_meta + deterministic tempdir integration tests. refs #3 --- .../plans/0001-symbol-meta-geometry-reader.md | 387 ++++++++++++++++++ 1 file changed, 387 insertions(+) create mode 100644 docs/plans/0001-symbol-meta-geometry-reader.md diff --git a/docs/plans/0001-symbol-meta-geometry-reader.md b/docs/plans/0001-symbol-meta-geometry-reader.md new file mode 100644 index 0000000..c240e53 --- /dev/null +++ b/docs/plans/0001-symbol-meta-geometry-reader.md @@ -0,0 +1,387 @@ +# Per-symbol geometry sidecar reader — Implementation Plan + +> **Parent spec:** `docs/specs/0001-symbol-meta-geometry-reader.md` +> +> **For agentic workers:** REQUIRED SUB-SKILL: use the `implement` +> skill to run this plan. Steps use `- [ ]` checkboxes for tracking. + +**Goal:** Add `DataServer::symbol_meta(symbol) -> Option`, +a reader that loads neutral price geometry from a `.meta.json` sidecar. + +**Architecture:** A new leaf module `src/meta.rs` owns the neutral +`InstrumentGeometry` struct and a pure `from_sidecar_bytes` parser (serde_json +`Value` extraction, absorbing scientific notation, `null`-for-non-finite, and +an unknown schema version). A new one-shot `&self` method on `DataServer` does +the single filesystem read via the existing `SymbolIndex::base_path`. The +reader is independent of the bar `FileCache` and streaming path. + +**Tech Stack:** Rust, `serde_json` (new direct dep), `tempfile` (existing +dev-dep) for the integration tests. + +**Files this plan creates or modifies:** + +- Create: `src/meta.rs` — `InstrumentGeometry`, `SUPPORTED_SCHEMA_VERSION`, + `from_sidecar_bytes`, private `finite`, and `#[cfg(test)] mod tests`. +- Modify: `src/lib.rs:21-23` — add `pub mod meta;` to the module-declaration block. +- Modify: `src/lib.rs:196-201` — add `symbol_meta` method to `impl DataServer`, + after `file_count`, before `stream_m1`. +- Modify: `src/lib.rs` (`#[cfg(test)] mod tests`, near line 792) — add a + `write_meta_file` helper and two integration tests. +- Modify: `Cargo.toml:7-10` — add `serde_json = "1"` to `[dependencies]`. +- Modify: `README.md:8` — update the dependency statement to include `serde_json`. +- Test: `src/meta.rs` unit tests — pin each absorbed parser quirk. +- Test: `src/lib.rs` integration tests — `symbol_meta` over a tempdir. + +**Out-of-scope (not touched):** `tests/data_server.rs` (its `/mnt`-skip-guard +idiom is unsuited to the deterministic tempdir tests); `Cargo.lock` (git-ignored +per the library convention — do NOT `git add` it even though `cargo build` +regenerates it). + +--- + +### Task 1: `meta` module — neutral geometry struct + sidecar parser + +**Files:** +- Modify: `Cargo.toml:7-10` +- Modify: `README.md:8` +- Create: `src/meta.rs` +- Modify: `src/lib.rs:21-23` + +- [ ] **Step 1: Add the `serde_json` dependency and update the README** + +In `Cargo.toml`, change the `[dependencies]` block from: + +```toml +[dependencies] +chrono = "0.4" +regex = "1.10" +zip = "2" +``` + +to: + +```toml +[dependencies] +chrono = "0.4" +regex = "1.10" +serde_json = "1" +zip = "2" +``` + +In `README.md`, change line 8 from: + +```markdown +Standalone leaf crate — depends only on `chrono`, `regex`, `zip`. +``` + +to: + +```markdown +Leaf crate — depends only on `chrono`, `regex`, `serde_json`, `zip`. +``` + +- [ ] **Step 2: Write `src/meta.rs` with the type, a stub parser, and the unit tests** + +Create `src/meta.rs` with this exact content (the parser body is a stub at this +step so the "parses to `Some`" tests fail RED): + +```rust +//! Reader for per-symbol `.meta.json` geometry sidecars. +//! +//! Turns the provider's flat JSON sidecar into neutral, broker-agnostic +//! price-interpretation geometry. Every format quirk (scientific-notation +//! numbers, `null` for a non-finite value, an unknown schema version) is +//! absorbed here so nothing provider-specific reaches a consumer. + +use serde_json::Value; + +/// The only sidecar schema version this reader understands. +pub const SUPPORTED_SCHEMA_VERSION: u64 = 2; + +/// Neutral, broker-agnostic price-interpretation geometry for one symbol. +/// +/// Intrinsic to the historical price series and effectively constant over its +/// span. Time-varying / account-dependent conditions are deliberately excluded. +#[derive(Debug, Clone, PartialEq)] +pub struct InstrumentGeometry { + /// Number of decimal places in a quoted price (e.g. 5 for EURUSD). + pub digits: u32, + /// Price increment of one pip (e.g. 0.0001 for EURUSD). + pub pip_size: f64, + /// Smallest representable price increment (e.g. 1e-05 for EURUSD). + pub tick_size: f64, + /// Contract size of one lot in base units (e.g. 100000 for EURUSD). + pub lot_size: f64, + /// Base asset code, passed through verbatim from the sidecar. + pub base: String, + /// Quote (price) currency code, passed through verbatim. + pub quote: String, +} + +impl InstrumentGeometry { + /// Parse sidecar JSON bytes into neutral geometry. + /// + /// Returns `None` if the JSON is malformed, the schema version is + /// unsupported, or any core field is missing / `null` / non-finite. + pub(crate) fn from_sidecar_bytes(bytes: &[u8]) -> Option { + // STUB — replaced in Step 4. + let _ = bytes; + None + } +} + +/// A finite `f64` from a JSON number, or `None` for missing / null / non-finite. +fn finite(v: Option<&Value>) -> Option { + let n = v?.as_f64()?; + n.is_finite().then_some(n) +} + +#[cfg(test)] +mod tests { + use super::*; + + const EURUSD: &str = r#"{ + "schemaVersion": 2, + "symbol": "EURUSD", + "generatedAtUtc": "2026-06-25T11:26:37Z", + "generator": "MS_SimpleExport (cTrader.Automate)", + "description": "Euro vs US Dollar", + "digits": 5, + "pipSize": 0.0001, + "tickSize": 1E-05, + "lotSize": 100000, + "baseAsset": "EUR", + "quoteAsset": "USD" + }"#; + + #[test] + fn parses_canonical_sidecar_with_scientific_notation_and_integer_lot() { + let g = InstrumentGeometry::from_sidecar_bytes(EURUSD.as_bytes()).unwrap(); + assert_eq!( + g, + InstrumentGeometry { + digits: 5, + pip_size: 0.0001, + tick_size: 1e-05, + lot_size: 100_000.0, + base: "EUR".to_string(), + quote: "USD".to_string(), + } + ); + } + + #[test] + fn integer_valued_geometry_parses_as_float() { + // GER40-shaped: pipSize 1, digits 1, lotSize 1. + let json = r#"{"schemaVersion":2,"digits":1,"pipSize":1,"tickSize":0.1,"lotSize":1,"baseAsset":"DE40","quoteAsset":"EUR"}"#; + let g = InstrumentGeometry::from_sidecar_bytes(json.as_bytes()).unwrap(); + assert_eq!(g.pip_size, 1.0); + assert_eq!(g.digits, 1); + assert_eq!(g.lot_size, 1.0); + } + + #[test] + fn provider_base_codes_pass_through_verbatim() { + let nas = r#"{"schemaVersion":2,"digits":1,"pipSize":1,"tickSize":0.1,"lotSize":1,"baseAsset":"USTEC","quoteAsset":"USD"}"#; + assert_eq!( + InstrumentGeometry::from_sidecar_bytes(nas.as_bytes()).unwrap().base, + "USTEC" + ); + let aapl = r#"{"schemaVersion":2,"digits":2,"pipSize":0.01,"tickSize":0.01,"lotSize":1,"baseAsset":"AAPL.US","quoteAsset":"USD"}"#; + assert_eq!( + InstrumentGeometry::from_sidecar_bytes(aapl.as_bytes()).unwrap().base, + "AAPL.US" + ); + } + + #[test] + fn unsupported_schema_version_yields_none() { + let json = r#"{"schemaVersion":99,"digits":5,"pipSize":0.0001,"tickSize":1E-05,"lotSize":100000,"baseAsset":"EUR","quoteAsset":"USD"}"#; + assert!(InstrumentGeometry::from_sidecar_bytes(json.as_bytes()).is_none()); + } + + #[test] + fn null_core_field_yields_none() { + let json = r#"{"schemaVersion":2,"digits":5,"pipSize":0.0001,"tickSize":1E-05,"lotSize":null,"baseAsset":"EUR","quoteAsset":"USD"}"#; + assert!(InstrumentGeometry::from_sidecar_bytes(json.as_bytes()).is_none()); + } + + #[test] + fn missing_core_field_yields_none() { + let json = r#"{"schemaVersion":2,"digits":5,"pipSize":0.0001,"tickSize":1E-05,"baseAsset":"EUR","quoteAsset":"USD"}"#; + assert!(InstrumentGeometry::from_sidecar_bytes(json.as_bytes()).is_none()); + } + + #[test] + fn malformed_json_yields_none() { + assert!(InstrumentGeometry::from_sidecar_bytes(b"not json").is_none()); + } +} +``` + +Then add the module declaration in `src/lib.rs`, changing the block at lines 21-23 +from: + +```rust +pub mod cache; +pub mod loader; +pub mod records; +``` + +to: + +```rust +pub mod cache; +pub mod loader; +pub mod meta; +pub mod records; +``` + +- [ ] **Step 3: Run the meta unit tests — verify RED** + +Run: `cargo test -p data-server --lib meta::` +Expected: the suite COMPILES and the 7 `meta::tests::*` tests run; the three +that expect `Some` FAIL (`parses_canonical_sidecar_with_scientific_notation_and_integer_lot`, +`integer_valued_geometry_parses_as_float`, `provider_base_codes_pass_through_verbatim`) +with `called Option::unwrap() on a None value` (the stub returns `None`). The +four `*_yields_none` / `malformed_*` tests pass against the stub. + +- [ ] **Step 4: Implement the real parser** + +In `src/meta.rs`, replace the stub body of `from_sidecar_bytes`: + +```rust + // STUB — replaced in Step 4. + let _ = bytes; + None +``` + +with: + +```rust + let v: Value = serde_json::from_slice(bytes).ok()?; + if v.get("schemaVersion").and_then(Value::as_u64)? != SUPPORTED_SCHEMA_VERSION { + return None; + } + Some(Self { + digits: v.get("digits").and_then(Value::as_u64)? as u32, + pip_size: finite(v.get("pipSize"))?, + tick_size: finite(v.get("tickSize"))?, + lot_size: finite(v.get("lotSize"))?, + base: v.get("baseAsset").and_then(Value::as_str)?.to_string(), + quote: v.get("quoteAsset").and_then(Value::as_str)?.to_string(), + }) +``` + +- [ ] **Step 5: Run the meta unit tests — verify GREEN** + +Run: `cargo test -p data-server --lib meta::` +Expected: PASS — all 7 `meta::tests::*` tests green. + +--- + +### Task 2: `DataServer::symbol_meta` method + integration tests + +**Files:** +- Modify: `src/lib.rs:196-201` (add method to `impl DataServer`) +- Modify: `src/lib.rs` (`#[cfg(test)] mod tests`, near line 792) + +- [ ] **Step 1: Write the integration tests (RED)** + +In `src/lib.rs`, inside `#[cfg(test)] mod tests` (after the existing +`write_m1_file` helper, before the first `#[test]`), add a helper: + +```rust + /// Writes a `.meta.json` sidecar into `dir` so the reader can + /// load it. The body is written verbatim. + fn write_meta_file(dir: &Path, symbol: &str, body: &str) { + std::fs::write(dir.join(format!("{symbol}.meta.json")), body).unwrap(); + } +``` + +Then add two tests at the end of the same `mod tests` (before its closing `}`): + +```rust + #[test] + fn test_symbol_meta_reads_sidecar() { + let dir = tempfile::tempdir().unwrap(); + write_meta_file( + dir.path(), + "EURUSD", + r#"{ + "schemaVersion": 2, "digits": 5, "pipSize": 0.0001, + "tickSize": 1E-05, "lotSize": 100000, + "baseAsset": "EUR", "quoteAsset": "USD" + }"#, + ); + let server = DataServer::new(dir.path()); + let g = server.symbol_meta("EURUSD").unwrap(); + assert_eq!(g.digits, 5); + assert_eq!(g.pip_size, 0.0001); + assert_eq!(g.tick_size, 1e-05); + assert_eq!(g.lot_size, 100_000.0); + assert_eq!(g.base, "EUR"); + assert_eq!(g.quote, "USD"); + // Issue's worked derivation: pip_value_per_lot = lot_size * pip_size. + assert!((g.lot_size * g.pip_size - 10.0).abs() < 1e-9); + } + + #[test] + fn test_symbol_meta_missing_sidecar_is_none() { + let dir = tempfile::tempdir().unwrap(); + // A symbol with a bar file but no sidecar -> None. + write_m1_file(dir.path(), "GBPUSD", 2020, 6); + let server = DataServer::new(dir.path()); + assert!(server.symbol_meta("GBPUSD").is_none()); + // An entirely unknown symbol -> None. + assert!(server.symbol_meta("NOPE").is_none()); + } +``` + +- [ ] **Step 2: Run the integration tests — verify RED** + +Run: `cargo test -p data-server --lib test_symbol_meta` +Expected: COMPILE ERROR — `no method named symbol_meta found for ... DataServer`. +(The method does not exist yet; this is the RED state. The two +`test_symbol_meta_*` tests are the ones gated.) + +- [ ] **Step 3: Add the `symbol_meta` method** + +In `src/lib.rs`, inside `impl DataServer`, immediately after the `file_count` +method (which ends at line 200, before `stream_m1` at line 202), insert: + +```rust + /// Loads the neutral price geometry for a symbol from its + /// `.meta.json` sidecar, or `None` if no usable sidecar exists. + /// + /// A missing sidecar is normal (a symbol may have bars but no sidecar) and + /// returns `None` silently. A present-but-unusable sidecar (malformed, + /// unsupported schema version, or an incomplete / `null` / non-finite core + /// field) returns `None` after a one-line stderr warning. The returned + /// geometry is always complete; the consumer never sees a partial struct. + pub fn symbol_meta(&self, symbol: &str) -> Option { + let path = self.index.base_path.join(format!("{symbol}.meta.json")); + let bytes = std::fs::read(&path).ok()?; // missing / unreadable -> silent None + let geo = meta::InstrumentGeometry::from_sidecar_bytes(&bytes); + if geo.is_none() { + eprintln!( + "Warning: {} present but unusable (bad/incomplete geometry)", + path.display() + ); + } + geo + } +``` + +- [ ] **Step 4: Run the integration tests — verify GREEN** + +Run: `cargo test -p data-server --lib test_symbol_meta` +Expected: PASS — `test_symbol_meta_reads_sidecar` and +`test_symbol_meta_missing_sidecar_is_none` both green. + +- [ ] **Step 5: Run the full suite — verify no regression** + +Run: `cargo test -p data-server` +Expected: PASS — all tests green (existing suite + 7 meta unit tests + 2 +`symbol_meta` integration tests). No test is filtered out; the result line +reports the full count.