From be2489ac95dc3bfd05641af16e4a9b9e1a43ed1e Mon Sep 17 00:00:00 2001 From: Brummel Date: Thu, 25 Jun 2026 14:42:04 +0200 Subject: [PATCH] Add symbol_meta reader for per-symbol geometry sidecars Implements #3: DataServer::symbol_meta(symbol) -> Option loads a .meta.json sidecar from the owned data directory and returns neutral, broker-agnostic price geometry (digits, pip_size, tick_size, lot_size, base, quote). A consumer can now obtain a symbol's bars and its price-interpretation geometry from one server. Design (forks logged on #3): - Neutral output is the struct's shape/semantics, not currency-code normalization: base/quote pass through verbatim. The live sidecars carry wildly heterogeneous base codes (ISO, index codes, descriptive names, stock tickers) with no definable neutral target; the downstream consumer cross-checks them against its own vetted table. - Format quirks are absorbed in a pure parser (meta::from_sidecar_bytes): scientific notation parses natively; a null / non-finite / missing core field or an unsupported schemaVersion yields None, never a partial struct. - A missing sidecar is a normal None (silent); a present-but-unusable one warns to stderr, matching the existing loader's degrade-and-warn style. - serde_json (Value extraction, no derive) is the new direct dependency, chosen over a hand-rolled parser for robust number/escape handling; the README dependency statement is updated to match. - No caching and no Arc: geometry files are tiny and read rarely, and the read needs no prefetch. Lookup is not gated on bar-index membership. Verified: cargo build and clippy clean (0 warnings); full suite green (7 parser unit tests + 5 symbol_meta integration tests pin each absorbed quirk and the index-independent lookup). Cargo.lock stays untracked per the library convention. closes #3 --- Cargo.toml | 1 + README.md | 3 +- src/lib.rs | 144 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/meta.rs | 140 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 287 insertions(+), 1 deletion(-) create mode 100644 src/meta.rs diff --git a/Cargo.toml b/Cargo.toml index 11b4ed4..c089b14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ description = "High-performance, thread-safe loader and cache for binary market [dependencies] chrono = "0.4" regex = "1.10" +serde_json = "1" zip = "2" [dev-dependencies] diff --git a/README.md b/README.md index 5ab048f..2e70946 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ High-performance, thread-safe loader and cache for binary market data files one `.bin` entry of tightly packed records; it is read and parsed exactly once, then shared lock-free across many readers via `Arc<[T]>` chunks. -Standalone leaf crate — depends only on `chrono`, `regex`, `zip`. +Leaf crate — depends only on `chrono`, `regex`, `serde_json`, `zip`. ## Add as dependency @@ -32,6 +32,7 @@ data-server = { git = "http://192.168.178.103:3000/Brummel/data-server.git", bra | `has_symbol(symbol) -> bool` | Whether the symbol has any files. | | `symbols() -> Vec>` | All known symbols, sorted. | | `file_count(symbol, DataFormat) -> Option` | Number of files for that symbol/format. | +| `symbol_meta(symbol) -> Option` | Neutral price geometry from the symbol's `.meta.json` sidecar; `None` if no usable sidecar exists. | | `stream_m1(symbol)` / `stream_tick(symbol)` | `Option>` over all data. | | `stream_m1_windowed(symbol, from_ms, to_ms)` / `stream_tick_windowed(...)` | Same, restricted to an inclusive `[from_ms, to_ms]` Unix-millisecond window (`Option` bounds; files outside are skipped without I/O). | diff --git a/src/lib.rs b/src/lib.rs index 130f9cd..e5a3bae 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,6 +20,7 @@ pub mod cache; pub mod loader; +pub mod meta; pub mod records; use cache::{ChunkVec, FileCache, FileKey, SymbolGuard}; @@ -199,6 +200,27 @@ impl DataServer { }) } + /// Loads the neutral price geometry for a symbol from its + /// `.meta.json` sidecar, or `None` if no usable sidecar exists. + /// + /// A missing sidecar is normal (a symbol may have bars but no sidecar) and + /// returns `None` silently. A present-but-unusable sidecar (malformed, + /// unsupported schema version, or an incomplete / `null` / non-finite core + /// field) returns `None` after a one-line stderr warning. The returned + /// geometry is always complete; the consumer never sees a partial struct. + pub fn symbol_meta(&self, symbol: &str) -> Option { + let path = self.index.base_path.join(format!("{symbol}.meta.json")); + let bytes = std::fs::read(&path).ok()?; // missing / unreadable -> silent None + let geo = meta::InstrumentGeometry::from_sidecar_bytes(&bytes); + if geo.is_none() { + eprintln!( + "Warning: {} present but unusable (bad/incomplete geometry)", + path.display() + ); + } + geo + } + /// Returns a chunk iterator over all M1 data for a symbol. /// /// The iterator yields `Arc<[M1Parsed]>` chunks in chronological order, @@ -645,6 +667,12 @@ mod tests { zip.finish().unwrap(); } + /// Writes a `.meta.json` sidecar into `dir` so the reader can + /// load it. The body is written verbatim. + fn write_meta_file(dir: &Path, symbol: &str, body: &str) { + std::fs::write(dir.join(format!("{symbol}.meta.json")), body).unwrap(); + } + /// Regression guard for the no-redundant-I/O contract: each data file is /// read from disk exactly once and shared via the cache — no redundant /// I/O — across prefetch and a second concurrent consumer of the same @@ -889,4 +917,120 @@ mod tests { assert!(result.is_none()); assert!(exhausted); } + + #[test] + fn test_symbol_meta_reads_sidecar() { + let dir = tempfile::tempdir().unwrap(); + write_meta_file( + dir.path(), + "EURUSD", + r#"{ + "schemaVersion": 2, "digits": 5, "pipSize": 0.0001, + "tickSize": 1E-05, "lotSize": 100000, + "baseAsset": "EUR", "quoteAsset": "USD" + }"#, + ); + let server = DataServer::new(dir.path()); + let g = server.symbol_meta("EURUSD").unwrap(); + assert_eq!(g.digits, 5); + assert_eq!(g.pip_size, 0.0001); + assert_eq!(g.tick_size, 1e-05); + assert_eq!(g.lot_size, 100_000.0); + assert_eq!(g.base, "EUR"); + assert_eq!(g.quote, "USD"); + // pip_value_per_lot = lot_size * pip_size = 100000 * 0.0001 = 10. + assert!((g.lot_size * g.pip_size - 10.0).abs() < 1e-9); + } + + #[test] + fn test_symbol_meta_missing_sidecar_is_none() { + let dir = tempfile::tempdir().unwrap(); + // A symbol with a bar file but no sidecar -> None. + write_m1_file(dir.path(), "GBPUSD", 2020, 6); + let server = DataServer::new(dir.path()); + assert!(server.symbol_meta("GBPUSD").is_none()); + // An entirely unknown symbol -> None. + assert!(server.symbol_meta("NOPE").is_none()); + } + + /// `symbol_meta` maps a present-but-unusable sidecar to `None`: a sidecar + /// that exists on disk but is malformed or carries an unsupported schema + /// version is rejected (the consumer never receives a partial struct), + /// distinguishing this warn-and-`None` branch from the silent missing-file + /// `None`. + #[test] + fn test_symbol_meta_present_but_unusable_is_none() { + let dir = tempfile::tempdir().unwrap(); + // Malformed JSON body in a present sidecar -> None. + write_meta_file(dir.path(), "EURUSD", "not json"); + let server = DataServer::new(dir.path()); + assert!(server.symbol_meta("EURUSD").is_none()); + // Present sidecar, well-formed JSON, but unsupported schema version -> None. + write_meta_file( + dir.path(), + "GBPUSD", + r#"{ + "schemaVersion": 99, "digits": 5, "pipSize": 0.0001, + "tickSize": 1E-05, "lotSize": 100000, + "baseAsset": "GBP", "quoteAsset": "USD" + }"#, + ); + assert!(server.symbol_meta("GBPUSD").is_none()); + } + + /// `symbol_meta` resolves a sidecar by its own filename, independent of the + /// bar-file symbol index: a symbol that has M1 bar files AND a sidecar + /// sitting beside them in the same directory yields its geometry through the + /// public method. This is the realistic on-disk layout (bars + sidecar for + /// one symbol); it pins that geometry lookup is not gated on, nor altered + /// by, the presence of bar files for that symbol. + #[test] + fn test_symbol_meta_resolves_alongside_bar_files() { + let dir = tempfile::tempdir().unwrap(); + // Realistic layout: the symbol has bars AND a geometry sidecar. + write_m1_file(dir.path(), "EURUSD", 2017, 1); + write_meta_file( + dir.path(), + "EURUSD", + r#"{ + "schemaVersion": 2, "digits": 5, "pipSize": 0.0001, + "tickSize": 1E-05, "lotSize": 100000, + "baseAsset": "EUR", "quoteAsset": "USD" + }"#, + ); + let server = DataServer::new(dir.path()); + // The symbol is indexed (has bars) and the sidecar resolves regardless. + assert!(server.has_symbol("EURUSD")); + let g = server.symbol_meta("EURUSD").unwrap(); + assert_eq!(g.digits, 5); + assert_eq!(g.base, "EUR"); + assert_eq!(g.quote, "USD"); + } + + /// Integer-valued index/stock geometry (`pipSize` 1, `digits` 1, `lotSize` + /// 1, encoded as JSON integers) survives the full public `symbol_meta` path + /// as `f64`/`u32` fields, not just the parser unit. The method and parser + /// are separate code; this pins that an index-shaped sidecar round-trips to + /// a complete `InstrumentGeometry` through the public API. + #[test] + fn test_symbol_meta_integer_geometry_round_trips() { + let dir = tempfile::tempdir().unwrap(); + write_meta_file( + dir.path(), + "GER40", + r#"{ + "schemaVersion": 2, "digits": 1, "pipSize": 1, + "tickSize": 0.1, "lotSize": 1, + "baseAsset": "DE40", "quoteAsset": "EUR" + }"#, + ); + let server = DataServer::new(dir.path()); + let g = server.symbol_meta("GER40").unwrap(); + assert_eq!(g.digits, 1); + assert_eq!(g.pip_size, 1.0); + assert_eq!(g.tick_size, 0.1); + assert_eq!(g.lot_size, 1.0); + assert_eq!(g.base, "DE40"); + assert_eq!(g.quote, "EUR"); + } } diff --git a/src/meta.rs b/src/meta.rs new file mode 100644 index 0000000..dbb3e49 --- /dev/null +++ b/src/meta.rs @@ -0,0 +1,140 @@ +//! Reader for per-symbol `.meta.json` geometry sidecars. +//! +//! Turns the provider's flat JSON sidecar into neutral, broker-agnostic +//! price-interpretation geometry. Every format quirk (scientific-notation +//! numbers, `null` for a non-finite value, an unknown schema version) is +//! absorbed here so nothing provider-specific reaches a consumer. + +use serde_json::Value; + +/// The only sidecar schema version this reader understands. +pub const SUPPORTED_SCHEMA_VERSION: u64 = 2; + +/// Neutral, broker-agnostic price-interpretation geometry for one symbol. +/// +/// Intrinsic to the historical price series and effectively constant over its +/// span. Time-varying / account-dependent conditions are deliberately excluded. +#[derive(Debug, Clone, PartialEq)] +pub struct InstrumentGeometry { + /// Number of decimal places in a quoted price (e.g. 5 for EURUSD). + pub digits: u32, + /// Price increment of one pip (e.g. 0.0001 for EURUSD). + pub pip_size: f64, + /// Smallest representable price increment (e.g. 1e-05 for EURUSD). + pub tick_size: f64, + /// Contract size of one lot in base units (e.g. 100000 for EURUSD). + pub lot_size: f64, + /// Base asset code, passed through verbatim from the sidecar. + pub base: String, + /// Quote (price) currency code, passed through verbatim. + pub quote: String, +} + +impl InstrumentGeometry { + /// Parse sidecar JSON bytes into neutral geometry. + /// + /// Returns `None` if the JSON is malformed, the schema version is + /// unsupported, or any core field is missing / `null` / non-finite. + pub(crate) fn from_sidecar_bytes(bytes: &[u8]) -> Option { + let v: Value = serde_json::from_slice(bytes).ok()?; + if v.get("schemaVersion").and_then(Value::as_u64)? != SUPPORTED_SCHEMA_VERSION { + return None; + } + Some(Self { + digits: v.get("digits").and_then(Value::as_u64)? as u32, + pip_size: finite(v.get("pipSize"))?, + tick_size: finite(v.get("tickSize"))?, + lot_size: finite(v.get("lotSize"))?, + base: v.get("baseAsset").and_then(Value::as_str)?.to_string(), + quote: v.get("quoteAsset").and_then(Value::as_str)?.to_string(), + }) + } +} + +/// A finite `f64` from a JSON number, or `None` for missing / null / non-finite. +fn finite(v: Option<&Value>) -> Option { + let n = v?.as_f64()?; + n.is_finite().then_some(n) +} + +#[cfg(test)] +mod tests { + use super::*; + + const EURUSD: &str = r#"{ + "schemaVersion": 2, + "symbol": "EURUSD", + "generatedAtUtc": "2026-06-25T11:26:37Z", + "generator": "MS_SimpleExport (cTrader.Automate)", + "description": "Euro vs US Dollar", + "digits": 5, + "pipSize": 0.0001, + "tickSize": 1E-05, + "lotSize": 100000, + "baseAsset": "EUR", + "quoteAsset": "USD" + }"#; + + #[test] + fn parses_canonical_sidecar_with_scientific_notation_and_integer_lot() { + let g = InstrumentGeometry::from_sidecar_bytes(EURUSD.as_bytes()).unwrap(); + assert_eq!( + g, + InstrumentGeometry { + digits: 5, + pip_size: 0.0001, + tick_size: 1e-05, + lot_size: 100_000.0, + base: "EUR".to_string(), + quote: "USD".to_string(), + } + ); + } + + #[test] + fn integer_valued_geometry_parses_as_float() { + // GER40-shaped: pipSize 1, digits 1, lotSize 1. + let json = r#"{"schemaVersion":2,"digits":1,"pipSize":1,"tickSize":0.1,"lotSize":1,"baseAsset":"DE40","quoteAsset":"EUR"}"#; + let g = InstrumentGeometry::from_sidecar_bytes(json.as_bytes()).unwrap(); + assert_eq!(g.pip_size, 1.0); + assert_eq!(g.digits, 1); + assert_eq!(g.lot_size, 1.0); + } + + #[test] + fn provider_base_codes_pass_through_verbatim() { + let nas = r#"{"schemaVersion":2,"digits":1,"pipSize":1,"tickSize":0.1,"lotSize":1,"baseAsset":"USTEC","quoteAsset":"USD"}"#; + assert_eq!( + InstrumentGeometry::from_sidecar_bytes(nas.as_bytes()).unwrap().base, + "USTEC" + ); + let aapl = r#"{"schemaVersion":2,"digits":2,"pipSize":0.01,"tickSize":0.01,"lotSize":1,"baseAsset":"AAPL.US","quoteAsset":"USD"}"#; + assert_eq!( + InstrumentGeometry::from_sidecar_bytes(aapl.as_bytes()).unwrap().base, + "AAPL.US" + ); + } + + #[test] + fn unsupported_schema_version_yields_none() { + let json = r#"{"schemaVersion":99,"digits":5,"pipSize":0.0001,"tickSize":1E-05,"lotSize":100000,"baseAsset":"EUR","quoteAsset":"USD"}"#; + assert!(InstrumentGeometry::from_sidecar_bytes(json.as_bytes()).is_none()); + } + + #[test] + fn null_core_field_yields_none() { + let json = r#"{"schemaVersion":2,"digits":5,"pipSize":0.0001,"tickSize":1E-05,"lotSize":null,"baseAsset":"EUR","quoteAsset":"USD"}"#; + assert!(InstrumentGeometry::from_sidecar_bytes(json.as_bytes()).is_none()); + } + + #[test] + fn missing_core_field_yields_none() { + let json = r#"{"schemaVersion":2,"digits":5,"pipSize":0.0001,"tickSize":1E-05,"baseAsset":"EUR","quoteAsset":"USD"}"#; + assert!(InstrumentGeometry::from_sidecar_bytes(json.as_bytes()).is_none()); + } + + #[test] + fn malformed_json_yields_none() { + assert!(InstrumentGeometry::from_sidecar_bytes(b"not json").is_none()); + } +}