feat: Add data server for market data loading

This commit introduces a new data server module (`src/ast/data_server`) designed for high-performance, thread-safe loading and caching of market data. Key components: - `DataServer`: Manages symbol indexing and delegates loading/caching. - `SymbolIndex`: Scans the data directory and maintains a sorted index of data files per symbol. - `FileCache`: Implements a thread-safe cache using `RwLock` and a loading guard to prevent duplicate work. - `loader`: Handles ZIP decompression and binary record parsing from `.bin` files. - `records`: Defines raw and parsed data structures for M1 and tick data. - `SymbolChunkIter`: Provides an iterator over pre-parsed data chunks, prefetching subsequent files. This architecture allows multiple VM threads to access market data concurrently without redundant I/O, mirroring the strategy used in the original Delphi `TDataServer`.
2026-03-29 20:28:55 +02:00
parent 1f3fb40bcc
commit 38f657076b
8 changed files with 1601 additions and 0 deletions
@@ -0,0 +1,189 @@
+//! Binary record types for Pepperstone tick data files.
+//!
+//! The raw types (`RawM1Record`, `RawTickRecord`) mirror the Delphi `packed record`
+//! layout used by the C# data exporter. The parsed types (`M1Parsed`, `TickParsed`)
+//! hold converted values ready for consumption by the VM.
+
+/// Delphi `TDateTime` epoch offset: days between 1899-12-30 and 1970-01-01.
+const DELPHI_EPOCH_OFFSET_DAYS: f64 = 25569.0;
+/// Milliseconds per day.
+const MS_PER_DAY: f64 = 86_400_000.0;
+
+/// Converts a Delphi `TDateTime` (f64 days since 1899-12-30) to Unix milliseconds.
+///
+/// Compatible with the existing `Value::DateTime(i64)` representation.
+#[inline]
+pub fn delphi_to_unix_ms(dt: f64) -> i64 {
+    ((dt - DELPHI_EPOCH_OFFSET_DAYS) * MS_PER_DAY) as i64
+}
+
+/// Raw M1 (minute) record as stored on disk. 48 bytes, packed.
+///
+/// Matches the Delphi `TM1Record = packed record` layout exactly:
+/// `Time: Double; Open: Double; High: Double; Low: Double; Close: Double; Spread: Single; Volume: Integer;`
+#[repr(C, packed)]
+#[derive(Clone, Copy)]
+pub struct RawM1Record {
+    pub time: f64,
+    pub open: f64,
+    pub high: f64,
+    pub low: f64,
+    pub close: f64,
+    pub spread: f32,
+    pub volume: i32,
+}
+
+/// Raw tick record as stored on disk. 24 bytes, packed.
+///
+/// Matches the Delphi `TTickRecord = packed record` layout:
+/// `Time: Double; Ask: Double; Bid: Double;`
+#[repr(C, packed)]
+#[derive(Clone, Copy)]
+pub struct RawTickRecord {
+    pub time: f64,
+    pub ask: f64,
+    pub bid: f64,
+}
+
+/// Parsed M1 record with converted field types.
+///
+/// All fields are native-width (`f64`/`i64`) for direct mapping to `Value` variants.
+/// `time_ms` is Unix milliseconds (converted from Delphi TDateTime).
+#[derive(Debug, Clone, Copy)]
+pub struct M1Parsed {
+    pub time_ms: i64,
+    pub open: f64,
+    pub high: f64,
+    pub low: f64,
+    pub close: f64,
+    pub spread: f64,
+    pub volume: i64,
+}
+
+/// Parsed tick record with converted field types.
+#[derive(Debug, Clone, Copy)]
+pub struct TickParsed {
+    pub time_ms: i64,
+    pub ask: f64,
+    pub bid: f64,
+}
+
+/// Distinguishes the two supported data file formats.
+#[derive(Debug, Clone, Copy, Hash, Eq, PartialEq)]
+pub enum DataFormat {
+    M1,
+    Tick,
+}
+
+impl M1Parsed {
+    /// Converts a raw on-disk record to the parsed representation.
+    ///
+    /// Uses `read_unaligned` because `RawM1Record` is `packed` and field
+    /// access would be unaligned on architectures that require alignment.
+    #[inline]
+    pub fn from_raw(raw: &RawM1Record) -> Self {
+        // SAFETY: `raw` points to a valid `RawM1Record`. We use `read_unaligned`
+        // to safely read potentially unaligned fields from the packed struct.
+        unsafe {
+            Self {
+                time_ms: delphi_to_unix_ms(std::ptr::addr_of!(raw.time).read_unaligned()),
+                open: std::ptr::addr_of!(raw.open).read_unaligned(),
+                high: std::ptr::addr_of!(raw.high).read_unaligned(),
+                low: std::ptr::addr_of!(raw.low).read_unaligned(),
+                close: std::ptr::addr_of!(raw.close).read_unaligned(),
+                spread: std::ptr::addr_of!(raw.spread).read_unaligned() as f64,
+                volume: std::ptr::addr_of!(raw.volume).read_unaligned() as i64,
+            }
+        }
+    }
+}
+
+impl TickParsed {
+    /// Converts a raw on-disk tick record to the parsed representation.
+    #[inline]
+    pub fn from_raw(raw: &RawTickRecord) -> Self {
+        unsafe {
+            Self {
+                time_ms: delphi_to_unix_ms(std::ptr::addr_of!(raw.time).read_unaligned()),
+                ask: std::ptr::addr_of!(raw.ask).read_unaligned(),
+                bid: std::ptr::addr_of!(raw.bid).read_unaligned(),
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_delphi_datetime_conversion() {
+        // 2017-03-01 00:00:00 UTC = Unix timestamp 1488326400000 ms
+        // Delphi TDateTime for 2017-03-01 = 42795.0
+        let delphi_dt = 42795.0;
+        let unix_ms = delphi_to_unix_ms(delphi_dt);
+        // Allow 1ms tolerance for floating-point rounding
+        assert!(
+            (unix_ms - 1_488_326_400_000).abs() <= 1,
+            "Expected ~1488326400000, got {unix_ms}"
+        );
+    }
+
+    #[test]
+    fn test_delphi_epoch() {
+        // The Unix epoch (1970-01-01) in Delphi TDateTime is exactly 25569.0
+        let unix_ms = delphi_to_unix_ms(25569.0);
+        assert_eq!(unix_ms, 0);
+    }
+
+    #[test]
+    fn test_raw_m1_record_size() {
+        assert_eq!(std::mem::size_of::<RawM1Record>(), 48);
+    }
+
+    #[test]
+    fn test_raw_tick_record_size() {
+        assert_eq!(std::mem::size_of::<RawTickRecord>(), 24);
+    }
+
+    #[test]
+    fn test_m1_from_raw() {
+        // Construct a raw record with known values
+        let raw = RawM1Record {
+            time: 42795.0, // 2017-03-01
+            open: 1.05,
+            high: 1.06,
+            low: 1.04,
+            close: 1.055,
+            spread: 0.5_f32,
+            volume: 1000,
+        };
+        let parsed = M1Parsed::from_raw(&raw);
+        assert!((parsed.time_ms - 1_488_326_400_000).abs() <= 1);
+        assert_eq!(parsed.open, 1.05);
+        assert_eq!(parsed.high, 1.06);
+        assert_eq!(parsed.low, 1.04);
+        assert_eq!(parsed.close, 1.055);
+        assert!((parsed.spread - 0.5).abs() < f64::EPSILON);
+        assert_eq!(parsed.volume, 1000);
+    }
+
+    #[test]
+    fn test_tick_from_raw() {
+        let raw = RawTickRecord {
+            time: 42795.5, // 2017-03-01 12:00:00
+            ask: 1.05684,
+            bid: 1.05689,
+        };
+        let parsed = TickParsed::from_raw(&raw);
+        assert_eq!(parsed.ask, 1.05684);
+        assert_eq!(parsed.bid, 1.05689);
+        // 12:00:00 = half a day = 43200000 ms offset
+        let expected_ms = 1_488_326_400_000 + 43_200_000;
+        assert!(
+            (parsed.time_ms - expected_ms).abs() <= 1,
+            "Expected ~{expected_ms}, got {}",
+            parsed.time_ms
+        );
+    }
+}