RustAst/src/ast/data_server/records.rs

//! Binary record types for Pepperstone tick data files.
//!
//! The raw types (`RawM1Record`, `RawTickRecord`) mirror the Delphi `packed record`
//! layout used by the C# data exporter. The parsed types (`M1Parsed`, `TickParsed`)
//! hold converted values ready for consumption by the VM.

/// Delphi `TDateTime` epoch offset: days between 1899-12-30 and 1970-01-01.
const DELPHI_EPOCH_OFFSET_DAYS: f64 = 25569.0;
/// Milliseconds per day.
const MS_PER_DAY: f64 = 86_400_000.0;

/// Converts a Delphi `TDateTime` (f64 days since 1899-12-30) to Unix milliseconds.
///
/// Compatible with the existing `Value::DateTime(i64)` representation.
#[inline]
pub fn delphi_to_unix_ms(dt: f64) -> i64 {
    ((dt - DELPHI_EPOCH_OFFSET_DAYS) * MS_PER_DAY) as i64
}

/// Extracts the (year, month) from a Unix-millisecond timestamp.
///
/// Used for coarse file-level filtering: data files are named `SYMBOL_YYYY_MM.ext`,
/// so we can skip entire files that fall outside the requested time window.
#[inline]
pub fn unix_ms_to_year_month(ms: i64) -> (u16, u8) {
    use chrono::Datelike;
    let dt = chrono::DateTime::from_timestamp_millis(ms)
        .expect("unix_ms_to_year_month: invalid timestamp");
    (dt.year() as u16, dt.month() as u8)
}

/// Common timestamp access for parsed record types.
///
/// Enables generic time-window filtering in [`super::SymbolChunkIter`] without
/// duplicating the logic for M1 and tick records.
pub trait HasTimestamp {
    fn time_ms(&self) -> i64;
}

/// Raw M1 (minute) record as stored on disk. 48 bytes, packed.
///
/// Matches the Delphi `TM1Record = packed record` layout exactly:
/// `Time: Double; Open: Double; High: Double; Low: Double; Close: Double; Spread: Single; Volume: Integer;`
#[repr(C, packed)]
#[derive(Clone, Copy)]
pub struct RawM1Record {
    pub time: f64,
    pub open: f64,
    pub high: f64,
    pub low: f64,
    pub close: f64,
    pub spread: f32,
    pub volume: i32,
}

/// Raw tick record as stored on disk. 24 bytes, packed.
///
/// Matches the Delphi `TTickRecord = packed record` layout:
/// `Time: Double; Ask: Double; Bid: Double;`
#[repr(C, packed)]
#[derive(Clone, Copy)]
pub struct RawTickRecord {
    pub time: f64,
    pub ask: f64,
    pub bid: f64,
}

/// Parsed M1 record with converted field types.
///
/// All fields are native-width (`f64`/`i64`) for direct mapping to `Value` variants.
/// `time_ms` is Unix milliseconds (converted from Delphi TDateTime).
#[derive(Debug, Clone, Copy)]
pub struct M1Parsed {
    pub time_ms: i64,
    pub open: f64,
    pub high: f64,
    pub low: f64,
    pub close: f64,
    pub spread: f64,
    pub volume: i64,
}

/// Parsed tick record with converted field types.
#[derive(Debug, Clone, Copy)]
pub struct TickParsed {
    pub time_ms: i64,
    pub ask: f64,
    pub bid: f64,
}

/// Distinguishes the two supported data file formats.
#[derive(Debug, Clone, Copy, Hash, Eq, PartialEq)]
pub enum DataFormat {
    M1,
    Tick,
}

impl HasTimestamp for M1Parsed {
    #[inline]
    fn time_ms(&self) -> i64 {
        self.time_ms
    }
}

impl HasTimestamp for TickParsed {
    #[inline]
    fn time_ms(&self) -> i64 {
        self.time_ms
    }
}

impl M1Parsed {
    /// Converts a raw on-disk record to the parsed representation.
    ///
    /// Uses `read_unaligned` because `RawM1Record` is `packed` and field
    /// access would be unaligned on architectures that require alignment.
    #[inline]
    pub fn from_raw(raw: &RawM1Record) -> Self {
        // SAFETY: `raw` points to a valid `RawM1Record`. We use `read_unaligned`
        // to safely read potentially unaligned fields from the packed struct.
        unsafe {
            Self {
                time_ms: delphi_to_unix_ms(std::ptr::addr_of!(raw.time).read_unaligned()),
                open: std::ptr::addr_of!(raw.open).read_unaligned(),
                high: std::ptr::addr_of!(raw.high).read_unaligned(),
                low: std::ptr::addr_of!(raw.low).read_unaligned(),
                close: std::ptr::addr_of!(raw.close).read_unaligned(),
                spread: std::ptr::addr_of!(raw.spread).read_unaligned() as f64,
                volume: std::ptr::addr_of!(raw.volume).read_unaligned() as i64,
            }
        }
    }
}

impl TickParsed {
    /// Converts a raw on-disk tick record to the parsed representation.
    #[inline]
    pub fn from_raw(raw: &RawTickRecord) -> Self {
        unsafe {
            Self {
                time_ms: delphi_to_unix_ms(std::ptr::addr_of!(raw.time).read_unaligned()),
                ask: std::ptr::addr_of!(raw.ask).read_unaligned(),
                bid: std::ptr::addr_of!(raw.bid).read_unaligned(),
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_delphi_datetime_conversion() {
        // 2017-03-01 00:00:00 UTC = Unix timestamp 1488326400000 ms
        // Delphi TDateTime for 2017-03-01 = 42795.0
        let delphi_dt = 42795.0;
        let unix_ms = delphi_to_unix_ms(delphi_dt);
        // Allow 1ms tolerance for floating-point rounding
        assert!(
            (unix_ms - 1_488_326_400_000).abs() <= 1,
            "Expected ~1488326400000, got {unix_ms}"
        );
    }

    #[test]
    fn test_delphi_epoch() {
        // The Unix epoch (1970-01-01) in Delphi TDateTime is exactly 25569.0
        let unix_ms = delphi_to_unix_ms(25569.0);
        assert_eq!(unix_ms, 0);
    }

    #[test]
    fn test_unix_ms_to_year_month() {
        // 2017-03-01 00:00:00 UTC = 1488326400000 ms
        assert_eq!(unix_ms_to_year_month(1_488_326_400_000), (2017, 3));
        // 2020-12-31 23:59:59.999 UTC
        assert_eq!(unix_ms_to_year_month(1_609_459_199_999), (2020, 12));
        // Unix epoch
        assert_eq!(unix_ms_to_year_month(0), (1970, 1));
    }

    #[test]
    fn test_raw_m1_record_size() {
        assert_eq!(std::mem::size_of::<RawM1Record>(), 48);
    }

    #[test]
    fn test_raw_tick_record_size() {
        assert_eq!(std::mem::size_of::<RawTickRecord>(), 24);
    }

    #[test]
    fn test_m1_from_raw() {
        // Construct a raw record with known values
        let raw = RawM1Record {
            time: 42795.0, // 2017-03-01
            open: 1.05,
            high: 1.06,
            low: 1.04,
            close: 1.055,
            spread: 0.5_f32,
            volume: 1000,
        };
        let parsed = M1Parsed::from_raw(&raw);
        assert!((parsed.time_ms - 1_488_326_400_000).abs() <= 1);
        assert_eq!(parsed.open, 1.05);
        assert_eq!(parsed.high, 1.06);
        assert_eq!(parsed.low, 1.04);
        assert_eq!(parsed.close, 1.055);
        assert!((parsed.spread - 0.5).abs() < f64::EPSILON);
        assert_eq!(parsed.volume, 1000);
    }

    #[test]
    fn test_tick_from_raw() {
        let raw = RawTickRecord {
            time: 42795.5, // 2017-03-01 12:00:00
            ask: 1.05684,
            bid: 1.05689,
        };
        let parsed = TickParsed::from_raw(&raw);
        assert_eq!(parsed.ask, 1.05684);
        assert_eq!(parsed.bid, 1.05689);
        // 12:00:00 = half a day = 43200000 ms offset
        let expected_ms = 1_488_326_400_000 + 43_200_000;
        assert!(
            (parsed.time_ms - expected_ms).abs() <= 1,
            "Expected ~{expected_ms}, got {}",
            parsed.time_ms
        );
    }
}