diff --git a/src/ast/data_server/mod.rs b/src/ast/data_server/mod.rs index eae5083..909aef1 100644 --- a/src/ast/data_server/mod.rs +++ b/src/ast/data_server/mod.rs @@ -23,7 +23,7 @@ pub mod loader; pub mod records; use cache::{ChunkVec, FileCache, FileKey}; -use records::{DataFormat, M1Parsed, TickParsed}; +use records::{DataFormat, HasTimestamp, M1Parsed, TickParsed}; use regex::Regex; use std::collections::HashMap; @@ -170,6 +170,11 @@ impl DataServer { } } + /// Returns `true` if the symbol exists in the index (has any data files). + pub fn has_symbol(&self, symbol: &str) -> bool { + self.index.symbols.contains_key(symbol) + } + /// Returns a sorted list of all known symbol names. pub fn symbols(&self) -> Vec> { let mut syms: Vec<_> = self.index.symbols.keys().cloned().collect(); @@ -190,19 +195,21 @@ impl DataServer { /// The iterator yields `Arc<[M1Parsed]>` chunks in chronological order, /// prefetching the next file while the current one is consumed. pub fn stream_m1(self: &Arc, symbol: &str) -> Option> { - let symbol_arc = self.index.symbols.get_key_value(symbol)?.0.clone(); - let files: Vec = self - .index - .symbols - .get(symbol)? - .iter() - .filter(|f| f.format == DataFormat::M1) - .cloned() - .collect(); + self.stream_m1_windowed(symbol, None, None) + } - if files.is_empty() { - return None; - } + /// Returns a chunk iterator over M1 data within an optional time window. + /// + /// Files outside `[from_ms, to_ms]` are skipped entirely (no I/O). + /// Records in boundary files are filtered by exact timestamp. + pub fn stream_m1_windowed( + self: &Arc, + symbol: &str, + from_ms: Option, + to_ms: Option, + ) -> Option> { + let symbol_arc = self.index.symbols.get_key_value(symbol)?.0.clone(); + let files = self.filter_files(symbol, DataFormat::M1, from_ms, to_ms)?; // Eagerly load the first file and prefetch the second self.ensure_m1_loaded(&files[0]); @@ -219,24 +226,26 @@ impl DataServer { file_idx: 0, chunk_idx: 0, current_file_chunks: None, + from_ms, + to_ms, + exhausted: false, }) } /// Returns a chunk iterator over all tick data for a symbol. pub fn stream_tick(self: &Arc, symbol: &str) -> Option> { - let symbol_arc = self.index.symbols.get_key_value(symbol)?.0.clone(); - let files: Vec = self - .index - .symbols - .get(symbol)? - .iter() - .filter(|f| f.format == DataFormat::Tick) - .cloned() - .collect(); + self.stream_tick_windowed(symbol, None, None) + } - if files.is_empty() { - return None; - } + /// Returns a chunk iterator over tick data within an optional time window. + pub fn stream_tick_windowed( + self: &Arc, + symbol: &str, + from_ms: Option, + to_ms: Option, + ) -> Option> { + let symbol_arc = self.index.symbols.get_key_value(symbol)?.0.clone(); + let files = self.filter_files(symbol, DataFormat::Tick, from_ms, to_ms)?; self.ensure_tick_loaded(&files[0]); if files.len() > 1 { @@ -252,9 +261,52 @@ impl DataServer { file_idx: 0, chunk_idx: 0, current_file_chunks: None, + from_ms, + to_ms, + exhausted: false, }) } + /// Filters and returns file keys for a symbol within an optional time window. + /// + /// Files are selected by format and then narrowed by `(year, month)` range + /// derived from the timestamps. Returns `None` if no matching files exist. + fn filter_files( + &self, + symbol: &str, + format: DataFormat, + from_ms: Option, + to_ms: Option, + ) -> Option> { + let from_ym = from_ms.map(records::unix_ms_to_year_month); + let to_ym = to_ms.map(records::unix_ms_to_year_month); + + let files: Vec = self + .index + .symbols + .get(symbol)? + .iter() + .filter(|f| f.format == format) + .filter(|f| { + if let Some((y, m)) = from_ym { + (f.year, f.month) >= (y, m) + } else { + true + } + }) + .filter(|f| { + if let Some((y, m)) = to_ym { + (f.year, f.month) <= (y, m) + } else { + true + } + }) + .cloned() + .collect(); + + if files.is_empty() { None } else { Some(files) } + } + // -- M1 loading ---------------------------------------------------------- /// Ensures the given M1 file is loaded into the cache (blocking). @@ -362,6 +414,75 @@ pub struct SymbolChunkIter { /// Cached reference to the current file's chunks to avoid repeated /// cache lookups within the same file. current_file_chunks: Option>, + /// Inclusive lower bound for record timestamps. Records before this + /// are skipped in boundary files. + from_ms: Option, + /// Inclusive upper bound for record timestamps. When a record exceeds + /// this, iteration stops immediately. + to_ms: Option, + /// Set to `true` when a `to_ms` boundary is crossed, preventing + /// further chunk loading. + exhausted: bool, +} + +/// Filters a chunk of records by an optional `[from_ms, to_ms]` time window. +/// +/// Returns `None` if the entire chunk falls outside the window, or a new +/// `Arc<[T]>` containing only the matching records. Returns `Some(original)` +/// unchanged when no filtering is needed (the common case for middle files). +/// +/// When a record exceeds `to_ms`, returns the truncated slice and sets +/// `exhausted` to `true` so the iterator can stop. +fn filter_chunk( + chunk: &Arc<[T]>, + from_ms: Option, + to_ms: Option, + exhausted: &mut bool, +) -> Option> { + // Fast path: no filtering needed + if from_ms.is_none() && to_ms.is_none() { + return Some(Arc::clone(chunk)); + } + + let from = from_ms.unwrap_or(i64::MIN); + let to = to_ms.unwrap_or(i64::MAX); + + // Quick rejection: if the last record is before `from`, skip the whole chunk + if let Some(last) = chunk.last() + && last.time_ms() < from + { + return None; + } + + // Quick rejection: if the first record is past `to`, we're done + if let Some(first) = chunk.first() + && first.time_ms() > to + { + *exhausted = true; + return None; + } + + // Quick acceptance: entire chunk is within bounds + let first_ok = chunk.first().is_none_or(|r| r.time_ms() >= from); + let last_ok = chunk.last().is_none_or(|r| r.time_ms() <= to); + if first_ok && last_ok { + return Some(Arc::clone(chunk)); + } + + // Partial filtering: collect matching records + let mut filtered = Vec::new(); + for rec in chunk.iter() { + let t = rec.time_ms(); + if t > to { + *exhausted = true; + break; + } + if t >= from { + filtered.push(rec.clone()); + } + } + + if filtered.is_empty() { None } else { Some(Arc::from(filtered)) } } impl Drop for SymbolChunkIter { @@ -372,8 +493,15 @@ impl Drop for SymbolChunkIter { impl SymbolChunkIter { /// Returns the next chunk of M1 records, or `None` when exhausted. + /// + /// When a time window is set, records in boundary chunks are filtered + /// and iteration stops early once `to_ms` is exceeded. pub fn next_chunk(&mut self) -> Option> { loop { + if self.exhausted { + return None; + } + // Try to get chunks for the current file if self.current_file_chunks.is_none() && self.file_idx < self.files.len() { let key = &self.files[self.file_idx]; @@ -385,9 +513,14 @@ impl SymbolChunkIter { let chunks = self.current_file_chunks.as_ref()?; if self.chunk_idx < chunks.len() { - let chunk = chunks[self.chunk_idx].clone(); + let raw_chunk = &chunks[self.chunk_idx]; self.chunk_idx += 1; - return Some(chunk); + + match filter_chunk(raw_chunk, self.from_ms, self.to_ms, &mut self.exhausted) { + Some(filtered) => return Some(filtered), + None if self.exhausted => return None, + None => continue, // chunk was before `from_ms`, skip it + } } // Current file exhausted — advance to next file @@ -411,6 +544,10 @@ impl SymbolChunkIter { /// Returns the next chunk of tick records, or `None` when exhausted. pub fn next_chunk(&mut self) -> Option> { loop { + if self.exhausted { + return None; + } + if self.current_file_chunks.is_none() && self.file_idx < self.files.len() { let key = &self.files[self.file_idx]; self.server.ensure_tick_loaded(key); @@ -421,9 +558,14 @@ impl SymbolChunkIter { let chunks = self.current_file_chunks.as_ref()?; if self.chunk_idx < chunks.len() { - let chunk = chunks[self.chunk_idx].clone(); + let raw_chunk = &chunks[self.chunk_idx]; self.chunk_idx += 1; - return Some(chunk); + + match filter_chunk(raw_chunk, self.from_ms, self.to_ms, &mut self.exhausted) { + Some(filtered) => return Some(filtered), + None if self.exhausted => return None, + None => continue, + } } self.file_idx += 1; @@ -530,4 +672,93 @@ mod tests { assert_eq!(syms[0].as_ref(), "EURUSD"); assert_eq!(syms[1].as_ref(), "GBPUSD"); } + + #[test] + fn test_filter_files_by_time_window() { + let dir = tempfile::tempdir().unwrap(); + for name in [ + "EURUSD_2019_06.m1", + "EURUSD_2020_01.m1", + "EURUSD_2020_06.m1", + "EURUSD_2020_12.m1", + "EURUSD_2021_03.m1", + ] { + std::fs::write(dir.path().join(name), b"").unwrap(); + } + + let server = DataServer::new(dir.path()); + + // Full range — all 5 files + let all = server.filter_files("EURUSD", DataFormat::M1, None, None).unwrap(); + assert_eq!(all.len(), 5); + + // 2020-01-01 to 2020-12-31 — should include 2020_01, 2020_06, 2020_12 + let from = chrono::NaiveDate::from_ymd_opt(2020, 1, 1).unwrap() + .and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp_millis(); + let to = chrono::NaiveDate::from_ymd_opt(2020, 12, 31).unwrap() + .and_hms_opt(23, 59, 59).unwrap().and_utc().timestamp_millis(); + let windowed = server.filter_files("EURUSD", DataFormat::M1, Some(from), Some(to)).unwrap(); + assert_eq!(windowed.len(), 3); + assert_eq!(windowed[0].month, 1); + assert_eq!(windowed[1].month, 6); + assert_eq!(windowed[2].month, 12); + + // Only from — 2020-06 onwards + let from_june = chrono::NaiveDate::from_ymd_opt(2020, 6, 15).unwrap() + .and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp_millis(); + let from_only = server.filter_files("EURUSD", DataFormat::M1, Some(from_june), None).unwrap(); + assert_eq!(from_only.len(), 3); // 2020_06, 2020_12, 2021_03 + + // Window outside all data — returns None + let future = chrono::NaiveDate::from_ymd_opt(2025, 1, 1).unwrap() + .and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp_millis(); + assert!(server.filter_files("EURUSD", DataFormat::M1, Some(future), None).is_none()); + } + + #[test] + fn test_filter_chunk_boundaries() { + // Create a chunk with timestamps 100, 200, 300, 400, 500 + let chunk: Arc<[M1Parsed]> = Arc::from(vec![ + M1Parsed { time_ms: 100, open: 1.0, high: 1.0, low: 1.0, close: 1.0, spread: 0.0, volume: 0 }, + M1Parsed { time_ms: 200, open: 1.0, high: 1.0, low: 1.0, close: 1.0, spread: 0.0, volume: 0 }, + M1Parsed { time_ms: 300, open: 1.0, high: 1.0, low: 1.0, close: 1.0, spread: 0.0, volume: 0 }, + M1Parsed { time_ms: 400, open: 1.0, high: 1.0, low: 1.0, close: 1.0, spread: 0.0, volume: 0 }, + M1Parsed { time_ms: 500, open: 1.0, high: 1.0, low: 1.0, close: 1.0, spread: 0.0, volume: 0 }, + ]); + + let mut exhausted = false; + + // No filter — returns original chunk + let result = filter_chunk(&chunk, None, None, &mut exhausted); + assert_eq!(result.unwrap().len(), 5); + assert!(!exhausted); + + // from=250 — skips 100, 200 + let result = filter_chunk(&chunk, Some(250), None, &mut exhausted); + assert_eq!(result.unwrap().len(), 3); // 300, 400, 500 + + // to=350 — keeps 100, 200, 300, sets exhausted + exhausted = false; + let result = filter_chunk(&chunk, None, Some(350), &mut exhausted); + assert_eq!(result.unwrap().len(), 3); // 100, 200, 300 + assert!(exhausted); + + // from=200, to=400 — keeps 200, 300, 400 + exhausted = false; + let result = filter_chunk(&chunk, Some(200), Some(400), &mut exhausted); + assert_eq!(result.unwrap().len(), 3); // 200, 300, 400 + assert!(exhausted); // 500 > 400 + + // Entirely before from — returns None + exhausted = false; + let result = filter_chunk(&chunk, Some(600), None, &mut exhausted); + assert!(result.is_none()); + assert!(!exhausted); + + // Entirely after to — returns None and sets exhausted + exhausted = false; + let result = filter_chunk(&chunk, None, Some(50), &mut exhausted); + assert!(result.is_none()); + assert!(exhausted); + } } diff --git a/src/ast/data_server/records.rs b/src/ast/data_server/records.rs index 8c10f9b..b79569c 100644 --- a/src/ast/data_server/records.rs +++ b/src/ast/data_server/records.rs @@ -17,6 +17,26 @@ pub fn delphi_to_unix_ms(dt: f64) -> i64 { ((dt - DELPHI_EPOCH_OFFSET_DAYS) * MS_PER_DAY) as i64 } +/// Extracts the (year, month) from a Unix-millisecond timestamp. +/// +/// Used for coarse file-level filtering: data files are named `SYMBOL_YYYY_MM.ext`, +/// so we can skip entire files that fall outside the requested time window. +#[inline] +pub fn unix_ms_to_year_month(ms: i64) -> (u16, u8) { + use chrono::Datelike; + let dt = chrono::DateTime::from_timestamp_millis(ms) + .expect("unix_ms_to_year_month: invalid timestamp"); + (dt.year() as u16, dt.month() as u8) +} + +/// Common timestamp access for parsed record types. +/// +/// Enables generic time-window filtering in [`super::SymbolChunkIter`] without +/// duplicating the logic for M1 and tick records. +pub trait HasTimestamp { + fn time_ms(&self) -> i64; +} + /// Raw M1 (minute) record as stored on disk. 48 bytes, packed. /// /// Matches the Delphi `TM1Record = packed record` layout exactly: @@ -75,6 +95,20 @@ pub enum DataFormat { Tick, } +impl HasTimestamp for M1Parsed { + #[inline] + fn time_ms(&self) -> i64 { + self.time_ms + } +} + +impl HasTimestamp for TickParsed { + #[inline] + fn time_ms(&self) -> i64 { + self.time_ms + } +} + impl M1Parsed { /// Converts a raw on-disk record to the parsed representation. /// @@ -136,6 +170,16 @@ mod tests { assert_eq!(unix_ms, 0); } + #[test] + fn test_unix_ms_to_year_month() { + // 2017-03-01 00:00:00 UTC = 1488326400000 ms + assert_eq!(unix_ms_to_year_month(1_488_326_400_000), (2017, 3)); + // 2020-12-31 23:59:59.999 UTC + assert_eq!(unix_ms_to_year_month(1_609_459_199_999), (2020, 12)); + // Unix epoch + assert_eq!(unix_ms_to_year_month(0), (1970, 1)); + } + #[test] fn test_raw_m1_record_size() { assert_eq!(std::mem::size_of::(), 48); diff --git a/src/ast/rtl/data_streams.rs b/src/ast/rtl/data_streams.rs index 37464f9..0d6e1fd 100644 --- a/src/ast/rtl/data_streams.rs +++ b/src/ast/rtl/data_streams.rs @@ -36,7 +36,7 @@ fn register_m1_stream(env: &Environment) { env.register_native_fn( "create-m1-stream", StaticType::Function(Box::new(Signature { - params: StaticType::Tuple(vec![StaticType::Text]), + params: StaticType::Any, ret: ret_type, })), Purity::Impure, @@ -46,12 +46,24 @@ fn register_m1_stream(env: &Environment) { _ => panic!("create-m1-stream: first argument must be a string"), }; + let from_ms = args.get(1).map(|v| match v { + Value::DateTime(ms) => *ms, + _ => panic!("create-m1-stream: 'from' must be a DateTime"), + }); + let to_ms = args.get(2).map(|v| match v { + Value::DateTime(ms) => *ms, + _ => panic!("create-m1-stream: 'to' must be a DateTime"), + }); + let server = get_data_server() .expect("create-m1-stream: DataServer not initialized. Call init_data_server() first."); - let mut iter = server - .stream_m1(symbol) - .unwrap_or_else(|| panic!("create-m1-stream: symbol '{}' not found", symbol)); + // Unknown symbol → Void (no stream created) + if !server.has_symbol(symbol) { + return Value::Void; + } + + let mut iter = server.stream_m1_windowed(symbol, from_ms, to_ms); // Create the pipeline entry point let root_stream = Rc::new(RootStream::new()); @@ -61,8 +73,8 @@ fn register_m1_stream(env: &Environment) { element_type: StaticType::Record(layout.clone()), }; - // Pre-fetch the first chunk - let mut current_chunk = iter.next_chunk(); + // Pre-fetch the first chunk (None if time window is empty) + let mut current_chunk = iter.as_mut().and_then(|it| it.next_chunk()); let mut pos = 0; let generator = move || -> bool { @@ -93,7 +105,7 @@ fn register_m1_stream(env: &Environment) { } // Advance to next chunk (triggers prefetch internally) - current_chunk = iter.next_chunk(); + current_chunk = iter.as_mut().and_then(|it| it.next_chunk()); pos = 0; } }; @@ -103,8 +115,11 @@ fn register_m1_stream(env: &Environment) { }, ) .doc("Creates a stream of M1 (minute) OHLCV bars from tick data files.") - .description("Reads pre-cached binary data from the DataServer. The symbol must match a directory entry (e.g. \"EURUSD\", \"AAPL.US\").") - .examples(&["(def ohlc (create-m1-stream \"EURUSD\"))"]); + .description("Reads pre-cached binary data from the DataServer. Optional from/to DateTime arguments restrict the time window (millisecond precision). Returns Void if the symbol is unknown. Returns an empty (immediately exhausted) stream if the symbol exists but the time window contains no data.") + .examples(&[ + "(def ohlc (create-m1-stream \"EURUSD\"))", + "(def ohlc (create-m1-stream \"EURUSD\" (date \"2020-01-01\") (date \"2020-12-31\")))", + ]); } // -- Tick Stream ------------------------------------------------------------- @@ -123,7 +138,7 @@ fn register_tick_stream(env: &Environment) { env.register_native_fn( "create-tick-stream", StaticType::Function(Box::new(Signature { - params: StaticType::Tuple(vec![StaticType::Text]), + params: StaticType::Any, ret: ret_type, })), Purity::Impure, @@ -133,12 +148,24 @@ fn register_tick_stream(env: &Environment) { _ => panic!("create-tick-stream: first argument must be a string"), }; + let from_ms = args.get(1).map(|v| match v { + Value::DateTime(ms) => *ms, + _ => panic!("create-tick-stream: 'from' must be a DateTime"), + }); + let to_ms = args.get(2).map(|v| match v { + Value::DateTime(ms) => *ms, + _ => panic!("create-tick-stream: 'to' must be a DateTime"), + }); + let server = get_data_server() .expect("create-tick-stream: DataServer not initialized."); - let mut iter = server - .stream_tick(symbol) - .unwrap_or_else(|| panic!("create-tick-stream: symbol '{}' not found", symbol)); + // Unknown symbol → Void (no stream created) + if !server.has_symbol(symbol) { + return Value::Void; + } + + let mut iter = server.stream_tick_windowed(symbol, from_ms, to_ms); let root_stream = Rc::new(RootStream::new()); let layout = tick_layout.clone(); @@ -147,7 +174,8 @@ fn register_tick_stream(env: &Environment) { element_type: StaticType::Record(layout.clone()), }; - let mut current_chunk = iter.next_chunk(); + // Pre-fetch the first chunk (None if time window is empty) + let mut current_chunk = iter.as_mut().and_then(|it| it.next_chunk()); let mut pos = 0; let generator = move || -> bool { @@ -173,7 +201,7 @@ fn register_tick_stream(env: &Environment) { return true; } - current_chunk = iter.next_chunk(); + current_chunk = iter.as_mut().and_then(|it| it.next_chunk()); pos = 0; } }; @@ -183,6 +211,9 @@ fn register_tick_stream(env: &Environment) { }, ) .doc("Creates a stream of tick data (ask/bid) from tick data files.") - .description("Reads pre-cached binary data from the DataServer. The symbol must match a directory entry.") - .examples(&["(def ticks (create-tick-stream \"EURUSD\"))"]); + .description("Reads pre-cached binary data from the DataServer. Optional from/to DateTime arguments restrict the time window (millisecond precision). Returns Void if the symbol is unknown. Returns an empty (immediately exhausted) stream if the symbol exists but the time window contains no data.") + .examples(&[ + "(def ticks (create-tick-stream \"EURUSD\"))", + "(def ticks (create-tick-stream \"EURUSD\" (date \"2020-01-01\") (date \"2020-12-31\")))", + ]); }