Refactor data streams to support time windows

This commit introduces time windowing functionality to the M1 and tick
data streams.

Key changes:

- **`DataServer::stream_m1_windowed` and
  `DataServer::stream_tick_windowed`**: New methods added to
  `DataServer` to allow streaming data within a specified `from_ms` and
  `to_ms` range.
- **`SymbolChunkIter` modifications**: The `SymbolChunkIter` struct now
  includes `from_ms`, `to_ms`, and `exhausted` fields to manage time
  window filtering.
- **`filter_chunk` helper function**: A new utility function to
  efficiently filter individual data chunks based on the time window.
  This function also handles early termination when `to_ms` is exceeded.
- **`HasTimestamp` trait**: Introduced to provide a common interface for
  accessing timestamps across different record types (`M1Parsed`,
  `TickParsed`), enabling generic filtering.
- **`filter_files` method**: Added to `DataServer` to pre-filter file
  keys based on the time window before creating iterators. This
  optimizes file loading by skipping files that fall entirely outside
  the desired range.
- **`create-m1-stream` and `create-tick-stream`**: The native functions
  in `data_streams.rs` are updated to accept optional `DateTime`
  arguments for `from` and `to` timestamps. They now also handle unknown
  symbols by returning `Value::Void` and return an empty stream if the
  time window results in no data.
- **Documentation**: Updated doc comments for the stream creation
  functions to reflect the new time windowing capabilities and to
  clarify behavior for unknown symbols and empty time windows.

These changes significantly enhance the flexibility of data streaming,
allowing users to query specific time ranges of M1 and tick data more
efficiently.
This commit is contained in:
2026-03-29 22:03:33 +02:00
parent 1f68371920
commit e82d48d1cb
3 changed files with 352 additions and 46 deletions
+260 -29
View File
@@ -23,7 +23,7 @@ pub mod loader;
pub mod records;
use cache::{ChunkVec, FileCache, FileKey};
use records::{DataFormat, M1Parsed, TickParsed};
use records::{DataFormat, HasTimestamp, M1Parsed, TickParsed};
use regex::Regex;
use std::collections::HashMap;
@@ -170,6 +170,11 @@ impl DataServer {
}
}
/// Returns `true` if the symbol exists in the index (has any data files).
pub fn has_symbol(&self, symbol: &str) -> bool {
self.index.symbols.contains_key(symbol)
}
/// Returns a sorted list of all known symbol names.
pub fn symbols(&self) -> Vec<Arc<str>> {
let mut syms: Vec<_> = self.index.symbols.keys().cloned().collect();
@@ -190,19 +195,21 @@ impl DataServer {
/// The iterator yields `Arc<[M1Parsed]>` chunks in chronological order,
/// prefetching the next file while the current one is consumed.
pub fn stream_m1(self: &Arc<Self>, symbol: &str) -> Option<SymbolChunkIter<M1Parsed>> {
let symbol_arc = self.index.symbols.get_key_value(symbol)?.0.clone();
let files: Vec<FileKey> = self
.index
.symbols
.get(symbol)?
.iter()
.filter(|f| f.format == DataFormat::M1)
.cloned()
.collect();
self.stream_m1_windowed(symbol, None, None)
}
if files.is_empty() {
return None;
}
/// Returns a chunk iterator over M1 data within an optional time window.
///
/// Files outside `[from_ms, to_ms]` are skipped entirely (no I/O).
/// Records in boundary files are filtered by exact timestamp.
pub fn stream_m1_windowed(
self: &Arc<Self>,
symbol: &str,
from_ms: Option<i64>,
to_ms: Option<i64>,
) -> Option<SymbolChunkIter<M1Parsed>> {
let symbol_arc = self.index.symbols.get_key_value(symbol)?.0.clone();
let files = self.filter_files(symbol, DataFormat::M1, from_ms, to_ms)?;
// Eagerly load the first file and prefetch the second
self.ensure_m1_loaded(&files[0]);
@@ -219,24 +226,26 @@ impl DataServer {
file_idx: 0,
chunk_idx: 0,
current_file_chunks: None,
from_ms,
to_ms,
exhausted: false,
})
}
/// Returns a chunk iterator over all tick data for a symbol.
pub fn stream_tick(self: &Arc<Self>, symbol: &str) -> Option<SymbolChunkIter<TickParsed>> {
let symbol_arc = self.index.symbols.get_key_value(symbol)?.0.clone();
let files: Vec<FileKey> = self
.index
.symbols
.get(symbol)?
.iter()
.filter(|f| f.format == DataFormat::Tick)
.cloned()
.collect();
self.stream_tick_windowed(symbol, None, None)
}
if files.is_empty() {
return None;
}
/// Returns a chunk iterator over tick data within an optional time window.
pub fn stream_tick_windowed(
self: &Arc<Self>,
symbol: &str,
from_ms: Option<i64>,
to_ms: Option<i64>,
) -> Option<SymbolChunkIter<TickParsed>> {
let symbol_arc = self.index.symbols.get_key_value(symbol)?.0.clone();
let files = self.filter_files(symbol, DataFormat::Tick, from_ms, to_ms)?;
self.ensure_tick_loaded(&files[0]);
if files.len() > 1 {
@@ -252,9 +261,52 @@ impl DataServer {
file_idx: 0,
chunk_idx: 0,
current_file_chunks: None,
from_ms,
to_ms,
exhausted: false,
})
}
/// Filters and returns file keys for a symbol within an optional time window.
///
/// Files are selected by format and then narrowed by `(year, month)` range
/// derived from the timestamps. Returns `None` if no matching files exist.
fn filter_files(
&self,
symbol: &str,
format: DataFormat,
from_ms: Option<i64>,
to_ms: Option<i64>,
) -> Option<Vec<FileKey>> {
let from_ym = from_ms.map(records::unix_ms_to_year_month);
let to_ym = to_ms.map(records::unix_ms_to_year_month);
let files: Vec<FileKey> = self
.index
.symbols
.get(symbol)?
.iter()
.filter(|f| f.format == format)
.filter(|f| {
if let Some((y, m)) = from_ym {
(f.year, f.month) >= (y, m)
} else {
true
}
})
.filter(|f| {
if let Some((y, m)) = to_ym {
(f.year, f.month) <= (y, m)
} else {
true
}
})
.cloned()
.collect();
if files.is_empty() { None } else { Some(files) }
}
// -- M1 loading ----------------------------------------------------------
/// Ensures the given M1 file is loaded into the cache (blocking).
@@ -362,6 +414,75 @@ pub struct SymbolChunkIter<T> {
/// Cached reference to the current file's chunks to avoid repeated
/// cache lookups within the same file.
current_file_chunks: Option<ChunkVec<T>>,
/// Inclusive lower bound for record timestamps. Records before this
/// are skipped in boundary files.
from_ms: Option<i64>,
/// Inclusive upper bound for record timestamps. When a record exceeds
/// this, iteration stops immediately.
to_ms: Option<i64>,
/// Set to `true` when a `to_ms` boundary is crossed, preventing
/// further chunk loading.
exhausted: bool,
}
/// Filters a chunk of records by an optional `[from_ms, to_ms]` time window.
///
/// Returns `None` if the entire chunk falls outside the window, or a new
/// `Arc<[T]>` containing only the matching records. Returns `Some(original)`
/// unchanged when no filtering is needed (the common case for middle files).
///
/// When a record exceeds `to_ms`, returns the truncated slice and sets
/// `exhausted` to `true` so the iterator can stop.
fn filter_chunk<T: HasTimestamp + Clone>(
chunk: &Arc<[T]>,
from_ms: Option<i64>,
to_ms: Option<i64>,
exhausted: &mut bool,
) -> Option<Arc<[T]>> {
// Fast path: no filtering needed
if from_ms.is_none() && to_ms.is_none() {
return Some(Arc::clone(chunk));
}
let from = from_ms.unwrap_or(i64::MIN);
let to = to_ms.unwrap_or(i64::MAX);
// Quick rejection: if the last record is before `from`, skip the whole chunk
if let Some(last) = chunk.last()
&& last.time_ms() < from
{
return None;
}
// Quick rejection: if the first record is past `to`, we're done
if let Some(first) = chunk.first()
&& first.time_ms() > to
{
*exhausted = true;
return None;
}
// Quick acceptance: entire chunk is within bounds
let first_ok = chunk.first().is_none_or(|r| r.time_ms() >= from);
let last_ok = chunk.last().is_none_or(|r| r.time_ms() <= to);
if first_ok && last_ok {
return Some(Arc::clone(chunk));
}
// Partial filtering: collect matching records
let mut filtered = Vec::new();
for rec in chunk.iter() {
let t = rec.time_ms();
if t > to {
*exhausted = true;
break;
}
if t >= from {
filtered.push(rec.clone());
}
}
if filtered.is_empty() { None } else { Some(Arc::from(filtered)) }
}
impl<T> Drop for SymbolChunkIter<T> {
@@ -372,8 +493,15 @@ impl<T> Drop for SymbolChunkIter<T> {
impl SymbolChunkIter<M1Parsed> {
/// Returns the next chunk of M1 records, or `None` when exhausted.
///
/// When a time window is set, records in boundary chunks are filtered
/// and iteration stops early once `to_ms` is exceeded.
pub fn next_chunk(&mut self) -> Option<Arc<[M1Parsed]>> {
loop {
if self.exhausted {
return None;
}
// Try to get chunks for the current file
if self.current_file_chunks.is_none() && self.file_idx < self.files.len() {
let key = &self.files[self.file_idx];
@@ -385,9 +513,14 @@ impl SymbolChunkIter<M1Parsed> {
let chunks = self.current_file_chunks.as_ref()?;
if self.chunk_idx < chunks.len() {
let chunk = chunks[self.chunk_idx].clone();
let raw_chunk = &chunks[self.chunk_idx];
self.chunk_idx += 1;
return Some(chunk);
match filter_chunk(raw_chunk, self.from_ms, self.to_ms, &mut self.exhausted) {
Some(filtered) => return Some(filtered),
None if self.exhausted => return None,
None => continue, // chunk was before `from_ms`, skip it
}
}
// Current file exhausted — advance to next file
@@ -411,6 +544,10 @@ impl SymbolChunkIter<TickParsed> {
/// Returns the next chunk of tick records, or `None` when exhausted.
pub fn next_chunk(&mut self) -> Option<Arc<[TickParsed]>> {
loop {
if self.exhausted {
return None;
}
if self.current_file_chunks.is_none() && self.file_idx < self.files.len() {
let key = &self.files[self.file_idx];
self.server.ensure_tick_loaded(key);
@@ -421,9 +558,14 @@ impl SymbolChunkIter<TickParsed> {
let chunks = self.current_file_chunks.as_ref()?;
if self.chunk_idx < chunks.len() {
let chunk = chunks[self.chunk_idx].clone();
let raw_chunk = &chunks[self.chunk_idx];
self.chunk_idx += 1;
return Some(chunk);
match filter_chunk(raw_chunk, self.from_ms, self.to_ms, &mut self.exhausted) {
Some(filtered) => return Some(filtered),
None if self.exhausted => return None,
None => continue,
}
}
self.file_idx += 1;
@@ -530,4 +672,93 @@ mod tests {
assert_eq!(syms[0].as_ref(), "EURUSD");
assert_eq!(syms[1].as_ref(), "GBPUSD");
}
#[test]
fn test_filter_files_by_time_window() {
let dir = tempfile::tempdir().unwrap();
for name in [
"EURUSD_2019_06.m1",
"EURUSD_2020_01.m1",
"EURUSD_2020_06.m1",
"EURUSD_2020_12.m1",
"EURUSD_2021_03.m1",
] {
std::fs::write(dir.path().join(name), b"").unwrap();
}
let server = DataServer::new(dir.path());
// Full range — all 5 files
let all = server.filter_files("EURUSD", DataFormat::M1, None, None).unwrap();
assert_eq!(all.len(), 5);
// 2020-01-01 to 2020-12-31 — should include 2020_01, 2020_06, 2020_12
let from = chrono::NaiveDate::from_ymd_opt(2020, 1, 1).unwrap()
.and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp_millis();
let to = chrono::NaiveDate::from_ymd_opt(2020, 12, 31).unwrap()
.and_hms_opt(23, 59, 59).unwrap().and_utc().timestamp_millis();
let windowed = server.filter_files("EURUSD", DataFormat::M1, Some(from), Some(to)).unwrap();
assert_eq!(windowed.len(), 3);
assert_eq!(windowed[0].month, 1);
assert_eq!(windowed[1].month, 6);
assert_eq!(windowed[2].month, 12);
// Only from — 2020-06 onwards
let from_june = chrono::NaiveDate::from_ymd_opt(2020, 6, 15).unwrap()
.and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp_millis();
let from_only = server.filter_files("EURUSD", DataFormat::M1, Some(from_june), None).unwrap();
assert_eq!(from_only.len(), 3); // 2020_06, 2020_12, 2021_03
// Window outside all data — returns None
let future = chrono::NaiveDate::from_ymd_opt(2025, 1, 1).unwrap()
.and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp_millis();
assert!(server.filter_files("EURUSD", DataFormat::M1, Some(future), None).is_none());
}
#[test]
fn test_filter_chunk_boundaries() {
// Create a chunk with timestamps 100, 200, 300, 400, 500
let chunk: Arc<[M1Parsed]> = Arc::from(vec![
M1Parsed { time_ms: 100, open: 1.0, high: 1.0, low: 1.0, close: 1.0, spread: 0.0, volume: 0 },
M1Parsed { time_ms: 200, open: 1.0, high: 1.0, low: 1.0, close: 1.0, spread: 0.0, volume: 0 },
M1Parsed { time_ms: 300, open: 1.0, high: 1.0, low: 1.0, close: 1.0, spread: 0.0, volume: 0 },
M1Parsed { time_ms: 400, open: 1.0, high: 1.0, low: 1.0, close: 1.0, spread: 0.0, volume: 0 },
M1Parsed { time_ms: 500, open: 1.0, high: 1.0, low: 1.0, close: 1.0, spread: 0.0, volume: 0 },
]);
let mut exhausted = false;
// No filter — returns original chunk
let result = filter_chunk(&chunk, None, None, &mut exhausted);
assert_eq!(result.unwrap().len(), 5);
assert!(!exhausted);
// from=250 — skips 100, 200
let result = filter_chunk(&chunk, Some(250), None, &mut exhausted);
assert_eq!(result.unwrap().len(), 3); // 300, 400, 500
// to=350 — keeps 100, 200, 300, sets exhausted
exhausted = false;
let result = filter_chunk(&chunk, None, Some(350), &mut exhausted);
assert_eq!(result.unwrap().len(), 3); // 100, 200, 300
assert!(exhausted);
// from=200, to=400 — keeps 200, 300, 400
exhausted = false;
let result = filter_chunk(&chunk, Some(200), Some(400), &mut exhausted);
assert_eq!(result.unwrap().len(), 3); // 200, 300, 400
assert!(exhausted); // 500 > 400
// Entirely before from — returns None
exhausted = false;
let result = filter_chunk(&chunk, Some(600), None, &mut exhausted);
assert!(result.is_none());
assert!(!exhausted);
// Entirely after to — returns None and sets exhausted
exhausted = false;
let result = filter_chunk(&chunk, None, Some(50), &mut exhausted);
assert!(result.is_none());
assert!(exhausted);
}
}
+44
View File
@@ -17,6 +17,26 @@ pub fn delphi_to_unix_ms(dt: f64) -> i64 {
((dt - DELPHI_EPOCH_OFFSET_DAYS) * MS_PER_DAY) as i64
}
/// Extracts the (year, month) from a Unix-millisecond timestamp.
///
/// Used for coarse file-level filtering: data files are named `SYMBOL_YYYY_MM.ext`,
/// so we can skip entire files that fall outside the requested time window.
#[inline]
pub fn unix_ms_to_year_month(ms: i64) -> (u16, u8) {
use chrono::Datelike;
let dt = chrono::DateTime::from_timestamp_millis(ms)
.expect("unix_ms_to_year_month: invalid timestamp");
(dt.year() as u16, dt.month() as u8)
}
/// Common timestamp access for parsed record types.
///
/// Enables generic time-window filtering in [`super::SymbolChunkIter`] without
/// duplicating the logic for M1 and tick records.
pub trait HasTimestamp {
fn time_ms(&self) -> i64;
}
/// Raw M1 (minute) record as stored on disk. 48 bytes, packed.
///
/// Matches the Delphi `TM1Record = packed record` layout exactly:
@@ -75,6 +95,20 @@ pub enum DataFormat {
Tick,
}
impl HasTimestamp for M1Parsed {
#[inline]
fn time_ms(&self) -> i64 {
self.time_ms
}
}
impl HasTimestamp for TickParsed {
#[inline]
fn time_ms(&self) -> i64 {
self.time_ms
}
}
impl M1Parsed {
/// Converts a raw on-disk record to the parsed representation.
///
@@ -136,6 +170,16 @@ mod tests {
assert_eq!(unix_ms, 0);
}
#[test]
fn test_unix_ms_to_year_month() {
// 2017-03-01 00:00:00 UTC = 1488326400000 ms
assert_eq!(unix_ms_to_year_month(1_488_326_400_000), (2017, 3));
// 2020-12-31 23:59:59.999 UTC
assert_eq!(unix_ms_to_year_month(1_609_459_199_999), (2020, 12));
// Unix epoch
assert_eq!(unix_ms_to_year_month(0), (1970, 1));
}
#[test]
fn test_raw_m1_record_size() {
assert_eq!(std::mem::size_of::<RawM1Record>(), 48);
+48 -17
View File
@@ -36,7 +36,7 @@ fn register_m1_stream(env: &Environment) {
env.register_native_fn(
"create-m1-stream",
StaticType::Function(Box::new(Signature {
params: StaticType::Tuple(vec![StaticType::Text]),
params: StaticType::Any,
ret: ret_type,
})),
Purity::Impure,
@@ -46,12 +46,24 @@ fn register_m1_stream(env: &Environment) {
_ => panic!("create-m1-stream: first argument must be a string"),
};
let from_ms = args.get(1).map(|v| match v {
Value::DateTime(ms) => *ms,
_ => panic!("create-m1-stream: 'from' must be a DateTime"),
});
let to_ms = args.get(2).map(|v| match v {
Value::DateTime(ms) => *ms,
_ => panic!("create-m1-stream: 'to' must be a DateTime"),
});
let server = get_data_server()
.expect("create-m1-stream: DataServer not initialized. Call init_data_server() first.");
let mut iter = server
.stream_m1(symbol)
.unwrap_or_else(|| panic!("create-m1-stream: symbol '{}' not found", symbol));
// Unknown symbol → Void (no stream created)
if !server.has_symbol(symbol) {
return Value::Void;
}
let mut iter = server.stream_m1_windowed(symbol, from_ms, to_ms);
// Create the pipeline entry point
let root_stream = Rc::new(RootStream::new());
@@ -61,8 +73,8 @@ fn register_m1_stream(env: &Environment) {
element_type: StaticType::Record(layout.clone()),
};
// Pre-fetch the first chunk
let mut current_chunk = iter.next_chunk();
// Pre-fetch the first chunk (None if time window is empty)
let mut current_chunk = iter.as_mut().and_then(|it| it.next_chunk());
let mut pos = 0;
let generator = move || -> bool {
@@ -93,7 +105,7 @@ fn register_m1_stream(env: &Environment) {
}
// Advance to next chunk (triggers prefetch internally)
current_chunk = iter.next_chunk();
current_chunk = iter.as_mut().and_then(|it| it.next_chunk());
pos = 0;
}
};
@@ -103,8 +115,11 @@ fn register_m1_stream(env: &Environment) {
},
)
.doc("Creates a stream of M1 (minute) OHLCV bars from tick data files.")
.description("Reads pre-cached binary data from the DataServer. The symbol must match a directory entry (e.g. \"EURUSD\", \"AAPL.US\").")
.examples(&["(def ohlc (create-m1-stream \"EURUSD\"))"]);
.description("Reads pre-cached binary data from the DataServer. Optional from/to DateTime arguments restrict the time window (millisecond precision). Returns Void if the symbol is unknown. Returns an empty (immediately exhausted) stream if the symbol exists but the time window contains no data.")
.examples(&[
"(def ohlc (create-m1-stream \"EURUSD\"))",
"(def ohlc (create-m1-stream \"EURUSD\" (date \"2020-01-01\") (date \"2020-12-31\")))",
]);
}
// -- Tick Stream -------------------------------------------------------------
@@ -123,7 +138,7 @@ fn register_tick_stream(env: &Environment) {
env.register_native_fn(
"create-tick-stream",
StaticType::Function(Box::new(Signature {
params: StaticType::Tuple(vec![StaticType::Text]),
params: StaticType::Any,
ret: ret_type,
})),
Purity::Impure,
@@ -133,12 +148,24 @@ fn register_tick_stream(env: &Environment) {
_ => panic!("create-tick-stream: first argument must be a string"),
};
let from_ms = args.get(1).map(|v| match v {
Value::DateTime(ms) => *ms,
_ => panic!("create-tick-stream: 'from' must be a DateTime"),
});
let to_ms = args.get(2).map(|v| match v {
Value::DateTime(ms) => *ms,
_ => panic!("create-tick-stream: 'to' must be a DateTime"),
});
let server = get_data_server()
.expect("create-tick-stream: DataServer not initialized.");
let mut iter = server
.stream_tick(symbol)
.unwrap_or_else(|| panic!("create-tick-stream: symbol '{}' not found", symbol));
// Unknown symbol → Void (no stream created)
if !server.has_symbol(symbol) {
return Value::Void;
}
let mut iter = server.stream_tick_windowed(symbol, from_ms, to_ms);
let root_stream = Rc::new(RootStream::new());
let layout = tick_layout.clone();
@@ -147,7 +174,8 @@ fn register_tick_stream(env: &Environment) {
element_type: StaticType::Record(layout.clone()),
};
let mut current_chunk = iter.next_chunk();
// Pre-fetch the first chunk (None if time window is empty)
let mut current_chunk = iter.as_mut().and_then(|it| it.next_chunk());
let mut pos = 0;
let generator = move || -> bool {
@@ -173,7 +201,7 @@ fn register_tick_stream(env: &Environment) {
return true;
}
current_chunk = iter.next_chunk();
current_chunk = iter.as_mut().and_then(|it| it.next_chunk());
pos = 0;
}
};
@@ -183,6 +211,9 @@ fn register_tick_stream(env: &Environment) {
},
)
.doc("Creates a stream of tick data (ask/bid) from tick data files.")
.description("Reads pre-cached binary data from the DataServer. The symbol must match a directory entry.")
.examples(&["(def ticks (create-tick-stream \"EURUSD\"))"]);
.description("Reads pre-cached binary data from the DataServer. Optional from/to DateTime arguments restrict the time window (millisecond precision). Returns Void if the symbol is unknown. Returns an empty (immediately exhausted) stream if the symbol exists but the time window contains no data.")
.examples(&[
"(def ticks (create-tick-stream \"EURUSD\"))",
"(def ticks (create-tick-stream \"EURUSD\" (date \"2020-01-01\") (date \"2020-12-31\")))",
]);
}