Refactor data streams to support time windows
This commit introduces time windowing functionality to the M1 and tick data streams. Key changes: - **`DataServer::stream_m1_windowed` and `DataServer::stream_tick_windowed`**: New methods added to `DataServer` to allow streaming data within a specified `from_ms` and `to_ms` range. - **`SymbolChunkIter` modifications**: The `SymbolChunkIter` struct now includes `from_ms`, `to_ms`, and `exhausted` fields to manage time window filtering. - **`filter_chunk` helper function**: A new utility function to efficiently filter individual data chunks based on the time window. This function also handles early termination when `to_ms` is exceeded. - **`HasTimestamp` trait**: Introduced to provide a common interface for accessing timestamps across different record types (`M1Parsed`, `TickParsed`), enabling generic filtering. - **`filter_files` method**: Added to `DataServer` to pre-filter file keys based on the time window before creating iterators. This optimizes file loading by skipping files that fall entirely outside the desired range. - **`create-m1-stream` and `create-tick-stream`**: The native functions in `data_streams.rs` are updated to accept optional `DateTime` arguments for `from` and `to` timestamps. They now also handle unknown symbols by returning `Value::Void` and return an empty stream if the time window results in no data. - **Documentation**: Updated doc comments for the stream creation functions to reflect the new time windowing capabilities and to clarify behavior for unknown symbols and empty time windows. These changes significantly enhance the flexibility of data streaming, allowing users to query specific time ranges of M1 and tick data more efficiently.
This commit is contained in:
+260
-29
@@ -23,7 +23,7 @@ pub mod loader;
|
|||||||
pub mod records;
|
pub mod records;
|
||||||
|
|
||||||
use cache::{ChunkVec, FileCache, FileKey};
|
use cache::{ChunkVec, FileCache, FileKey};
|
||||||
use records::{DataFormat, M1Parsed, TickParsed};
|
use records::{DataFormat, HasTimestamp, M1Parsed, TickParsed};
|
||||||
|
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
@@ -170,6 +170,11 @@ impl DataServer {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns `true` if the symbol exists in the index (has any data files).
|
||||||
|
pub fn has_symbol(&self, symbol: &str) -> bool {
|
||||||
|
self.index.symbols.contains_key(symbol)
|
||||||
|
}
|
||||||
|
|
||||||
/// Returns a sorted list of all known symbol names.
|
/// Returns a sorted list of all known symbol names.
|
||||||
pub fn symbols(&self) -> Vec<Arc<str>> {
|
pub fn symbols(&self) -> Vec<Arc<str>> {
|
||||||
let mut syms: Vec<_> = self.index.symbols.keys().cloned().collect();
|
let mut syms: Vec<_> = self.index.symbols.keys().cloned().collect();
|
||||||
@@ -190,19 +195,21 @@ impl DataServer {
|
|||||||
/// The iterator yields `Arc<[M1Parsed]>` chunks in chronological order,
|
/// The iterator yields `Arc<[M1Parsed]>` chunks in chronological order,
|
||||||
/// prefetching the next file while the current one is consumed.
|
/// prefetching the next file while the current one is consumed.
|
||||||
pub fn stream_m1(self: &Arc<Self>, symbol: &str) -> Option<SymbolChunkIter<M1Parsed>> {
|
pub fn stream_m1(self: &Arc<Self>, symbol: &str) -> Option<SymbolChunkIter<M1Parsed>> {
|
||||||
let symbol_arc = self.index.symbols.get_key_value(symbol)?.0.clone();
|
self.stream_m1_windowed(symbol, None, None)
|
||||||
let files: Vec<FileKey> = self
|
}
|
||||||
.index
|
|
||||||
.symbols
|
|
||||||
.get(symbol)?
|
|
||||||
.iter()
|
|
||||||
.filter(|f| f.format == DataFormat::M1)
|
|
||||||
.cloned()
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
if files.is_empty() {
|
/// Returns a chunk iterator over M1 data within an optional time window.
|
||||||
return None;
|
///
|
||||||
}
|
/// Files outside `[from_ms, to_ms]` are skipped entirely (no I/O).
|
||||||
|
/// Records in boundary files are filtered by exact timestamp.
|
||||||
|
pub fn stream_m1_windowed(
|
||||||
|
self: &Arc<Self>,
|
||||||
|
symbol: &str,
|
||||||
|
from_ms: Option<i64>,
|
||||||
|
to_ms: Option<i64>,
|
||||||
|
) -> Option<SymbolChunkIter<M1Parsed>> {
|
||||||
|
let symbol_arc = self.index.symbols.get_key_value(symbol)?.0.clone();
|
||||||
|
let files = self.filter_files(symbol, DataFormat::M1, from_ms, to_ms)?;
|
||||||
|
|
||||||
// Eagerly load the first file and prefetch the second
|
// Eagerly load the first file and prefetch the second
|
||||||
self.ensure_m1_loaded(&files[0]);
|
self.ensure_m1_loaded(&files[0]);
|
||||||
@@ -219,24 +226,26 @@ impl DataServer {
|
|||||||
file_idx: 0,
|
file_idx: 0,
|
||||||
chunk_idx: 0,
|
chunk_idx: 0,
|
||||||
current_file_chunks: None,
|
current_file_chunks: None,
|
||||||
|
from_ms,
|
||||||
|
to_ms,
|
||||||
|
exhausted: false,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Returns a chunk iterator over all tick data for a symbol.
|
/// Returns a chunk iterator over all tick data for a symbol.
|
||||||
pub fn stream_tick(self: &Arc<Self>, symbol: &str) -> Option<SymbolChunkIter<TickParsed>> {
|
pub fn stream_tick(self: &Arc<Self>, symbol: &str) -> Option<SymbolChunkIter<TickParsed>> {
|
||||||
let symbol_arc = self.index.symbols.get_key_value(symbol)?.0.clone();
|
self.stream_tick_windowed(symbol, None, None)
|
||||||
let files: Vec<FileKey> = self
|
}
|
||||||
.index
|
|
||||||
.symbols
|
|
||||||
.get(symbol)?
|
|
||||||
.iter()
|
|
||||||
.filter(|f| f.format == DataFormat::Tick)
|
|
||||||
.cloned()
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
if files.is_empty() {
|
/// Returns a chunk iterator over tick data within an optional time window.
|
||||||
return None;
|
pub fn stream_tick_windowed(
|
||||||
}
|
self: &Arc<Self>,
|
||||||
|
symbol: &str,
|
||||||
|
from_ms: Option<i64>,
|
||||||
|
to_ms: Option<i64>,
|
||||||
|
) -> Option<SymbolChunkIter<TickParsed>> {
|
||||||
|
let symbol_arc = self.index.symbols.get_key_value(symbol)?.0.clone();
|
||||||
|
let files = self.filter_files(symbol, DataFormat::Tick, from_ms, to_ms)?;
|
||||||
|
|
||||||
self.ensure_tick_loaded(&files[0]);
|
self.ensure_tick_loaded(&files[0]);
|
||||||
if files.len() > 1 {
|
if files.len() > 1 {
|
||||||
@@ -252,9 +261,52 @@ impl DataServer {
|
|||||||
file_idx: 0,
|
file_idx: 0,
|
||||||
chunk_idx: 0,
|
chunk_idx: 0,
|
||||||
current_file_chunks: None,
|
current_file_chunks: None,
|
||||||
|
from_ms,
|
||||||
|
to_ms,
|
||||||
|
exhausted: false,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Filters and returns file keys for a symbol within an optional time window.
|
||||||
|
///
|
||||||
|
/// Files are selected by format and then narrowed by `(year, month)` range
|
||||||
|
/// derived from the timestamps. Returns `None` if no matching files exist.
|
||||||
|
fn filter_files(
|
||||||
|
&self,
|
||||||
|
symbol: &str,
|
||||||
|
format: DataFormat,
|
||||||
|
from_ms: Option<i64>,
|
||||||
|
to_ms: Option<i64>,
|
||||||
|
) -> Option<Vec<FileKey>> {
|
||||||
|
let from_ym = from_ms.map(records::unix_ms_to_year_month);
|
||||||
|
let to_ym = to_ms.map(records::unix_ms_to_year_month);
|
||||||
|
|
||||||
|
let files: Vec<FileKey> = self
|
||||||
|
.index
|
||||||
|
.symbols
|
||||||
|
.get(symbol)?
|
||||||
|
.iter()
|
||||||
|
.filter(|f| f.format == format)
|
||||||
|
.filter(|f| {
|
||||||
|
if let Some((y, m)) = from_ym {
|
||||||
|
(f.year, f.month) >= (y, m)
|
||||||
|
} else {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.filter(|f| {
|
||||||
|
if let Some((y, m)) = to_ym {
|
||||||
|
(f.year, f.month) <= (y, m)
|
||||||
|
} else {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.cloned()
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
if files.is_empty() { None } else { Some(files) }
|
||||||
|
}
|
||||||
|
|
||||||
// -- M1 loading ----------------------------------------------------------
|
// -- M1 loading ----------------------------------------------------------
|
||||||
|
|
||||||
/// Ensures the given M1 file is loaded into the cache (blocking).
|
/// Ensures the given M1 file is loaded into the cache (blocking).
|
||||||
@@ -362,6 +414,75 @@ pub struct SymbolChunkIter<T> {
|
|||||||
/// Cached reference to the current file's chunks to avoid repeated
|
/// Cached reference to the current file's chunks to avoid repeated
|
||||||
/// cache lookups within the same file.
|
/// cache lookups within the same file.
|
||||||
current_file_chunks: Option<ChunkVec<T>>,
|
current_file_chunks: Option<ChunkVec<T>>,
|
||||||
|
/// Inclusive lower bound for record timestamps. Records before this
|
||||||
|
/// are skipped in boundary files.
|
||||||
|
from_ms: Option<i64>,
|
||||||
|
/// Inclusive upper bound for record timestamps. When a record exceeds
|
||||||
|
/// this, iteration stops immediately.
|
||||||
|
to_ms: Option<i64>,
|
||||||
|
/// Set to `true` when a `to_ms` boundary is crossed, preventing
|
||||||
|
/// further chunk loading.
|
||||||
|
exhausted: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Filters a chunk of records by an optional `[from_ms, to_ms]` time window.
|
||||||
|
///
|
||||||
|
/// Returns `None` if the entire chunk falls outside the window, or a new
|
||||||
|
/// `Arc<[T]>` containing only the matching records. Returns `Some(original)`
|
||||||
|
/// unchanged when no filtering is needed (the common case for middle files).
|
||||||
|
///
|
||||||
|
/// When a record exceeds `to_ms`, returns the truncated slice and sets
|
||||||
|
/// `exhausted` to `true` so the iterator can stop.
|
||||||
|
fn filter_chunk<T: HasTimestamp + Clone>(
|
||||||
|
chunk: &Arc<[T]>,
|
||||||
|
from_ms: Option<i64>,
|
||||||
|
to_ms: Option<i64>,
|
||||||
|
exhausted: &mut bool,
|
||||||
|
) -> Option<Arc<[T]>> {
|
||||||
|
// Fast path: no filtering needed
|
||||||
|
if from_ms.is_none() && to_ms.is_none() {
|
||||||
|
return Some(Arc::clone(chunk));
|
||||||
|
}
|
||||||
|
|
||||||
|
let from = from_ms.unwrap_or(i64::MIN);
|
||||||
|
let to = to_ms.unwrap_or(i64::MAX);
|
||||||
|
|
||||||
|
// Quick rejection: if the last record is before `from`, skip the whole chunk
|
||||||
|
if let Some(last) = chunk.last()
|
||||||
|
&& last.time_ms() < from
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Quick rejection: if the first record is past `to`, we're done
|
||||||
|
if let Some(first) = chunk.first()
|
||||||
|
&& first.time_ms() > to
|
||||||
|
{
|
||||||
|
*exhausted = true;
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Quick acceptance: entire chunk is within bounds
|
||||||
|
let first_ok = chunk.first().is_none_or(|r| r.time_ms() >= from);
|
||||||
|
let last_ok = chunk.last().is_none_or(|r| r.time_ms() <= to);
|
||||||
|
if first_ok && last_ok {
|
||||||
|
return Some(Arc::clone(chunk));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Partial filtering: collect matching records
|
||||||
|
let mut filtered = Vec::new();
|
||||||
|
for rec in chunk.iter() {
|
||||||
|
let t = rec.time_ms();
|
||||||
|
if t > to {
|
||||||
|
*exhausted = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if t >= from {
|
||||||
|
filtered.push(rec.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if filtered.is_empty() { None } else { Some(Arc::from(filtered)) }
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<T> Drop for SymbolChunkIter<T> {
|
impl<T> Drop for SymbolChunkIter<T> {
|
||||||
@@ -372,8 +493,15 @@ impl<T> Drop for SymbolChunkIter<T> {
|
|||||||
|
|
||||||
impl SymbolChunkIter<M1Parsed> {
|
impl SymbolChunkIter<M1Parsed> {
|
||||||
/// Returns the next chunk of M1 records, or `None` when exhausted.
|
/// Returns the next chunk of M1 records, or `None` when exhausted.
|
||||||
|
///
|
||||||
|
/// When a time window is set, records in boundary chunks are filtered
|
||||||
|
/// and iteration stops early once `to_ms` is exceeded.
|
||||||
pub fn next_chunk(&mut self) -> Option<Arc<[M1Parsed]>> {
|
pub fn next_chunk(&mut self) -> Option<Arc<[M1Parsed]>> {
|
||||||
loop {
|
loop {
|
||||||
|
if self.exhausted {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
// Try to get chunks for the current file
|
// Try to get chunks for the current file
|
||||||
if self.current_file_chunks.is_none() && self.file_idx < self.files.len() {
|
if self.current_file_chunks.is_none() && self.file_idx < self.files.len() {
|
||||||
let key = &self.files[self.file_idx];
|
let key = &self.files[self.file_idx];
|
||||||
@@ -385,9 +513,14 @@ impl SymbolChunkIter<M1Parsed> {
|
|||||||
let chunks = self.current_file_chunks.as_ref()?;
|
let chunks = self.current_file_chunks.as_ref()?;
|
||||||
|
|
||||||
if self.chunk_idx < chunks.len() {
|
if self.chunk_idx < chunks.len() {
|
||||||
let chunk = chunks[self.chunk_idx].clone();
|
let raw_chunk = &chunks[self.chunk_idx];
|
||||||
self.chunk_idx += 1;
|
self.chunk_idx += 1;
|
||||||
return Some(chunk);
|
|
||||||
|
match filter_chunk(raw_chunk, self.from_ms, self.to_ms, &mut self.exhausted) {
|
||||||
|
Some(filtered) => return Some(filtered),
|
||||||
|
None if self.exhausted => return None,
|
||||||
|
None => continue, // chunk was before `from_ms`, skip it
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Current file exhausted — advance to next file
|
// Current file exhausted — advance to next file
|
||||||
@@ -411,6 +544,10 @@ impl SymbolChunkIter<TickParsed> {
|
|||||||
/// Returns the next chunk of tick records, or `None` when exhausted.
|
/// Returns the next chunk of tick records, or `None` when exhausted.
|
||||||
pub fn next_chunk(&mut self) -> Option<Arc<[TickParsed]>> {
|
pub fn next_chunk(&mut self) -> Option<Arc<[TickParsed]>> {
|
||||||
loop {
|
loop {
|
||||||
|
if self.exhausted {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
if self.current_file_chunks.is_none() && self.file_idx < self.files.len() {
|
if self.current_file_chunks.is_none() && self.file_idx < self.files.len() {
|
||||||
let key = &self.files[self.file_idx];
|
let key = &self.files[self.file_idx];
|
||||||
self.server.ensure_tick_loaded(key);
|
self.server.ensure_tick_loaded(key);
|
||||||
@@ -421,9 +558,14 @@ impl SymbolChunkIter<TickParsed> {
|
|||||||
let chunks = self.current_file_chunks.as_ref()?;
|
let chunks = self.current_file_chunks.as_ref()?;
|
||||||
|
|
||||||
if self.chunk_idx < chunks.len() {
|
if self.chunk_idx < chunks.len() {
|
||||||
let chunk = chunks[self.chunk_idx].clone();
|
let raw_chunk = &chunks[self.chunk_idx];
|
||||||
self.chunk_idx += 1;
|
self.chunk_idx += 1;
|
||||||
return Some(chunk);
|
|
||||||
|
match filter_chunk(raw_chunk, self.from_ms, self.to_ms, &mut self.exhausted) {
|
||||||
|
Some(filtered) => return Some(filtered),
|
||||||
|
None if self.exhausted => return None,
|
||||||
|
None => continue,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
self.file_idx += 1;
|
self.file_idx += 1;
|
||||||
@@ -530,4 +672,93 @@ mod tests {
|
|||||||
assert_eq!(syms[0].as_ref(), "EURUSD");
|
assert_eq!(syms[0].as_ref(), "EURUSD");
|
||||||
assert_eq!(syms[1].as_ref(), "GBPUSD");
|
assert_eq!(syms[1].as_ref(), "GBPUSD");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_filter_files_by_time_window() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
for name in [
|
||||||
|
"EURUSD_2019_06.m1",
|
||||||
|
"EURUSD_2020_01.m1",
|
||||||
|
"EURUSD_2020_06.m1",
|
||||||
|
"EURUSD_2020_12.m1",
|
||||||
|
"EURUSD_2021_03.m1",
|
||||||
|
] {
|
||||||
|
std::fs::write(dir.path().join(name), b"").unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
let server = DataServer::new(dir.path());
|
||||||
|
|
||||||
|
// Full range — all 5 files
|
||||||
|
let all = server.filter_files("EURUSD", DataFormat::M1, None, None).unwrap();
|
||||||
|
assert_eq!(all.len(), 5);
|
||||||
|
|
||||||
|
// 2020-01-01 to 2020-12-31 — should include 2020_01, 2020_06, 2020_12
|
||||||
|
let from = chrono::NaiveDate::from_ymd_opt(2020, 1, 1).unwrap()
|
||||||
|
.and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp_millis();
|
||||||
|
let to = chrono::NaiveDate::from_ymd_opt(2020, 12, 31).unwrap()
|
||||||
|
.and_hms_opt(23, 59, 59).unwrap().and_utc().timestamp_millis();
|
||||||
|
let windowed = server.filter_files("EURUSD", DataFormat::M1, Some(from), Some(to)).unwrap();
|
||||||
|
assert_eq!(windowed.len(), 3);
|
||||||
|
assert_eq!(windowed[0].month, 1);
|
||||||
|
assert_eq!(windowed[1].month, 6);
|
||||||
|
assert_eq!(windowed[2].month, 12);
|
||||||
|
|
||||||
|
// Only from — 2020-06 onwards
|
||||||
|
let from_june = chrono::NaiveDate::from_ymd_opt(2020, 6, 15).unwrap()
|
||||||
|
.and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp_millis();
|
||||||
|
let from_only = server.filter_files("EURUSD", DataFormat::M1, Some(from_june), None).unwrap();
|
||||||
|
assert_eq!(from_only.len(), 3); // 2020_06, 2020_12, 2021_03
|
||||||
|
|
||||||
|
// Window outside all data — returns None
|
||||||
|
let future = chrono::NaiveDate::from_ymd_opt(2025, 1, 1).unwrap()
|
||||||
|
.and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp_millis();
|
||||||
|
assert!(server.filter_files("EURUSD", DataFormat::M1, Some(future), None).is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_filter_chunk_boundaries() {
|
||||||
|
// Create a chunk with timestamps 100, 200, 300, 400, 500
|
||||||
|
let chunk: Arc<[M1Parsed]> = Arc::from(vec![
|
||||||
|
M1Parsed { time_ms: 100, open: 1.0, high: 1.0, low: 1.0, close: 1.0, spread: 0.0, volume: 0 },
|
||||||
|
M1Parsed { time_ms: 200, open: 1.0, high: 1.0, low: 1.0, close: 1.0, spread: 0.0, volume: 0 },
|
||||||
|
M1Parsed { time_ms: 300, open: 1.0, high: 1.0, low: 1.0, close: 1.0, spread: 0.0, volume: 0 },
|
||||||
|
M1Parsed { time_ms: 400, open: 1.0, high: 1.0, low: 1.0, close: 1.0, spread: 0.0, volume: 0 },
|
||||||
|
M1Parsed { time_ms: 500, open: 1.0, high: 1.0, low: 1.0, close: 1.0, spread: 0.0, volume: 0 },
|
||||||
|
]);
|
||||||
|
|
||||||
|
let mut exhausted = false;
|
||||||
|
|
||||||
|
// No filter — returns original chunk
|
||||||
|
let result = filter_chunk(&chunk, None, None, &mut exhausted);
|
||||||
|
assert_eq!(result.unwrap().len(), 5);
|
||||||
|
assert!(!exhausted);
|
||||||
|
|
||||||
|
// from=250 — skips 100, 200
|
||||||
|
let result = filter_chunk(&chunk, Some(250), None, &mut exhausted);
|
||||||
|
assert_eq!(result.unwrap().len(), 3); // 300, 400, 500
|
||||||
|
|
||||||
|
// to=350 — keeps 100, 200, 300, sets exhausted
|
||||||
|
exhausted = false;
|
||||||
|
let result = filter_chunk(&chunk, None, Some(350), &mut exhausted);
|
||||||
|
assert_eq!(result.unwrap().len(), 3); // 100, 200, 300
|
||||||
|
assert!(exhausted);
|
||||||
|
|
||||||
|
// from=200, to=400 — keeps 200, 300, 400
|
||||||
|
exhausted = false;
|
||||||
|
let result = filter_chunk(&chunk, Some(200), Some(400), &mut exhausted);
|
||||||
|
assert_eq!(result.unwrap().len(), 3); // 200, 300, 400
|
||||||
|
assert!(exhausted); // 500 > 400
|
||||||
|
|
||||||
|
// Entirely before from — returns None
|
||||||
|
exhausted = false;
|
||||||
|
let result = filter_chunk(&chunk, Some(600), None, &mut exhausted);
|
||||||
|
assert!(result.is_none());
|
||||||
|
assert!(!exhausted);
|
||||||
|
|
||||||
|
// Entirely after to — returns None and sets exhausted
|
||||||
|
exhausted = false;
|
||||||
|
let result = filter_chunk(&chunk, None, Some(50), &mut exhausted);
|
||||||
|
assert!(result.is_none());
|
||||||
|
assert!(exhausted);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,6 +17,26 @@ pub fn delphi_to_unix_ms(dt: f64) -> i64 {
|
|||||||
((dt - DELPHI_EPOCH_OFFSET_DAYS) * MS_PER_DAY) as i64
|
((dt - DELPHI_EPOCH_OFFSET_DAYS) * MS_PER_DAY) as i64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Extracts the (year, month) from a Unix-millisecond timestamp.
|
||||||
|
///
|
||||||
|
/// Used for coarse file-level filtering: data files are named `SYMBOL_YYYY_MM.ext`,
|
||||||
|
/// so we can skip entire files that fall outside the requested time window.
|
||||||
|
#[inline]
|
||||||
|
pub fn unix_ms_to_year_month(ms: i64) -> (u16, u8) {
|
||||||
|
use chrono::Datelike;
|
||||||
|
let dt = chrono::DateTime::from_timestamp_millis(ms)
|
||||||
|
.expect("unix_ms_to_year_month: invalid timestamp");
|
||||||
|
(dt.year() as u16, dt.month() as u8)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Common timestamp access for parsed record types.
|
||||||
|
///
|
||||||
|
/// Enables generic time-window filtering in [`super::SymbolChunkIter`] without
|
||||||
|
/// duplicating the logic for M1 and tick records.
|
||||||
|
pub trait HasTimestamp {
|
||||||
|
fn time_ms(&self) -> i64;
|
||||||
|
}
|
||||||
|
|
||||||
/// Raw M1 (minute) record as stored on disk. 48 bytes, packed.
|
/// Raw M1 (minute) record as stored on disk. 48 bytes, packed.
|
||||||
///
|
///
|
||||||
/// Matches the Delphi `TM1Record = packed record` layout exactly:
|
/// Matches the Delphi `TM1Record = packed record` layout exactly:
|
||||||
@@ -75,6 +95,20 @@ pub enum DataFormat {
|
|||||||
Tick,
|
Tick,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl HasTimestamp for M1Parsed {
|
||||||
|
#[inline]
|
||||||
|
fn time_ms(&self) -> i64 {
|
||||||
|
self.time_ms
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HasTimestamp for TickParsed {
|
||||||
|
#[inline]
|
||||||
|
fn time_ms(&self) -> i64 {
|
||||||
|
self.time_ms
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl M1Parsed {
|
impl M1Parsed {
|
||||||
/// Converts a raw on-disk record to the parsed representation.
|
/// Converts a raw on-disk record to the parsed representation.
|
||||||
///
|
///
|
||||||
@@ -136,6 +170,16 @@ mod tests {
|
|||||||
assert_eq!(unix_ms, 0);
|
assert_eq!(unix_ms, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_unix_ms_to_year_month() {
|
||||||
|
// 2017-03-01 00:00:00 UTC = 1488326400000 ms
|
||||||
|
assert_eq!(unix_ms_to_year_month(1_488_326_400_000), (2017, 3));
|
||||||
|
// 2020-12-31 23:59:59.999 UTC
|
||||||
|
assert_eq!(unix_ms_to_year_month(1_609_459_199_999), (2020, 12));
|
||||||
|
// Unix epoch
|
||||||
|
assert_eq!(unix_ms_to_year_month(0), (1970, 1));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_raw_m1_record_size() {
|
fn test_raw_m1_record_size() {
|
||||||
assert_eq!(std::mem::size_of::<RawM1Record>(), 48);
|
assert_eq!(std::mem::size_of::<RawM1Record>(), 48);
|
||||||
|
|||||||
+48
-17
@@ -36,7 +36,7 @@ fn register_m1_stream(env: &Environment) {
|
|||||||
env.register_native_fn(
|
env.register_native_fn(
|
||||||
"create-m1-stream",
|
"create-m1-stream",
|
||||||
StaticType::Function(Box::new(Signature {
|
StaticType::Function(Box::new(Signature {
|
||||||
params: StaticType::Tuple(vec![StaticType::Text]),
|
params: StaticType::Any,
|
||||||
ret: ret_type,
|
ret: ret_type,
|
||||||
})),
|
})),
|
||||||
Purity::Impure,
|
Purity::Impure,
|
||||||
@@ -46,12 +46,24 @@ fn register_m1_stream(env: &Environment) {
|
|||||||
_ => panic!("create-m1-stream: first argument must be a string"),
|
_ => panic!("create-m1-stream: first argument must be a string"),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let from_ms = args.get(1).map(|v| match v {
|
||||||
|
Value::DateTime(ms) => *ms,
|
||||||
|
_ => panic!("create-m1-stream: 'from' must be a DateTime"),
|
||||||
|
});
|
||||||
|
let to_ms = args.get(2).map(|v| match v {
|
||||||
|
Value::DateTime(ms) => *ms,
|
||||||
|
_ => panic!("create-m1-stream: 'to' must be a DateTime"),
|
||||||
|
});
|
||||||
|
|
||||||
let server = get_data_server()
|
let server = get_data_server()
|
||||||
.expect("create-m1-stream: DataServer not initialized. Call init_data_server() first.");
|
.expect("create-m1-stream: DataServer not initialized. Call init_data_server() first.");
|
||||||
|
|
||||||
let mut iter = server
|
// Unknown symbol → Void (no stream created)
|
||||||
.stream_m1(symbol)
|
if !server.has_symbol(symbol) {
|
||||||
.unwrap_or_else(|| panic!("create-m1-stream: symbol '{}' not found", symbol));
|
return Value::Void;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut iter = server.stream_m1_windowed(symbol, from_ms, to_ms);
|
||||||
|
|
||||||
// Create the pipeline entry point
|
// Create the pipeline entry point
|
||||||
let root_stream = Rc::new(RootStream::new());
|
let root_stream = Rc::new(RootStream::new());
|
||||||
@@ -61,8 +73,8 @@ fn register_m1_stream(env: &Environment) {
|
|||||||
element_type: StaticType::Record(layout.clone()),
|
element_type: StaticType::Record(layout.clone()),
|
||||||
};
|
};
|
||||||
|
|
||||||
// Pre-fetch the first chunk
|
// Pre-fetch the first chunk (None if time window is empty)
|
||||||
let mut current_chunk = iter.next_chunk();
|
let mut current_chunk = iter.as_mut().and_then(|it| it.next_chunk());
|
||||||
let mut pos = 0;
|
let mut pos = 0;
|
||||||
|
|
||||||
let generator = move || -> bool {
|
let generator = move || -> bool {
|
||||||
@@ -93,7 +105,7 @@ fn register_m1_stream(env: &Environment) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Advance to next chunk (triggers prefetch internally)
|
// Advance to next chunk (triggers prefetch internally)
|
||||||
current_chunk = iter.next_chunk();
|
current_chunk = iter.as_mut().and_then(|it| it.next_chunk());
|
||||||
pos = 0;
|
pos = 0;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -103,8 +115,11 @@ fn register_m1_stream(env: &Environment) {
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
.doc("Creates a stream of M1 (minute) OHLCV bars from tick data files.")
|
.doc("Creates a stream of M1 (minute) OHLCV bars from tick data files.")
|
||||||
.description("Reads pre-cached binary data from the DataServer. The symbol must match a directory entry (e.g. \"EURUSD\", \"AAPL.US\").")
|
.description("Reads pre-cached binary data from the DataServer. Optional from/to DateTime arguments restrict the time window (millisecond precision). Returns Void if the symbol is unknown. Returns an empty (immediately exhausted) stream if the symbol exists but the time window contains no data.")
|
||||||
.examples(&["(def ohlc (create-m1-stream \"EURUSD\"))"]);
|
.examples(&[
|
||||||
|
"(def ohlc (create-m1-stream \"EURUSD\"))",
|
||||||
|
"(def ohlc (create-m1-stream \"EURUSD\" (date \"2020-01-01\") (date \"2020-12-31\")))",
|
||||||
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
// -- Tick Stream -------------------------------------------------------------
|
// -- Tick Stream -------------------------------------------------------------
|
||||||
@@ -123,7 +138,7 @@ fn register_tick_stream(env: &Environment) {
|
|||||||
env.register_native_fn(
|
env.register_native_fn(
|
||||||
"create-tick-stream",
|
"create-tick-stream",
|
||||||
StaticType::Function(Box::new(Signature {
|
StaticType::Function(Box::new(Signature {
|
||||||
params: StaticType::Tuple(vec![StaticType::Text]),
|
params: StaticType::Any,
|
||||||
ret: ret_type,
|
ret: ret_type,
|
||||||
})),
|
})),
|
||||||
Purity::Impure,
|
Purity::Impure,
|
||||||
@@ -133,12 +148,24 @@ fn register_tick_stream(env: &Environment) {
|
|||||||
_ => panic!("create-tick-stream: first argument must be a string"),
|
_ => panic!("create-tick-stream: first argument must be a string"),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let from_ms = args.get(1).map(|v| match v {
|
||||||
|
Value::DateTime(ms) => *ms,
|
||||||
|
_ => panic!("create-tick-stream: 'from' must be a DateTime"),
|
||||||
|
});
|
||||||
|
let to_ms = args.get(2).map(|v| match v {
|
||||||
|
Value::DateTime(ms) => *ms,
|
||||||
|
_ => panic!("create-tick-stream: 'to' must be a DateTime"),
|
||||||
|
});
|
||||||
|
|
||||||
let server = get_data_server()
|
let server = get_data_server()
|
||||||
.expect("create-tick-stream: DataServer not initialized.");
|
.expect("create-tick-stream: DataServer not initialized.");
|
||||||
|
|
||||||
let mut iter = server
|
// Unknown symbol → Void (no stream created)
|
||||||
.stream_tick(symbol)
|
if !server.has_symbol(symbol) {
|
||||||
.unwrap_or_else(|| panic!("create-tick-stream: symbol '{}' not found", symbol));
|
return Value::Void;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut iter = server.stream_tick_windowed(symbol, from_ms, to_ms);
|
||||||
|
|
||||||
let root_stream = Rc::new(RootStream::new());
|
let root_stream = Rc::new(RootStream::new());
|
||||||
let layout = tick_layout.clone();
|
let layout = tick_layout.clone();
|
||||||
@@ -147,7 +174,8 @@ fn register_tick_stream(env: &Environment) {
|
|||||||
element_type: StaticType::Record(layout.clone()),
|
element_type: StaticType::Record(layout.clone()),
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut current_chunk = iter.next_chunk();
|
// Pre-fetch the first chunk (None if time window is empty)
|
||||||
|
let mut current_chunk = iter.as_mut().and_then(|it| it.next_chunk());
|
||||||
let mut pos = 0;
|
let mut pos = 0;
|
||||||
|
|
||||||
let generator = move || -> bool {
|
let generator = move || -> bool {
|
||||||
@@ -173,7 +201,7 @@ fn register_tick_stream(env: &Environment) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
current_chunk = iter.next_chunk();
|
current_chunk = iter.as_mut().and_then(|it| it.next_chunk());
|
||||||
pos = 0;
|
pos = 0;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -183,6 +211,9 @@ fn register_tick_stream(env: &Environment) {
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
.doc("Creates a stream of tick data (ask/bid) from tick data files.")
|
.doc("Creates a stream of tick data (ask/bid) from tick data files.")
|
||||||
.description("Reads pre-cached binary data from the DataServer. The symbol must match a directory entry.")
|
.description("Reads pre-cached binary data from the DataServer. Optional from/to DateTime arguments restrict the time window (millisecond precision). Returns Void if the symbol is unknown. Returns an empty (immediately exhausted) stream if the symbol exists but the time window contains no data.")
|
||||||
.examples(&["(def ticks (create-tick-stream \"EURUSD\"))"]);
|
.examples(&[
|
||||||
|
"(def ticks (create-tick-stream \"EURUSD\"))",
|
||||||
|
"(def ticks (create-tick-stream \"EURUSD\" (date \"2020-01-01\") (date \"2020-12-31\")))",
|
||||||
|
]);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user