Refactor FileCache to use Condvar for efficient waits
The `FileCache` now uses `std::sync::Condvar` to signal when files have finished loading. This replaces the previous spin-wait mechanism, significantly improving efficiency by allowing waiting threads to sleep until explicitly woken up. The `wait_for_load` method now blocks on the `Condvar`, and the `insert_m1` and `insert_tick` methods call `notify_all` after updating the cache and removing the loading guard. This ensures that all waiting threads are woken up and can re-check the cache. Additionally, `ensure_m1_loaded` and `ensure_tick_loaded` in `DataServer` have been simplified to directly call `wait_for_load` when another thread is already loading the file, removing the redundant synchronous loading logic. Refactor FileCache to use Condvar
This commit is contained in:
@@ -0,0 +1,188 @@
|
||||
//! RTL registration for file-based market data streams.
|
||||
//!
|
||||
//! Bridges the `Arc`-based [`DataServer`] cache with the `Rc`-based VM world
|
||||
//! by registering generator closures that read pre-parsed chunks and convert
|
||||
//! them into `Value::Record` items for `RootStream::tick()`.
|
||||
|
||||
use crate::ast::data_server::get_data_server;
|
||||
use crate::ast::environment::Environment;
|
||||
use crate::ast::rtl::streams::{RootStream, StreamNode};
|
||||
use crate::ast::types::{Keyword, Purity, RecordLayout, Signature, StaticType, Value};
|
||||
use std::rc::Rc;
|
||||
|
||||
/// Registers `create-m1-stream` and `create-tick-stream` as RTL functions.
|
||||
pub fn register(env: &Environment) {
|
||||
register_m1_stream(env);
|
||||
register_tick_stream(env);
|
||||
}
|
||||
|
||||
// -- M1 Stream ---------------------------------------------------------------
|
||||
|
||||
fn register_m1_stream(env: &Environment) {
|
||||
let generators = env.pipeline_generators.clone();
|
||||
|
||||
let m1_layout = RecordLayout::get_or_create(vec![
|
||||
(Keyword::intern("time"), StaticType::DateTime),
|
||||
(Keyword::intern("open"), StaticType::Float),
|
||||
(Keyword::intern("high"), StaticType::Float),
|
||||
(Keyword::intern("low"), StaticType::Float),
|
||||
(Keyword::intern("close"), StaticType::Float),
|
||||
(Keyword::intern("spread"), StaticType::Float),
|
||||
(Keyword::intern("volume"), StaticType::Int),
|
||||
]);
|
||||
|
||||
let ret_type = StaticType::Stream(Box::new(StaticType::Record(m1_layout.clone())));
|
||||
|
||||
env.register_native_fn(
|
||||
"create-m1-stream",
|
||||
StaticType::Function(Box::new(Signature {
|
||||
params: StaticType::Tuple(vec![StaticType::Text]),
|
||||
ret: ret_type,
|
||||
})),
|
||||
Purity::Impure,
|
||||
move |args: &[Value]| {
|
||||
let symbol = match &args[0] {
|
||||
Value::Text(s) => s.as_ref(),
|
||||
_ => panic!("create-m1-stream: first argument must be a string"),
|
||||
};
|
||||
|
||||
let server = get_data_server()
|
||||
.expect("create-m1-stream: DataServer not initialized. Call init_data_server() first.");
|
||||
|
||||
let mut iter = server
|
||||
.stream_m1(symbol)
|
||||
.unwrap_or_else(|| panic!("create-m1-stream: symbol '{}' not found", symbol));
|
||||
|
||||
// Create the pipeline entry point
|
||||
let root_stream = Rc::new(RootStream::new());
|
||||
let layout = m1_layout.clone();
|
||||
let stream_node = StreamNode {
|
||||
inner: root_stream.clone(),
|
||||
element_type: StaticType::Record(layout.clone()),
|
||||
};
|
||||
|
||||
// Pre-fetch the first chunk
|
||||
let mut current_chunk = iter.next_chunk();
|
||||
let mut pos = 0;
|
||||
|
||||
let generator = move || -> bool {
|
||||
loop {
|
||||
let chunk = match ¤t_chunk {
|
||||
Some(c) => c,
|
||||
None => return false,
|
||||
};
|
||||
|
||||
if pos < chunk.len() {
|
||||
let rec = &chunk[pos];
|
||||
pos += 1;
|
||||
|
||||
let value = Value::Record(
|
||||
layout.clone(),
|
||||
Rc::new(vec![
|
||||
Value::DateTime(rec.time_ms),
|
||||
Value::Float(rec.open),
|
||||
Value::Float(rec.high),
|
||||
Value::Float(rec.low),
|
||||
Value::Float(rec.close),
|
||||
Value::Float(rec.spread),
|
||||
Value::Int(rec.volume),
|
||||
]),
|
||||
);
|
||||
root_stream.tick(value);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Advance to next chunk (triggers prefetch internally)
|
||||
current_chunk = iter.next_chunk();
|
||||
pos = 0;
|
||||
}
|
||||
};
|
||||
|
||||
generators.borrow_mut().push(Box::new(generator));
|
||||
Value::Stream(Rc::new(stream_node))
|
||||
},
|
||||
)
|
||||
.doc("Creates a stream of M1 (minute) OHLCV bars from tick data files.")
|
||||
.description("Reads pre-cached binary data from the DataServer. The symbol must match a directory entry (e.g. \"EURUSD\", \"AAPL.US\").")
|
||||
.examples(&["(def ohlc (create-m1-stream \"EURUSD\"))"]);
|
||||
}
|
||||
|
||||
// -- Tick Stream -------------------------------------------------------------
|
||||
|
||||
fn register_tick_stream(env: &Environment) {
|
||||
let generators = env.pipeline_generators.clone();
|
||||
|
||||
let tick_layout = RecordLayout::get_or_create(vec![
|
||||
(Keyword::intern("time"), StaticType::DateTime),
|
||||
(Keyword::intern("ask"), StaticType::Float),
|
||||
(Keyword::intern("bid"), StaticType::Float),
|
||||
]);
|
||||
|
||||
let ret_type = StaticType::Stream(Box::new(StaticType::Record(tick_layout.clone())));
|
||||
|
||||
env.register_native_fn(
|
||||
"create-tick-stream",
|
||||
StaticType::Function(Box::new(Signature {
|
||||
params: StaticType::Tuple(vec![StaticType::Text]),
|
||||
ret: ret_type,
|
||||
})),
|
||||
Purity::Impure,
|
||||
move |args: &[Value]| {
|
||||
let symbol = match &args[0] {
|
||||
Value::Text(s) => s.as_ref(),
|
||||
_ => panic!("create-tick-stream: first argument must be a string"),
|
||||
};
|
||||
|
||||
let server = get_data_server()
|
||||
.expect("create-tick-stream: DataServer not initialized.");
|
||||
|
||||
let mut iter = server
|
||||
.stream_tick(symbol)
|
||||
.unwrap_or_else(|| panic!("create-tick-stream: symbol '{}' not found", symbol));
|
||||
|
||||
let root_stream = Rc::new(RootStream::new());
|
||||
let layout = tick_layout.clone();
|
||||
let stream_node = StreamNode {
|
||||
inner: root_stream.clone(),
|
||||
element_type: StaticType::Record(layout.clone()),
|
||||
};
|
||||
|
||||
let mut current_chunk = iter.next_chunk();
|
||||
let mut pos = 0;
|
||||
|
||||
let generator = move || -> bool {
|
||||
loop {
|
||||
let chunk = match ¤t_chunk {
|
||||
Some(c) => c,
|
||||
None => return false,
|
||||
};
|
||||
|
||||
if pos < chunk.len() {
|
||||
let rec = &chunk[pos];
|
||||
pos += 1;
|
||||
|
||||
let value = Value::Record(
|
||||
layout.clone(),
|
||||
Rc::new(vec![
|
||||
Value::DateTime(rec.time_ms),
|
||||
Value::Float(rec.ask),
|
||||
Value::Float(rec.bid),
|
||||
]),
|
||||
);
|
||||
root_stream.tick(value);
|
||||
return true;
|
||||
}
|
||||
|
||||
current_chunk = iter.next_chunk();
|
||||
pos = 0;
|
||||
}
|
||||
};
|
||||
|
||||
generators.borrow_mut().push(Box::new(generator));
|
||||
Value::Stream(Rc::new(stream_node))
|
||||
},
|
||||
)
|
||||
.doc("Creates a stream of tick data (ask/bid) from tick data files.")
|
||||
.description("Reads pre-cached binary data from the DataServer. The symbol must match a directory entry.")
|
||||
.examples(&["(def ticks (create-tick-stream \"EURUSD\"))"]);
|
||||
}
|
||||
Reference in New Issue
Block a user