perf(ingest,cli): derive archive bounds from the monthly file index — probe_window drain retired
probe_window drained a source's entire close column just to learn its first and last bar timestamp; both live callers (open_real_source and campaign_window_ms on the campaign trunk) now resolve through aura-ingest's new archive_extent(): list the symbol's SYM_YYYY_MM.m1 files, load only the boundary months via the existing loader seam, and walk across gap months forward/backward — O(2 file loads) typical instead of O(archive). Empty-window refusal semantics are unchanged (pinned). Two characterization tests captured the resolved windows against the OLD implementation over the synthetic per-test archive and stay green unchanged after the swap. Measured: the probe itself drops 366ms -> 13ms (~28x) on the full 2014-2026 GER40 archive; end-to-end command wall-clock moves only ~2-3% because the sim loop dominates — the fix removes an O(archive-size) term, not the dominant cost. closes #252
This commit is contained in:
Generated
+1
@@ -182,6 +182,7 @@ dependencies = [
|
|||||||
"chrono",
|
"chrono",
|
||||||
"chrono-tz",
|
"chrono-tz",
|
||||||
"data-server",
|
"data-server",
|
||||||
|
"zip",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|||||||
@@ -639,17 +639,8 @@ fn probe_window(
|
|||||||
to_ms: Option<i64>,
|
to_ms: Option<i64>,
|
||||||
env: &project::Env,
|
env: &project::Env,
|
||||||
) -> (Timestamp, Timestamp) {
|
) -> (Timestamp, Timestamp) {
|
||||||
let mut probe = aura_ingest::open_columns(server, symbol, from_ms, to_ms, &[aura_ingest::M1Field::Close])
|
aura_ingest::archive_extent(server, std::path::Path::new(&env.data_path()), symbol, from_ms, to_ms)
|
||||||
.unwrap_or_else(|| no_data_in_window(symbol, from_ms, to_ms, env))
|
.unwrap_or_else(|| no_data_in_window(symbol, from_ms, to_ms, env))
|
||||||
.pop()
|
|
||||||
.expect("open_columns yields one source per requested field");
|
|
||||||
let first =
|
|
||||||
aura_engine::Source::peek(probe.as_ref()).unwrap_or_else(|| no_data_in_window(symbol, from_ms, to_ms, env));
|
|
||||||
let mut last = first;
|
|
||||||
while let Some((t, _)) = aura_engine::Source::next(&mut *probe) {
|
|
||||||
last = t;
|
|
||||||
}
|
|
||||||
(first, last)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Open the real M1 sources for a recorded symbol over an optional window — one
|
/// Open the real M1 sources for a recorded symbol over an optional window — one
|
||||||
|
|||||||
@@ -5694,6 +5694,141 @@ fn shipped_r_sma_example_reproduces_the_builtin_grade() {
|
|||||||
assert!(stdout.contains("\"n_trades\":3"), "{stdout}");
|
assert!(stdout.contains("\"n_trades\":3"), "{stdout}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Characterization pin (#252, byte-identity anchor before `probe_window`'s
|
||||||
|
/// internals swap from a full-window drain to `aura_ingest::archive_extent`'s
|
||||||
|
/// boundary-month derivation). Captured against the PRE-#252 drain-based code
|
||||||
|
/// over `fresh_project_with_data`'s hermetic two-symbol synthetic archive
|
||||||
|
/// (SYMA spans 2024-01..08, SYMB 2024-03..06, weekday 08:00-16:00 UTC bars):
|
||||||
|
/// the resolved `campaign_window_ms(full_window)` span persists verbatim into
|
||||||
|
/// the generated campaign document's `data.windows[0]`, exactly like the
|
||||||
|
/// sibling `generalize_without_explicit_window_resolves_the_intersection_of_all_symbols`
|
||||||
|
/// probe. Three cases: SYMA's and SYMB's full (no-window) spans, and an
|
||||||
|
/// explicit `--from`/`--to` sub-window straddling three month files
|
||||||
|
/// (2024-03-15..2024-05-15, so `filter_files` alone would load 3 files under
|
||||||
|
/// the old drain). The first bar of each captured span carries a known -1ms
|
||||||
|
/// float round-trip artifact (the synthetic fixture's Delphi-`TDateTime`
|
||||||
|
/// pack/unpack in `data-server`'s own record format, not a `probe_window` bug)
|
||||||
|
/// — this test does not explain it, only pins it exactly, since the
|
||||||
|
/// post-swap code must reproduce the identical bytes (same underlying decoder,
|
||||||
|
/// only fewer files touched to reach them).
|
||||||
|
#[test]
|
||||||
|
fn probe_window_resolves_the_pinned_span_before_the_252_swap() {
|
||||||
|
let (cwd, _g) = fresh_project_with_data();
|
||||||
|
let fixture = format!("{}/examples/r_sma.json", env!("CARGO_MANIFEST_DIR"));
|
||||||
|
let window_of = |name: &str| -> (i64, i64) {
|
||||||
|
let dir = cwd.join("runs").join("campaigns");
|
||||||
|
let path = std::fs::read_dir(&dir)
|
||||||
|
.expect("campaigns dir exists after a run")
|
||||||
|
.filter_map(|e| e.ok())
|
||||||
|
.map(|e| e.path())
|
||||||
|
.find(|p| {
|
||||||
|
let doc: serde_json::Value =
|
||||||
|
serde_json::from_str(&std::fs::read_to_string(p).expect("read campaign doc"))
|
||||||
|
.expect("campaign doc parses as JSON");
|
||||||
|
doc["name"].as_str() == Some(name)
|
||||||
|
})
|
||||||
|
.unwrap_or_else(|| panic!("no campaign document named {name:?} in {}", dir.display()));
|
||||||
|
let doc: serde_json::Value =
|
||||||
|
serde_json::from_str(&std::fs::read_to_string(&path).expect("read campaign doc"))
|
||||||
|
.expect("campaign doc parses as JSON");
|
||||||
|
let w = &doc["data"]["windows"][0];
|
||||||
|
(
|
||||||
|
w["from_ms"].as_i64().expect("from_ms is an integer"),
|
||||||
|
w["to_ms"].as_i64().expect("to_ms is an integer"),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let sweep = |symbol: &str, name: &str, from: Option<&str>, to: Option<&str>| {
|
||||||
|
let mut args = vec![
|
||||||
|
"sweep", fixture.as_str(), "--real", symbol,
|
||||||
|
"--axis", "sma_signal.fast.length=3", "--axis", "sma_signal.slow.length=12",
|
||||||
|
"--name", name,
|
||||||
|
];
|
||||||
|
if let Some(f) = from {
|
||||||
|
args.push("--from");
|
||||||
|
args.push(f);
|
||||||
|
}
|
||||||
|
if let Some(t) = to {
|
||||||
|
args.push("--to");
|
||||||
|
args.push(t);
|
||||||
|
}
|
||||||
|
let out = std::process::Command::new(env!("CARGO_BIN_EXE_aura"))
|
||||||
|
.args(&args)
|
||||||
|
.current_dir(&cwd)
|
||||||
|
.output()
|
||||||
|
.expect("spawn aura sweep");
|
||||||
|
assert_eq!(
|
||||||
|
out.status.code(), Some(0),
|
||||||
|
"window-probe sweep {name:?} must run to exit 0: {}",
|
||||||
|
String::from_utf8_lossy(&out.stderr)
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
sweep("SYMA", "probe-252-no-window-syma", None, None);
|
||||||
|
assert_eq!(
|
||||||
|
window_of("probe-252-no-window-syma"),
|
||||||
|
(1_704_095_999_999, 1_725_033_540_000),
|
||||||
|
"SYMA's full (no-window) resolved span"
|
||||||
|
);
|
||||||
|
|
||||||
|
sweep("SYMB", "probe-252-no-window-symb", None, None);
|
||||||
|
assert_eq!(
|
||||||
|
window_of("probe-252-no-window-symb"),
|
||||||
|
(1_709_279_999_999, 1_719_590_340_000),
|
||||||
|
"SYMB's full (no-window) resolved span"
|
||||||
|
);
|
||||||
|
|
||||||
|
// 2024-03-15 00:00 UTC .. 2024-05-15 00:00 UTC: straddles the March, April,
|
||||||
|
// and May files (three boundary months under the old file-range drain).
|
||||||
|
sweep("SYMA", "probe-252-straddle", Some("1710460800000"), Some("1715731200000"));
|
||||||
|
assert_eq!(
|
||||||
|
window_of("probe-252-straddle"),
|
||||||
|
(1_710_489_599_999, 1_715_702_340_000),
|
||||||
|
"explicit sub-window straddling month boundaries, clipped to the archive's actual bars"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Characterization pin (#252, hostless zero-bar case): a window that falls
|
||||||
|
/// entirely on a weekend inside a COVERED month (2024-03-02/03, both inside
|
||||||
|
/// SYMA's Jan-Aug 2024 span, weekday-only bars) still refuses with the SAME
|
||||||
|
/// exit-1 "no data ... in the requested window" message `probe_window`'s
|
||||||
|
/// drain-based emptiness check raises today — `archive_extent` must reach the
|
||||||
|
/// identical conclusion by finding zero matching months in range, not by
|
||||||
|
/// draining a stream, and the refusal wording is part of what must not move
|
||||||
|
/// (mirrors `run_real_empty_window_refusal_names_the_window_not_the_symbol`,
|
||||||
|
/// hostless here since the property (an empty resolved sub-window still
|
||||||
|
/// refuses) needs no full archive, only a covered-but-empty range).
|
||||||
|
#[test]
|
||||||
|
fn probe_window_still_refuses_a_hostless_zero_bar_window() {
|
||||||
|
let (cwd, _g) = fresh_project_with_data();
|
||||||
|
let fixture = format!("{}/examples/r_sma.json", env!("CARGO_MANIFEST_DIR"));
|
||||||
|
let out = std::process::Command::new(env!("CARGO_BIN_EXE_aura"))
|
||||||
|
.args([
|
||||||
|
"sweep", &fixture, "--real", "SYMA",
|
||||||
|
"--axis", "sma_signal.fast.length=3", "--axis", "sma_signal.slow.length=12",
|
||||||
|
"--name", "probe-252-zero-bars",
|
||||||
|
"--from", "1709337600000", // 2024-03-02 00:00 UTC (Saturday)
|
||||||
|
"--to", "1709424000000", // 2024-03-03 00:00 UTC (Sunday)
|
||||||
|
])
|
||||||
|
.current_dir(&cwd)
|
||||||
|
.output()
|
||||||
|
.expect("spawn aura sweep");
|
||||||
|
assert_eq!(
|
||||||
|
out.status.code(), Some(1),
|
||||||
|
"a zero-bar window inside covered archive months still refuses with exit 1: stderr={}",
|
||||||
|
String::from_utf8_lossy(&out.stderr)
|
||||||
|
);
|
||||||
|
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||||
|
assert!(
|
||||||
|
stderr.contains("no data") && stderr.contains("window"),
|
||||||
|
"must name the empty window, not a missing symbol: {stderr}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!cwd.join("runs").join("campaigns").exists()
|
||||||
|
|| std::fs::read_dir(cwd.join("runs").join("campaigns")).unwrap().count() == 0,
|
||||||
|
"a refused zero-bar window must not register a generated campaign document"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
/// `aura run` on the shipped closed r-meanrev example (#159 cut 3) actually loads
|
/// `aura run` on the shipped closed r-meanrev example (#159 cut 3) actually loads
|
||||||
/// and grades through the real, separately-linked `aura` binary — not merely
|
/// and grades through the real, separately-linked `aura` binary — not merely
|
||||||
/// in-process, as `r_meanrev_example_loaded_runs_identically_to_the_carved_signal`
|
/// in-process, as `r_meanrev_example_loaded_runs_identically_to_the_carved_signal`
|
||||||
|
|||||||
@@ -16,6 +16,13 @@ data-server = { git = "http://192.168.178.103:3000/Brummel/data-server.git", bra
|
|||||||
aura-engine = { path = "../aura-engine" }
|
aura-engine = { path = "../aura-engine" }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
|
# zip: hand-packs a tiny synthetic M1 archive (lib.rs's own unit tests, #252's
|
||||||
|
# archive_extent gap-walk coverage) in the exact on-disk format data-server
|
||||||
|
# reads — already transitive via the data-server dependency above (a normal,
|
||||||
|
# non-dev dependency there); declared directly here so test code may `use
|
||||||
|
# zip::...` (mirrors aura-cli's tests/common/mod.rs, no new dependency
|
||||||
|
# actually enters the build graph).
|
||||||
|
zip = "2"
|
||||||
# the integration tests bootstrap a sample harness + fold a RunReport
|
# the integration tests bootstrap a sample harness + fold a RunReport
|
||||||
aura-std = { path = "../aura-std" }
|
aura-std = { path = "../aura-std" }
|
||||||
# chrono / chrono-tz build the Berlin-local-wall-clock window bounds + the
|
# chrono / chrono-tz build the Berlin-local-wall-clock window bounds + the
|
||||||
|
|||||||
@@ -23,6 +23,7 @@
|
|||||||
use aura_core::{Scalar, Timestamp};
|
use aura_core::{Scalar, Timestamp};
|
||||||
use data_server::records::M1Parsed;
|
use data_server::records::M1Parsed;
|
||||||
use data_server::SymbolChunkIter;
|
use data_server::SymbolChunkIter;
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
/// Re-export of the data-server archive entry points (#81): a real-data source
|
/// Re-export of the data-server archive entry points (#81): a real-data source
|
||||||
/// builds from `aura-ingest` alone — a consumer never names the external
|
/// builds from `aura-ingest` alone — a consumer never names the external
|
||||||
@@ -411,10 +412,168 @@ pub fn instrument_geometry(server: &Arc<DataServer>, symbol: &str) -> Option<Ins
|
|||||||
server.symbol_meta(symbol)
|
server.symbol_meta(symbol)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Days since 1970-01-01 for the first of a proleptic-Gregorian civil
|
||||||
|
/// `(year, month, day)` — Howard Hinnant's closed-form `days_from_civil`, so
|
||||||
|
/// [`month_start_ms`] needs no calendar library. Already precedented in this
|
||||||
|
/// codebase (`aura-cli`'s `tests/common::synthetic_data` carries the identical
|
||||||
|
/// formula for its own synthetic archive generator); duplicated here rather
|
||||||
|
/// than shared because that copy lives in a test-only module of a different
|
||||||
|
/// crate.
|
||||||
|
fn days_from_civil(y: i64, m: u32, d: u32) -> i64 {
|
||||||
|
let y = if m <= 2 { y - 1 } else { y };
|
||||||
|
let era = if y >= 0 { y } else { y - 399 } / 400;
|
||||||
|
let yoe = (y - era * 400) as u64; // [0, 399]
|
||||||
|
let mp = (u64::from(m) + 9) % 12; // [0, 11]: Mar=0 .. Feb=11
|
||||||
|
let doy = (153 * mp + 2) / 5 + u64::from(d) - 1; // [0, 365]
|
||||||
|
let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; // [0, 146096]
|
||||||
|
era * 146_097 + doe as i64 - 719_468
|
||||||
|
}
|
||||||
|
|
||||||
|
/// UTC epoch-ms of `(year, month)`'s first instant — matches
|
||||||
|
/// `data_server::records::unix_ms_to_year_month`'s own UTC calendar (chrono's
|
||||||
|
/// `DateTime<Utc>`), the reference [`archive_extent`] derives a
|
||||||
|
/// single-candidate-month probe window against.
|
||||||
|
fn month_start_ms(year: u16, month: u8) -> i64 {
|
||||||
|
days_from_civil(i64::from(year), u32::from(month), 1) * 86_400_000
|
||||||
|
}
|
||||||
|
|
||||||
|
/// UTC epoch-ms of the instant immediately after `(year, month)`'s last
|
||||||
|
/// millisecond (the exclusive upper bound [`archive_extent`] clamps a
|
||||||
|
/// forward-search probe's `to_ms` against, so a single query loads at most
|
||||||
|
/// one month's file).
|
||||||
|
fn month_end_ms_exclusive(year: u16, month: u8) -> i64 {
|
||||||
|
let (ny, nm) = if month == 12 { (year + 1, 1) } else { (year, month + 1) };
|
||||||
|
month_start_ms(ny, nm)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The sorted `(year, month)` pairs that exist as `<symbol>_YYYY_MM.m1` files
|
||||||
|
/// directly under `data_path` — mirrors data-server's own file-naming
|
||||||
|
/// convention (`SymbolIndex::scan`, already load-bearing in its loader) via a
|
||||||
|
/// plain directory listing scoped to the one already-known `symbol`, so this
|
||||||
|
/// needs no general symbol-boundary regex and reads no private index (#252:
|
||||||
|
/// no new data-server surface). Empty when `data_path` doesn't exist or holds
|
||||||
|
/// no matching file for `symbol` — the same "no archive for this symbol"
|
||||||
|
/// condition an absent `DataServer` symbol represents.
|
||||||
|
fn list_m1_months(data_path: &Path, symbol: &str) -> Vec<(u16, u8)> {
|
||||||
|
let prefix = format!("{symbol}_");
|
||||||
|
let mut months = Vec::new();
|
||||||
|
let Ok(entries) = std::fs::read_dir(data_path) else {
|
||||||
|
return months;
|
||||||
|
};
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
let file_name = entry.file_name();
|
||||||
|
let Some(name) = file_name.to_str() else { continue };
|
||||||
|
let Some(rest) = name.strip_prefix(prefix.as_str()) else { continue };
|
||||||
|
let Some(year_month) = rest.strip_suffix(".m1") else { continue };
|
||||||
|
let Some((year_str, month_str)) = year_month.split_once('_') else { continue };
|
||||||
|
if year_str.len() != 4
|
||||||
|
|| month_str.len() != 2
|
||||||
|
|| !year_str.bytes().all(|b| b.is_ascii_digit())
|
||||||
|
|| !month_str.bytes().all(|b| b.is_ascii_digit())
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let (Ok(year), Ok(month)) = (year_str.parse::<u16>(), month_str.parse::<u8>()) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if (1..=12).contains(&month) {
|
||||||
|
months.push((year, month));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
months.sort_unstable();
|
||||||
|
months
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Derive the archive's `(first, last)` bar timestamp inside `[from_ms, to_ms]`
|
||||||
|
/// (inclusive Unix-ms, `None` = open-ended) at O(1) file loads per boundary
|
||||||
|
/// instead of draining every bar in the window (#252). Lists the symbol's
|
||||||
|
/// monthly files directly off `data_path` ([`list_m1_months`]), then loads
|
||||||
|
/// ONLY the boundary month through the existing `stream_m1_windowed`/
|
||||||
|
/// [`M1FieldSource`] seam — walking forward (for the first bar) or backward
|
||||||
|
/// (for the last) across gap months whose candidate file yields nothing in
|
||||||
|
/// range, exactly like the cross-file skip-ahead a full-window drain already
|
||||||
|
/// did internally, just narrowed to one file per candidate instead of every
|
||||||
|
/// file in the window. `server` is the caller's own `Arc<DataServer>` (shares
|
||||||
|
/// its file cache — the two boundary files loaded here are exactly the ones a
|
||||||
|
/// subsequent real run reads first/last, so this is not wasted work, only
|
||||||
|
/// reordered). Returns `None` exactly when no month in `[from_ms, to_ms]` has
|
||||||
|
/// a bar — no archive for the symbol, or a window with zero matching bars —
|
||||||
|
/// the same refusal condition a drain-based emptiness check detects.
|
||||||
|
pub fn archive_extent(
|
||||||
|
server: &Arc<DataServer>,
|
||||||
|
data_path: &Path,
|
||||||
|
symbol: &str,
|
||||||
|
from_ms: Option<i64>,
|
||||||
|
to_ms: Option<i64>,
|
||||||
|
) -> Option<(Timestamp, Timestamp)> {
|
||||||
|
let months = list_m1_months(data_path, symbol);
|
||||||
|
if months.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let from_ym = from_ms.map(data_server::records::unix_ms_to_year_month);
|
||||||
|
let to_ym = to_ms.map(data_server::records::unix_ms_to_year_month);
|
||||||
|
let in_range = |&(y, m): &(u16, u8)| {
|
||||||
|
from_ym.is_none_or(|fym| (y, m) >= fym) && to_ym.is_none_or(|tym| (y, m) <= tym)
|
||||||
|
};
|
||||||
|
|
||||||
|
// Forward walk: the first candidate month (ascending) whose file, probed
|
||||||
|
// over just that month's span, yields a bar — a gap or an empty boundary
|
||||||
|
// file simply advances to the next one.
|
||||||
|
let first = months.iter().filter(|ym| in_range(ym)).find_map(|&(y, m)| {
|
||||||
|
let month_cap = month_end_ms_exclusive(y, m) - 1;
|
||||||
|
let probe_to = to_ms.map_or(month_cap, |t| t.min(month_cap));
|
||||||
|
let src = M1FieldSource::open(server, symbol, from_ms, Some(probe_to), M1Field::Close)?;
|
||||||
|
aura_engine::Source::peek(&src)
|
||||||
|
})?;
|
||||||
|
|
||||||
|
// Backward walk: the last candidate month (descending) whose file,
|
||||||
|
// probed over just that month's span, yields a bar — fully drained (a
|
||||||
|
// single file) to find its own last matching record.
|
||||||
|
let last = months.iter().rev().filter(|ym| in_range(ym)).find_map(|&(y, m)| {
|
||||||
|
let month_floor = month_start_ms(y, m);
|
||||||
|
let probe_from = from_ms.map_or(month_floor, |f| f.max(month_floor));
|
||||||
|
let mut src = M1FieldSource::open(server, symbol, Some(probe_from), to_ms, M1Field::Close)?;
|
||||||
|
let mut last_ts = aura_engine::Source::peek(&src)?;
|
||||||
|
while let Some((t, _)) = aura_engine::Source::next(&mut src) {
|
||||||
|
last_ts = t;
|
||||||
|
}
|
||||||
|
Some(last_ts)
|
||||||
|
})?;
|
||||||
|
|
||||||
|
Some((first, last))
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
/// A unique scratch directory under the OS temp root, removed on drop —
|
||||||
|
/// mirrors `aura-cli`'s own `tests/common::mint_tempdir` pattern (unique
|
||||||
|
/// name via pid + an atomic counter) so this crate's tests need no
|
||||||
|
/// `tempfile` dev-dependency for a handful of throwaway fixture dirs.
|
||||||
|
struct ScratchDir(std::path::PathBuf);
|
||||||
|
|
||||||
|
impl ScratchDir {
|
||||||
|
fn new(tag: &str) -> Self {
|
||||||
|
static COUNTER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
|
||||||
|
let n = COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||||
|
let dir = std::env::temp_dir()
|
||||||
|
.join(format!("aura-ingest-test-{tag}-{}-{n}", std::process::id()));
|
||||||
|
std::fs::create_dir_all(&dir).expect("create scratch dir");
|
||||||
|
Self(dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn path(&self) -> &std::path::Path {
|
||||||
|
&self.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for ScratchDir {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let _ = std::fs::remove_dir_all(&self.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn instrument_geometry_none_for_unknown_symbol() {
|
fn instrument_geometry_none_for_unknown_symbol() {
|
||||||
// A symbol with no sidecar yields None — the accessor surfaces the reader's
|
// A symbol with no sidecar yields None — the accessor surfaces the reader's
|
||||||
@@ -447,6 +606,159 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// [`month_start_ms`] anchored against the same 2017-03-01 UTC reference
|
||||||
|
/// [`unix_ms_to_epoch_ns_scales_ms_to_ns`] pins, plus the Feb/Dec month-end
|
||||||
|
/// edges [`month_end_ms_exclusive`] must get right (leap-year length, and
|
||||||
|
/// the December -> next-January year rollover) — the two properties
|
||||||
|
/// [`archive_extent`]'s per-candidate-month probe bound depends on.
|
||||||
|
#[test]
|
||||||
|
fn month_start_and_end_ms_bracket_known_months() {
|
||||||
|
assert_eq!(month_start_ms(2017, 3), 1_488_326_400_000);
|
||||||
|
// 2024 is a leap year: February runs through the 29th, so March 1
|
||||||
|
// 00:00 UTC is exactly 29 days after February 1 00:00 UTC.
|
||||||
|
assert_eq!(
|
||||||
|
month_end_ms_exclusive(2024, 2) - month_start_ms(2024, 2),
|
||||||
|
29 * 86_400_000
|
||||||
|
);
|
||||||
|
// 2023 is not a leap year: 28 days.
|
||||||
|
assert_eq!(
|
||||||
|
month_end_ms_exclusive(2023, 2) - month_start_ms(2023, 2),
|
||||||
|
28 * 86_400_000
|
||||||
|
);
|
||||||
|
// December rolls over into the next year's January.
|
||||||
|
assert_eq!(month_end_ms_exclusive(2024, 12), month_start_ms(2025, 1));
|
||||||
|
// A month's end is exclusive: the next month starts immediately after.
|
||||||
|
assert_eq!(month_end_ms_exclusive(2024, 3), month_start_ms(2024, 4));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// [`list_m1_months`] scans `data_path` for `<symbol>_YYYY_MM.m1` files —
|
||||||
|
/// mirroring data-server's own `SymbolIndex::scan` regex (`test_symbol_index_scan`
|
||||||
|
/// in its `lib.rs`) but scoped to one already-known symbol, so it needs no
|
||||||
|
/// general-purpose parser. Sorted ascending, and never confused by another
|
||||||
|
/// symbol's file or a non-M1 extension sharing the same directory.
|
||||||
|
#[test]
|
||||||
|
fn list_m1_months_lists_and_sorts_the_one_symbols_files() {
|
||||||
|
let dir = ScratchDir::new("list-months");
|
||||||
|
for name in [
|
||||||
|
"EURUSD_2020_06.m1",
|
||||||
|
"EURUSD_2019_12.m1",
|
||||||
|
"EURUSD_2020_01.m1",
|
||||||
|
"EURUSD_2020_01.tick", // different format, same symbol+month: ignored
|
||||||
|
"GBPUSD_2020_01.m1", // different symbol: ignored
|
||||||
|
"EURUSD_2020_1.m1", // malformed month (one digit): ignored
|
||||||
|
"not_a_data_file.txt",
|
||||||
|
] {
|
||||||
|
std::fs::write(dir.path().join(name), b"").expect("write stub file");
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
list_m1_months(dir.path(), "EURUSD"),
|
||||||
|
vec![(2019, 12), (2020, 1), (2020, 6)]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn list_m1_months_empty_for_unknown_symbol_or_missing_dir() {
|
||||||
|
let dir = ScratchDir::new("list-months-empty");
|
||||||
|
std::fs::write(dir.path().join("EURUSD_2020_01.m1"), b"").expect("write stub file");
|
||||||
|
assert!(list_m1_months(dir.path(), "NOSYM").is_empty());
|
||||||
|
assert!(list_m1_months(Path::new("/nonexistent-archive-path"), "EURUSD").is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Writes a minimal one-record `<symbol>_YYYY_MM.m1` zip at `unix_ms_time`
|
||||||
|
/// (mirrors `data-server`'s own `write_m1_file` test helper — duplicated
|
||||||
|
/// here, not imported, since that helper lives in its private `#[cfg(test)]`
|
||||||
|
/// module). Deliberately zip/binary-shaped, not a stub, since
|
||||||
|
/// [`archive_extent`] must actually load this file through the real
|
||||||
|
/// `stream_m1_windowed`/`M1FieldSource` seam, not just see its filename.
|
||||||
|
fn write_one_bar_month(dir: &Path, symbol: &str, year: u16, month: u8, unix_ms_time: i64) {
|
||||||
|
use std::io::Write;
|
||||||
|
const DELPHI_EPOCH_OFFSET_DAYS: f64 = 25569.0;
|
||||||
|
const MS_PER_DAY: f64 = 86_400_000.0;
|
||||||
|
let dt = unix_ms_time as f64 / MS_PER_DAY + DELPHI_EPOCH_OFFSET_DAYS;
|
||||||
|
let mut rec = [0u8; 48];
|
||||||
|
rec[0..8].copy_from_slice(&dt.to_le_bytes());
|
||||||
|
rec[8..16].copy_from_slice(&1.0_f64.to_le_bytes()); // open
|
||||||
|
rec[16..24].copy_from_slice(&1.0_f64.to_le_bytes()); // high
|
||||||
|
rec[24..32].copy_from_slice(&1.0_f64.to_le_bytes()); // low
|
||||||
|
rec[32..40].copy_from_slice(&1.0_f64.to_le_bytes()); // close
|
||||||
|
rec[40..44].copy_from_slice(&0.0_f32.to_le_bytes()); // spread
|
||||||
|
rec[44..48].copy_from_slice(&0_i32.to_le_bytes()); // volume
|
||||||
|
let path = dir.join(format!("{symbol}_{year:04}_{month:02}.m1"));
|
||||||
|
let file = std::fs::File::create(&path).expect("create synthetic m1 zip");
|
||||||
|
let mut zip = zip::ZipWriter::new(file);
|
||||||
|
zip.start_file::<String, ()>(format!("{symbol}.bin"), Default::default())
|
||||||
|
.expect("start synthetic m1 zip entry");
|
||||||
|
zip.write_all(&rec).expect("write synthetic m1 record");
|
||||||
|
zip.finish().expect("finish synthetic m1 zip");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Property (#252): a requested window whose starting month has NO archive
|
||||||
|
/// file (a gap) still resolves the first bar, by walking forward to the
|
||||||
|
/// next available month that actually has one — not refusing outright.
|
||||||
|
/// Three files (Jan, MISSING Feb, Mar 2021), each holding one bar; a
|
||||||
|
/// window opening in the missing February must land on March's bar.
|
||||||
|
#[test]
|
||||||
|
fn archive_extent_walks_forward_across_a_gap_month() {
|
||||||
|
let dir = ScratchDir::new("extent-forward-gap");
|
||||||
|
let jan_ms = month_start_ms(2021, 1) + 3_600_000; // 2021-01-01 01:00 UTC
|
||||||
|
let mar_ms = month_start_ms(2021, 3) + 3_600_000; // 2021-03-01 01:00 UTC
|
||||||
|
write_one_bar_month(dir.path(), "GAPSYM", 2021, 1, jan_ms);
|
||||||
|
write_one_bar_month(dir.path(), "GAPSYM", 2021, 3, mar_ms);
|
||||||
|
let server = Arc::new(DataServer::new(dir.path()));
|
||||||
|
|
||||||
|
// From = the start of the missing February; no upper bound.
|
||||||
|
let from_ms = month_start_ms(2021, 2);
|
||||||
|
let (first, last) = archive_extent(&server, dir.path(), "GAPSYM", Some(from_ms), None)
|
||||||
|
.expect("March's bar is reachable by walking past the empty February");
|
||||||
|
assert_eq!(first, unix_ms_to_epoch_ns(mar_ms));
|
||||||
|
assert_eq!(last, unix_ms_to_epoch_ns(mar_ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Property (#252): symmetric to the forward walk, an upper bound landing
|
||||||
|
/// in a gap month walks BACKWARD to find the last bar of the nearest
|
||||||
|
/// earlier available month.
|
||||||
|
#[test]
|
||||||
|
fn archive_extent_walks_backward_across_a_gap_month() {
|
||||||
|
let dir = ScratchDir::new("extent-backward-gap");
|
||||||
|
let jan_ms = month_start_ms(2021, 1) + 3_600_000;
|
||||||
|
let mar_ms = month_start_ms(2021, 3) + 3_600_000;
|
||||||
|
write_one_bar_month(dir.path(), "GAPSYM", 2021, 1, jan_ms);
|
||||||
|
write_one_bar_month(dir.path(), "GAPSYM", 2021, 3, mar_ms);
|
||||||
|
let server = Arc::new(DataServer::new(dir.path()));
|
||||||
|
|
||||||
|
// To = the end of the missing February; no lower bound.
|
||||||
|
let to_ms = month_end_ms_exclusive(2021, 2) - 1;
|
||||||
|
let (first, last) = archive_extent(&server, dir.path(), "GAPSYM", None, Some(to_ms))
|
||||||
|
.expect("January's bar is reachable by walking back past the empty February");
|
||||||
|
assert_eq!(first, unix_ms_to_epoch_ns(jan_ms));
|
||||||
|
assert_eq!(last, unix_ms_to_epoch_ns(jan_ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Property (#252): an unknown symbol (no monthly files at all) yields
|
||||||
|
/// `None` — the same file-level-absence contract [`open_columns`] and
|
||||||
|
/// [`instrument_geometry`] already share.
|
||||||
|
#[test]
|
||||||
|
fn archive_extent_none_for_unknown_symbol() {
|
||||||
|
let dir = ScratchDir::new("extent-unknown-symbol");
|
||||||
|
let server = Arc::new(DataServer::new(dir.path()));
|
||||||
|
assert!(archive_extent(&server, dir.path(), "NOSYM", None, None).is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Property (#252): a window entirely before the archive's first month (or
|
||||||
|
/// entirely after its last) yields `None` rather than snapping to the
|
||||||
|
/// nearest available month — [`archive_extent`]'s `in_range` bound must
|
||||||
|
/// reject every candidate, not just prefer the closest one.
|
||||||
|
#[test]
|
||||||
|
fn archive_extent_none_when_window_misses_every_month() {
|
||||||
|
let dir = ScratchDir::new("extent-misses-every-month");
|
||||||
|
write_one_bar_month(dir.path(), "GAPSYM", 2021, 3, month_start_ms(2021, 3) + 3_600_000);
|
||||||
|
let server = Arc::new(DataServer::new(dir.path()));
|
||||||
|
// Entirely before the one archived month.
|
||||||
|
assert!(archive_extent(&server, dir.path(), "GAPSYM", None, Some(month_start_ms(2021, 1))).is_none());
|
||||||
|
// Entirely after it.
|
||||||
|
assert!(archive_extent(&server, dir.path(), "GAPSYM", Some(month_start_ms(2021, 5)), None).is_none());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn open_columns_propagates_file_level_absence_as_none() {
|
fn open_columns_propagates_file_level_absence_as_none() {
|
||||||
// The open_ohlc contract, generalized: file-level absence (unknown
|
// The open_ohlc contract, generalized: file-level absence (unknown
|
||||||
|
|||||||
Reference in New Issue
Block a user