feat: Add data server for market data loading

This commit introduces a new data server module (`src/ast/data_server`)
designed for high-performance, thread-safe loading and caching of market
data.

Key components:
- `DataServer`: Manages symbol indexing and delegates loading/caching.
- `SymbolIndex`: Scans the data directory and maintains a sorted index
  of data files per symbol.
- `FileCache`: Implements a thread-safe cache using `RwLock` and a
  loading guard to prevent duplicate work.
- `loader`: Handles ZIP decompression and binary record parsing from
  `.bin` files.
- `records`: Defines raw and parsed data structures for M1 and tick
  data.
- `SymbolChunkIter`: Provides an iterator over pre-parsed data chunks,
  prefetching subsequent files.

This architecture allows multiple VM threads to access market data
concurrently without redundant I/O, mirroring the strategy used in the
original Delphi `TDataServer`.
This commit is contained in:
2026-03-29 20:28:55 +02:00
parent 1f3fb40bcc
commit 38f657076b
8 changed files with 1601 additions and 0 deletions
+108
View File
@@ -0,0 +1,108 @@
using System;
using System.IO;
using System.IO.Compression;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using cAlgo.API;
namespace cAlgo.Robots
{
public enum ExportMode { M1, Tick }
[Robot(AccessRights = AccessRights.FullAccess)]
public class MS_SimpleExport : Robot
{
[StructLayout(LayoutKind.Sequential, Pack = 1)]
private struct M1Record {
public double Time;
public double O, H, L, C;
public float S;
public int V;
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
private struct TickRecord {
public double Time;
public double Ask, Bid;
}
[Parameter("Export Mode", DefaultValue = ExportMode.M1)]
public ExportMode Mode { get; set; }
[Parameter("Output File (ZIP)", DefaultValue = "")]
public string OutputFile { get; set; }
[Parameter("Is Probing", DefaultValue = false)]
public bool IsProbing { get; set; }
private List<M1Record> _m1Buffer = new List<M1Record>();
private List<TickRecord> _tickBuffer = new List<TickRecord>();
protected override void OnStart() {
// No price factor needed for double/float export
}
protected override void OnTick() {
if (Mode != ExportMode.Tick) return;
if (IsProbing) {
Print("DATA_FOUND");
Stop();
return;
}
_tickBuffer.Add(new TickRecord {
Time = Server.TimeInUtc.ToOADate(),
Ask = Symbol.Ask,
Bid = Symbol.Bid
});
}
protected override void OnBar() {
if (Mode != ExportMode.M1) return;
if (IsProbing) {
Print("DATA_FOUND");
Stop();
return;
}
var bar = Bars.Last(1);
_m1Buffer.Add(new M1Record {
Time = bar.OpenTime.ToOADate(),
O = bar.Open,
H = bar.High,
L = bar.Low,
C = bar.Close,
S = (float)Symbol.Spread,
V = (int)bar.TickVolume
});
}
protected override void OnStop() {
if (IsProbing || string.IsNullOrEmpty(OutputFile)) return;
try {
using var fileStream = new FileStream(OutputFile, FileMode.Create);
using var archive = new ZipArchive(fileStream, ZipArchiveMode.Create);
var entry = archive.CreateEntry($"{Symbol.Name}.bin", CompressionLevel.Fastest);
using var entryStream = entry.Open();
if (Mode == ExportMode.M1) {
ReadOnlySpan<M1Record> span = CollectionsMarshal.AsSpan(_m1Buffer);
ReadOnlySpan<byte> byteSpan = MemoryMarshal.Cast<M1Record, byte>(span);
if (!byteSpan.IsEmpty) entryStream.Write(byteSpan);
} else {
ReadOnlySpan<TickRecord> span = CollectionsMarshal.AsSpan(_tickBuffer);
ReadOnlySpan<byte> byteSpan = MemoryMarshal.Cast<TickRecord, byte>(span);
if (!byteSpan.IsEmpty) entryStream.Write(byteSpan);
}
Print("EXPORT_SUCCESS");
} catch (Exception ex) {
Print("ERROR: {0}", ex.Message);
}
}
}
}