feat: Add data server for market data loading
This commit introduces a new data server module (`src/ast/data_server`) designed for high-performance, thread-safe loading and caching of market data. Key components: - `DataServer`: Manages symbol indexing and delegates loading/caching. - `SymbolIndex`: Scans the data directory and maintains a sorted index of data files per symbol. - `FileCache`: Implements a thread-safe cache using `RwLock` and a loading guard to prevent duplicate work. - `loader`: Handles ZIP decompression and binary record parsing from `.bin` files. - `records`: Defines raw and parsed data structures for M1 and tick data. - `SymbolChunkIter`: Provides an iterator over pre-parsed data chunks, prefetching subsequent files. This architecture allows multiple VM threads to access market data concurrently without redundant I/O, mirroring the strategy used in the original Delphi `TDataServer`.
This commit is contained in:
Executable
+108
@@ -0,0 +1,108 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.InteropServices;
|
||||
using cAlgo.API;
|
||||
|
||||
namespace cAlgo.Robots
|
||||
{
|
||||
public enum ExportMode { M1, Tick }
|
||||
|
||||
[Robot(AccessRights = AccessRights.FullAccess)]
|
||||
public class MS_SimpleExport : Robot
|
||||
{
|
||||
[StructLayout(LayoutKind.Sequential, Pack = 1)]
|
||||
private struct M1Record {
|
||||
public double Time;
|
||||
public double O, H, L, C;
|
||||
public float S;
|
||||
public int V;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential, Pack = 1)]
|
||||
private struct TickRecord {
|
||||
public double Time;
|
||||
public double Ask, Bid;
|
||||
}
|
||||
|
||||
[Parameter("Export Mode", DefaultValue = ExportMode.M1)]
|
||||
public ExportMode Mode { get; set; }
|
||||
|
||||
[Parameter("Output File (ZIP)", DefaultValue = "")]
|
||||
public string OutputFile { get; set; }
|
||||
|
||||
[Parameter("Is Probing", DefaultValue = false)]
|
||||
public bool IsProbing { get; set; }
|
||||
|
||||
private List<M1Record> _m1Buffer = new List<M1Record>();
|
||||
private List<TickRecord> _tickBuffer = new List<TickRecord>();
|
||||
|
||||
protected override void OnStart() {
|
||||
// No price factor needed for double/float export
|
||||
}
|
||||
|
||||
protected override void OnTick() {
|
||||
if (Mode != ExportMode.Tick) return;
|
||||
|
||||
if (IsProbing) {
|
||||
Print("DATA_FOUND");
|
||||
Stop();
|
||||
return;
|
||||
}
|
||||
|
||||
_tickBuffer.Add(new TickRecord {
|
||||
Time = Server.TimeInUtc.ToOADate(),
|
||||
Ask = Symbol.Ask,
|
||||
Bid = Symbol.Bid
|
||||
});
|
||||
}
|
||||
|
||||
protected override void OnBar() {
|
||||
if (Mode != ExportMode.M1) return;
|
||||
|
||||
if (IsProbing) {
|
||||
Print("DATA_FOUND");
|
||||
Stop();
|
||||
return;
|
||||
}
|
||||
|
||||
var bar = Bars.Last(1);
|
||||
_m1Buffer.Add(new M1Record {
|
||||
Time = bar.OpenTime.ToOADate(),
|
||||
O = bar.Open,
|
||||
H = bar.High,
|
||||
L = bar.Low,
|
||||
C = bar.Close,
|
||||
S = (float)Symbol.Spread,
|
||||
V = (int)bar.TickVolume
|
||||
});
|
||||
}
|
||||
|
||||
protected override void OnStop() {
|
||||
if (IsProbing || string.IsNullOrEmpty(OutputFile)) return;
|
||||
|
||||
try {
|
||||
using var fileStream = new FileStream(OutputFile, FileMode.Create);
|
||||
using var archive = new ZipArchive(fileStream, ZipArchiveMode.Create);
|
||||
var entry = archive.CreateEntry($"{Symbol.Name}.bin", CompressionLevel.Fastest);
|
||||
|
||||
using var entryStream = entry.Open();
|
||||
|
||||
if (Mode == ExportMode.M1) {
|
||||
ReadOnlySpan<M1Record> span = CollectionsMarshal.AsSpan(_m1Buffer);
|
||||
ReadOnlySpan<byte> byteSpan = MemoryMarshal.Cast<M1Record, byte>(span);
|
||||
if (!byteSpan.IsEmpty) entryStream.Write(byteSpan);
|
||||
} else {
|
||||
ReadOnlySpan<TickRecord> span = CollectionsMarshal.AsSpan(_tickBuffer);
|
||||
ReadOnlySpan<byte> byteSpan = MemoryMarshal.Cast<TickRecord, byte>(span);
|
||||
if (!byteSpan.IsEmpty) entryStream.Write(byteSpan);
|
||||
}
|
||||
|
||||
Print("EXPORT_SUCCESS");
|
||||
} catch (Exception ex) {
|
||||
Print("ERROR: {0}", ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user