38f657076b
This commit introduces a new data server module (`src/ast/data_server`) designed for high-performance, thread-safe loading and caching of market data. Key components: - `DataServer`: Manages symbol indexing and delegates loading/caching. - `SymbolIndex`: Scans the data directory and maintains a sorted index of data files per symbol. - `FileCache`: Implements a thread-safe cache using `RwLock` and a loading guard to prevent duplicate work. - `loader`: Handles ZIP decompression and binary record parsing from `.bin` files. - `records`: Defines raw and parsed data structures for M1 and tick data. - `SymbolChunkIter`: Provides an iterator over pre-parsed data chunks, prefetching subsequent files. This architecture allows multiple VM threads to access market data concurrently without redundant I/O, mirroring the strategy used in the original Delphi `TDataServer`.
108 lines
3.5 KiB
C#
Executable File
108 lines
3.5 KiB
C#
Executable File
using System;
|
|
using System.IO;
|
|
using System.IO.Compression;
|
|
using System.Collections.Generic;
|
|
using System.Runtime.InteropServices;
|
|
using cAlgo.API;
|
|
|
|
namespace cAlgo.Robots
|
|
{
|
|
public enum ExportMode { M1, Tick }
|
|
|
|
[Robot(AccessRights = AccessRights.FullAccess)]
|
|
public class MS_SimpleExport : Robot
|
|
{
|
|
[StructLayout(LayoutKind.Sequential, Pack = 1)]
|
|
private struct M1Record {
|
|
public double Time;
|
|
public double O, H, L, C;
|
|
public float S;
|
|
public int V;
|
|
}
|
|
|
|
[StructLayout(LayoutKind.Sequential, Pack = 1)]
|
|
private struct TickRecord {
|
|
public double Time;
|
|
public double Ask, Bid;
|
|
}
|
|
|
|
[Parameter("Export Mode", DefaultValue = ExportMode.M1)]
|
|
public ExportMode Mode { get; set; }
|
|
|
|
[Parameter("Output File (ZIP)", DefaultValue = "")]
|
|
public string OutputFile { get; set; }
|
|
|
|
[Parameter("Is Probing", DefaultValue = false)]
|
|
public bool IsProbing { get; set; }
|
|
|
|
private List<M1Record> _m1Buffer = new List<M1Record>();
|
|
private List<TickRecord> _tickBuffer = new List<TickRecord>();
|
|
|
|
protected override void OnStart() {
|
|
// No price factor needed for double/float export
|
|
}
|
|
|
|
protected override void OnTick() {
|
|
if (Mode != ExportMode.Tick) return;
|
|
|
|
if (IsProbing) {
|
|
Print("DATA_FOUND");
|
|
Stop();
|
|
return;
|
|
}
|
|
|
|
_tickBuffer.Add(new TickRecord {
|
|
Time = Server.TimeInUtc.ToOADate(),
|
|
Ask = Symbol.Ask,
|
|
Bid = Symbol.Bid
|
|
});
|
|
}
|
|
|
|
protected override void OnBar() {
|
|
if (Mode != ExportMode.M1) return;
|
|
|
|
if (IsProbing) {
|
|
Print("DATA_FOUND");
|
|
Stop();
|
|
return;
|
|
}
|
|
|
|
var bar = Bars.Last(1);
|
|
_m1Buffer.Add(new M1Record {
|
|
Time = bar.OpenTime.ToOADate(),
|
|
O = bar.Open,
|
|
H = bar.High,
|
|
L = bar.Low,
|
|
C = bar.Close,
|
|
S = (float)Symbol.Spread,
|
|
V = (int)bar.TickVolume
|
|
});
|
|
}
|
|
|
|
protected override void OnStop() {
|
|
if (IsProbing || string.IsNullOrEmpty(OutputFile)) return;
|
|
|
|
try {
|
|
using var fileStream = new FileStream(OutputFile, FileMode.Create);
|
|
using var archive = new ZipArchive(fileStream, ZipArchiveMode.Create);
|
|
var entry = archive.CreateEntry($"{Symbol.Name}.bin", CompressionLevel.Fastest);
|
|
|
|
using var entryStream = entry.Open();
|
|
|
|
if (Mode == ExportMode.M1) {
|
|
ReadOnlySpan<M1Record> span = CollectionsMarshal.AsSpan(_m1Buffer);
|
|
ReadOnlySpan<byte> byteSpan = MemoryMarshal.Cast<M1Record, byte>(span);
|
|
if (!byteSpan.IsEmpty) entryStream.Write(byteSpan);
|
|
} else {
|
|
ReadOnlySpan<TickRecord> span = CollectionsMarshal.AsSpan(_tickBuffer);
|
|
ReadOnlySpan<byte> byteSpan = MemoryMarshal.Cast<TickRecord, byte>(span);
|
|
if (!byteSpan.IsEmpty) entryStream.Write(byteSpan);
|
|
}
|
|
|
|
Print("EXPORT_SUCCESS");
|
|
} catch (Exception ex) {
|
|
Print("ERROR: {0}", ex.Message);
|
|
}
|
|
}
|
|
}
|
|
} |