Brummel/MycLib
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

DataStream updated

Browse Source
This commit is contained in:
Michael Schimmel
2026-01-21 14:09:26 +01:00
parent ceb6f13833
commit ef4583c2ae
9 changed files with 198 additions and 1280 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ASTPlayground/ASTPlayground.delphilsp.json
+1 -1
View File
File diff suppressed because one or more lines are too long
AuraTrader/AuraTrader.dpr
+1 -2
View File
@@ -9,8 +9,7 @@ uses
DynamicFMXControl in 'DynamicFMXControl.pas',
Myc.Trade.Pipeline.Impl in '..\Src\Myc.Trade.Pipeline.Impl.pas',
Myc.Trade.Indicators_v2 in '..\Src\Myc.Trade.Indicators_v2.pas',
Strategy2 in 'Strategy2.pas',
Myc.Trade.DataFeed in '..\Src\Myc.Trade.DataFeed.pas';
Strategy2 in 'Strategy2.pas';
{$R *.res}
AuraTrader/AuraTrader.dproj
+1 -8
View File
@@ -4,7 +4,7 @@
<ProjectVersion>20.3</ProjectVersion>
<FrameworkType>FMX</FrameworkType>
<Base>True</Base>
<Config Condition="'$(Config)'==''">Debug</Config>
<Config Condition="'$(Config)'==''">Release</Config>
<Platform Condition="'$(Platform)'==''">Win64</Platform>
<ProjectName Condition="'$(ProjectName)'==''">AuraTrader</ProjectName>
<TargetedPlatforms>3</TargetedPlatforms>
@@ -140,7 +140,6 @@
<DCCReference Include="..\Src\Myc.Trade.Pipeline.Impl.pas"/>
<DCCReference Include="..\Src\Myc.Trade.Indicators_v2.pas"/>
<DCCReference Include="Strategy2.pas"/>
<DCCReference Include="..\Src\Myc.Trade.DataFeed.pas"/>
<BuildConfiguration Include="Base">
<Key>Base</Key>
</BuildConfiguration>
@@ -190,12 +189,6 @@
<Overwrite>true</Overwrite>
</Platform>
</DeployFile>
<DeployFile LocalName="Win64\Debug\AuraTrader.rsm" Configuration="Debug" Class="DebugSymbols">
<Platform Name="Win64">
<RemoteName>AuraTrader.rsm</RemoteName>
<Overwrite>true</Overwrite>
</Platform>
</DeployFile>
<DeployFile LocalName="Win64\Release\AuraTrader.exe" Configuration="Release" Class="ProjectOutput">
<Platform Name="Win64">
<RemoteName>AuraTrader.exe</RemoteName>
AuraTrader/MainForm.fmx
-7
View File
@@ -204,13 +204,6 @@ object Form1: TForm1
Text = 'Button2'
OnClick = Button2Click
end
object Button3: TButton
Position.X = 64.000000000000000000
Position.Y = 152.000000000000000000
TabOrder = 2
Text = 'Button3'
OnClick = Button3Click
end
end
end
end
AuraTrader/MainForm.pas
-100
View File
@@ -56,7 +56,6 @@ uses
Myc.Trade.Indicators,
Myc.Trade.Indicators.Common,
Myc.Core.FileCache,
Myc.Trade.DataFeed,
StrategyTest,
TestMethodCallFromRecordParams;
@@ -89,7 +88,6 @@ type
Strat2Button: TSpeedButton;
Button1: TButton;
Button2: TButton;
Button3: TButton;
procedure FormCreate(Sender: TObject);
procedure FormDestroy(Sender: TObject);
procedure StopButtonClick(Sender: TObject);
@@ -97,7 +95,6 @@ type
procedure AddWorkspaceActionExecute(Sender: TObject);
procedure Button1Click(Sender: TObject);
procedure Button2Click(Sender: TObject);
procedure Button3Click(Sender: TObject);
procedure LogMemoChange(Sender: TObject);
procedure Strat2ButtonClick(Sender: TObject);
procedure TestActionExecute(Sender: TObject);
@@ -115,8 +112,6 @@ type
FProcessDone: TState;
FApplication: IAuraApplication;
FModulesItem: TTreeViewItem;
FFileCache: IDataFileCache<TArray<TScalarBatch>>;
FFeed: IScalarBatchLoader;
function SelectedSymbol: String;
procedure ExecuteStrategy(const Symbol: String; Timeframe: TTimeframe; const Consumer: IConsumer<TDataPoint<TOhlcItem>>);
@@ -197,8 +192,6 @@ procedure TForm1.FormCreate(Sender: TObject);
begin
FApplication := TMycAuraApplication.Create;
FFileCache := TDataFileCache<TArray<TScalarBatch>>.Create;
FModulesItem := TTreeViewItem.Create(Self);
FModulesItem.Text := 'Modules';
@@ -350,99 +343,6 @@ begin
end;
procedure TForm1.Button3Click(Sender: TObject);
begin
var symbol := SelectedSymbol;
if symbol = '' then
exit;
var layout := CurrLayout<TVertScrollBox>;
if layout = nil then
exit;
// 1. Setup Chart
var chart := TMycChart.Create(Self);
AlignControl(chart);
chart.Height := layout.ChildrenRect.Width * 9 / 16;
chart.Lookback.Value := 500000;
// 2. Define Schema (Keywords & Layout)
var kTime := TKeywordRegistry.Intern('Time');
var kClose := TKeywordRegistry.Intern('Close');
var def :=
TScalarRecord.TRegistry.Intern(
[
TPair<IKeyword, TScalar.TKind>.Create(kTime, TScalar.TKind.DateTime),
TPair<IKeyword, TScalar.TKind>.Create(kClose, TScalar.TKind.Float)
]
);
// 3. Build Pipeline
var iTime := def.IndexOf(kTime);
var iClose := def.IndexOf(kClose);
var terminated := TFlag.CreateObserver(FTerminate.Signal).State;
var path := '\\COFFEE\TickData\Pepperstone'; // Or FServer.Path if compatible
FFeed :=
TScalarBatchLoader.Create(
FFileCache,
path,
'.m1',
true,
SizeOf(TM1FileItem),
procedure(const Src: Pointer; Dst: TScalar.PValue)
var
Item: ^TM1FileItem;
begin
Item := Src;
// Field 0: Time (DateTime is stored as Double/OADate in TScalar)
Dst.AsDouble := Item.OADateTime;
// Move to next field in destination (TScalar values are contiguous)
Inc(Dst);
// Field 1: Close (Float)
// Convert Int64 price to Double using Digits: Price / 10^Digits
if Item.Digits > 0 then
Dst.AsDouble := Item.Close / Power(10, Item.Digits)
else
Dst.AsDouble := Item.Close;
end
);
// Converter: Transforms raw IRecordSeries (Columns) into TArray<TDataPoint>
var unpacker := TScalarBatchLoader.CreateTicker(def);
var ticker :=
unpacker.Producer.Chain<TDataPoint<Double>>(
function(const Tick: IScalarRecord): TDataPoint<Double>
begin
Result.Time := Tick[iTime].Value.Value.AsDouble;
Result.Data := Tick[iClose].Value.Value.AsDouble;
end
);
FFeed.BatchProducer.Chain(unpacker.Consumer);
// Link Chart to Ticker
chart
.SetXAxisSeries(TTimeframe.M, ticker.Chain<TDateTime>(function(const p: TDataPoint<Double>): TDateTime begin Result := p.Time end));
var panel := chart.AddPanel;
panel.AddDoubleSeries(
ticker.Chain<Double>(function(const p: TDataPoint<Double>): Double begin Result := p.Data end),
TAlphaColors.Orange
);
// Connect pipeline: Server -> Converter
FProcessDone := FFeed.ProcessData(symbol, terminated);
end;
function TForm1.CreateStrategy2(Timeframe: TTimeframe): IConsumer<TDataPoint<TOhlcItem>>;
type
TSignal = record
Src/Data/Myc.Data.Decimal.pas
+5 -10
View File
@@ -86,17 +86,12 @@ begin
var newValue := AValue.GetValue;
var scaleDiff := ANewScale - oldScale;
if scaleDiff > 0 then
if (scaleDiff > 0) then
begin
// Increasing scale: multiply by 10^scaleDiff, checking for overflow at each step.
for var i := 1 to scaleDiff do
begin
if newValue > (MAX_VALUE_61BIT div 10) then
var multiplier := PowersOf10[scaleDiff];
if (newValue > (MAX_VALUE_61BIT div multiplier)) or (newValue < (MIN_VALUE_61BIT div multiplier)) then
raise EOverflow.Create('Decimal scale up resulted in an overflow.');
if newValue < (MIN_VALUE_61BIT div 10) then
raise EOverflow.Create('Decimal scale up resulted in an overflow.');
newValue := newValue * 10;
end;
newValue := newValue * multiplier;
end
else
begin
@@ -380,7 +375,7 @@ begin
if val < 0 then
exit(-1);
if val > 0 then
exit(-1);
exit(1);
exit(0);
end;
Src/Data/Myc.Data.Pipeline.pas
+4 -2
View File
@@ -7,16 +7,18 @@ uses
Myc.Mutable,
Myc.Data.Series;
{$M+}
type
TTag = Pointer;
// A generic interface for components that consume data of type T.
IConsumer<T> = interface(IInvokable)
IConsumer<T> = interface
function Consume(const Value: T): TState;
end;
// A producer generates data and distributes it to linked consumers.
IProducer<T> = interface(IInvokable)
IProducer<T> = interface
function Link(const Consumer: IConsumer<T>): TTag;
procedure Unlink(Tag: TTag);
end;
Src/Myc.Trade.DataFeed.pas
-697
View File
@@ -1,697 +0,0 @@
unit Myc.Trade.DataFeed;
interface
uses
System.SysUtils,
System.Classes,
System.Generics.Collections,
System.Generics.Defaults,
System.IOUtils,
System.TimeSpan,
Myc.Futures,
Myc.Signals,
Myc.Mutable,
Myc.Data.Scalar,
Myc.Data.Keyword,
Myc.Data.Pipeline,
Myc.Data.Pipeline.Impl,
Myc.Core.FileCache;
type
// Callback to convert raw bytes from a file into a TScalar.TValue sequence
TImportConverter = reference to procedure(const Src: Pointer; Dst: TScalar.PValue);
// Represents a raw block of data read from disk
TScalarBatch = record
Data: TArray<TScalar.TValue>;
RecCount: Integer;
end;
IBroker = interface
function GetEquity: Double;
procedure Buy(Count: Integer);
procedure Sell(Count: Integer);
property Equity: Double read GetEquity;
end;
IScalarBatchLoader = interface
{$region 'private'}
function GetBatchProducer: TProducer<TScalarBatch>;
{$endregion}
function EnumerateSymbols: TFuture<TArray<String>>;
// Streaming processing: Reads data from disk and broadcasts it via BatchProducer
function ProcessData(const Symbol: String; const Terminated: TState): TState;
// Provides raw TScalarBatch events
property BatchProducer: TProducer<TScalarBatch> read GetBatchProducer;
end;
TScalarBatchLoader = class(TInterfacedObject, IScalarBatchLoader)
public
type
TDataFile = record
private
FExtension: String;
FPath: String;
FSymbol: String;
FYear: Integer;
FMonth: Integer;
FBasename: String;
function GetIsValid: Boolean;
public
constructor Create(const APath, ASymbol, AExtension: String; AYear, AMonth: Integer; const ABasename: String);
function GetFullFileName: string;
property BaseFileName: string read FBasename;
property IsValid: Boolean read GetIsValid;
property Extension: String read FExtension;
property Path: String read FPath;
property Symbol: String read FSymbol;
property Year: Integer read FYear;
property Month: Integer read FMonth;
end;
var
FPath: String;
FLookback: Int64;
FFileExtension: String;
FIsCompressed: Boolean;
FCachedFiles: IDataFileCache<TArray<TScalarBatch>>;
FSymbols: TFuture<TDictionary<String, TArray<TDataFile>>>;
// Internal producer for raw batches
FBatchProducer: TMycContainedProducer<TScalarBatch>;
class var
FLoadGate: TLatch;
function GetPath: String;
// Loading Logic
function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TScalarBatch>>;
function LoadDataFile(const DataFile: TDataFile): TFuture<TArray<TScalarBatch>>;
// Processing Logic
function ProcessFile(
const Files: TArray<TDataFile>;
Index: Integer;
const DataFile: TFuture<TArray<TScalarBatch>>;
const Terminated: TState
): TState;
function ProcessChunks(const Batches: TArray<TScalarBatch>; Terminated: TState): TState;
// Binary I/O
function ReadCompressedData(const InputStream: TStream): TArray<TScalarBatch>;
function ReadUncompressedData(const InputStream: TStream): TArray<TScalarBatch>;
// Helpers
function ParseFileName(const FileName: string): TDataFile;
procedure UpdateSymbols;
private
FRecordSize: Integer;
FConverter: TImportConverter;
function GetBatchProducer: TProducer<TScalarBatch>;
public
constructor Create(
const ACachedFiles: IDataFileCache<TArray<TScalarBatch>>;
const APath, AExtension: String;
IsCompressed: Boolean;
// RecordSize defines both the input bytes per record AND the output TValues per record
ARecordSize: Integer;
const AConverter: TImportConverter
);
destructor Destroy; override;
procedure AfterConstruction; override;
function EnumerateSymbols: TFuture<TArray<String>>;
// Streaming processing: Reads data from disk and broadcasts it via BatchTicker
function ProcessData(const Symbol: String; const Terminated: TState): TState;
class function CreateTicker(const Def: IScalarRecordDefinition): TConverter<TScalarBatch, IScalarRecord>; static;
property Path: String read GetPath;
property FileExtension: String read FFileExtension;
property IsCompressed: Boolean read FIsCompressed;
property Lookback: Int64 read FLookback write FLookback;
property BatchProducer: TProducer<TScalarBatch> read GetBatchProducer;
end;
implementation
uses
System.Zip,
System.Math,
System.StrUtils,
Myc.TaskManager;
const
BatchSize = 1024;
type
// Private implementation of the view logic (Cursor Pattern).
// This object is reused to avoid allocations per tick.
TScalarRecordView = class(TInterfacedObject, IScalarRecord)
private
FDef: IScalarRecordDefinition;
FData: TScalar.PValue;
// IKeywordMapping<TScalar.TValue> implementation
function IndexOf(const Key: IKeyword): Integer;
function GetCount: Integer;
function GetItems(Idx: Integer): TPair<IKeyword, TScalar>;
function GetFields(const Key: IKeyword): TScalar;
function GetDef: IScalarRecordDefinition;
public
constructor Create(const ADef: IScalarRecordDefinition);
procedure SetData(const AData: TScalar.PValue); inline;
end;
{ TScalarRecordView }
constructor TScalarRecordView.Create(const ADef: IScalarRecordDefinition);
begin
inherited Create;
FDef := ADef;
end;
procedure TScalarRecordView.SetData(const AData: TScalar.PValue);
begin
FData := AData;
end;
function TScalarRecordView.GetCount: Integer;
begin
Result := FDef.Count;
end;
function TScalarRecordView.GetDef: IScalarRecordDefinition;
begin
Result := FDef;
end;
function TScalarRecordView.GetFields(const Key: IKeyword): TScalar;
var
idx: Integer;
begin
idx := FDef.IndexOf(Key);
if idx < 0 then
exit(Default(TScalar));
// Return the raw TValue at the offset
Result.Kind := FDef[idx].Value;
Result.Value := TScalar.PValue(NativeUInt(FData) + NativeUInt(idx * SizeOf(TScalar.TValue)))^;
end;
function TScalarRecordView.GetItems(Idx: Integer): TPair<IKeyword, TScalar>;
begin
Result.Key := FDef[Idx].Key;
Result.Value.Kind := FDef[idx].Value;
Result.Value.Value := TScalar.PValue(NativeUInt(FData) + NativeUInt(Idx * SizeOf(TScalar.TValue)))^;
end;
function TScalarRecordView.IndexOf(const Key: IKeyword): Integer;
begin
Result := FDef.IndexOf(Key);
end;
{ TScalarBatchLoader.TDataFile }
constructor TScalarBatchLoader.TDataFile.Create(const APath, ASymbol, AExtension: String; AYear, AMonth: Integer; const ABasename: String);
begin
FPath := APath;
FSymbol := ASymbol;
FExtension := AExtension;
FYear := AYear;
FMonth := AMonth;
FBasename := ABasename;
end;
function TScalarBatchLoader.TDataFile.GetFullFileName: string;
begin
Result := TPath.Combine(FPath, FBasename + FExtension);
end;
function TScalarBatchLoader.TDataFile.GetIsValid: Boolean;
begin
Result := (FSymbol <> '') and (FYear > 0);
end;
{ TScalarBatchLoader }
constructor TScalarBatchLoader.Create(
const ACachedFiles: IDataFileCache<TArray<TScalarBatch>>;
const APath, AExtension: String;
IsCompressed: Boolean;
ARecordSize: Integer;
const AConverter: TImportConverter
);
begin
inherited Create;
FPath := APath;
FCachedFiles := ACachedFiles;
FFileExtension := AExtension;
FIsCompressed := IsCompressed;
FLookback := -1;
// RecordSize is used for both Input Bytes Stride AND Output TValues Count
FRecordSize := ARecordSize;
FConverter := AConverter;
Assert(FRecordSize > 0);
Assert(Assigned(FConverter));
FBatchProducer := TMycContainedProducer<TScalarBatch>.Create(Self);
FCachedFiles := TDataFileCache<TArray<TScalarBatch>>.Create;
end;
destructor TScalarBatchLoader.Destroy;
begin
FCachedFiles.Clear;
FBatchProducer.Free;
inherited Destroy;
end;
procedure TScalarBatchLoader.AfterConstruction;
begin
inherited;
UpdateSymbols;
end;
class function TScalarBatchLoader.CreateTicker(const Def: IScalarRecordDefinition): TConverter<TScalarBatch, IScalarRecord>;
var
viewObj: TScalarRecordView;
view: IScalarRecord;
begin
Result :=
TConverter<TScalarBatch, IScalarRecord>.CreateAggregation(
function(const Value: TScalarBatch; const Broadcast: TBroadcastFunc<IScalarRecord>): TState
begin
if (Length(Value.Data) > 0) and (Value.RecCount > 0) then
begin
var Stride := Length(Value.Data) div Value.RecCount;
var pCursor: TScalar.PValue := @Value.Data[0];
for var i := 0 to Value.RecCount - 1 do
begin
if view = nil then
begin
// Initialize reused View object (Cursor Pattern)
// This object implements IScalarRecord and is updated for each tick.
viewObj := TScalarRecordView.Create(Def);
view := viewObj as IScalarRecord;
end;
var rc := viewObj._AddRef;
try
// Broadcast tick
viewObj.SetData(pCursor);
Broadcast(view);
finally
// If client picked it up, we release it
if viewObj._Release >= rc then
view := nil;
end;
Inc(pCursor, Stride);
end;
end;
Result := TState.Null;
end
);
end;
function TScalarBatchLoader.GetPath: String;
begin
Result := FPath;
end;
function TScalarBatchLoader.GetBatchProducer: TProducer<TScalarBatch>;
begin
Result := FBatchProducer;
end;
procedure TScalarBatchLoader.UpdateSymbols;
begin
FSymbols :=
TFuture<TDictionary<String, TArray<TDataFile>>>.Construct(
FSymbols.Done,
function: TDictionary<String, TArray<TDataFile>>
var
fileNames: TArray<string>;
tempSymbolMap: TDictionary<String, TList<TDataFile>>;
dataFile: TDataFile;
symbolList: TList<TDataFile>;
begin
Result := TDictionary<String, TArray<TDataFile>>.Create;
tempSymbolMap := TDictionary<String, TList<TDataFile>>.Create;
try
if not TDirectory.Exists(FPath) then
exit;
fileNames := TDirectory.GetFiles(FPath);
for var currentFile in fileNames do
begin
dataFile := ParseFileName(currentFile);
if dataFile.IsValid then
begin
if not tempSymbolMap.TryGetValue(dataFile.Symbol, symbolList) then
begin
symbolList := TList<TDataFile>.Create;
tempSymbolMap.Add(dataFile.Symbol, symbolList);
end;
symbolList.Add(dataFile);
end;
end;
for var kvp in tempSymbolMap do
begin
symbolList := kvp.Value;
symbolList.Sort(
TComparer<TDataFile>.Construct(
function(const Left, Right: TDataFile): Integer
begin
if Left.Year < Right.Year then
Result := -1
else if Left.Year > Right.Year then
Result := 1
else
Result := Left.Month - Right.Month;
end
)
);
Result.Add(kvp.Key, symbolList.ToArray);
end;
finally
for var kvp in tempSymbolMap do
kvp.Value.Free;
tempSymbolMap.Free;
end;
end
);
FSymbols.Manage;
end;
function TScalarBatchLoader.EnumerateSymbols: TFuture<TArray<String>>;
begin
Result :=
FSymbols.Chain<TArray<String>>(
function(const Symbols: TDictionary<String, TArray<TDataFile>>): TArray<String> begin Result := Symbols.Keys.ToArray; end
);
end;
function TScalarBatchLoader.ParseFileName(const FileName: string): TDataFile;
var
path, fileNameNoPath, baseName, ext, symbol: string;
year, month: Integer;
parts, dateParts: TArray<string>;
begin
Result := Default(TDataFile);
path := TPath.GetDirectoryName(FileName);
fileNameNoPath := TPath.GetFileName(FileName);
ext := TPath.GetExtension(fileNameNoPath);
if not fileNameNoPath.EndsWith(FFileExtension, True) then
exit;
baseName := TPath.GetFileNameWithoutExtension(fileNameNoPath);
parts := baseName.Split(['_']);
if Length(parts) < 3 then
exit;
var startDatePart := parts[High(parts) - 1];
dateParts := startDatePart.Split(['-']);
if Length(dateParts) <> 2 then
exit;
if not TryStrToInt(dateParts[0], year) then
exit;
if not TryStrToInt(dateParts[1], month) then
exit;
if (month < 1) or (month > 12) or (year <= 0) then
exit;
symbol := string.Join('_', Copy(parts, 0, Length(parts) - 2));
if symbol = '' then
exit;
Result := TDataFile.Create(path, symbol, ext, year, month, baseName);
end;
function TScalarBatchLoader.LoadDataFile(const DataFile: TDataFile): TFuture<TArray<TScalarBatch>>;
begin
if DataFile.IsValid then
Result :=
FCachedFiles.GetOrAdd(
DataFile.GetFullFileName,
function(const Filename: String): TFuture<TArray<TScalarBatch>> begin Result := DoLoad(Filename, FLoadGate); end
)
else
Result := TFuture<TArray<TScalarBatch>>.Construct(TArray<TScalarBatch>(nil));
end;
function TScalarBatchLoader.DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TScalarBatch>>;
begin
Result := TFuture<TArray<TScalarBatch>>.Null;
if not TFile.Exists(FileName) then
exit;
var capFileName := FileName;
if FIsCompressed then
begin
Result :=
TFuture<TBytes>
.Construct(Gate.Enqueue, function: TBytes begin Result := TFile.ReadAllBytes(capFileName); end)
.Chain<TArray<TScalarBatch>>(
function(bytes: TBytes): TArray<TScalarBatch>
begin
var ms := TBytesStream.Create(bytes);
try
Result := ReadCompressedData(ms);
finally
ms.Free;
end;
end);
end
else
begin
Result :=
TFuture<TArray<TScalarBatch>>.Construct(
Gate.Enqueue,
function: TArray<TScalarBatch>
begin
var fs := TFileStream.Create(capFileName, fmOpenRead or fmShareDenyWrite);
try
Result := ReadUncompressedData(fs);
finally
fs.Free;
end;
end
);
end;
end;
function TScalarBatchLoader.ReadCompressedData(const InputStream: TStream): TArray<TScalarBatch>;
var
decompressionStream: TStream;
localHeader: TZipHeader;
entryIndex, i: Integer;
zip: TZipFile;
begin
Result := nil;
decompressionStream := nil;
zip := nil;
if not Assigned(InputStream) or (InputStream.Size = 0) then
exit;
try
InputStream.Position := 0;
zip := TZipFile.Create;
zip.Open(InputStream, TZipMode.zmRead);
if zip.FileCount = 0 then
exit;
entryIndex := -1;
for i := 0 to zip.FileCount - 1 do
begin
if zip.FileNames[i].EndsWith(FFileExtension, True) then
begin
entryIndex := i;
break;
end;
end;
if entryIndex = -1 then
entryIndex := 0;
zip.Read(entryIndex, decompressionStream, localHeader);
if not Assigned(decompressionStream) then
exit;
Result := ReadUncompressedData(decompressionStream);
finally
decompressionStream.Free;
zip.Free;
end;
end;
function TScalarBatchLoader.ReadUncompressedData(const InputStream: TStream): TArray<TScalarBatch>;
var
fileSize, totalRecords: Int64;
numBatches, batchIdx, k: Integer;
bytesToRead, valuesToWrite: Integer;
tempBuffer: TBytes;
pSrc: PByte;
pDst: TScalar.PValue;
begin
Result := nil;
InputStream.Position := 0;
fileSize := InputStream.Size;
if (fileSize = 0) or ((fileSize mod FRecordSize) <> 0) then
exit;
totalRecords := fileSize div FRecordSize;
if totalRecords = 0 then
exit;
numBatches := (totalRecords + BatchSize - 1) div BatchSize;
SetLength(Result, numBatches);
// Buffer for reading raw data chunks before conversion
SetLength(tempBuffer, BatchSize * FRecordSize);
for batchIdx := 0 to numBatches - 1 do
begin
var recordsInBatch := Min(BatchSize, totalRecords - (batchIdx * BatchSize));
Result[batchIdx].RecCount := recordsInBatch;
// FRecordSize determines both allocation count and input byte stride
valuesToWrite := recordsInBatch * FRecordSize;
SetLength(Result[batchIdx].Data, valuesToWrite);
bytesToRead := recordsInBatch * FRecordSize;
if InputStream.Read(tempBuffer, 0, bytesToRead) <> bytesToRead then
raise EReadError.Create('Unexpected end of stream');
pSrc := @tempBuffer[0];
pDst := @Result[batchIdx].Data[0];
// Always use the converter to transform raw bytes to TValues
for k := 0 to recordsInBatch - 1 do
begin
FConverter(pSrc, pDst);
Inc(pSrc, FRecordSize);
Inc(pDst, FRecordSize);
end;
end;
end;
function TScalarBatchLoader.ProcessData(const Symbol: String; const Terminated: TState): TState;
begin
var cTerminated := Terminated;
var cSymbol := Symbol;
// We fetch the list once, then iterate by index.
Result :=
FSymbols.Chain(
function(const AllSymbols: TDictionary<String, TArray<TDataFile>>): TState
var
files: TArray<TDataFile>;
begin
if AllSymbols.TryGetValue(cSymbol, files) and (Length(files) > 0) then
begin
// Start with Index 0
Result := ProcessFile(files, 0, LoadDataFile(files[0]), cTerminated);
end
else
begin
Result := TState.Null;
end;
end
);
end;
function TScalarBatchLoader.ProcessFile(
const Files: TArray<TDataFile>;
Index: Integer;
const DataFile: TFuture<TArray<TScalarBatch>>;
const Terminated: TState
): TState;
begin
// Check bounds and termination
if (Index < 0) or (Index > High(Files)) or Terminated.IsSet then
exit(DataFile.Done);
var currentInfo := Files[Index];
if not currentInfo.IsValid then
exit(DataFile.Done);
// Prefetch logic: Prepare the next file immediately (if available)
var nextIndex := Index + 1;
var nextFuture: TFuture<TArray<TScalarBatch>>;
if nextIndex <= High(Files) then
nextFuture := LoadDataFile(Files[nextIndex])
else
nextFuture := TFuture<TArray<TScalarBatch>>.Construct(TArray<TScalarBatch>(nil)); // End of stream
var cTerminated := Terminated;
// Process current data, then recurse to next index
Result :=
DataFile.Chain(
function(const Batches: TArray<TScalarBatch>): TState
begin
var done := ProcessChunks(Batches, Terminated);
// Recurse with Index + 1
Result :=
TaskManager.RunTask(done, function: TState begin Result := ProcessFile(Files, nextIndex, nextFuture, cTerminated); end);
end
);
end;
function TScalarBatchLoader.ProcessChunks(const Batches: TArray<TScalarBatch>; Terminated: TState): TState;
begin
var callProcess :=
function(Idx: Integer): TFunc<TState>
begin
var cBatch := Batches[Idx];
Result :=
function: TState
begin
if not Terminated.IsSet then
begin
// Broadcast the raw batch.
FBatchProducer.Broadcast(cBatch);
Result := TState.Null;
end;
end;
end;
Result := TState.Null;
for var i := 0 to High(Batches) do
if not Terminated.IsSet then
begin
var queue := TaskManager.RunTask(Result, callProcess(i));
Result := queue;
end;
end;
end.
Src/Myc.Trade.DataStream.pas
+185 -452
View File
@@ -16,6 +16,24 @@ uses
Myc.Core.FileCache;
type
// Match C# M1Record (Pack = 1, 48 Bytes)
TM1Record = packed record
Time: Double;
Open: Double;
High: Double;
Low: Double;
Close: Double;
Spread: Single;
Volume: Integer;
end;
// Match C# TickRecord (Pack = 1, 24 Bytes)
TTickRecord = packed record
Time: Double;
Ask: Double;
Bid: Double;
end;
// Interface for an instantiable data server.
IDataServer<T: record> = interface
procedure ClearCache;
@@ -23,7 +41,7 @@ type
function EnumerateSymbols: TFuture<TArray<String>>;
end;
// Represents metadata for a single data file.
// Metadata for a monthly data file (SYMBOL_YYYY_MM.EXT)
TDataFile = record
private
FExtension: String;
@@ -45,12 +63,11 @@ type
property Month: Integer read FMonth;
end;
// Generic server for loading and managing sequential time-series data from files.
TDataServer<T: record> = class(TInterfacedObject, IDataServer<T>)
// Generic server base handling the pipeline and preloading.
TDataServer<T: record; R: record> = class(TInterfacedObject, IDataServer<T>)
private
FCachedFiles: TDataFileCache<TArray<TArray<TDataPoint<T>>>>;
FPath: String;
// The cache now holds a sorted array of all files for each symbol.
FSymbols: TFuture<TDictionary<String, TArray<TDataFile>>>;
function GetPath: String;
@@ -72,10 +89,12 @@ type
FLoadGate: TLatch;
protected
// Used by cache to actually load an uncached file. Now virtual and abstract.
function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<T>>>>; virtual; abstract;
// Gets the next consecutive data file from the cached, sorted list.
// Maps the raw file record to the internal TDataPoint<T>
function MapRecord(const RecordBuffer: R): TDataPoint<T>; virtual; abstract;
function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<T>>>>;
function GetNextFile(const Curr: TDataFile): TDataFile; virtual;
function ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<T>>>;
function ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<T>>>;
public
constructor Create(const APath: String);
@@ -83,58 +102,27 @@ type
procedure AfterConstruction; override;
procedure ClearCache;
procedure UpdateSymbols;
// Scans a directory to find the oldest file for a specific symbol.
function FindFirstFile(const Symbol: string): TFuture<TDataFile>;
function ParseFileName(const FileName: string): TDataFile; virtual; abstract;
function ParseFileName(const FileName: string): TDataFile;
function EnumerateSymbols: TFuture<TArray<String>>;
// Load data file and split content into chunks.
function LoadDataFile(const DataFile: TDataFile): TFuture<TArray<TArray<TDataPoint<T>>>>;
function ProcessData(const Symbol: String; const Terminated: TState; const Processor: IConsumer<TArray<TDataPoint<T>>>): TState;
property Path: String read GetPath;
end;
TAskBidFileItem = packed record
Ask: Single;
Bid: Single;
end;
TAskBidFileServer = class(TDataServer<TAskBidFileItem>)
private
class function ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAskBidFileItem>>>;
class function ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAskBidFileItem>>>;
TTickFileServer = class(TDataServer<TTickRecord, TTickRecord>)
protected
function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TAskBidFileItem>>>>; override;
public
function ParseFileName(const FileName: string): TDataFile; override;
function MapRecord(const RecordBuffer: TTickRecord): TDataPoint<TTickRecord>; override;
end;
TM1FileItem = packed record
OADateTime: Double;
Open: Int64;
High: Int64;
Low: Int64;
Close: Int64;
Spread: Int64;
TickVolume: Integer;
Digits: Integer;
end;
// File server for cTrader M1 data. Note that the generic type is TOhlcItem,
// as the server converts the file data into standard OHLC items.
TM1FileServer = class(TDataServer<TOhlcItem>)
private
class function ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>;
class function ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>;
TM1FileServer = class(TDataServer<TOhlcItem, TM1Record>)
protected
function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TOhlcItem>>>>; override;
public
function ParseFileName(const FileName: string): TDataFile; override;
function MapRecord(const RecordBuffer: TM1Record): TDataPoint<TOhlcItem>; override;
end;
implementation
uses
System.Zip,
System.ZLib,
@@ -143,8 +131,7 @@ uses
Myc.TaskManager;
const
// The number of records per data chunk when loading files.
ChunkSize = 512;
ChunkSize = 1024;
{ TDataFile }
@@ -165,43 +152,42 @@ end;
function TDataFile.GetFullFileName: string;
begin
Result := TPath.Combine(FPath, GetBaseFileName + FExtension);
Result := TPath.Combine(FPath, FBasename + FExtension);
end;
function TDataFile.GetIsValid: Boolean;
begin
Result := (FSymbol <> '') and (FYear > 0);
Result := (FSymbol <> '') and (FYear > 0) and (FMonth >= 1) and (FMonth <= 12);
end;
{ TDataServer<T> }
{ TDataServer<T, R> }
constructor TDataServer<T>.Create(const APath: String);
constructor TDataServer<T, R>.Create(const APath: String);
begin
inherited Create;
FPath := APath;
FCachedFiles := TDataFileCache<TArray<TArray<TDataPoint<T>>>>.Create;
end;
destructor TDataServer<T>.Destroy;
destructor TDataServer<T, R>.Destroy;
begin
ClearCache;
FCachedFiles.Free;
inherited Destroy;
end;
procedure TDataServer<T>.AfterConstruction;
procedure TDataServer<T, R>.AfterConstruction;
begin
inherited;
UpdateSymbols;
end;
procedure TDataServer<T>.ClearCache;
procedure TDataServer<T, R>.ClearCache;
begin
FCachedFiles.Clear;
end;
procedure TDataServer<T>.UpdateSymbols;
procedure TDataServer<T, R>.UpdateSymbols;
begin
FSymbols :=
TFuture<TDictionary<String, TArray<TDataFile>>>.Construct(
@@ -241,10 +227,8 @@ begin
TComparer<TDataFile>.Construct(
function(const Left, Right: TDataFile): Integer
begin
if Left.Year < Right.Year then
Result := -1
else if Left.Year > Right.Year then
Result := 1
if Left.Year <> Right.Year then
Result := Left.Year - Right.Year
else
Result := Left.Month - Right.Month;
end
@@ -263,7 +247,7 @@ begin
FSymbols.Manage;
end;
function TDataServer<T>.EnumerateSymbols: TFuture<TArray<String>>;
function TDataServer<T, R>.EnumerateSymbols: TFuture<TArray<String>>;
begin
Result :=
FSymbols.Chain<TArray<String>>(
@@ -271,7 +255,7 @@ begin
);
end;
function TDataServer<T>.FindFirstFile(const Symbol: string): TFuture<TDataFile>;
function TDataServer<T, R>.FindFirstFile(const Symbol: string): TFuture<TDataFile>;
begin
var symName := Symbol;
Result :=
@@ -288,13 +272,13 @@ begin
);
end;
function TDataServer<T>.GetNextFile(const Curr: TDataFile): TDataFile;
function TDataServer<T, R>.GetNextFile(const Curr: TDataFile): TDataFile;
var
allSymbolFiles: TArray<TDataFile>;
foundIndex: Integer;
begin
Result := Default(TDataFile);
if not Curr.IsValid or not FSymbols.Done.IsSet then
if (not Curr.IsValid) or (not FSymbols.Done.IsSet) then
exit;
if not FSymbols.Value.TryGetValue(Curr.Symbol, allSymbolFiles) then
@@ -303,7 +287,7 @@ begin
foundIndex := -1;
for var i := 0 to High(allSymbolFiles) do
begin
if allSymbolFiles[i].GetBaseFileName() = Curr.GetBaseFileName() then
if allSymbolFiles[i].GetBaseFileName = Curr.GetBaseFileName then
begin
foundIndex := i;
break;
@@ -311,51 +295,137 @@ begin
end;
if (foundIndex <> -1) and (foundIndex < High(allSymbolFiles)) then
begin
Result := allSymbolFiles[foundIndex + 1];
end;
end;
function TDataServer<T>.GetPath: String;
function TDataServer<T, R>.GetPath: String;
begin
Result := FPath;
end;
function TDataServer<T>.ProcessData(
const Symbol: String;
const Terminated: TState;
const Processor: IConsumer<TArray<TDataPoint<T>>>
): TState;
function TDataServer<T, R>.ParseFileName(const FileName: string): TDataFile;
var
fileNameNoPath, baseName, symbol, ext: string;
parts: TArray<string>;
year, month: Integer;
begin
var cProc := Processor;
var cTerminated := Terminated;
Result := Default(TDataFile);
fileNameNoPath := TPath.GetFileName(FileName);
ext := TPath.GetExtension(fileNameNoPath);
baseName := TPath.GetFileNameWithoutExtension(fileNameNoPath);
Result :=
FindFirstFile(Symbol)
.Chain(
function(const FirstFileInfo: TDataFile): TState
begin
Result := ProcessFile(FirstFileInfo, LoadDataFile(FirstFileInfo), cTerminated, cProc);
end);
// Format: SYMBOL_YYYY_MM
parts := baseName.Split(['_']);
if Length(parts) < 3 then
exit;
// Use reverse indexing for year and month
if (not TryStrToInt(parts[High(parts)], month)) or (not TryStrToInt(parts[High(parts) - 1], year)) then
exit;
symbol := string.Join('_', Copy(parts, 0, Length(parts) - 2));
Result := TDataFile.Create(TPath.GetDirectoryName(FileName), symbol, ext, year, month, baseName);
end;
function TDataServer<T>.LoadDataFile(const DataFile: TDataFile): TFuture<TArray<TArray<TDataPoint<T>>>>;
function TDataServer<T, R>.LoadDataFile(const DataFile: TDataFile): TFuture<TArray<TArray<TDataPoint<T>>>>;
begin
if DataFile.IsValid then
Result :=
FCachedFiles.GetOrAdd(
DataFile.GetFullFileName,
function(const Filename: String): TFuture<TArray<TArray<TDataPoint<T>>>>
begin
// Pass the private load gate to the virtual DoLoad method.
Result := DoLoad(Filename, FLoadGate);
end
function(const Filename: String): TFuture<TArray<TArray<TDataPoint<T>>>> begin Result := DoLoad(Filename, FLoadGate); end
)
else
Result := TFuture<TArray<TArray<TDataPoint<T>>>>.Construct(TArray<TArray<TDataPoint<T>>>(nil));
end;
function TDataServer<T>.ProcessChunks(
function TDataServer<T, R>.DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<T>>>>;
begin
var capFileName := FileName;
Result :=
TFuture<TBytes>
.Construct(Gate.Enqueue, function: TBytes begin Result := TFile.ReadAllBytes(capFileName); end)
.Chain<TArray<TArray<TDataPoint<T>>>>(
function(Bytes: TBytes): TArray<TArray<TDataPoint<T>>>
var
compStream: TBytesStream;
begin
compStream := TBytesStream.Create(Bytes);
try
Result := ReadCompressedData(compStream);
finally
compStream.Free;
end;
end);
end;
function TDataServer<T, R>.ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<T>>>;
var
zip: TZipFile;
decompStream: TStream;
header: TZipHeader;
entryIdx: Integer;
begin
Result := nil;
InputStream.Position := 0;
zip := TZipFile.Create;
try
zip.Open(InputStream, TZipMode.zmRead);
entryIdx := -1;
// Search for the .bin entry created by C#
for var i := 0 to zip.FileCount - 1 do
begin
if zip.FileNames[i].EndsWith('.bin', true) then
begin
entryIdx := i;
break;
end;
end;
if entryIdx = -1 then
exit;
zip.Read(entryIdx, decompStream, header);
try
Result := ReadUncompressedData(decompStream);
finally
decompStream.Free;
end;
finally
zip.Free;
end;
end;
function TDataServer<T, R>.ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<T>>>;
var
totalRecords, chunkCount, i, j, recsInChunk: Integer;
rawBuffer: TArray<R>;
begin
Result := nil;
totalRecords := InputStream.Size div sizeof(R);
if totalRecords = 0 then
exit;
// Fast block read into memory
SetLength(rawBuffer, totalRecords);
InputStream.Position := 0;
InputStream.ReadBuffer(rawBuffer[0], InputStream.Size);
chunkCount := (totalRecords + ChunkSize - 1) div ChunkSize;
SetLength(Result, chunkCount);
for i := 0 to chunkCount - 1 do
begin
recsInChunk := Min(ChunkSize, totalRecords - (i * ChunkSize));
SetLength(Result[i], recsInChunk);
for j := 0 to recsInChunk - 1 do
begin
Result[i][j] := MapRecord(rawBuffer[i * ChunkSize + j]);
end;
end;
end;
function TDataServer<T, R>.ProcessChunks(
const DataChunks: TArray<TArray<TDataPoint<T>>>;
Terminated: TState;
Processor: IConsumer<TArray<TDataPoint<T>>>
@@ -374,36 +444,34 @@ begin
end;
for var i := 0 to High(DataChunks) do
begin
if not Terminated.IsSet then
begin
var q := TaskManager.RunTask(Result, callProcess(i));
Result := q;
end
end;
end;
end;
function TDataServer<T>.ProcessFile(
function TDataServer<T, R>.ProcessFile(
FileInfo: TDataFile;
const DataFile: TFuture<TArray<TArray<TDataPoint<T>>>>;
const Terminated: TState;
Processor: IConsumer<TArray<TDataPoint<T>>>
): TState;
begin
if not FileInfo.IsValid or Terminated.IsSet then
if (not FileInfo.IsValid) or (Terminated.IsSet) then
exit(DataFile.Done);
// Read ahead the next file, while processing the current one
var nextFileInfo := GetNextFile(FileInfo);
var nextFile := LoadDataFile(nextFileInfo);
var cTerminated := Terminated;
Result :=
DataFile.Chain(
function(const DataChunks: TArray<TArray<TDataPoint<T>>>): TState
begin
// Process each chunk, checking for termination between chunks.
var done := ProcessChunks(DataChunks, Terminated, Processor);
// Move to next file (which is currently preloading) once all Processors are done
var done := ProcessChunks(DataChunks, cTerminated, Processor);
Result :=
TaskManager
.RunTask(done, function: TState begin Result := ProcessFile(nextFileInfo, nextFile, cTerminated, Processor); end);
@@ -411,370 +479,35 @@ begin
);
end;
{ TAskBidFileServer }
function TAskBidFileServer.DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TAskBidFileItem>>>>;
begin
Result := TFuture<TArray<TArray<TDataPoint<TAskBidFileItem>>>>.Null;
if not TFile.Exists(FileName) then
exit;
var capFileName := FileName;
if FileName.EndsWith('_zip', True) then
function TDataServer<T, R>.ProcessData(
const Symbol: String;
const Terminated: TState;
const Processor: IConsumer<TArray<TDataPoint<T>>>
): TState;
begin
var cProc := Processor;
var cTerm := Terminated;
Result :=
TFuture<TBytes>
.Construct(Gate.Enqueue, function: TBytes begin Result := TFile.ReadAllBytes(capFileName); end)
.Chain<TArray<TArray<TDataPoint<TAskBidFileItem>>>>(
function(bytes: TBytes): TArray<TArray<TDataPoint<TAskBidFileItem>>>
FindFirstFile(Symbol)
.Chain(function(const Info: TDataFile): TState begin Result := ProcessFile(Info, LoadDataFile(Info), cTerm, cProc); end);
end;
{ TTickFileServer }
function TTickFileServer.MapRecord(const RecordBuffer: TTickRecord): TDataPoint<TTickRecord>;
begin
var compMemoryStream := TBytesStream.Create(bytes);
try
compMemoryStream.Position := 0;
Result := ReadCompressedData(compMemoryStream);
finally
compMemoryStream.Free;
end;
end);
end
else
begin
Result :=
TFuture<TArray<TArray<TDataPoint<TAskBidFileItem>>>>.Construct(
Gate.Enqueue,
function: TArray<TArray<TDataPoint<TAskBidFileItem>>>
begin
if TFile.Exists(capFileName) then
begin
var stream := TFileStream.Create(capFileName, fmOpenRead or fmShareDenyWrite);
try
Result := ReadUncompressedData(stream);
finally
stream.Free;
end;
end;
end
);
end;
end;
function TAskBidFileServer.ParseFileName(const FileName: string): TDataFile;
var
fileNameNoPath, nameForParsing, baseName, ext, path, symbol: string;
year, month: Integer;
parts: TArray<string>;
begin
Result := Default(TDataFile);
path := TPath.GetDirectoryName(FileName);
fileNameNoPath := TPath.GetFileName(FileName);
nameForParsing := fileNameNoPath;
ext := '';
if nameForParsing.EndsWith('_zip', True) then
begin
ext := '_zip';
nameForParsing := nameForParsing.Substring(0, nameForParsing.Length - 4);
end;
var fileExt := TPath.GetExtension(nameForParsing) + ext;
baseName := TPath.GetFileNameWithoutExtension(nameForParsing);
parts := baseName.Split(['_']);
if Length(parts) < 3 then
exit;
if not TryStrToInt(parts[High(parts)], month) then
exit;
if (month < 1) or (month > 12) then
exit;
if not TryStrToInt(parts[High(parts) - 1], year) then
exit;
if year <= 0 then
exit;
symbol := string.Join('_', Copy(parts, 0, Length(parts) - 2));
if symbol = '' then
exit;
Result := TDataFile.Create(path, symbol, fileExt, year, month, baseName);
end;
class function TAskBidFileServer.ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAskBidFileItem>>>;
var
decompressionStream: TStream;
localHeader: TZipHeader;
entryIndex, i: Integer;
zipFileInstance: TZipFile;
begin
Result := nil;
decompressionStream := nil;
zipFileInstance := nil;
try
InputStream.Position := 0;
zipFileInstance := TZipFile.Create;
zipFileInstance.Open(InputStream, TZipMode.zmRead);
if zipFileInstance.FileCount = 0 then
exit;
entryIndex := -1;
for i := 0 to zipFileInstance.FileCount - 1 do
if zipFileInstance.FileNames[i].EndsWith('.tab', True) then
begin
entryIndex := i;
break;
end;
if entryIndex = -1 then
entryIndex := 0;
zipFileInstance.Read(entryIndex, decompressionStream, localHeader);
if not Assigned(decompressionStream) then
exit;
Result := ReadUncompressedData(decompressionStream);
finally
decompressionStream.Free;
zipFileInstance.Free;
end;
end;
class function TAskBidFileServer.ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAskBidFileItem>>>;
type
TFileRecord = packed record
TimeStamp: TDateTime;
Data: TAskBidFileItem;
end;
var
fileSize: Int64;
totalRecordCount, bytesRead, chunkIndex, indexInChunk, recordIndex: Integer;
rec: TFileRecord;
begin
Result := nil;
InputStream.Position := 0;
fileSize := InputStream.Size;
if (fileSize = 0) or ((fileSize mod SizeOf(TFileRecord)) <> 0) then
exit;
totalRecordCount := fileSize div SizeOf(TFileRecord);
if totalRecordCount > 0 then
begin
var numChunks := (totalRecordCount + ChunkSize - 1) div ChunkSize;
SetLength(Result, numChunks);
chunkIndex := 0;
indexInChunk := 0;
SetLength(Result[chunkIndex], Min(ChunkSize, totalRecordCount));
for recordIndex := 0 to totalRecordCount - 1 do
begin
if indexInChunk >= ChunkSize then
begin
Inc(chunkIndex);
indexInChunk := 0;
var remainingRecords := totalRecordCount - (chunkIndex * ChunkSize);
SetLength(Result[chunkIndex], Min(ChunkSize, remainingRecords));
end;
bytesRead := InputStream.Read(rec, SizeOf(TFileRecord));
if bytesRead <> SizeOf(TFileRecord) then
raise EReadError.CreateFmt('Read error. Expected %d bytes, read %d.', [SizeOf(TFileRecord), bytesRead]);
// Corrected syntax: Assign the newly created record instance.
Result[chunkIndex][indexInChunk] := TDataPoint<TAskBidFileItem>.Create(rec.TimeStamp, rec.Data);
Inc(indexInChunk);
end;
end;
Result := TDataPoint<TTickRecord>.Create(RecordBuffer.Time, RecordBuffer);
end;
{ TM1FileServer }
function TM1FileServer.DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TOhlcItem>>>>;
begin
Result := TFuture<TArray<TArray<TDataPoint<TOhlcItem>>>>.Null;
if not TFile.Exists(FileName) then
exit;
var capFileName := FileName;
// M1 files are always compressed with the .m1 extension
if FileName.EndsWith('.m1', True) then
function TM1FileServer.MapRecord(const RecordBuffer: TM1Record): TDataPoint<TOhlcItem>;
begin
Result :=
TFuture<TBytes>
.Construct(Gate.Enqueue, function: TBytes begin Result := TFile.ReadAllBytes(capFileName); end)
.Chain<TArray<TArray<TDataPoint<TOhlcItem>>>>(
function(bytes: TBytes): TArray<TArray<TDataPoint<TOhlcItem>>>
begin
var compMemoryStream := TBytesStream.Create(bytes);
try
compMemoryStream.Position := 0;
Result := ReadCompressedData(compMemoryStream);
finally
compMemoryStream.Free;
end;
end);
end;
end;
function TM1FileServer.ParseFileName(const FileName: string): TDataFile;
var
path, fileNameNoPath, baseName, ext, symbol: string;
year, month: Integer;
parts, dateParts: TArray<string>;
begin
Result := Default(TDataFile);
path := TPath.GetDirectoryName(FileName);
fileNameNoPath := TPath.GetFileName(FileName);
// M1 files are compressed and have a .m1 extension
if not fileNameNoPath.EndsWith('.m1', True) then
exit;
ext := TPath.GetExtension(fileNameNoPath);
baseName := TPath.GetFileNameWithoutExtension(fileNameNoPath);
// Expected format: SYMBOL_YYYY-MM_YYYY-MM
parts := baseName.Split(['_']);
if Length(parts) < 3 then
exit; // Not enough parts for SYMBOL_START_END
// The symbol can contain underscores, so we take everything before the last two parts.
var startDatePart := parts[High(parts) - 1];
dateParts := startDatePart.Split(['-']);
if Length(dateParts) <> 2 then
exit; // Start date is not YYYY-MM
if not TryStrToInt(dateParts[0], year) then
exit;
if not TryStrToInt(dateParts[1], month) then
exit;
if (month < 1) or (month > 12) or (year <= 0) then
exit;
symbol := string.Join('_', Copy(parts, 0, Length(parts) - 2));
if symbol = '' then
exit;
// The TDataFile only represents the start date of the file batch.
Result := TDataFile.Create(path, symbol, ext, year, month, baseName);
end;
class function TM1FileServer.ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>;
var
decompressionStream: TStream;
localHeader: TZipHeader;
entryIndex, i: Integer;
zipFileInstance: TZipFile;
begin
Result := nil;
decompressionStream := nil;
zipFileInstance := nil;
// M1 files are now zip archives containing the actual data file.
if not Assigned(InputStream) or (InputStream.Size = 0) then
exit;
try
InputStream.Position := 0;
zipFileInstance := TZipFile.Create;
zipFileInstance.Open(InputStream, TZipMode.zmRead);
if zipFileInstance.FileCount = 0 then
exit;
// Find the data file entry within the archive.
// The C# code creates an entry with the same name as the base filename, ending in .m1.
entryIndex := -1;
for i := 0 to zipFileInstance.FileCount - 1 do
begin
if zipFileInstance.FileNames[i].EndsWith('.m1', True) then
begin
entryIndex := i;
break;
end;
end;
// Fallback to the first entry if no specific .m1 file is found inside.
if entryIndex = -1 then
entryIndex := 0;
zipFileInstance.Read(entryIndex, decompressionStream, localHeader);
if not Assigned(decompressionStream) then
exit;
// The decompressed stream now contains the raw binary data.
Result := ReadUncompressedData(decompressionStream);
finally
decompressionStream.Free;
zipFileInstance.Free;
end;
end;
class function TM1FileServer.ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>;
type
TFileRecord = TM1FileItem;
var
fileSize: Int64;
totalRecordCount, bytesRead, chunkIndex, indexInChunk, recordIndex: Integer;
rec: TFileRecord;
ohlc: TOhlcItem;
factor: Double;
begin
Result := nil;
InputStream.Position := 0;
fileSize := InputStream.Size;
if (fileSize = 0) or ((fileSize mod SizeOf(TFileRecord)) <> 0) then
exit;
totalRecordCount := fileSize div SizeOf(TFileRecord);
if totalRecordCount > 0 then
begin
var numChunks := (totalRecordCount + ChunkSize - 1) div ChunkSize;
SetLength(Result, numChunks);
chunkIndex := 0;
indexInChunk := 0;
SetLength(Result[chunkIndex], Min(ChunkSize, totalRecordCount));
for recordIndex := 0 to totalRecordCount - 1 do
begin
if indexInChunk >= ChunkSize then
begin
Inc(chunkIndex);
indexInChunk := 0;
var remainingRecords := totalRecordCount - (chunkIndex * ChunkSize);
SetLength(Result[chunkIndex], Min(ChunkSize, remainingRecords));
end;
bytesRead := InputStream.Read(rec, SizeOf(TFileRecord));
if bytesRead <> SizeOf(TFileRecord) then
raise EReadError.CreateFmt('Read error. Expected %d bytes, read %d.', [SizeOf(TFileRecord), bytesRead]);
// Convert from M1 file record to a standard OHLC item
if rec.Digits > 0 then
factor := Power(10, rec.Digits)
else
factor := 1.0;
ohlc.Open := rec.Open / factor;
ohlc.High := rec.High / factor;
ohlc.Low := rec.Low / factor;
ohlc.Close := rec.Close / factor;
ohlc.Volume := rec.TickVolume;
// Corrected syntax: Assign the newly created record instance.
Result[chunkIndex][indexInChunk] := TDataPoint<TOhlcItem>.Create(rec.OADateTime, ohlc);
Inc(indexInChunk);
end;
end;
TDataPoint<TOhlcItem>.Create(
RecordBuffer.Time,
TOhlcItem.Create(RecordBuffer.Open, RecordBuffer.High, RecordBuffer.Low, RecordBuffer.Close, RecordBuffer.Volume)
);
end;
end.