DataStream updated

AuraTrader/AuraTrader.dpr

@@ -9,8 +9,7 @@ uses
     DynamicFMXControl in 'DynamicFMXControl.pas',
     Myc.Trade.Pipeline.Impl in '..\Src\Myc.Trade.Pipeline.Impl.pas',
     Myc.Trade.Indicators_v2 in '..\Src\Myc.Trade.Indicators_v2.pas',
-    Strategy2 in 'Strategy2.pas',
+    Strategy2 in 'Strategy2.pas';
-    Myc.Trade.DataFeed in '..\Src\Myc.Trade.DataFeed.pas';
 
 {$R *.res}
 

AuraTrader/AuraTrader.dproj

@@ -4,7 +4,7 @@
         <ProjectVersion>20.3</ProjectVersion>
         <FrameworkType>FMX</FrameworkType>
         <Base>True</Base>
-        <Config Condition="'$(Config)'==''">Debug</Config>
+        <Config Condition="'$(Config)'==''">Release</Config>
         <Platform Condition="'$(Platform)'==''">Win64</Platform>
         <ProjectName Condition="'$(ProjectName)'==''">AuraTrader</ProjectName>
         <TargetedPlatforms>3</TargetedPlatforms>
@@ -140,7 +140,6 @@
         <DCCReference Include="..\Src\Myc.Trade.Pipeline.Impl.pas"/>
         <DCCReference Include="..\Src\Myc.Trade.Indicators_v2.pas"/>
         <DCCReference Include="Strategy2.pas"/>
-        <DCCReference Include="..\Src\Myc.Trade.DataFeed.pas"/>
         <BuildConfiguration Include="Base">
             <Key>Base</Key>
         </BuildConfiguration>
@@ -190,12 +189,6 @@
                         <Overwrite>true</Overwrite>
                     </Platform>
                 </DeployFile>
-                <DeployFile LocalName="Win64\Debug\AuraTrader.rsm" Configuration="Debug" Class="DebugSymbols">
-                    <Platform Name="Win64">
-                        <RemoteName>AuraTrader.rsm</RemoteName>
-                        <Overwrite>true</Overwrite>
-                    </Platform>
-                </DeployFile>
                 <DeployFile LocalName="Win64\Release\AuraTrader.exe" Configuration="Release" Class="ProjectOutput">
                     <Platform Name="Win64">
                         <RemoteName>AuraTrader.exe</RemoteName>

AuraTrader/MainForm.fmx

@@ -204,13 +204,6 @@ object Form1: TForm1
               Text = 'Button2'
               OnClick = Button2Click
             end
-            object Button3: TButton
-              Position.X = 64.000000000000000000
-              Position.Y = 152.000000000000000000
-              TabOrder = 2
-              Text = 'Button3'
-              OnClick = Button3Click
-            end
           end
         end
       end

AuraTrader/MainForm.pas

@@ -56,7 +56,6 @@ uses
     Myc.Trade.Indicators,
     Myc.Trade.Indicators.Common,
     Myc.Core.FileCache,
-    Myc.Trade.DataFeed,
     StrategyTest,
     TestMethodCallFromRecordParams;
 
@@ -89,7 +88,6 @@ type
         Strat2Button: TSpeedButton;
         Button1: TButton;
         Button2: TButton;
-        Button3: TButton;
         procedure FormCreate(Sender: TObject);
         procedure FormDestroy(Sender: TObject);
         procedure StopButtonClick(Sender: TObject);
@@ -97,7 +95,6 @@ type
         procedure AddWorkspaceActionExecute(Sender: TObject);
         procedure Button1Click(Sender: TObject);
         procedure Button2Click(Sender: TObject);
-        procedure Button3Click(Sender: TObject);
         procedure LogMemoChange(Sender: TObject);
         procedure Strat2ButtonClick(Sender: TObject);
         procedure TestActionExecute(Sender: TObject);
@@ -115,8 +112,6 @@ type
         FProcessDone: TState;
         FApplication: IAuraApplication;
         FModulesItem: TTreeViewItem;
-        FFileCache: IDataFileCache<TArray<TScalarBatch>>;
-        FFeed: IScalarBatchLoader;
         function SelectedSymbol: String;
         procedure ExecuteStrategy(const Symbol: String; Timeframe: TTimeframe; const Consumer: IConsumer<TDataPoint<TOhlcItem>>);
 
@@ -197,8 +192,6 @@ procedure TForm1.FormCreate(Sender: TObject);
 begin
     FApplication := TMycAuraApplication.Create;
 
-    FFileCache := TDataFileCache<TArray<TScalarBatch>>.Create;
-
     FModulesItem := TTreeViewItem.Create(Self);
     FModulesItem.Text := 'Modules';
 
@@ -350,99 +343,6 @@ begin
 
 end;
 
-procedure TForm1.Button3Click(Sender: TObject);
-begin
-    var symbol := SelectedSymbol;
-    if symbol = '' then
-        exit;
-
-    var layout := CurrLayout<TVertScrollBox>;
-    if layout = nil then
-        exit;
-
-    // 1. Setup Chart
-    var chart := TMycChart.Create(Self);
-    AlignControl(chart);
-    chart.Height := layout.ChildrenRect.Width * 9 / 16;
-    chart.Lookback.Value := 500000;
-
-    // 2. Define Schema (Keywords & Layout)
-    var kTime := TKeywordRegistry.Intern('Time');
-    var kClose := TKeywordRegistry.Intern('Close');
-
-    var def :=
-        TScalarRecord.TRegistry.Intern(
-            [
-                TPair<IKeyword, TScalar.TKind>.Create(kTime, TScalar.TKind.DateTime),
-                TPair<IKeyword, TScalar.TKind>.Create(kClose, TScalar.TKind.Float)
-            ]
-        );
-
-    // 3. Build Pipeline
-
-    var iTime := def.IndexOf(kTime);
-    var iClose := def.IndexOf(kClose);
-
-    var terminated := TFlag.CreateObserver(FTerminate.Signal).State;
-    var path := '\\COFFEE\TickData\Pepperstone'; // Or FServer.Path if compatible
-
-    FFeed :=
-        TScalarBatchLoader.Create(
-            FFileCache,
-            path,
-            '.m1',
-            true,
-            SizeOf(TM1FileItem),
-            procedure(const Src: Pointer; Dst: TScalar.PValue)
-            var
-                Item: ^TM1FileItem;
-            begin
-                Item := Src;
-
-                // Field 0: Time (DateTime is stored as Double/OADate in TScalar)
-                Dst.AsDouble := Item.OADateTime;
-
-                // Move to next field in destination (TScalar values are contiguous)
-                Inc(Dst);
-
-                // Field 1: Close (Float)
-                // Convert Int64 price to Double using Digits: Price / 10^Digits
-                if Item.Digits > 0 then
-                    Dst.AsDouble := Item.Close / Power(10, Item.Digits)
-                else
-                    Dst.AsDouble := Item.Close;
-            end
-        );
-
-    // Converter: Transforms raw IRecordSeries (Columns) into TArray<TDataPoint>
-
-    var unpacker := TScalarBatchLoader.CreateTicker(def);
-
-    var ticker :=
-        unpacker.Producer.Chain<TDataPoint<Double>>(
-            function(const Tick: IScalarRecord): TDataPoint<Double>
-            begin
-                Result.Time := Tick[iTime].Value.Value.AsDouble;
-                Result.Data := Tick[iClose].Value.Value.AsDouble;
-            end
-        );
-
-    FFeed.BatchProducer.Chain(unpacker.Consumer);
-
-    // Link Chart to Ticker
-    chart
-        .SetXAxisSeries(TTimeframe.M, ticker.Chain<TDateTime>(function(const p: TDataPoint<Double>): TDateTime begin Result := p.Time end));
-
-    var panel := chart.AddPanel;
-    panel.AddDoubleSeries(
-        ticker.Chain<Double>(function(const p: TDataPoint<Double>): Double begin Result := p.Data end),
-        TAlphaColors.Orange
-    );
-
-    // Connect pipeline: Server -> Converter
-    FProcessDone := FFeed.ProcessData(symbol, terminated);
-end;
-
 function TForm1.CreateStrategy2(Timeframe: TTimeframe): IConsumer<TDataPoint<TOhlcItem>>;
 type
     TSignal = record

Src/Data/Myc.Data.Decimal.pas

@@ -86,17 +86,12 @@ begin
     var newValue := AValue.GetValue;
     var scaleDiff := ANewScale - oldScale;
 
-    if scaleDiff > 0 then
+    if (scaleDiff > 0) then
     begin
-        // Increasing scale: multiply by 10^scaleDiff, checking for overflow at each step.
+        var multiplier := PowersOf10[scaleDiff];
-        for var i := 1 to scaleDiff do
+        if (newValue > (MAX_VALUE_61BIT div multiplier)) or (newValue < (MIN_VALUE_61BIT div multiplier)) then
-        begin
-            if newValue > (MAX_VALUE_61BIT div 10) then
             raise EOverflow.Create('Decimal scale up resulted in an overflow.');
-            if newValue < (MIN_VALUE_61BIT div 10) then
+        newValue := newValue * multiplier;
-                raise EOverflow.Create('Decimal scale up resulted in an overflow.');
-            newValue := newValue * 10;
-        end;
     end
     else
     begin
@@ -380,7 +375,7 @@ begin
     if val < 0 then
         exit(-1);
     if val > 0 then
-        exit(-1);
+        exit(1);
     exit(0);
 end;
 

Src/Data/Myc.Data.Pipeline.pas

@@ -7,16 +7,18 @@ uses
     Myc.Mutable,
     Myc.Data.Series;
 
+{$M+}
+
 type
     TTag = Pointer;
 
     // A generic interface for components that consume data of type T.
-    IConsumer<T> = interface(IInvokable)
+    IConsumer<T> = interface
         function Consume(const Value: T): TState;
     end;
 
     // A producer generates data and distributes it to linked consumers.
-    IProducer<T> = interface(IInvokable)
+    IProducer<T> = interface
         function Link(const Consumer: IConsumer<T>): TTag;
         procedure Unlink(Tag: TTag);
     end;

Src/Myc.Trade.DataFeed.pas

@@ -1,697 +0,0 @@
-unit Myc.Trade.DataFeed;
-
-interface
-
-uses
-    System.SysUtils,
-    System.Classes,
-    System.Generics.Collections,
-    System.Generics.Defaults,
-    System.IOUtils,
-    System.TimeSpan,
-    Myc.Futures,
-    Myc.Signals,
-    Myc.Mutable,
-    Myc.Data.Scalar,
-    Myc.Data.Keyword,
-    Myc.Data.Pipeline,
-    Myc.Data.Pipeline.Impl,
-    Myc.Core.FileCache;
-
-type
-    // Callback to convert raw bytes from a file into a TScalar.TValue sequence
-    TImportConverter = reference to procedure(const Src: Pointer; Dst: TScalar.PValue);
-
-    // Represents a raw block of data read from disk
-    TScalarBatch = record
-        Data: TArray<TScalar.TValue>;
-        RecCount: Integer;
-    end;
-
-    IBroker = interface
-        function GetEquity: Double;
-        procedure Buy(Count: Integer);
-        procedure Sell(Count: Integer);
-        property Equity: Double read GetEquity;
-    end;
-
-    IScalarBatchLoader = interface
-        {$region 'private'}
-        function GetBatchProducer: TProducer<TScalarBatch>;
-        {$endregion}
-
-        function EnumerateSymbols: TFuture<TArray<String>>;
-
-        // Streaming processing: Reads data from disk and broadcasts it via BatchProducer
-        function ProcessData(const Symbol: String; const Terminated: TState): TState;
-
-        // Provides raw TScalarBatch events
-        property BatchProducer: TProducer<TScalarBatch> read GetBatchProducer;
-    end;
-
-    TScalarBatchLoader = class(TInterfacedObject, IScalarBatchLoader)
-    public
-        type
-            TDataFile = record
-            private
-                FExtension: String;
-                FPath: String;
-                FSymbol: String;
-                FYear: Integer;
-                FMonth: Integer;
-                FBasename: String;
-                function GetIsValid: Boolean;
-            public
-                constructor Create(const APath, ASymbol, AExtension: String; AYear, AMonth: Integer; const ABasename: String);
-                function GetFullFileName: string;
-                property BaseFileName: string read FBasename;
-                property IsValid: Boolean read GetIsValid;
-                property Extension: String read FExtension;
-                property Path: String read FPath;
-                property Symbol: String read FSymbol;
-                property Year: Integer read FYear;
-                property Month: Integer read FMonth;
-            end;
-
-        var
-            FPath: String;
-            FLookback: Int64;
-            FFileExtension: String;
-            FIsCompressed: Boolean;
-
-            FCachedFiles: IDataFileCache<TArray<TScalarBatch>>;
-            FSymbols: TFuture<TDictionary<String, TArray<TDataFile>>>;
-
-            // Internal producer for raw batches
-            FBatchProducer: TMycContainedProducer<TScalarBatch>;
-
-        class var
-            FLoadGate: TLatch;
-
-        function GetPath: String;
-
-        // Loading Logic
-        function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TScalarBatch>>;
-        function LoadDataFile(const DataFile: TDataFile): TFuture<TArray<TScalarBatch>>;
-
-        // Processing Logic
-        function ProcessFile(
-            const Files: TArray<TDataFile>;
-            Index: Integer;
-            const DataFile: TFuture<TArray<TScalarBatch>>;
-            const Terminated: TState
-        ): TState;
-
-        function ProcessChunks(const Batches: TArray<TScalarBatch>; Terminated: TState): TState;
-
-        // Binary I/O
-        function ReadCompressedData(const InputStream: TStream): TArray<TScalarBatch>;
-        function ReadUncompressedData(const InputStream: TStream): TArray<TScalarBatch>;
-
-        // Helpers
-        function ParseFileName(const FileName: string): TDataFile;
-        procedure UpdateSymbols;
-
-    private
-        FRecordSize: Integer;
-        FConverter: TImportConverter;
-
-        function GetBatchProducer: TProducer<TScalarBatch>;
-
-    public
-        constructor Create(
-            const ACachedFiles: IDataFileCache<TArray<TScalarBatch>>;
-            const APath, AExtension: String;
-            IsCompressed: Boolean;
-            // RecordSize defines both the input bytes per record AND the output TValues per record
-            ARecordSize: Integer;
-            const AConverter: TImportConverter
-        );
-        destructor Destroy; override;
-        procedure AfterConstruction; override;
-
-        function EnumerateSymbols: TFuture<TArray<String>>;
-
-        // Streaming processing: Reads data from disk and broadcasts it via BatchTicker
-        function ProcessData(const Symbol: String; const Terminated: TState): TState;
-
-        class function CreateTicker(const Def: IScalarRecordDefinition): TConverter<TScalarBatch, IScalarRecord>; static;
-
-        property Path: String read GetPath;
-        property FileExtension: String read FFileExtension;
-        property IsCompressed: Boolean read FIsCompressed;
-        property Lookback: Int64 read FLookback write FLookback;
-        property BatchProducer: TProducer<TScalarBatch> read GetBatchProducer;
-    end;
-
-implementation
-
-uses
-    System.Zip,
-    System.Math,
-    System.StrUtils,
-    Myc.TaskManager;
-
-const
-    BatchSize = 1024;
-
-type
-    // Private implementation of the view logic (Cursor Pattern).
-    // This object is reused to avoid allocations per tick.
-    TScalarRecordView = class(TInterfacedObject, IScalarRecord)
-    private
-        FDef: IScalarRecordDefinition;
-        FData: TScalar.PValue;
-
-        // IKeywordMapping<TScalar.TValue> implementation
-        function IndexOf(const Key: IKeyword): Integer;
-        function GetCount: Integer;
-        function GetItems(Idx: Integer): TPair<IKeyword, TScalar>;
-        function GetFields(const Key: IKeyword): TScalar;
-        function GetDef: IScalarRecordDefinition;
-    public
-        constructor Create(const ADef: IScalarRecordDefinition);
-        procedure SetData(const AData: TScalar.PValue); inline;
-    end;
-
-{ TScalarRecordView }
-
-constructor TScalarRecordView.Create(const ADef: IScalarRecordDefinition);
-begin
-    inherited Create;
-    FDef := ADef;
-end;
-
-procedure TScalarRecordView.SetData(const AData: TScalar.PValue);
-begin
-    FData := AData;
-end;
-
-function TScalarRecordView.GetCount: Integer;
-begin
-    Result := FDef.Count;
-end;
-
-function TScalarRecordView.GetDef: IScalarRecordDefinition;
-begin
-    Result := FDef;
-end;
-
-function TScalarRecordView.GetFields(const Key: IKeyword): TScalar;
-var
-    idx: Integer;
-begin
-    idx := FDef.IndexOf(Key);
-    if idx < 0 then
-        exit(Default(TScalar));
-
-    // Return the raw TValue at the offset
-    Result.Kind := FDef[idx].Value;
-    Result.Value := TScalar.PValue(NativeUInt(FData) + NativeUInt(idx * SizeOf(TScalar.TValue)))^;
-end;
-
-function TScalarRecordView.GetItems(Idx: Integer): TPair<IKeyword, TScalar>;
-begin
-    Result.Key := FDef[Idx].Key;
-    Result.Value.Kind := FDef[idx].Value;
-    Result.Value.Value := TScalar.PValue(NativeUInt(FData) + NativeUInt(Idx * SizeOf(TScalar.TValue)))^;
-end;
-
-function TScalarRecordView.IndexOf(const Key: IKeyword): Integer;
-begin
-    Result := FDef.IndexOf(Key);
-end;
-
-{ TScalarBatchLoader.TDataFile }
-
-constructor TScalarBatchLoader.TDataFile.Create(const APath, ASymbol, AExtension: String; AYear, AMonth: Integer; const ABasename: String);
-begin
-    FPath := APath;
-    FSymbol := ASymbol;
-    FExtension := AExtension;
-    FYear := AYear;
-    FMonth := AMonth;
-    FBasename := ABasename;
-end;
-
-function TScalarBatchLoader.TDataFile.GetFullFileName: string;
-begin
-    Result := TPath.Combine(FPath, FBasename + FExtension);
-end;
-
-function TScalarBatchLoader.TDataFile.GetIsValid: Boolean;
-begin
-    Result := (FSymbol <> '') and (FYear > 0);
-end;
-
-{ TScalarBatchLoader }
-
-constructor TScalarBatchLoader.Create(
-    const ACachedFiles: IDataFileCache<TArray<TScalarBatch>>;
-    const APath, AExtension: String;
-    IsCompressed: Boolean;
-    ARecordSize: Integer;
-    const AConverter: TImportConverter
-);
-begin
-    inherited Create;
-    FPath := APath;
-    FCachedFiles := ACachedFiles;
-    FFileExtension := AExtension;
-    FIsCompressed := IsCompressed;
-    FLookback := -1;
-
-    // RecordSize is used for both Input Bytes Stride AND Output TValues Count
-    FRecordSize := ARecordSize;
-    FConverter := AConverter;
-
-    Assert(FRecordSize > 0);
-    Assert(Assigned(FConverter));
-
-    FBatchProducer := TMycContainedProducer<TScalarBatch>.Create(Self);
-    FCachedFiles := TDataFileCache<TArray<TScalarBatch>>.Create;
-end;
-
-destructor TScalarBatchLoader.Destroy;
-begin
-    FCachedFiles.Clear;
-    FBatchProducer.Free;
-    inherited Destroy;
-end;
-
-procedure TScalarBatchLoader.AfterConstruction;
-begin
-    inherited;
-    UpdateSymbols;
-end;
-
-class function TScalarBatchLoader.CreateTicker(const Def: IScalarRecordDefinition): TConverter<TScalarBatch, IScalarRecord>;
-var
-    viewObj: TScalarRecordView;
-    view: IScalarRecord;
-begin
-    Result :=
-        TConverter<TScalarBatch, IScalarRecord>.CreateAggregation(
-            function(const Value: TScalarBatch; const Broadcast: TBroadcastFunc<IScalarRecord>): TState
-            begin
-                if (Length(Value.Data) > 0) and (Value.RecCount > 0) then
-                begin
-                    var Stride := Length(Value.Data) div Value.RecCount;
-                    var pCursor: TScalar.PValue := @Value.Data[0];
-
-                    for var i := 0 to Value.RecCount - 1 do
-                    begin
-                        if view = nil then
-                        begin
-                            // Initialize reused View object (Cursor Pattern)
-                            // This object implements IScalarRecord and is updated for each tick.
-                            viewObj := TScalarRecordView.Create(Def);
-                            view := viewObj as IScalarRecord;
-                        end;
-
-                        var rc := viewObj._AddRef;
-                        try
-                            // Broadcast tick
-                            viewObj.SetData(pCursor);
-                            Broadcast(view);
-                        finally
-                            // If client picked it up, we release it
-                            if viewObj._Release >= rc then
-                                view := nil;
-                        end;
-
-                        Inc(pCursor, Stride);
-                    end;
-                end;
-                Result := TState.Null;
-            end
-        );
-end;
-
-function TScalarBatchLoader.GetPath: String;
-begin
-    Result := FPath;
-end;
-
-function TScalarBatchLoader.GetBatchProducer: TProducer<TScalarBatch>;
-begin
-    Result := FBatchProducer;
-end;
-
-procedure TScalarBatchLoader.UpdateSymbols;
-begin
-    FSymbols :=
-        TFuture<TDictionary<String, TArray<TDataFile>>>.Construct(
-            FSymbols.Done,
-            function: TDictionary<String, TArray<TDataFile>>
-            var
-                fileNames: TArray<string>;
-                tempSymbolMap: TDictionary<String, TList<TDataFile>>;
-                dataFile: TDataFile;
-                symbolList: TList<TDataFile>;
-            begin
-                Result := TDictionary<String, TArray<TDataFile>>.Create;
-                tempSymbolMap := TDictionary<String, TList<TDataFile>>.Create;
-                try
-                    if not TDirectory.Exists(FPath) then
-                        exit;
-
-                    fileNames := TDirectory.GetFiles(FPath);
-                    for var currentFile in fileNames do
-                    begin
-                        dataFile := ParseFileName(currentFile);
-                        if dataFile.IsValid then
-                        begin
-                            if not tempSymbolMap.TryGetValue(dataFile.Symbol, symbolList) then
-                            begin
-                                symbolList := TList<TDataFile>.Create;
-                                tempSymbolMap.Add(dataFile.Symbol, symbolList);
-                            end;
-                            symbolList.Add(dataFile);
-                        end;
-                    end;
-
-                    for var kvp in tempSymbolMap do
-                    begin
-                        symbolList := kvp.Value;
-                        symbolList.Sort(
-                            TComparer<TDataFile>.Construct(
-                                function(const Left, Right: TDataFile): Integer
-                                begin
-                                    if Left.Year < Right.Year then
-                                        Result := -1
-                                    else if Left.Year > Right.Year then
-                                        Result := 1
-                                    else
-                                        Result := Left.Month - Right.Month;
-                                end
-                            )
-                        );
-                        Result.Add(kvp.Key, symbolList.ToArray);
-                    end;
-                finally
-                    for var kvp in tempSymbolMap do
-                        kvp.Value.Free;
-                    tempSymbolMap.Free;
-                end;
-            end
-        );
-    FSymbols.Manage;
-end;
-
-function TScalarBatchLoader.EnumerateSymbols: TFuture<TArray<String>>;
-begin
-    Result :=
-        FSymbols.Chain<TArray<String>>(
-            function(const Symbols: TDictionary<String, TArray<TDataFile>>): TArray<String> begin Result := Symbols.Keys.ToArray; end
-        );
-end;
-
-function TScalarBatchLoader.ParseFileName(const FileName: string): TDataFile;
-var
-    path, fileNameNoPath, baseName, ext, symbol: string;
-    year, month: Integer;
-    parts, dateParts: TArray<string>;
-begin
-    Result := Default(TDataFile);
-    path := TPath.GetDirectoryName(FileName);
-    fileNameNoPath := TPath.GetFileName(FileName);
-
-    ext := TPath.GetExtension(fileNameNoPath);
-
-    if not fileNameNoPath.EndsWith(FFileExtension, True) then
-        exit;
-
-    baseName := TPath.GetFileNameWithoutExtension(fileNameNoPath);
-
-    parts := baseName.Split(['_']);
-    if Length(parts) < 3 then
-        exit;
-
-    var startDatePart := parts[High(parts) - 1];
-    dateParts := startDatePart.Split(['-']);
-    if Length(dateParts) <> 2 then
-        exit;
-
-    if not TryStrToInt(dateParts[0], year) then
-        exit;
-    if not TryStrToInt(dateParts[1], month) then
-        exit;
-
-    if (month < 1) or (month > 12) or (year <= 0) then
-        exit;
-
-    symbol := string.Join('_', Copy(parts, 0, Length(parts) - 2));
-    if symbol = '' then
-        exit;
-
-    Result := TDataFile.Create(path, symbol, ext, year, month, baseName);
-end;
-
-function TScalarBatchLoader.LoadDataFile(const DataFile: TDataFile): TFuture<TArray<TScalarBatch>>;
-begin
-    if DataFile.IsValid then
-        Result :=
-            FCachedFiles.GetOrAdd(
-                DataFile.GetFullFileName,
-                function(const Filename: String): TFuture<TArray<TScalarBatch>> begin Result := DoLoad(Filename, FLoadGate); end
-            )
-    else
-        Result := TFuture<TArray<TScalarBatch>>.Construct(TArray<TScalarBatch>(nil));
-end;
-
-function TScalarBatchLoader.DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TScalarBatch>>;
-begin
-    Result := TFuture<TArray<TScalarBatch>>.Null;
-    if not TFile.Exists(FileName) then
-        exit;
-
-    var capFileName := FileName;
-
-    if FIsCompressed then
-    begin
-        Result :=
-            TFuture<TBytes>
-                .Construct(Gate.Enqueue, function: TBytes begin Result := TFile.ReadAllBytes(capFileName); end)
-                .Chain<TArray<TScalarBatch>>(
-                    function(bytes: TBytes): TArray<TScalarBatch>
-                    begin
-                        var ms := TBytesStream.Create(bytes);
-                        try
-                            Result := ReadCompressedData(ms);
-                        finally
-                            ms.Free;
-                        end;
-                    end);
-    end
-    else
-    begin
-        Result :=
-            TFuture<TArray<TScalarBatch>>.Construct(
-                Gate.Enqueue,
-                function: TArray<TScalarBatch>
-                begin
-                    var fs := TFileStream.Create(capFileName, fmOpenRead or fmShareDenyWrite);
-                    try
-                        Result := ReadUncompressedData(fs);
-                    finally
-                        fs.Free;
-                    end;
-                end
-            );
-    end;
-end;
-
-function TScalarBatchLoader.ReadCompressedData(const InputStream: TStream): TArray<TScalarBatch>;
-var
-    decompressionStream: TStream;
-    localHeader: TZipHeader;
-    entryIndex, i: Integer;
-    zip: TZipFile;
-begin
-    Result := nil;
-    decompressionStream := nil;
-    zip := nil;
-
-    if not Assigned(InputStream) or (InputStream.Size = 0) then
-        exit;
-
-    try
-        InputStream.Position := 0;
-        zip := TZipFile.Create;
-        zip.Open(InputStream, TZipMode.zmRead);
-
-        if zip.FileCount = 0 then
-            exit;
-
-        entryIndex := -1;
-        for i := 0 to zip.FileCount - 1 do
-        begin
-            if zip.FileNames[i].EndsWith(FFileExtension, True) then
-            begin
-                entryIndex := i;
-                break;
-            end;
-        end;
-
-        if entryIndex = -1 then
-            entryIndex := 0;
-
-        zip.Read(entryIndex, decompressionStream, localHeader);
-        if not Assigned(decompressionStream) then
-            exit;
-
-        Result := ReadUncompressedData(decompressionStream);
-    finally
-        decompressionStream.Free;
-        zip.Free;
-    end;
-end;
-
-function TScalarBatchLoader.ReadUncompressedData(const InputStream: TStream): TArray<TScalarBatch>;
-var
-    fileSize, totalRecords: Int64;
-    numBatches, batchIdx, k: Integer;
-    bytesToRead, valuesToWrite: Integer;
-    tempBuffer: TBytes;
-    pSrc: PByte;
-    pDst: TScalar.PValue;
-begin
-    Result := nil;
-    InputStream.Position := 0;
-    fileSize := InputStream.Size;
-
-    if (fileSize = 0) or ((fileSize mod FRecordSize) <> 0) then
-        exit;
-
-    totalRecords := fileSize div FRecordSize;
-    if totalRecords = 0 then
-        exit;
-
-    numBatches := (totalRecords + BatchSize - 1) div BatchSize;
-    SetLength(Result, numBatches);
-
-    // Buffer for reading raw data chunks before conversion
-    SetLength(tempBuffer, BatchSize * FRecordSize);
-
-    for batchIdx := 0 to numBatches - 1 do
-    begin
-        var recordsInBatch := Min(BatchSize, totalRecords - (batchIdx * BatchSize));
-        Result[batchIdx].RecCount := recordsInBatch;
-
-        // FRecordSize determines both allocation count and input byte stride
-        valuesToWrite := recordsInBatch * FRecordSize;
-        SetLength(Result[batchIdx].Data, valuesToWrite);
-
-        bytesToRead := recordsInBatch * FRecordSize;
-        if InputStream.Read(tempBuffer, 0, bytesToRead) <> bytesToRead then
-            raise EReadError.Create('Unexpected end of stream');
-
-        pSrc := @tempBuffer[0];
-        pDst := @Result[batchIdx].Data[0];
-
-        // Always use the converter to transform raw bytes to TValues
-        for k := 0 to recordsInBatch - 1 do
-        begin
-            FConverter(pSrc, pDst);
-            Inc(pSrc, FRecordSize);
-            Inc(pDst, FRecordSize);
-        end;
-    end;
-end;
-
-function TScalarBatchLoader.ProcessData(const Symbol: String; const Terminated: TState): TState;
-begin
-    var cTerminated := Terminated;
-    var cSymbol := Symbol;
-
-    // We fetch the list once, then iterate by index.
-    Result :=
-        FSymbols.Chain(
-            function(const AllSymbols: TDictionary<String, TArray<TDataFile>>): TState
-            var
-                files: TArray<TDataFile>;
-            begin
-                if AllSymbols.TryGetValue(cSymbol, files) and (Length(files) > 0) then
-                begin
-                    // Start with Index 0
-                    Result := ProcessFile(files, 0, LoadDataFile(files[0]), cTerminated);
-                end
-                else
-                begin
-                    Result := TState.Null;
-                end;
-            end
-        );
-end;
-
-function TScalarBatchLoader.ProcessFile(
-    const Files: TArray<TDataFile>;
-    Index: Integer;
-    const DataFile: TFuture<TArray<TScalarBatch>>;
-    const Terminated: TState
-): TState;
-begin
-    // Check bounds and termination
-    if (Index < 0) or (Index > High(Files)) or Terminated.IsSet then
-        exit(DataFile.Done);
-
-    var currentInfo := Files[Index];
-    if not currentInfo.IsValid then
-        exit(DataFile.Done);
-
-    // Prefetch logic: Prepare the next file immediately (if available)
-    var nextIndex := Index + 1;
-    var nextFuture: TFuture<TArray<TScalarBatch>>;
-
-    if nextIndex <= High(Files) then
-        nextFuture := LoadDataFile(Files[nextIndex])
-    else
-        nextFuture := TFuture<TArray<TScalarBatch>>.Construct(TArray<TScalarBatch>(nil)); // End of stream
-
-    var cTerminated := Terminated;
-
-    // Process current data, then recurse to next index
-    Result :=
-        DataFile.Chain(
-            function(const Batches: TArray<TScalarBatch>): TState
-            begin
-                var done := ProcessChunks(Batches, Terminated);
-
-                // Recurse with Index + 1
-                Result :=
-                    TaskManager.RunTask(done, function: TState begin Result := ProcessFile(Files, nextIndex, nextFuture, cTerminated); end);
-            end
-        );
-end;
-
-function TScalarBatchLoader.ProcessChunks(const Batches: TArray<TScalarBatch>; Terminated: TState): TState;
-begin
-    var callProcess :=
-        function(Idx: Integer): TFunc<TState>
-        begin
-            var cBatch := Batches[Idx];
-
-            Result :=
-                function: TState
-                begin
-                    if not Terminated.IsSet then
-                    begin
-                        // Broadcast the raw batch.
-                        FBatchProducer.Broadcast(cBatch);
-                        Result := TState.Null;
-                    end;
-                end;
-        end;
-
-    Result := TState.Null;
-
-    for var i := 0 to High(Batches) do
-        if not Terminated.IsSet then
-        begin
-            var queue := TaskManager.RunTask(Result, callProcess(i));
-            Result := queue;
-        end;
-end;
-
-end.

Src/Myc.Trade.DataStream.pas

@@ -16,6 +16,24 @@ uses
     Myc.Core.FileCache;
 
 type
+    // Match C# M1Record (Pack = 1, 48 Bytes)
+    TM1Record = packed record
+        Time: Double;
+        Open: Double;
+        High: Double;
+        Low: Double;
+        Close: Double;
+        Spread: Single;
+        Volume: Integer;
+    end;
+
+    // Match C# TickRecord (Pack = 1, 24 Bytes)
+    TTickRecord = packed record
+        Time: Double;
+        Ask: Double;
+        Bid: Double;
+    end;
+
     // Interface for an instantiable data server.
     IDataServer<T: record> = interface
         procedure ClearCache;
@@ -23,7 +41,7 @@ type
         function EnumerateSymbols: TFuture<TArray<String>>;
     end;
 
-    // Represents metadata for a single data file.
+    // Metadata for a monthly data file (SYMBOL_YYYY_MM.EXT)
     TDataFile = record
     private
         FExtension: String;
@@ -45,12 +63,11 @@ type
         property Month: Integer read FMonth;
     end;
 
-    // Generic server for loading and managing sequential time-series data from files.
+    // Generic server base handling the pipeline and preloading.
-    TDataServer<T: record> = class(TInterfacedObject, IDataServer<T>)
+    TDataServer<T: record; R: record> = class(TInterfacedObject, IDataServer<T>)
     private
         FCachedFiles: TDataFileCache<TArray<TArray<TDataPoint<T>>>>;
         FPath: String;
-        // The cache now holds a sorted array of all files for each symbol.
         FSymbols: TFuture<TDictionary<String, TArray<TDataFile>>>;
         function GetPath: String;
 
@@ -72,10 +89,12 @@ type
             FLoadGate: TLatch;
 
     protected
-        // Used by cache to actually load an uncached file. Now virtual and abstract.
+        // Maps the raw file record to the internal TDataPoint<T>
-        function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<T>>>>; virtual; abstract;
+        function MapRecord(const RecordBuffer: R): TDataPoint<T>; virtual; abstract;
-        // Gets the next consecutive data file from the cached, sorted list.
+        function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<T>>>>;
         function GetNextFile(const Curr: TDataFile): TDataFile; virtual;
+        function ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<T>>>;
+        function ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<T>>>;
 
     public
         constructor Create(const APath: String);
@@ -83,58 +102,27 @@ type
         procedure AfterConstruction; override;
         procedure ClearCache;
         procedure UpdateSymbols;
-        // Scans a directory to find the oldest file for a specific symbol.
         function FindFirstFile(const Symbol: string): TFuture<TDataFile>;
-        function ParseFileName(const FileName: string): TDataFile; virtual; abstract;
+        function ParseFileName(const FileName: string): TDataFile;
         function EnumerateSymbols: TFuture<TArray<String>>;
-
-        // Load data file and split content into chunks.
         function LoadDataFile(const DataFile: TDataFile): TFuture<TArray<TArray<TDataPoint<T>>>>;
-
         function ProcessData(const Symbol: String; const Terminated: TState; const Processor: IConsumer<TArray<TDataPoint<T>>>): TState;
 
         property Path: String read GetPath;
     end;
 
-    TAskBidFileItem = packed record
+    TTickFileServer = class(TDataServer<TTickRecord, TTickRecord>)
-        Ask: Single;
-        Bid: Single;
-    end;
-
-    TAskBidFileServer = class(TDataServer<TAskBidFileItem>)
-    private
-        class function ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAskBidFileItem>>>;
-        class function ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAskBidFileItem>>>;
     protected
-        function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TAskBidFileItem>>>>; override;
+        function MapRecord(const RecordBuffer: TTickRecord): TDataPoint<TTickRecord>; override;
-    public
-        function ParseFileName(const FileName: string): TDataFile; override;
     end;
 
-    TM1FileItem = packed record
+    TM1FileServer = class(TDataServer<TOhlcItem, TM1Record>)
-        OADateTime: Double;
-        Open: Int64;
-        High: Int64;
-        Low: Int64;
-        Close: Int64;
-        Spread: Int64;
-        TickVolume: Integer;
-        Digits: Integer;
-    end;
-
-    // File server for cTrader M1 data. Note that the generic type is TOhlcItem,
-    // as the server converts the file data into standard OHLC items.
-    TM1FileServer = class(TDataServer<TOhlcItem>)
-    private
-        class function ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>;
-        class function ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>;
     protected
-        function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TOhlcItem>>>>; override;
+        function MapRecord(const RecordBuffer: TM1Record): TDataPoint<TOhlcItem>; override;
-    public
-        function ParseFileName(const FileName: string): TDataFile; override;
     end;
 
 implementation
+
 uses
     System.Zip,
     System.ZLib,
@@ -143,8 +131,7 @@ uses
     Myc.TaskManager;
 
 const
-    // The number of records per data chunk when loading files.
+    ChunkSize = 1024;
-    ChunkSize = 512;
 
 { TDataFile }
 
@@ -165,43 +152,42 @@ end;
 
 function TDataFile.GetFullFileName: string;
 begin
-    Result := TPath.Combine(FPath, GetBaseFileName + FExtension);
+    Result := TPath.Combine(FPath, FBasename + FExtension);
 end;
 
 function TDataFile.GetIsValid: Boolean;
 begin
-    Result := (FSymbol <> '') and (FYear > 0);
+    Result := (FSymbol <> '') and (FYear > 0) and (FMonth >= 1) and (FMonth <= 12);
 end;
 
-{ TDataServer<T> }
+{ TDataServer<T, R> }
 
-constructor TDataServer<T>.Create(const APath: String);
+constructor TDataServer<T, R>.Create(const APath: String);
 begin
     inherited Create;
     FPath := APath;
-
     FCachedFiles := TDataFileCache<TArray<TArray<TDataPoint<T>>>>.Create;
 end;
 
-destructor TDataServer<T>.Destroy;
+destructor TDataServer<T, R>.Destroy;
 begin
     ClearCache;
     FCachedFiles.Free;
     inherited Destroy;
 end;
 
-procedure TDataServer<T>.AfterConstruction;
+procedure TDataServer<T, R>.AfterConstruction;
 begin
     inherited;
     UpdateSymbols;
 end;
 
-procedure TDataServer<T>.ClearCache;
+procedure TDataServer<T, R>.ClearCache;
 begin
     FCachedFiles.Clear;
 end;
 
-procedure TDataServer<T>.UpdateSymbols;
+procedure TDataServer<T, R>.UpdateSymbols;
 begin
     FSymbols :=
         TFuture<TDictionary<String, TArray<TDataFile>>>.Construct(
@@ -241,10 +227,8 @@ begin
                             TComparer<TDataFile>.Construct(
                                 function(const Left, Right: TDataFile): Integer
                                 begin
-                                    if Left.Year < Right.Year then
+                                    if Left.Year <> Right.Year then
-                                        Result := -1
+                                        Result := Left.Year - Right.Year
-                                    else if Left.Year > Right.Year then
-                                        Result := 1
                                     else
                                         Result := Left.Month - Right.Month;
                                 end
@@ -263,7 +247,7 @@ begin
     FSymbols.Manage;
 end;
 
-function TDataServer<T>.EnumerateSymbols: TFuture<TArray<String>>;
+function TDataServer<T, R>.EnumerateSymbols: TFuture<TArray<String>>;
 begin
     Result :=
         FSymbols.Chain<TArray<String>>(
@@ -271,7 +255,7 @@ begin
         );
 end;
 
-function TDataServer<T>.FindFirstFile(const Symbol: string): TFuture<TDataFile>;
+function TDataServer<T, R>.FindFirstFile(const Symbol: string): TFuture<TDataFile>;
 begin
     var symName := Symbol;
     Result :=
@@ -288,13 +272,13 @@ begin
         );
 end;
 
-function TDataServer<T>.GetNextFile(const Curr: TDataFile): TDataFile;
+function TDataServer<T, R>.GetNextFile(const Curr: TDataFile): TDataFile;
 var
     allSymbolFiles: TArray<TDataFile>;
     foundIndex: Integer;
 begin
     Result := Default(TDataFile);
-    if not Curr.IsValid or not FSymbols.Done.IsSet then
+    if (not Curr.IsValid) or (not FSymbols.Done.IsSet) then
         exit;
 
     if not FSymbols.Value.TryGetValue(Curr.Symbol, allSymbolFiles) then
@@ -303,7 +287,7 @@ begin
     foundIndex := -1;
     for var i := 0 to High(allSymbolFiles) do
     begin
-        if allSymbolFiles[i].GetBaseFileName() = Curr.GetBaseFileName() then
+        if allSymbolFiles[i].GetBaseFileName = Curr.GetBaseFileName then
         begin
             foundIndex := i;
             break;
@@ -311,51 +295,137 @@ begin
     end;
 
     if (foundIndex <> -1) and (foundIndex < High(allSymbolFiles)) then
-    begin
         Result := allSymbolFiles[foundIndex + 1];
-    end;
 end;
 
-function TDataServer<T>.GetPath: String;
+function TDataServer<T, R>.GetPath: String;
 begin
     Result := FPath;
 end;
 
-function TDataServer<T>.ProcessData(
+function TDataServer<T, R>.ParseFileName(const FileName: string): TDataFile;
-    const Symbol: String;
+var
-    const Terminated: TState;
+    fileNameNoPath, baseName, symbol, ext: string;
-    const Processor: IConsumer<TArray<TDataPoint<T>>>
+    parts: TArray<string>;
-): TState;
+    year, month: Integer;
 begin
-    var cProc := Processor;
+    Result := Default(TDataFile);
-    var cTerminated := Terminated;
+    fileNameNoPath := TPath.GetFileName(FileName);
+    ext := TPath.GetExtension(fileNameNoPath);
+    baseName := TPath.GetFileNameWithoutExtension(fileNameNoPath);
 
-    Result :=
+    // Format: SYMBOL_YYYY_MM
-        FindFirstFile(Symbol)
+    parts := baseName.Split(['_']);
-            .Chain(
+    if Length(parts) < 3 then
-                function(const FirstFileInfo: TDataFile): TState
+        exit;
-                begin
+
-                    Result := ProcessFile(FirstFileInfo, LoadDataFile(FirstFileInfo), cTerminated, cProc);
+    // Use reverse indexing for year and month
-                end);
+    if (not TryStrToInt(parts[High(parts)], month)) or (not TryStrToInt(parts[High(parts) - 1], year)) then
+        exit;
+
+    symbol := string.Join('_', Copy(parts, 0, Length(parts) - 2));
+    Result := TDataFile.Create(TPath.GetDirectoryName(FileName), symbol, ext, year, month, baseName);
 end;
 
-function TDataServer<T>.LoadDataFile(const DataFile: TDataFile): TFuture<TArray<TArray<TDataPoint<T>>>>;
+function TDataServer<T, R>.LoadDataFile(const DataFile: TDataFile): TFuture<TArray<TArray<TDataPoint<T>>>>;
 begin
     if DataFile.IsValid then
         Result :=
             FCachedFiles.GetOrAdd(
                 DataFile.GetFullFileName,
-                function(const Filename: String): TFuture<TArray<TArray<TDataPoint<T>>>>
+                function(const Filename: String): TFuture<TArray<TArray<TDataPoint<T>>>> begin Result := DoLoad(Filename, FLoadGate); end
-                begin
-                    // Pass the private load gate to the virtual DoLoad method.
-                    Result := DoLoad(Filename, FLoadGate);
-                end
             )
     else
         Result := TFuture<TArray<TArray<TDataPoint<T>>>>.Construct(TArray<TArray<TDataPoint<T>>>(nil));
 end;
 
-function TDataServer<T>.ProcessChunks(
+function TDataServer<T, R>.DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<T>>>>;
+begin
+    var capFileName := FileName;
+    Result :=
+        TFuture<TBytes>
+            .Construct(Gate.Enqueue, function: TBytes begin Result := TFile.ReadAllBytes(capFileName); end)
+            .Chain<TArray<TArray<TDataPoint<T>>>>(
+                function(Bytes: TBytes): TArray<TArray<TDataPoint<T>>>
+                var
+                    compStream: TBytesStream;
+                begin
+                    compStream := TBytesStream.Create(Bytes);
+                    try
+                        Result := ReadCompressedData(compStream);
+                    finally
+                        compStream.Free;
+                    end;
+                end);
+end;
+
+function TDataServer<T, R>.ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<T>>>;
+var
+    zip: TZipFile;
+    decompStream: TStream;
+    header: TZipHeader;
+    entryIdx: Integer;
+begin
+    Result := nil;
+    InputStream.Position := 0;
+    zip := TZipFile.Create;
+    try
+        zip.Open(InputStream, TZipMode.zmRead);
+        entryIdx := -1;
+        // Search for the .bin entry created by C#
+        for var i := 0 to zip.FileCount - 1 do
+        begin
+            if zip.FileNames[i].EndsWith('.bin', true) then
+            begin
+                entryIdx := i;
+                break;
+            end;
+        end;
+
+        if entryIdx = -1 then
+            exit;
+
+        zip.Read(entryIdx, decompStream, header);
+        try
+            Result := ReadUncompressedData(decompStream);
+        finally
+            decompStream.Free;
+        end;
+    finally
+        zip.Free;
+    end;
+end;
+
+function TDataServer<T, R>.ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<T>>>;
+var
+    totalRecords, chunkCount, i, j, recsInChunk: Integer;
+    rawBuffer: TArray<R>;
+begin
+    Result := nil;
+    totalRecords := InputStream.Size div sizeof(R);
+    if totalRecords = 0 then
+        exit;
+
+    // Fast block read into memory
+    SetLength(rawBuffer, totalRecords);
+    InputStream.Position := 0;
+    InputStream.ReadBuffer(rawBuffer[0], InputStream.Size);
+
+    chunkCount := (totalRecords + ChunkSize - 1) div ChunkSize;
+    SetLength(Result, chunkCount);
+
+    for i := 0 to chunkCount - 1 do
+    begin
+        recsInChunk := Min(ChunkSize, totalRecords - (i * ChunkSize));
+        SetLength(Result[i], recsInChunk);
+        for j := 0 to recsInChunk - 1 do
+        begin
+            Result[i][j] := MapRecord(rawBuffer[i * ChunkSize + j]);
+        end;
+    end;
+end;
+
+function TDataServer<T, R>.ProcessChunks(
     const DataChunks: TArray<TArray<TDataPoint<T>>>;
     Terminated: TState;
     Processor: IConsumer<TArray<TDataPoint<T>>>
@@ -374,36 +444,34 @@ begin
         end;
 
     for var i := 0 to High(DataChunks) do
+    begin
         if not Terminated.IsSet then
         begin
             var q := TaskManager.RunTask(Result, callProcess(i));
             Result := q;
-        end
+        end;
+    end;
 end;
 
-function TDataServer<T>.ProcessFile(
+function TDataServer<T, R>.ProcessFile(
     FileInfo: TDataFile;
     const DataFile: TFuture<TArray<TArray<TDataPoint<T>>>>;
     const Terminated: TState;
     Processor: IConsumer<TArray<TDataPoint<T>>>
 ): TState;
 begin
-    if not FileInfo.IsValid or Terminated.IsSet then
+    if (not FileInfo.IsValid) or (Terminated.IsSet) then
         exit(DataFile.Done);
 
-    // Read ahead the next file, while processing the current one
     var nextFileInfo := GetNextFile(FileInfo);
     var nextFile := LoadDataFile(nextFileInfo);
-
     var cTerminated := Terminated;
+
     Result :=
         DataFile.Chain(
             function(const DataChunks: TArray<TArray<TDataPoint<T>>>): TState
             begin
-                // Process each chunk, checking for termination between chunks.
+                var done := ProcessChunks(DataChunks, cTerminated, Processor);
-                var done := ProcessChunks(DataChunks, Terminated, Processor);
-
-                // Move to next file (which is currently preloading) once all Processors are done
                 Result :=
                     TaskManager
                         .RunTask(done, function: TState begin Result := ProcessFile(nextFileInfo, nextFile, cTerminated, Processor); end);
@@ -411,370 +479,35 @@ begin
         );
 end;
 
-{ TAskBidFileServer }
+function TDataServer<T, R>.ProcessData(
-
+    const Symbol: String;
-function TAskBidFileServer.DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TAskBidFileItem>>>>;
+    const Terminated: TState;
+    const Processor: IConsumer<TArray<TDataPoint<T>>>
+): TState;
 begin
-    Result := TFuture<TArray<TArray<TDataPoint<TAskBidFileItem>>>>.Null;
+    var cProc := Processor;
-
+    var cTerm := Terminated;
-    if not TFile.Exists(FileName) then
-        exit;
-
-    var capFileName := FileName;
-    if FileName.EndsWith('_zip', True) then
-    begin
     Result :=
-            TFuture<TBytes>
+        FindFirstFile(Symbol)
-                .Construct(Gate.Enqueue, function: TBytes begin Result := TFile.ReadAllBytes(capFileName); end)
+            .Chain(function(const Info: TDataFile): TState begin Result := ProcessFile(Info, LoadDataFile(Info), cTerm, cProc); end);
-                .Chain<TArray<TArray<TDataPoint<TAskBidFileItem>>>>(
-                    function(bytes: TBytes): TArray<TArray<TDataPoint<TAskBidFileItem>>>
-                    begin
-                        var compMemoryStream := TBytesStream.Create(bytes);
-                        try
-                            compMemoryStream.Position := 0;
-                            Result := ReadCompressedData(compMemoryStream);
-                        finally
-                            compMemoryStream.Free;
-                        end;
-                    end);
-    end
-    else
-    begin
-        Result :=
-            TFuture<TArray<TArray<TDataPoint<TAskBidFileItem>>>>.Construct(
-                Gate.Enqueue,
-                function: TArray<TArray<TDataPoint<TAskBidFileItem>>>
-                begin
-                    if TFile.Exists(capFileName) then
-                    begin
-                        var stream := TFileStream.Create(capFileName, fmOpenRead or fmShareDenyWrite);
-                        try
-                            Result := ReadUncompressedData(stream);
-                        finally
-                            stream.Free;
-                        end;
-                    end;
-                end
-            );
-    end;
 end;
 
-function TAskBidFileServer.ParseFileName(const FileName: string): TDataFile;
+{ TTickFileServer }
-var
+
-    fileNameNoPath, nameForParsing, baseName, ext, path, symbol: string;
+function TTickFileServer.MapRecord(const RecordBuffer: TTickRecord): TDataPoint<TTickRecord>;
-    year, month: Integer;
-    parts: TArray<string>;
 begin
-    Result := Default(TDataFile);
+    Result := TDataPoint<TTickRecord>.Create(RecordBuffer.Time, RecordBuffer);
-
-    path := TPath.GetDirectoryName(FileName);
-    fileNameNoPath := TPath.GetFileName(FileName);
-    nameForParsing := fileNameNoPath;
-
-    ext := '';
-    if nameForParsing.EndsWith('_zip', True) then
-    begin
-        ext := '_zip';
-        nameForParsing := nameForParsing.Substring(0, nameForParsing.Length - 4);
-    end;
-
-    var fileExt := TPath.GetExtension(nameForParsing) + ext;
-    baseName := TPath.GetFileNameWithoutExtension(nameForParsing);
-
-    parts := baseName.Split(['_']);
-
-    if Length(parts) < 3 then
-        exit;
-
-    if not TryStrToInt(parts[High(parts)], month) then
-        exit;
-
-    if (month < 1) or (month > 12) then
-        exit;
-
-    if not TryStrToInt(parts[High(parts) - 1], year) then
-        exit;
-
-    if year <= 0 then
-        exit;
-
-    symbol := string.Join('_', Copy(parts, 0, Length(parts) - 2));
-
-    if symbol = '' then
-        exit;
-
-    Result := TDataFile.Create(path, symbol, fileExt, year, month, baseName);
-end;
-
-class function TAskBidFileServer.ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAskBidFileItem>>>;
-var
-    decompressionStream: TStream;
-    localHeader: TZipHeader;
-    entryIndex, i: Integer;
-    zipFileInstance: TZipFile;
-begin
-    Result := nil;
-    decompressionStream := nil;
-    zipFileInstance := nil;
-    try
-        InputStream.Position := 0;
-        zipFileInstance := TZipFile.Create;
-        zipFileInstance.Open(InputStream, TZipMode.zmRead);
-
-        if zipFileInstance.FileCount = 0 then
-            exit;
-
-        entryIndex := -1;
-        for i := 0 to zipFileInstance.FileCount - 1 do
-            if zipFileInstance.FileNames[i].EndsWith('.tab', True) then
-            begin
-                entryIndex := i;
-                break;
-            end;
-
-        if entryIndex = -1 then
-            entryIndex := 0;
-
-        zipFileInstance.Read(entryIndex, decompressionStream, localHeader);
-        if not Assigned(decompressionStream) then
-            exit;
-        Result := ReadUncompressedData(decompressionStream);
-    finally
-        decompressionStream.Free;
-        zipFileInstance.Free;
-    end;
-end;
-
-class function TAskBidFileServer.ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAskBidFileItem>>>;
-type
-    TFileRecord = packed record
-        TimeStamp: TDateTime;
-        Data: TAskBidFileItem;
-    end;
-var
-    fileSize: Int64;
-    totalRecordCount, bytesRead, chunkIndex, indexInChunk, recordIndex: Integer;
-    rec: TFileRecord;
-begin
-    Result := nil;
-    InputStream.Position := 0;
-    fileSize := InputStream.Size;
-    if (fileSize = 0) or ((fileSize mod SizeOf(TFileRecord)) <> 0) then
-        exit;
-
-    totalRecordCount := fileSize div SizeOf(TFileRecord);
-    if totalRecordCount > 0 then
-    begin
-        var numChunks := (totalRecordCount + ChunkSize - 1) div ChunkSize;
-        SetLength(Result, numChunks);
-        chunkIndex := 0;
-        indexInChunk := 0;
-        SetLength(Result[chunkIndex], Min(ChunkSize, totalRecordCount));
-
-        for recordIndex := 0 to totalRecordCount - 1 do
-        begin
-            if indexInChunk >= ChunkSize then
-            begin
-                Inc(chunkIndex);
-                indexInChunk := 0;
-                var remainingRecords := totalRecordCount - (chunkIndex * ChunkSize);
-                SetLength(Result[chunkIndex], Min(ChunkSize, remainingRecords));
-            end;
-
-            bytesRead := InputStream.Read(rec, SizeOf(TFileRecord));
-            if bytesRead <> SizeOf(TFileRecord) then
-                raise EReadError.CreateFmt('Read error. Expected %d bytes, read %d.', [SizeOf(TFileRecord), bytesRead]);
-
-            // Corrected syntax: Assign the newly created record instance.
-            Result[chunkIndex][indexInChunk] := TDataPoint<TAskBidFileItem>.Create(rec.TimeStamp, rec.Data);
-            Inc(indexInChunk);
-        end;
-    end;
 end;
 
 { TM1FileServer }
 
-function TM1FileServer.DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TOhlcItem>>>>;
+function TM1FileServer.MapRecord(const RecordBuffer: TM1Record): TDataPoint<TOhlcItem>;
 begin
-    Result := TFuture<TArray<TArray<TDataPoint<TOhlcItem>>>>.Null;
-
-    if not TFile.Exists(FileName) then
-        exit;
-
-    var capFileName := FileName;
-    // M1 files are always compressed with the .m1 extension
-    if FileName.EndsWith('.m1', True) then
-    begin
     Result :=
-            TFuture<TBytes>
+        TDataPoint<TOhlcItem>.Create(
-                .Construct(Gate.Enqueue, function: TBytes begin Result := TFile.ReadAllBytes(capFileName); end)
+            RecordBuffer.Time,
-                .Chain<TArray<TArray<TDataPoint<TOhlcItem>>>>(
+            TOhlcItem.Create(RecordBuffer.Open, RecordBuffer.High, RecordBuffer.Low, RecordBuffer.Close, RecordBuffer.Volume)
-                    function(bytes: TBytes): TArray<TArray<TDataPoint<TOhlcItem>>>
+        );
-                    begin
-                        var compMemoryStream := TBytesStream.Create(bytes);
-                        try
-                            compMemoryStream.Position := 0;
-                            Result := ReadCompressedData(compMemoryStream);
-                        finally
-                            compMemoryStream.Free;
-                        end;
-                    end);
-    end;
-end;
-
-function TM1FileServer.ParseFileName(const FileName: string): TDataFile;
-var
-    path, fileNameNoPath, baseName, ext, symbol: string;
-    year, month: Integer;
-    parts, dateParts: TArray<string>;
-begin
-    Result := Default(TDataFile);
-    path := TPath.GetDirectoryName(FileName);
-    fileNameNoPath := TPath.GetFileName(FileName);
-
-    // M1 files are compressed and have a .m1 extension
-    if not fileNameNoPath.EndsWith('.m1', True) then
-        exit;
-
-    ext := TPath.GetExtension(fileNameNoPath);
-    baseName := TPath.GetFileNameWithoutExtension(fileNameNoPath);
-
-    // Expected format: SYMBOL_YYYY-MM_YYYY-MM
-    parts := baseName.Split(['_']);
-    if Length(parts) < 3 then
-        exit; // Not enough parts for SYMBOL_START_END
-
-    // The symbol can contain underscores, so we take everything before the last two parts.
-    var startDatePart := parts[High(parts) - 1];
-
-    dateParts := startDatePart.Split(['-']);
-    if Length(dateParts) <> 2 then
-        exit; // Start date is not YYYY-MM
-
-    if not TryStrToInt(dateParts[0], year) then
-        exit;
-
-    if not TryStrToInt(dateParts[1], month) then
-        exit;
-
-    if (month < 1) or (month > 12) or (year <= 0) then
-        exit;
-
-    symbol := string.Join('_', Copy(parts, 0, Length(parts) - 2));
-    if symbol = '' then
-        exit;
-
-    // The TDataFile only represents the start date of the file batch.
-    Result := TDataFile.Create(path, symbol, ext, year, month, baseName);
-end;
-
-class function TM1FileServer.ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>;
-var
-    decompressionStream: TStream;
-    localHeader: TZipHeader;
-    entryIndex, i: Integer;
-    zipFileInstance: TZipFile;
-begin
-    Result := nil;
-    decompressionStream := nil;
-    zipFileInstance := nil;
-
-    // M1 files are now zip archives containing the actual data file.
-    if not Assigned(InputStream) or (InputStream.Size = 0) then
-        exit;
-
-    try
-        InputStream.Position := 0;
-        zipFileInstance := TZipFile.Create;
-        zipFileInstance.Open(InputStream, TZipMode.zmRead);
-
-        if zipFileInstance.FileCount = 0 then
-            exit;
-
-        // Find the data file entry within the archive.
-        // The C# code creates an entry with the same name as the base filename, ending in .m1.
-        entryIndex := -1;
-        for i := 0 to zipFileInstance.FileCount - 1 do
-        begin
-            if zipFileInstance.FileNames[i].EndsWith('.m1', True) then
-            begin
-                entryIndex := i;
-                break;
-            end;
-        end;
-
-        // Fallback to the first entry if no specific .m1 file is found inside.
-        if entryIndex = -1 then
-            entryIndex := 0;
-
-        zipFileInstance.Read(entryIndex, decompressionStream, localHeader);
-        if not Assigned(decompressionStream) then
-            exit;
-
-        // The decompressed stream now contains the raw binary data.
-        Result := ReadUncompressedData(decompressionStream);
-    finally
-        decompressionStream.Free;
-        zipFileInstance.Free;
-    end;
-end;
-
-class function TM1FileServer.ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>;
-type
-    TFileRecord = TM1FileItem;
-var
-    fileSize: Int64;
-    totalRecordCount, bytesRead, chunkIndex, indexInChunk, recordIndex: Integer;
-    rec: TFileRecord;
-    ohlc: TOhlcItem;
-    factor: Double;
-begin
-    Result := nil;
-    InputStream.Position := 0;
-    fileSize := InputStream.Size;
-    if (fileSize = 0) or ((fileSize mod SizeOf(TFileRecord)) <> 0) then
-        exit;
-
-    totalRecordCount := fileSize div SizeOf(TFileRecord);
-    if totalRecordCount > 0 then
-    begin
-        var numChunks := (totalRecordCount + ChunkSize - 1) div ChunkSize;
-        SetLength(Result, numChunks);
-        chunkIndex := 0;
-        indexInChunk := 0;
-        SetLength(Result[chunkIndex], Min(ChunkSize, totalRecordCount));
-
-        for recordIndex := 0 to totalRecordCount - 1 do
-        begin
-            if indexInChunk >= ChunkSize then
-            begin
-                Inc(chunkIndex);
-                indexInChunk := 0;
-                var remainingRecords := totalRecordCount - (chunkIndex * ChunkSize);
-                SetLength(Result[chunkIndex], Min(ChunkSize, remainingRecords));
-            end;
-
-            bytesRead := InputStream.Read(rec, SizeOf(TFileRecord));
-            if bytesRead <> SizeOf(TFileRecord) then
-                raise EReadError.CreateFmt('Read error. Expected %d bytes, read %d.', [SizeOf(TFileRecord), bytesRead]);
-
-            // Convert from M1 file record to a standard OHLC item
-            if rec.Digits > 0 then
-                factor := Power(10, rec.Digits)
-            else
-                factor := 1.0;
-
-            ohlc.Open := rec.Open / factor;
-            ohlc.High := rec.High / factor;
-            ohlc.Low := rec.Low / factor;
-            ohlc.Close := rec.Close / factor;
-            ohlc.Volume := rec.TickVolume;
-
-            // Corrected syntax: Assign the newly created record instance.
-            Result[chunkIndex][indexInChunk] := TDataPoint<TOhlcItem>.Create(rec.OADateTime, ohlc);
-            Inc(indexInChunk);
-        end;
-    end;
 end;
 
 end.