implementing DataStream

Src/Myc.Test.Trade.DataStream.pas

@@ -1,4 +1,4 @@
-unit Myc.Test.Trade.DataServer;
+unit Myc.Test.Trade.DataStream;
 
 interface
 
@@ -19,8 +19,8 @@ type
     [IgnoreMemoryLeaks(true)]
     TTest_TABFileServer_Equivalence = class(TObject)
     private
-        FServer: IDataServer<TAskBidItem>;
+        FServer: IAuraDataServer<TAskBidItem>;
-        FStream: IDataStream<TDataPoint<TAskBidItem>>;
+        FStream: IDataStream<TAskBidItem>;
     public
         [SetupFixture]
         procedure SetupFixture;
@@ -42,7 +42,8 @@ uses
 
 const
     // The reference data file for this test.
-    C_TEST_FILENAME = '\\COFFEE\TickData\Pepperstone\GER40_2025_03.tab_zip';
+    C_TEST_PATH = '\\COFFEE\TickData\Pepperstone';
+    C_TEST_SYMBOL = 'GER40';
     // The chunk size for fetching data from the server.
     C_MAX_FETCH = 200;
 
@@ -50,7 +51,7 @@ const
 
 procedure TTest_TABFileServer_Equivalence.SetupFixture;
 begin
-    FServer := TAskBidServer.Create;
+    FServer := TAuraTABFileServer.Create(C_TEST_PATH);
     // TAskBid.ClearCache ensures a clean state for data series loading.
     FServer.ClearCache;
 end;
@@ -66,7 +67,7 @@ end;
 procedure TTest_TABFileServer_Equivalence.Setup;
 begin
     // Initializes the data server with a specific test file.
-    FStream := TAuraTABFileStream.Create(C_TEST_FILENAME, FServer);
+    FStream := FServer.CreateStream(C_TEST_SYMBOL);
 end;
 
 procedure TTest_TABFileServer_Equivalence.TearDown;
@@ -84,7 +85,12 @@ var
     i: Int64;
 begin
     // Loads the expected data directly using TAskBid.LoadDataSeries.WaitFor.
-    var ExpectedData := FServer.LoadDataSeries(C_TEST_FILENAME).WaitFor;
+
+    var Filename := TAuraTABFileServer.FindFirstDataFile(C_TEST_PATH, C_TEST_SYMBOL);
+
+    Assert.IsNotEmpty(Filename);
+
+    var ExpectedData := FServer.LoadDataSeries(Filename).WaitFor;
     SetLength(Dst, Length(ExpectedData));
 
     var cnt: Int64 := 0;

Src/Myc.Trade.DataStream.pas

@@ -6,7 +6,7 @@ unit Myc.Trade.DataStream;
 
   This unit contains the core components for handling historical data:
   - TDataRecord: A generic record for a single time-stamped data entry.
-  - IDataServer/TDataServer: A server component responsible for loading and caching
+  - IDataServer/TAuraDataServer: A server component responsible for loading and caching
     series of data files from disk. Each server instance manages its own cache,
     but all instances share a central load gate.
   - IDataStream/TDataStream: An interface representing a stream of data points that
@@ -39,7 +39,7 @@ type
     IDataStream<T> = interface
         ['{A6E246A2-E84E-49AB-A63E-333E561E488C}']
         function GetIsLiveData: TMutable<Boolean>;
-        function GetChunk(var Data: array of T): Integer;
+        function GetChunk(var Data: array of TDataPoint<T>): Integer;
         property IsLiveData: TMutable<Boolean> read GetIsLiveData;
     end;
 
@@ -47,7 +47,7 @@ type
     TDataStream<T> = class(TInterfacedObject, IDataStream<T>)
     protected
         function GetIsLiveData: TMutable<Boolean>; virtual; abstract;
-        function GetChunk(var Data: array of T): Integer; virtual; abstract;
+        function GetChunk(var Data: array of TDataPoint<T>): Integer; virtual; abstract;
     end;
 
     // Represents a factory for creating IDataStream instances.
@@ -65,16 +65,25 @@ type
     // Interface for an instantiable data server.
     IDataServer<T: record> = interface
         ['{1F8E5A9D-E92A-44C1-9F3F-C4B82A6E94B3}']
-        function LoadDataFile(const FileName: string): TFuture<TArray<TDataRecord<T>>>;
+        function CreateStream(const Symbol: String): IDataStream<T>;
-        function LoadDataSeries(const FileName: string): TFuture<TArray<TDataRecord<T>>>;
         procedure ClearCache;
     end;
 
+    IAuraDataServer<T: record> = interface(IDataServer<T>)
+        ['{481E27DE-DC58-49AF-B8A8-B6316980C423}']
+        function LoadDataFile(const FileName: string): TFuture<TArray<TDataRecord<T>>>;
+        function LoadDataSeries(const FileName: string): TFuture<TArray<TDataRecord<T>>>;
+        function EnumerateAssetFiles: TArray<String>;
+    end;
+
+    // Aura files
+
     // Generic server for loading and managing sequential time-series data from files.
-    TDataServer<T: record> = class(TInterfacedObject, IDataServer<T>)
+    TAuraDataServer<T: record> = class(TInterfacedObject, IDataServer<T>, IAuraDataServer<T>)
-    strict private
+    private
         type
             TCachedFile = class
+            public
                 Name: String;
                 Age: TDateTime;
                 LastUsed: TDateTime;
@@ -83,32 +92,47 @@ type
     private
         // The cache is per-instance.
         FCachedFiles: TDictionary<String, TCachedFile>;
+        FPath: String;
+        function GetPath: String;
+
     strict private
-        // The load gate is shared across all instances of TDataServer.
+        // The load gate is shared across all instances of TAuraDataServer.
-        class var FLoadGate: TLatch;
+        class var
+            FLoadGate: TLatch;
 
     protected
         class function ReadCompressedData(const InputStream: TStream): TArray<TDataRecord<T>>; static;
         class function ReadUncompressedData(const InputStream: TStream): TArray<TDataRecord<T>>; static;
 
     public
-        constructor Create;
+        constructor Create(const APath: String);
         destructor Destroy; override;
 
+        class function TryParseFileName(
+            const FileName: string;
+            out PathValue, SymbolValue: string;
+            out YearValue, MonthValue: Integer
+        ): Boolean; virtual; abstract;
+
+        class function FindFirstDataFile(const Path, Symbol: string): string;
+        class function FindNextDataFile(const FileName: string): string;
+        class function FindOldestFilesPerSymbol(const DirectoryPath: string): TArray<String>;
+
         // IDataServer<T> implementation
+        function CreateStream(const Symbol: String): IDataStream<T>; virtual; abstract;
         procedure ClearCache;
+        function EnumerateAssetFiles: TArray<String>;
+
         function LoadDataFile(const FileName: string): TFuture<TArray<TDataRecord<T>>>;
         function LoadDataSeries(const FileName: string): TFuture<TArray<TDataRecord<T>>>;
+
+        property Path: String read GetPath;
     end;
 
-    // A specialized TDataServer for handling Ask/Bid tick data.
+    // Implements a data stream that reads from Aura-specific historical data files.
-    TAskBidServer = class(TDataServer<TAskBidItem>)
+    TAuraFileStream<T: record> = class(TDataStream<T>, IDataStream<T>)
-    end;
-
-    // Implements a data stream that reads data from Aura-specific historical data files.
-    TAuraFileStream<T: record> = class(TDataStream<TDataPoint<T>>, IDataStream<TDataPoint<T>>)
     private
-        FDataServer: IDataServer<T>;
+        FDataServer: TAuraDataServer<T>;
         FIsLiveData: TMutable<Boolean>.IWriteable;
         FCurrentFileName: string;
         FCurrentData: TFuture<TArray<TDataRecord<T>>>;
@@ -121,16 +145,23 @@ type
         function GetChunk(var Data: array of TDataPoint<T>): Integer; override;
         function GetIsLiveData: TMutable<Boolean>; override;
     public
-        constructor Create(const AFilename: String; const ADataServer: IDataServer<T>);
+        constructor Create(ADataServer: TAuraDataServer<T>; const AFilename: String);
         procedure AfterConstruction; override;
     end;
 
-    // Specialized TAuraFileStream for TAskBidItem data.
+    // Aura tick data file Ask-Bid
-    TAuraTABFileStream = TAuraFileStream<TAskBidItem>;
 
-function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue, MonthValue: Integer): Boolean;
+    TAuraTABFileServer = class(TAuraDataServer<TAskBidItem>)
-function FindOldestFilesPerSymbol(const DirectoryPath: string): TArray<String>;
+    public
-function FindNextDataFile(const FileName: string): string;
+        // IDataServer<T> implementation
+        function CreateStream(const Symbol: String): IDataStream<TAskBidItem>; override;
+
+        class function TryParseFileName(
+            const FileName: string;
+            out PathValue, SymbolValue: string;
+            out YearValue, MonthValue: Integer
+        ): Boolean; override;
+    end;
 
 var
     IsMemoryLow: TFunc<Boolean>;
@@ -140,27 +171,150 @@ implementation
 uses
     System.Zip,
     System.Math,
+    System.StrUtils,
     Myc.TaskManager;
 
-constructor TDataServer<T>.Create;
+constructor TAuraDataServer<T>.Create(const APath: String);
 begin
     inherited Create;
     // Initialize the per-instance cache.
     FCachedFiles := TObjectDictionary<String, TCachedFile>.Create([doOwnsValues]);
+    FPath := APath;
 end;
 
-destructor TDataServer<T>.Destroy;
+destructor TAuraDataServer<T>.Destroy;
 begin
+    ClearCache;
     FCachedFiles.Free;
     inherited Destroy;
 end;
 
-procedure TDataServer<T>.ClearCache;
+procedure TAuraDataServer<T>.ClearCache;
 begin
+    for var cF in FCachedFiles.Values do
+        cF.Data.WaitFor;
+
     FCachedFiles.Clear;
 end;
 
-function TDataServer<T>.LoadDataFile(const FileName: string): TFuture<TArray<TDataRecord<T>>>;
+class function TAuraDataServer<T>.FindFirstDataFile(const Path, Symbol: string): string;
+var
+    oldestFiles: TArray<string>;
+    fileName: string;
+    pathValue, symbolValue: string;
+    yearValue, monthValue: Integer;
+begin
+    oldestFiles := FindOldestFilesPerSymbol(Path);
+    for fileName in oldestFiles do
+    begin
+        if TryParseFileName(fileName, pathValue, symbolValue, yearValue, monthValue) then
+        begin
+            if SameText(symbolValue, Symbol) then
+                exit(fileName);
+        end;
+    end;
+    Result := '';
+end;
+
+class function TAuraDataServer<T>.FindNextDataFile(const FileName: string): string;
+var
+    pathValue, symbolValue: string;
+    yearValue, monthValue: Integer;
+    nextBaseName, compressedPath, uncompressedPath: string;
+begin
+    if not TryParseFileName(FileName, pathValue, symbolValue, yearValue, monthValue) then
+        exit('');
+
+    Inc(monthValue);
+    if monthValue > 12 then
+    begin
+        monthValue := 1;
+        Inc(yearValue);
+    end;
+
+    nextBaseName := Format('%s_%.4d_%.2d.tab', [symbolValue, yearValue, monthValue]);
+
+    compressedPath := TPath.Combine(pathValue, nextBaseName + '_zip');
+    if TFile.Exists(compressedPath) then
+        exit(compressedPath);
+
+    uncompressedPath := TPath.Combine(pathValue, nextBaseName);
+    if TFile.Exists(uncompressedPath) then
+        exit(uncompressedPath);
+
+    Result := '';
+end;
+
+class function TAuraDataServer<T>.FindOldestFilesPerSymbol(const DirectoryPath: string): TArray<String>;
+type
+    TTrackedFileInfo = record
+        FullFileName: string;
+        Year: Integer;
+        Month: Integer;
+    end;
+var
+    oldestFilesPerSymbol: TDictionary<string, TTrackedFileInfo>;
+    fileNames: TArray<string>;
+    currentFile: string;
+    parsedPath: string;
+    parsedSymbol: string;
+    parsedYear: Integer;
+    parsedMonth: Integer;
+    trackedInfo: TTrackedFileInfo;
+begin
+    oldestFilesPerSymbol := TDictionary<string, TTrackedFileInfo>.Create;
+    try
+        if not TDirectory.Exists(DirectoryPath) then
+            exit(nil);
+
+        fileNames := TDirectory.GetFiles(DirectoryPath);
+        for currentFile in fileNames do
+        begin
+            if TryParseFileName(currentFile, parsedPath, parsedSymbol, parsedYear, parsedMonth) then
+            begin
+                if oldestFilesPerSymbol.TryGetValue(parsedSymbol, trackedInfo) then
+                begin
+                    if (parsedYear < trackedInfo.Year) or ((parsedYear = trackedInfo.Year) and (parsedMonth < trackedInfo.Month)) then
+                    begin
+                        trackedInfo.FullFileName := currentFile;
+                        trackedInfo.Year := parsedYear;
+                        trackedInfo.Month := parsedMonth;
+                        oldestFilesPerSymbol.AddOrSetValue(parsedSymbol, trackedInfo);
+                    end;
+                end
+                else
+                begin
+                    trackedInfo.FullFileName := currentFile;
+                    trackedInfo.Year := parsedYear;
+                    trackedInfo.Month := parsedMonth;
+                    oldestFilesPerSymbol.Add(parsedSymbol, trackedInfo);
+                end;
+            end;
+        end;
+
+        SetLength(Result, oldestFilesPerSymbol.Count);
+        var n := 0;
+        for trackedInfo in oldestFilesPerSymbol.Values do
+        begin
+            Result[n] := trackedInfo.FullFileName;
+            inc(n);
+        end;
+    finally
+        oldestFilesPerSymbol.Free;
+    end;
+end;
+
+function TAuraDataServer<T>.EnumerateAssetFiles: TArray<String>;
+begin
+    Result := FindOldestFilesPerSymbol(FPath);
+end;
+
+function TAuraDataServer<T>.GetPath: String;
+begin
+    Result := FPath;
+end;
+
+function TAuraDataServer<T>.LoadDataFile(const FileName: string): TFuture<TArray<TDataRecord<T>>>;
 var
     parsedPath, parsedSymbol: string;
     parsedYear, parsedMonth: Integer;
@@ -175,9 +329,10 @@ begin
         if IsMemoryLow() then
         begin
             var fL := FCachedFiles.Values.ToArray;
-            TArray.Sort<TCachedFile>(fL,
+            TArray.Sort<TCachedFile>(
-                TComparer<TCachedFile>.Construct(function(const Left, Right: TCachedFile): Integer
+                fL,
-                    begin Result := Sign(Left.LastUsed - Right.LastUsed); end)
+                TComparer<TCachedFile>
+                    .Construct(function(const Left, Right: TCachedFile): Integer begin Result := Sign(Left.LastUsed - Right.LastUsed); end)
             );
             for var i := 0 to High(fL) do
             begin
@@ -208,47 +363,46 @@ begin
     var capFileName := FileName;
     if FileName.EndsWith('_zip', True) then
     begin
-        Result := TFuture<TBytes>.Construct(
+        Result :=
-            TLatch.Enqueue(FLoadGate), // Use shared class var FLoadGate
+            TFuture<TBytes>
-            function: TBytes
+                .Construct(
-            begin
+                    TLatch.Enqueue(FLoadGate), // Use shared class var FLoadGate
-                Result := TFile.ReadAllBytes(capFileName);
+                    function: TBytes begin Result := TFile.ReadAllBytes(capFileName); end)
-            end
+                .Chain<TArray<TDataRecord<T>>>(
-        ).Chain<TArray<TDataRecord<T>>>(
+                    function(bytes: TBytes): TArray<TDataRecord<T>>
-            function(bytes: TBytes): TArray<TDataRecord<T>>
+                    begin
-            begin
+                        var zipMemoryStream := TBytesStream.Create(bytes);
-                var zipMemoryStream := TBytesStream.Create(bytes);
+                        try
-                try
+                            zipMemoryStream.Position := 0;
-                    zipMemoryStream.Position := 0;
+                            Result := ReadCompressedData(zipMemoryStream);
-                    Result := ReadCompressedData(zipMemoryStream);
+                        finally
-                finally
+                            zipMemoryStream.Free;
-                    zipMemoryStream.Free;
+                        end;
-                end;
+                    end);
-            end
-        );
     end
     else if TFile.Exists(FileName) then
     begin
-        Result := TFuture<TArray<TDataRecord<T>>>.Construct(
+        Result :=
-            TLatch.Enqueue(FLoadGate), // Use shared class var FLoadGate
+            TFuture<TArray<TDataRecord<T>>>.Construct(
-            function: TArray<TDataRecord<T>>
+                TLatch.Enqueue(FLoadGate), // Use shared class var FLoadGate
-            begin
+                function: TArray<TDataRecord<T>>
-                if TFile.Exists(capFileName) then
                 begin
-                    var stream := TFileStream.Create(capFileName, fmOpenRead or fmShareDenyWrite);
+                    if TFile.Exists(capFileName) then
-                    try
+                    begin
+                        var stream := TFileStream.Create(capFileName, fmOpenRead or fmShareDenyWrite);
                         try
-                            Result := ReadUncompressedData(stream);
+                            try
-                        except
+                                Result := ReadUncompressedData(stream);
-                            on E: EReadError do
+                            except
-                                raise EReadError.CreateFmt('File %s: %s', [capFileName, E.Message]);
+                                on E: EReadError do
+                                    raise EReadError.CreateFmt('File %s: %s', [capFileName, E.Message]);
+                            end;
+                        finally
+                            stream.Free;
                         end;
-                    finally
-                        stream.Free;
                     end;
-                end;
+                end
-            end
+            );
-        );
     end;
 
     if Assigned(IsMemoryLow) then
@@ -264,7 +418,7 @@ begin
     end;
 end;
 
-function TDataServer<T>.LoadDataSeries(const FileName: string): TFuture<TArray<TDataRecord<T>>>;
+function TAuraDataServer<T>.LoadDataSeries(const FileName: string): TFuture<TArray<TDataRecord<T>>>;
 var
     loadedState: TState;
     loadedFiles: TArray<TFuture<TArray<TDataRecord<T>>>>;
@@ -326,34 +480,35 @@ begin
         tabFiles.Free;
     end;
 
-    Result := TFuture<TArray<TDataRecord<T>>>.Construct(
+    Result :=
-        loadedState,
+        TFuture<TArray<TDataRecord<T>>>.Construct(
-        function: TArray<TDataRecord<T>>
+            loadedState,
-        begin
+            function: TArray<TDataRecord<T>>
-            var tickList := TList<TDataRecord<T>>.Create;
+            begin
-            try
+                var tickList := TList<TDataRecord<T>>.Create;
-                var overallLastTabTickTime: TDateTime := 0;
+                try
-                var cnt := 0;
+                    var overallLastTabTickTime: TDateTime := 0;
-                for var P in loadedFiles do
+                    var cnt := 0;
-                    inc(cnt, Length(P.Value));
+                    for var P in loadedFiles do
-                tickList.Capacity := cnt;
+                        inc(cnt, Length(P.Value));
+                    tickList.Capacity := cnt;
 
-                for var P in loadedFiles do
+                    for var P in loadedFiles do
-                    for var iterTickData in P.Value do
+                        for var iterTickData in P.Value do
-                        if overallLastTabTickTime < iterTickData.TimeStamp then
+                            if overallLastTabTickTime < iterTickData.TimeStamp then
-                        begin
+                            begin
-                            tickList.Add(iterTickData);
+                                tickList.Add(iterTickData);
-                            overallLastTabTickTime := iterTickData.TimeStamp;
+                                overallLastTabTickTime := iterTickData.TimeStamp;
-                        end;
+                            end;
-                Result := tickList.ToArray;
+                    Result := tickList.ToArray;
-            finally
+                finally
-                tickList.Free;
+                    tickList.Free;
-            end;
+                end;
-        end
+            end
-    );
+        );
 end;
 
-class function TDataServer<T>.ReadCompressedData(const InputStream: TStream): TArray<TDataRecord<T>>;
+class function TAuraDataServer<T>.ReadCompressedData(const InputStream: TStream): TArray<TDataRecord<T>>;
 var
     decompressionStream: TStream;
     localHeader: TZipHeader;
@@ -392,7 +547,7 @@ begin
     end;
 end;
 
-class function TDataServer<T>.ReadUncompressedData(const InputStream: TStream): TArray<TDataRecord<T>>;
+class function TAuraDataServer<T>.ReadUncompressedData(const InputStream: TStream): TArray<TDataRecord<T>>;
 var
     fileSize: Int64;
     recordCount, bytesRead: Integer;
@@ -415,7 +570,7 @@ end;
 
 { TAuraFileStream<T> }
 
-constructor TAuraFileStream<T>.Create(const AFilename: String; const ADataServer: IDataServer<T>);
+constructor TAuraFileStream<T>.Create(ADataServer: TAuraDataServer<T>; const AFilename: String);
 begin
     inherited Create;
     Assert(Assigned(ADataServer));
@@ -432,7 +587,7 @@ begin
     FCurrentIdx := 0;
     FIsLiveData.SetValue(false);
     FLastTimeStamp := 0;
-    FNextFileName := FindNextDataFile(FCurrentFileName);
+    FNextFileName := FDataServer.FindNextDataFile(FCurrentFileName);
     if FNextFileName <> '' then
         FNextData := FDataServer.LoadDataFile(FNextFileName)
     else
@@ -459,7 +614,7 @@ begin
             FCurrPosInFile := 0;
             if FCurrentFileName <> '' then
             begin
-                FNextFileName := FindNextDataFile(FCurrentFileName);
+                FNextFileName := FDataServer.FindNextDataFile(FCurrentFileName);
                 if FNextFileName <> '' then
                     FNextData := FDataServer.LoadDataFile(FNextFileName)
                 else
@@ -495,9 +650,28 @@ begin
     Result := FIsLiveData;
 end;
 
-{ Helper Functions }
+{ TAuraTABFileServer }
 
-function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue, MonthValue: Integer): Boolean;
+function TAuraTABFileServer.CreateStream(const Symbol: String): IDataStream<TAskBidItem>;
+var
+    fileName: string;
+begin
+    // FindFirstDataFile uses the server's path and the given symbol to locate the initial data file.
+    fileName := FindFirstDataFile(FPath, Symbol);
+
+    // Exit if no file is found or if it is not a .tab file, preventing stream creation for invalid sources.
+    if (fileName = '') or (not TPath.GetExtension(fileName).StartsWith('.tab')) then
+        exit(nil);
+
+    // Creates a new stream instance, passing itself as the data server and the found filename.
+    Result := TAuraFileStream<TAskBidItem>.Create(Self, fileName);
+end;
+
+class function TAuraTABFileServer.TryParseFileName(
+    const FileName: string;
+    out PathValue, SymbolValue: string;
+    out YearValue, MonthValue: Integer
+): Boolean;
 var
     fileNameNoPath: string;
     nameForParsing: string;
@@ -563,92 +737,4 @@ begin
     Result := True;
 end;
 
-function FindOldestFilesPerSymbol(const DirectoryPath: string): TArray<String>;
-type
-    TTrackedFileInfo = record
-        FullFileName: string;
-        Year: Integer;
-        Month: Integer;
-    end;
-var
-    oldestFilesPerSymbol: TDictionary<string, TTrackedFileInfo>;
-    fileNames: TArray<string>;
-    currentFile: string;
-    parsedPath: string;
-    parsedSymbol: string;
-    parsedYear: Integer;
-    parsedMonth: Integer;
-    trackedInfo: TTrackedFileInfo;
-begin
-    oldestFilesPerSymbol := TDictionary<string, TTrackedFileInfo>.Create;
-    try
-        if not TDirectory.Exists(DirectoryPath) then
-            exit(nil);
-
-        fileNames := TDirectory.GetFiles(DirectoryPath);
-        for currentFile in fileNames do
-        begin
-            if TryParseFileName(currentFile, parsedPath, parsedSymbol, parsedYear, parsedMonth) then
-            begin
-                if oldestFilesPerSymbol.TryGetValue(parsedSymbol, trackedInfo) then
-                begin
-                    if (parsedYear < trackedInfo.Year) or ((parsedYear = trackedInfo.Year) and (parsedMonth < trackedInfo.Month)) then
-                    begin
-                        trackedInfo.FullFileName := currentFile;
-                        trackedInfo.Year := parsedYear;
-                        trackedInfo.Month := parsedMonth;
-                        oldestFilesPerSymbol.AddOrSetValue(parsedSymbol, trackedInfo);
-                    end;
-                end
-                else
-                begin
-                    trackedInfo.FullFileName := currentFile;
-                    trackedInfo.Year := parsedYear;
-                    trackedInfo.Month := parsedMonth;
-                    oldestFilesPerSymbol.Add(parsedSymbol, trackedInfo);
-                end;
-            end;
-        end;
-
-        SetLength(Result, oldestFilesPerSymbol.Count);
-        var n := 0;
-        for trackedInfo in oldestFilesPerSymbol.Values do
-        begin
-            Result[n] := trackedInfo.FullFileName;
-            inc(n);
-        end;
-    finally
-        oldestFilesPerSymbol.Free;
-    end;
-end;
-
-function FindNextDataFile(const FileName: string): string;
-var
-    pathValue, symbolValue: string;
-    yearValue, monthValue: Integer;
-    nextBaseName, compressedPath, uncompressedPath: string;
-begin
-    if not TryParseFileName(FileName, pathValue, symbolValue, yearValue, monthValue) then
-        exit('');
-
-    Inc(monthValue);
-    if monthValue > 12 then
-    begin
-        monthValue := 1;
-        Inc(yearValue);
-    end;
-
-    nextBaseName := Format('%s_%.4d_%.2d.tab', [symbolValue, yearValue, monthValue]);
-
-    compressedPath := TPath.Combine(pathValue, nextBaseName + '_zip');
-    if TFile.Exists(compressedPath) then
-        exit(compressedPath);
-
-    uncompressedPath := TPath.Combine(pathValue, nextBaseName);
-    if TFile.Exists(uncompressedPath) then
-        exit(uncompressedPath);
-
-    Result := '';
-end;
-
 end.

Test/MycTests.dpr

@@ -5,37 +5,38 @@ program MycTests;
 {$ENDIF}
 {$STRONGLINKTYPES ON}
 uses
-  FastMM5,
+    FastMM5,
-  DUnitX.MemoryLeakMonitor.FastMM5,
+    DUnitX.MemoryLeakMonitor.FastMM5,
-  System.SysUtils,
+    System.SysUtils,
-  {$IFDEF TESTINSIGHT}
+{$IFDEF TESTINSIGHT}
-  TestInsight.DUnitX,
+    TestInsight.DUnitX,
-  {$ELSE}
+{$ELSE}
-  DUnitX.Loggers.Console,
+    DUnitX.Loggers.Console,
-  {$ENDIF }
+{$ENDIF }
-  DUnitX.TestFramework,
+    DUnitX.TestFramework,
-  TestNotifier in 'TestNotifier.pas',
+    TestNotifier in 'TestNotifier.pas',
-  TestNotifier_Threading in 'TestNotifier_Threading.pas' {/TestNotifier_ChaosStress in 'TestNotifier_ChaosStress.pas',},
+    TestNotifier_Threading in 'TestNotifier_Threading.pas' {/TestNotifier_ChaosStress in 'TestNotifier_ChaosStress.pas',},
-  TestNotifier_ChaosStress in 'TestNotifier_ChaosStress.pas',
+    TestNotifier_ChaosStress in 'TestNotifier_ChaosStress.pas',
-  Myc.Core.Tasks in '..\Src\Myc.Core.Tasks.pas',
+    Myc.Core.Tasks in '..\Src\Myc.Core.Tasks.pas',
-  TestTasks in 'TestTasks.pas',
+    TestTasks in 'TestTasks.pas',
-  Myc.Futures in '..\Src\Myc.Futures.pas',
+    Myc.Futures in '..\Src\Myc.Futures.pas',
-  TestCoreFutures in 'TestCoreFutures.pas',
+    TestCoreFutures in 'TestCoreFutures.pas',
-  Myc.TaskManager in '..\Src\Myc.TaskManager.pas',
+    Myc.TaskManager in '..\Src\Myc.TaskManager.pas',
-  Myc.Core.Futures in '..\Src\Myc.Core.Futures.pas',
+    Myc.Core.Futures in '..\Src\Myc.Core.Futures.pas',
-  Myc.Core.Signals in '..\Src\Myc.Core.Signals.pas',
+    Myc.Core.Signals in '..\Src\Myc.Core.Signals.pas',
-  TestFutures in 'TestFutures.pas',
+    TestFutures in 'TestFutures.pas',
-  Myc.Lazy in '..\Src\Myc.Lazy.pas',
+    Myc.Lazy in '..\Src\Myc.Lazy.pas',
-  Myc.Core.Lazy in '..\Src\Myc.Core.Lazy.pas',
+    Myc.Core.Lazy in '..\Src\Myc.Core.Lazy.pas',
-  Myc.Test.Core.Lazy in '..\Src\Myc.Test.Core.Lazy.pas',
+    Myc.Test.Core.Lazy in '..\Src\Myc.Test.Core.Lazy.pas',
-  Myc.Test.Lazy in '..\Src\Myc.Test.Lazy.pas',
+    Myc.Test.Lazy in '..\Src\Myc.Test.Lazy.pas',
-  Myc.Test.Core.Atomic in '..\Src\Myc.Test.Core.Atomic.pas',
+    Myc.Test.Core.Atomic in '..\Src\Myc.Test.Core.Atomic.pas',
-  Myc.Trade.Ticker in '..\Src\Myc.Trade.Ticker.pas',
+    Myc.Trade.Ticker in '..\Src\Myc.Trade.Ticker.pas',
-  Myc.Test.Signals.Latch in '..\Src\Myc.Test.Signals.Latch.pas',
+    Myc.Test.Signals.Latch in '..\Src\Myc.Test.Signals.Latch.pas',
-  Myc.Test.Signals.Dirty in '..\Src\Myc.Test.Signals.Dirty.pas',
+    Myc.Test.Signals.Dirty in '..\Src\Myc.Test.Signals.Dirty.pas',
-  Myc.Trade.DataPoint in '..\Src\Myc.Trade.DataPoint.pas',
+    Myc.Trade.DataPoint in '..\Src\Myc.Trade.DataPoint.pas',
-  Myc.Trade.Node in '..\Src\Myc.Trade.Node.pas',
+    Myc.Trade.Node in '..\Src\Myc.Trade.Node.pas',
-  Myc.Trade.DataStream in '..\Src\Myc.Trade.DataStream.pas';
+    Myc.Trade.DataStream in '..\Src\Myc.Trade.DataStream.pas',
+    Myc.Test.Trade.DataStream in '..\Src\Myc.Test.Trade.DataStream.pas';
 
 { keep comment here to protect the following conditional from being removed by the IDE when adding a unit }
 {$IFNDEF TESTINSIGHT}

Test/MycTests.dproj

@@ -136,6 +136,7 @@ $(PreBuildEvent)]]></PreBuildEvent>
         <DCCReference Include="..\Src\Myc.Trade.DataPoint.pas"/>
         <DCCReference Include="..\Src\Myc.Trade.Node.pas"/>
         <DCCReference Include="..\Src\Myc.Trade.DataStream.pas"/>
+        <DCCReference Include="..\Src\Myc.Test.Trade.DataStream.pas"/>
         <BuildConfiguration Include="Base">
             <Key>Base</Key>
         </BuildConfiguration>