Streamlining DataServer

Src/Myc.Trade.DataSeries.pas

@@ -45,7 +45,8 @@ uses
     System.Generics.Defaults,
     System.IOUtils,
     Myc.Futures,
-    Myc.Signals;
+    Myc.Signals,
+    Myc.Trade.DataPoint;
 
 type
     // Generic class for loading and managing sequential time-series data.
@@ -54,13 +55,13 @@ type
         type
             // Represents a single data point with a timestamp and generic data.
             TDataPoint = packed record
-                OADateTime: TDateTime;
+                TimeStamp: TDateTime;
                 Data: T;
             end;
 
     strict private
         type
-            TCachedFile = record
+            TCachedFile = class
                 Name: String;
                 Age: TDateTime;
                 LastUsed: TDateTime;
@@ -68,6 +69,7 @@ type
             end;
         class var
             FCachedFiles: TDictionary<String, TCachedFile>;
+            FLoadGate: TLatch;
 
         class constructor CreateClass;
         class destructor DestroyClass;
@@ -76,16 +78,12 @@ type
         class function ReadCompressedData(const InputStream: TStream): TArray<TDataPoint>; static;
         class function ReadUncompressedData(const InputStream: TStream): TArray<TDataPoint>; static;
     public
+        class procedure ClearCache;
         // Asynchronously loads a single data file with caching support.
-        class function LoadDataFile(const LoadGate: TState; const FileName: string): TFuture<TArray<TDataPoint>>; static;
+        class function LoadDataFile(const FileName: string): TFuture<TArray<TDataPoint>>; static;
         // Asynchronously loads a complete chronological series of data files.
         class function LoadDataSeries(const FileName: string): TFuture<TArray<TDataPoint>>; static;
-    end;
 
-    // A specific data record structure for Ask and Bid prices.
-    TAskBidItem = packed record
-        Ask: Single;
-        Bid: Single;
     end;
 
     // A specialized TDataSeries for handling Ask/Bid tick data.
@@ -315,7 +313,7 @@ end;
 
 class destructor TDataSeries<T>.CreateClass;
 begin
-    FCachedFiles := TDictionary<String, TCachedFile>.Create;
+    FCachedFiles := TObjectDictionary<String, TCachedFile>.Create([doOwnsValues]);
 end;
 
 class destructor TDataSeries<T>.DestroyClass;
@@ -323,9 +321,14 @@ begin
     FCachedFiles.Free;
 end;
 
+class procedure TDataSeries<T>.ClearCache;
+begin
+    FCachedFiles.Clear;
+end;
+
 // Asynchronously reads tick data from a single specified file.
 // Returns a TFuture that will provide an array of TAskBidSeries.
-class function TDataSeries<T>.LoadDataFile(const LoadGate: TState; const FileName: string): TFuture<TArray<TDataPoint>>;
+class function TDataSeries<T>.LoadDataFile(const FileName: string): TFuture<TArray<TDataPoint>>;
 var
     parsedPath, parsedSymbol: string;
     parsedYear, parsedMonth: Integer;
@@ -351,24 +354,24 @@ begin
             begin
                 if not IsMemoryLow then
                     break;
-                FCachedFiles.Remove(fL[i].Name);
+                if fL[i].Name <> FileName then
+                    FCachedFiles.Remove(fL[i].Name);
             end;
         end;
 
         var cachedFile: TCachedFile;
         if FCachedFiles.TryGetValue(Filename, cachedFile) then
         begin
-            FCachedFiles.Remove(FileName);
-
             var age: TDateTime;
             if FileAge(Filename, age, true) then
             begin
                 if age = cachedFile.Age then
                 begin
                     cachedFile.LastUsed := Now;
-                    FCachedFiles.Add(Filename, cachedFile);
                     exit(cachedFile.Data);
-                end;
+                end
+                else
+                    FCachedFiles.Remove(FileName);
             end;
         end;
     end;
@@ -379,7 +382,7 @@ begin
         Result :=
             TFuture<TBytes>
                 .Construct(
-                    LoadGate,
+                    TLatch.Enqueue(FLoadGate),
                     function: TBytes
                     begin
                         Result := TFile.ReadAllBytes(capFileName); // Read all bytes of the .zip file
@@ -401,7 +404,7 @@ begin
         var capFileName := FileName;
         Result :=
             TFuture<TArray<TDataPoint>>.Construct(
-                LoadGate,
+                TLatch.Enqueue(FLoadGate),
                 function: TArray<TDataPoint>
                 begin
                     if TFile.Exists(capFileName) then
@@ -426,7 +429,7 @@ begin
 
     if Assigned(IsMemoryLow) then
     begin
-        var cachedFile: TCachedFile;
+        var cachedFile := TCachedFile.Create;
         cachedFile.Name := Filename;
         if FileAge(Filename, cachedFile.Age, true) then
         begin
@@ -486,7 +489,7 @@ begin
             // Create load tasks for the historical .tab files.
             for currentFile in tabFiles do
             begin
-                var data := LoadDataFile(TLatch.Enqueue(LoadGate), currentFile);
+                var data := LoadDataFile(currentFile);
                 loadedFileList.Add(data);
                 loadStates.Add(data.Done);
             end;
@@ -495,7 +498,7 @@ begin
             liveData := TFuture<TArray<TDataPoint>>.Null;
             if liveFilePath <> '' then
             begin
-                liveData := LoadDataFile(TLatch.Enqueue(LoadGate), liveFilePath);
+                liveData := LoadDataFile(liveFilePath);
                 loadedFileList.Add(liveData);
                 loadStates.Add(liveData.Done);
             end;
@@ -529,10 +532,10 @@ begin
                     for var P in loadedFiles do
                     begin
                         for var iterTickData in P.Value do
-                            if overallLastTabTickTime < iterTickData.OADateTime then
+                            if overallLastTabTickTime < iterTickData.TimeStamp then
                             begin
                                 tickList.Add(iterTickData);
-                                overallLastTabTickTime := iterTickData.OADateTime;
+                                overallLastTabTickTime := iterTickData.TimeStamp;
                             end;
                     end;
                     Result := tickList.ToArray;