Global LoadCache for DataSeries

Src/Myc.Trade.DataSeries.pas

@@ -40,6 +40,7 @@ uses
     System.SysUtils,
     System.Classes,
     System.Generics.Collections,
+    System.Generics.Defaults,
     System.IOUtils,
     Myc.Futures,
     Myc.Signals;
@@ -51,6 +52,21 @@ type
                 OADateTime: Double;
                 Data: T;
             end;
+
+    strict private
+        type
+            TCachedFile = record
+                Name: String;
+                Age: TDateTime;
+                LastUsed: TDateTime;
+                Data: TFuture<TArray<TDataPoint>>;
+            end;
+        class var
+            FCachedFiles: TDictionary<String, TCachedFile>;
+
+        class constructor CreateClass;
+        class destructor DestroyClass;
+
     protected
         class function ReadCompressedData(const InputStream: TStream): TArray<TDataPoint>; static;
         class function ReadUncompressedData(const InputStream: TStream): TArray<TDataPoint>; static;
@@ -74,10 +90,14 @@ function TryParseFileName(const FileName: string; out PathValue, SymbolValue: st
 // Finds the full filename of the oldest (=first) file for each symbol in the path.
 function FindOldestFilesPerSymbol(const DirectoryPath: string): TArray<String>;
 
+var
+    IsMemoryLow: TFunc<Boolean>;
+
 implementation
 
 uses
     System.Zip,
+    System.Math,
     Myc.TaskManager;
 
 // TryParseFileName parses filenames like "Symbol_Year_MM.Extension" or "Symbol_Year_MM.Extension_zip"
@@ -237,6 +257,21 @@ begin
     end;
 end;
 
+function FindNextFile(const Filename: String): String;
+begin
+
+end;
+
+class destructor TDataSeries<T>.CreateClass;
+begin
+    FCachedFiles := TDictionary<String, TCachedFile>.Create;
+end;
+
+class destructor TDataSeries<T>.DestroyClass;
+begin
+    FCachedFiles.Free;
+end;
+
 // Asynchronously reads tick data from a single specified file.
 // Returns a TFuture that will provide an array of TAskBidSeries.
 class function TDataSeries<T>.LoadDataFile(var LoadGate: TLatch; const FileName: string): TFuture<TArray<TDataPoint>>;
@@ -250,6 +285,43 @@ begin
     if not TryParseFileName(FileName, parsedPath, parsedSymbol, parsedYear, parsedMonth) then
         exit;
 
+    if Assigned(IsMemoryLow) then
+    begin
+        if IsMemoryLow then
+        begin
+            var fL := FCachedFiles.Values.ToArray;
+            TArray.Sort<TCachedFile>(
+                fL,
+                TComparer<TCachedFile>
+                    .Construct(function(const Left, Right: TCachedFile): Integer begin Result := Sign(Left.LastUsed - Right.LastUsed); end)
+            );
+
+            for var i := 0 to High(fL) do
+            begin
+                FCachedFiles.Remove(fL[i].Name);
+                if not IsMemoryLow then
+                    break;
+            end;
+        end;
+
+        var cachedFile: TCachedFile;
+        if FCachedFiles.TryGetValue(Filename, cachedFile) then
+        begin
+            FCachedFiles.Remove(FileName);
+
+            var age: TDateTime;
+            if FileAge(Filename, age, true) then
+            begin
+                if age = cachedFile.Age then
+                begin
+                    cachedFile.LastUsed := Now;
+                    FCachedFiles.Add(Filename, cachedFile);
+                    exit(cachedFile.Data);
+                end;
+            end;
+        end;
+    end;
+
     if FileName.EndsWith('_zip', True) then
     begin
         var zipFileBytes := TFile.ReadAllBytes(FileName); // Read all bytes of the .zip file
@@ -303,6 +375,19 @@ begin
                 end
             );
     end;
+
+    if Assigned(IsMemoryLow) then
+    begin
+        var cachedFile: TCachedFile;
+        cachedFile.Name := Filename;
+        var age: TDateTime;
+        if FileAge(Filename, cachedFile.Age, true) then
+        begin
+            cachedFile.Data := Result;
+            cachedFile.LastUsed := Now;
+            FCachedFiles.Add(Filename, cachedFile);
+        end;
+    end;
 end;
 
 class function TDataSeries<T>.LoadDataSeries(const FileName: string): TFuture<TArray<TDataPoint>>;