Data file handling
This commit is contained in:
Michael Schimmel
+306
-341
@@ -26,10 +26,10 @@ uses
|
||||
Myc.Futures,
|
||||
Myc.Signals,
|
||||
Myc.Lazy,
|
||||
Myc.Trade.DataPoint;
|
||||
Myc.Trade.DataPoint,
|
||||
Myc.Core.FileCache;
|
||||
|
||||
type
|
||||
|
||||
// Represents a generic data stream capable of providing sequential data chunks.
|
||||
// IsHistory:
|
||||
// - true, if this stream is a history stream. Once HasData becomes false, it reached it's end and will not provide more data.
|
||||
@@ -39,13 +39,16 @@ type
|
||||
['{A6E246A2-E84E-49AB-A63E-333E561E488C}']
|
||||
function GetHasData: TSignal;
|
||||
function GetChunk(var Data: array of TDataPoint<T>): Integer;
|
||||
function GetSymbol: String;
|
||||
function IsHistory: Boolean;
|
||||
property HasData: TSignal read GetHasData;
|
||||
property Symbol: String read GetSymbol;
|
||||
end;
|
||||
|
||||
// Abstract base class for IDataStream implementations.
|
||||
TDataStream<T> = class(TInterfacedObject, IDataStream<T>)
|
||||
protected
|
||||
function GetSymbol: String; virtual; abstract;
|
||||
function GetHasData: TSignal; virtual; abstract;
|
||||
function GetChunk(var Data: array of TDataPoint<T>): Integer; virtual; abstract;
|
||||
function IsHistory: Boolean; virtual; abstract;
|
||||
@@ -79,31 +82,31 @@ type
|
||||
IAuraDataServer<T: record> = interface(IDataServer<T>)
|
||||
['{481E27DE-DC58-49AF-B8A8-B6316980C423}']
|
||||
function LoadDataFile(const FileName: string): TFuture<TArray<TDataPoint<T>>>;
|
||||
function LoadDataSeries(const FileName: string): TFuture<TArray<TDataPoint<T>>>;
|
||||
function EnumerateAssetFiles: TArray<String>;
|
||||
function EnumerateAssets: TArray<String>;
|
||||
end;
|
||||
|
||||
// Aura files
|
||||
|
||||
// Generic server for loading and managing sequential time-series data from files.
|
||||
TAuraDataServer<T: record> = class(TInterfacedObject, IDataServer<T>, IAuraDataServer<T>)
|
||||
public type
|
||||
TDataFile = record
|
||||
Extension: String;
|
||||
Path, Symbol: String;
|
||||
Year, Month: Integer;
|
||||
function GetBaseFileName: string;
|
||||
function GetFullFileName: string;
|
||||
function GetIsValid: Boolean;
|
||||
property IsValid: Boolean read GetIsValid;
|
||||
end;
|
||||
|
||||
private
|
||||
type
|
||||
TCachedFile = class
|
||||
public
|
||||
Name: String;
|
||||
Age: TDateTime;
|
||||
LastUsed: TDateTime;
|
||||
Data: TFuture<TArray<TDataPoint<T>>>;
|
||||
end;
|
||||
private
|
||||
// The cache is per-instance.
|
||||
FCachedFiles: TDictionary<String, TCachedFile>;
|
||||
FCachedFiles: TDataFileCache<TArray<TDataPoint<T>>>;
|
||||
FPath: String;
|
||||
function DoLoad(const FileName: string): TFuture<TArray<TDataPoint<T>>>;
|
||||
function GetPath: String;
|
||||
|
||||
strict private
|
||||
// The load gate is shared across all instances of TAuraDataServer.
|
||||
class var
|
||||
FLoadGate: TLatch;
|
||||
|
||||
@@ -115,65 +118,58 @@ type
|
||||
constructor Create(const APath: String);
|
||||
destructor Destroy; override;
|
||||
|
||||
class function TryParseFileName(
|
||||
const FileName: string;
|
||||
out PathValue, SymbolValue: string;
|
||||
out YearValue, MonthValue: Integer
|
||||
): Boolean; virtual; abstract;
|
||||
class function TryParseFileName(const FileName: string; out DataFile: TDataFile): Boolean; virtual; abstract;
|
||||
|
||||
class function FindFirstDataFile(const Path, Symbol: string): string;
|
||||
class function FindNextDataFile(const FileName: string): string;
|
||||
class function FindOldestFilesPerSymbol(const DirectoryPath: string): TArray<String>;
|
||||
function FindFirstDataFile(const Symbol: string): TDataFile;
|
||||
class function FindNextDataFile(const CurrentFile: TDataFile): TDataFile;
|
||||
class function FindOldestFilesPerSymbol(const DirectoryPath: string): TArray<TDataFile>;
|
||||
|
||||
// IDataServer<T> implementation
|
||||
function CreateStream(const Symbol: String): IDataStream<T>; virtual; abstract;
|
||||
procedure ClearCache;
|
||||
function EnumerateAssetFiles: TArray<String>;
|
||||
|
||||
function EnumerateAssets: TArray<String>;
|
||||
function LoadDataFile(const FileName: string): TFuture<TArray<TDataPoint<T>>>;
|
||||
function LoadDataSeries(const FileName: string): TFuture<TArray<TDataPoint<T>>>;
|
||||
|
||||
property Path: String read GetPath;
|
||||
end;
|
||||
|
||||
// Implements a data stream that reads from Aura-specific historical data files.
|
||||
TAuraFileStream<T: record> = class(TDataStream<T>)
|
||||
TAuraFileStream<T: record> = class(TDataStream<T>, IDataStream<T>)
|
||||
private
|
||||
type
|
||||
// Changed: Renamed "File" to "FileInfo" for clarity.
|
||||
TDataStream = record
|
||||
FileInfo: TAuraDataServer<T>.TDataFile;
|
||||
Data: TFuture<TArray<TDataPoint<T>>>;
|
||||
end;
|
||||
private
|
||||
FDataServer: TAuraDataServer<T>;
|
||||
FSymbol: String;
|
||||
FHasData: TEvent;
|
||||
FCurrentFileName: string;
|
||||
FCurrentData: TFuture<TArray<TDataPoint<T>>>;
|
||||
FNextFileName: string;
|
||||
FNextData: TFuture<TArray<TDataPoint<T>>>;
|
||||
FCurrent: TFuture<TDataStream>;
|
||||
FNext: TFuture<TDataStream>;
|
||||
FCurrPosInFile: Int64;
|
||||
FLastTimeStamp: TDateTime;
|
||||
procedure PreloadNext;
|
||||
protected
|
||||
function GetSymbol: String; override;
|
||||
function GetHasData: TSignal; override;
|
||||
function GetChunk(var Data: array of TDataPoint<T>): Integer; override;
|
||||
function IsHistory: Boolean; override;
|
||||
public
|
||||
constructor Create(ADataServer: TAuraDataServer<T>; const AFilename: String);
|
||||
constructor Create(ADataServer: TAuraDataServer<T>; const ASymbol: String);
|
||||
destructor Destroy; override;
|
||||
procedure AfterConstruction; override;
|
||||
procedure BeforeDestruction; override;
|
||||
end;
|
||||
|
||||
// Aura tick data file Ask-Bid
|
||||
|
||||
TAuraTABFileServer = class(TAuraDataServer<TAskBidItem>)
|
||||
public
|
||||
// IDataServer<T> implementation
|
||||
function CreateStream(const Symbol: String): IDataStream<TAskBidItem>; override;
|
||||
|
||||
class function TryParseFileName(
|
||||
const FileName: string;
|
||||
out PathValue, SymbolValue: string;
|
||||
out YearValue, MonthValue: Integer
|
||||
): Boolean; override;
|
||||
function LoadDataSeries(const FileName: string): TFuture<TArray<TDataPoint<TAskBidItem>>>;
|
||||
class function TryParseFileName(const FileName: string; out DataFile: TAuraDataServer<TAskBidItem>.TDataFile): Boolean; override;
|
||||
end;
|
||||
|
||||
var
|
||||
IsMemoryLow: TFunc<Boolean>;
|
||||
|
||||
implementation
|
||||
|
||||
uses
|
||||
@@ -182,12 +178,33 @@ uses
|
||||
System.StrUtils,
|
||||
Myc.TaskManager;
|
||||
|
||||
{ TAuraDataServer<T>.TDataFile }
|
||||
|
||||
function TAuraDataServer<T>.TDataFile.GetBaseFileName: string;
|
||||
begin
|
||||
Result := Format('%s_%.4d_%.2d', [Symbol, Year, Month]);
|
||||
end;
|
||||
|
||||
function TAuraDataServer<T>.TDataFile.GetFullFileName: string;
|
||||
begin
|
||||
Result := TPath.Combine(Path, GetBaseFileName + Extension);
|
||||
end;
|
||||
|
||||
function TAuraDataServer<T>.TDataFile.GetIsValid: Boolean;
|
||||
begin
|
||||
Result := (Symbol <> '') and (Year > 0);
|
||||
end;
|
||||
|
||||
{ TAuraDataServer<T> }
|
||||
|
||||
constructor TAuraDataServer<T>.Create(const APath: String);
|
||||
begin
|
||||
inherited Create;
|
||||
// Initialize the per-instance cache.
|
||||
FCachedFiles := TObjectDictionary<String, TCachedFile>.Create([doOwnsValues]);
|
||||
FPath := APath;
|
||||
|
||||
FCachedFiles :=
|
||||
TDataFileCache<TArray<TDataPoint<T>>>
|
||||
.Create(function(const Filename: String): TFuture<TArray<TDataPoint<T>>> begin Result := DoLoad(Filename); end);
|
||||
end;
|
||||
|
||||
destructor TAuraDataServer<T>.Destroy;
|
||||
@@ -199,78 +216,59 @@ end;
|
||||
|
||||
procedure TAuraDataServer<T>.ClearCache;
|
||||
begin
|
||||
for var cF in FCachedFiles.Values do
|
||||
cF.Data.WaitFor;
|
||||
|
||||
FCachedFiles.Clear;
|
||||
end;
|
||||
|
||||
class function TAuraDataServer<T>.FindFirstDataFile(const Path, Symbol: string): string;
|
||||
function TAuraDataServer<T>.FindFirstDataFile(const Symbol: string): TDataFile;
|
||||
var
|
||||
oldestFiles: TArray<string>;
|
||||
fileName: string;
|
||||
pathValue, symbolValue: string;
|
||||
yearValue, monthValue: Integer;
|
||||
oldestFiles: TArray<TDataFile>;
|
||||
fileInfo: TDataFile;
|
||||
begin
|
||||
oldestFiles := FindOldestFilesPerSymbol(Path);
|
||||
for fileName in oldestFiles do
|
||||
oldestFiles := FindOldestFilesPerSymbol(FPath);
|
||||
for fileInfo in oldestFiles do
|
||||
begin
|
||||
if TryParseFileName(fileName, pathValue, symbolValue, yearValue, monthValue) then
|
||||
begin
|
||||
if SameText(symbolValue, Symbol) then
|
||||
exit(fileName);
|
||||
end;
|
||||
if SameText(fileInfo.Symbol, Symbol) then
|
||||
exit(fileInfo);
|
||||
end;
|
||||
Result := '';
|
||||
Result := Default(TDataFile);
|
||||
end;
|
||||
|
||||
class function TAuraDataServer<T>.FindNextDataFile(const FileName: string): string;
|
||||
class function TAuraDataServer<T>.FindNextDataFile(const CurrentFile: TDataFile): TDataFile;
|
||||
var
|
||||
pathValue, symbolValue: string;
|
||||
yearValue, monthValue: Integer;
|
||||
nextBaseName, compressedPath, uncompressedPath: string;
|
||||
dataFileToProbe: TDataFile;
|
||||
begin
|
||||
if not TryParseFileName(FileName, pathValue, symbolValue, yearValue, monthValue) then
|
||||
exit('');
|
||||
Result := Default(TDataFile);
|
||||
|
||||
Inc(monthValue);
|
||||
if monthValue > 12 then
|
||||
if not CurrentFile.IsValid then
|
||||
exit;
|
||||
|
||||
// Changed: Simplified logic, no reparsing needed.
|
||||
dataFileToProbe := CurrentFile;
|
||||
Inc(dataFileToProbe.Month);
|
||||
if (dataFileToProbe.Month > 12) then
|
||||
begin
|
||||
monthValue := 1;
|
||||
Inc(yearValue);
|
||||
dataFileToProbe.Month := 1;
|
||||
Inc(dataFileToProbe.Year);
|
||||
end;
|
||||
|
||||
nextBaseName := Format('%s_%.4d_%.2d.tab', [symbolValue, yearValue, monthValue]);
|
||||
dataFileToProbe.Extension := '.tab_zip';
|
||||
if TFile.Exists(dataFileToProbe.GetFullFileName) then
|
||||
exit(dataFileToProbe);
|
||||
|
||||
compressedPath := TPath.Combine(pathValue, nextBaseName + '_zip');
|
||||
if TFile.Exists(compressedPath) then
|
||||
exit(compressedPath);
|
||||
|
||||
uncompressedPath := TPath.Combine(pathValue, nextBaseName);
|
||||
if TFile.Exists(uncompressedPath) then
|
||||
exit(uncompressedPath);
|
||||
|
||||
Result := '';
|
||||
dataFileToProbe.Extension := '.tab';
|
||||
if TFile.Exists(dataFileToProbe.GetFullFileName) then
|
||||
exit(dataFileToProbe);
|
||||
end;
|
||||
|
||||
class function TAuraDataServer<T>.FindOldestFilesPerSymbol(const DirectoryPath: string): TArray<String>;
|
||||
type
|
||||
TTrackedFileInfo = record
|
||||
FullFileName: string;
|
||||
Year: Integer;
|
||||
Month: Integer;
|
||||
end;
|
||||
class function TAuraDataServer<T>.FindOldestFilesPerSymbol(const DirectoryPath: string): TArray<TDataFile>;
|
||||
var
|
||||
oldestFilesPerSymbol: TDictionary<string, TTrackedFileInfo>;
|
||||
oldestFilesPerSymbol: TDictionary<string, TDataFile>;
|
||||
fileNames: TArray<string>;
|
||||
currentFile: string;
|
||||
parsedPath: string;
|
||||
parsedSymbol: string;
|
||||
parsedYear: Integer;
|
||||
parsedMonth: Integer;
|
||||
trackedInfo: TTrackedFileInfo;
|
||||
dataFile: TDataFile;
|
||||
trackedInfo: TDataFile;
|
||||
begin
|
||||
oldestFilesPerSymbol := TDictionary<string, TTrackedFileInfo>.Create;
|
||||
oldestFilesPerSymbol := TDictionary<string, TDataFile>.Create;
|
||||
try
|
||||
if not TDirectory.Exists(DirectoryPath) then
|
||||
exit(nil);
|
||||
@@ -278,43 +276,37 @@ begin
|
||||
fileNames := TDirectory.GetFiles(DirectoryPath);
|
||||
for currentFile in fileNames do
|
||||
begin
|
||||
if TryParseFileName(currentFile, parsedPath, parsedSymbol, parsedYear, parsedMonth) then
|
||||
if TryParseFileName(currentFile, dataFile) then
|
||||
begin
|
||||
if oldestFilesPerSymbol.TryGetValue(parsedSymbol, trackedInfo) then
|
||||
if oldestFilesPerSymbol.TryGetValue(dataFile.Symbol, trackedInfo) then
|
||||
begin
|
||||
if (parsedYear < trackedInfo.Year) or ((parsedYear = trackedInfo.Year) and (parsedMonth < trackedInfo.Month)) then
|
||||
if (dataFile.Year < trackedInfo.Year) or ((dataFile.Year = trackedInfo.Year) and (dataFile.Month < trackedInfo.Month)) then
|
||||
begin
|
||||
trackedInfo.FullFileName := currentFile;
|
||||
trackedInfo.Year := parsedYear;
|
||||
trackedInfo.Month := parsedMonth;
|
||||
oldestFilesPerSymbol.AddOrSetValue(parsedSymbol, trackedInfo);
|
||||
oldestFilesPerSymbol.AddOrSetValue(dataFile.Symbol, dataFile);
|
||||
end;
|
||||
end
|
||||
else
|
||||
begin
|
||||
trackedInfo.FullFileName := currentFile;
|
||||
trackedInfo.Year := parsedYear;
|
||||
trackedInfo.Month := parsedMonth;
|
||||
oldestFilesPerSymbol.Add(parsedSymbol, trackedInfo);
|
||||
oldestFilesPerSymbol.Add(dataFile.Symbol, dataFile);
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
|
||||
SetLength(Result, oldestFilesPerSymbol.Count);
|
||||
var n := 0;
|
||||
for trackedInfo in oldestFilesPerSymbol.Values do
|
||||
begin
|
||||
Result[n] := trackedInfo.FullFileName;
|
||||
inc(n);
|
||||
end;
|
||||
Result := oldestFilesPerSymbol.Values.ToArray;
|
||||
finally
|
||||
oldestFilesPerSymbol.Free;
|
||||
end;
|
||||
end;
|
||||
|
||||
function TAuraDataServer<T>.EnumerateAssetFiles: TArray<String>;
|
||||
function TAuraDataServer<T>.EnumerateAssets: TArray<String>;
|
||||
var
|
||||
dataFiles: TArray<TDataFile>;
|
||||
i: Integer;
|
||||
begin
|
||||
Result := FindOldestFilesPerSymbol(FPath);
|
||||
dataFiles := FindOldestFilesPerSymbol(FPath);
|
||||
SetLength(Result, Length(dataFiles));
|
||||
for i := 0 to High(dataFiles) do
|
||||
Result[i] := dataFiles[i].Symbol;
|
||||
end;
|
||||
|
||||
function TAuraDataServer<T>.GetPath: String;
|
||||
@@ -322,59 +314,20 @@ begin
|
||||
Result := FPath;
|
||||
end;
|
||||
|
||||
function TAuraDataServer<T>.LoadDataFile(const FileName: string): TFuture<TArray<TDataPoint<T>>>;
|
||||
var
|
||||
parsedPath, parsedSymbol: string;
|
||||
parsedYear, parsedMonth: Integer;
|
||||
function TAuraDataServer<T>.DoLoad(const FileName: string): TFuture<TArray<TDataPoint<T>>>;
|
||||
begin
|
||||
Result := TFuture<TArray<TDataPoint<T>>>.Null;
|
||||
|
||||
if not TryParseFileName(FileName, parsedPath, parsedSymbol, parsedYear, parsedMonth) then
|
||||
if not TFile.Exists(FileName) then
|
||||
exit;
|
||||
|
||||
if Assigned(IsMemoryLow) then
|
||||
begin
|
||||
if IsMemoryLow() then
|
||||
begin
|
||||
var fL := FCachedFiles.Values.ToArray;
|
||||
TArray.Sort<TCachedFile>(
|
||||
fL,
|
||||
TComparer<TCachedFile>
|
||||
.Construct(function(const Left, Right: TCachedFile): Integer begin Result := Sign(Left.LastUsed - Right.LastUsed); end)
|
||||
);
|
||||
for var i := 0 to High(fL) do
|
||||
begin
|
||||
if not IsMemoryLow() then
|
||||
break;
|
||||
if fL[i].Name <> FileName then
|
||||
FCachedFiles.Remove(fL[i].Name);
|
||||
end;
|
||||
end;
|
||||
|
||||
var cachedFile: TCachedFile;
|
||||
if FCachedFiles.TryGetValue(Filename, cachedFile) then
|
||||
begin
|
||||
var age: TDateTime;
|
||||
if FileAge(Filename, age, true) then
|
||||
begin
|
||||
if age = cachedFile.Age then
|
||||
begin
|
||||
cachedFile.LastUsed := Now;
|
||||
exit(cachedFile.Data);
|
||||
end
|
||||
else
|
||||
FCachedFiles.Remove(FileName);
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
|
||||
var capFileName := FileName;
|
||||
if FileName.EndsWith('_zip', True) then
|
||||
begin
|
||||
Result :=
|
||||
TFuture<TBytes>
|
||||
.Construct(
|
||||
TLatch.Enqueue(FLoadGate), // Use shared class var FLoadGate
|
||||
TLatch.Enqueue(FLoadGate),
|
||||
function: TBytes begin Result := TFile.ReadAllBytes(capFileName); end)
|
||||
.Chain<TArray<TDataPoint<T>>>(
|
||||
function(bytes: TBytes): TArray<TDataPoint<T>>
|
||||
@@ -388,11 +341,11 @@ begin
|
||||
end;
|
||||
end);
|
||||
end
|
||||
else if TFile.Exists(FileName) then
|
||||
else
|
||||
begin
|
||||
Result :=
|
||||
TFuture<TArray<TDataPoint<T>>>.Construct(
|
||||
TLatch.Enqueue(FLoadGate), // Use shared class var FLoadGate
|
||||
TLatch.Enqueue(FLoadGate),
|
||||
function: TArray<TDataPoint<T>>
|
||||
begin
|
||||
if TFile.Exists(capFileName) then
|
||||
@@ -412,108 +365,11 @@ begin
|
||||
end
|
||||
);
|
||||
end;
|
||||
|
||||
if Assigned(IsMemoryLow) then
|
||||
begin
|
||||
var cachedFile := TCachedFile.Create;
|
||||
cachedFile.Name := Filename;
|
||||
if FileAge(Filename, cachedFile.Age, true) then
|
||||
begin
|
||||
cachedFile.Data := Result;
|
||||
cachedFile.LastUsed := Now;
|
||||
FCachedFiles.Add(Filename, cachedFile);
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
|
||||
function TAuraDataServer<T>.LoadDataSeries(const FileName: string): TFuture<TArray<TDataPoint<T>>>;
|
||||
var
|
||||
loadedState: TState;
|
||||
loadedFiles: TArray<TFuture<TArray<TDataPoint<T>>>>;
|
||||
liveData: TFuture<TArray<TDataPoint<T>>>;
|
||||
tabFiles: TList<string>;
|
||||
currentFile: string;
|
||||
liveFilePath: string;
|
||||
pathName, symbolName: string;
|
||||
yearValue, monthValue: Integer;
|
||||
function TAuraDataServer<T>.LoadDataFile(const FileName: string): TFuture<TArray<TDataPoint<T>>>;
|
||||
begin
|
||||
tabFiles := TList<string>.Create;
|
||||
try
|
||||
currentFile := FileName;
|
||||
if TFile.Exists(currentFile) then
|
||||
while currentFile <> '' do
|
||||
begin
|
||||
tabFiles.Add(currentFile);
|
||||
currentFile := FindNextDataFile(currentFile);
|
||||
end;
|
||||
|
||||
liveFilePath := '';
|
||||
if tabFiles.Count > 0 then
|
||||
begin
|
||||
var lastTabFile := tabFiles.Last;
|
||||
if TryParseFileName(lastTabFile, pathName, symbolName, yearValue, monthValue) then
|
||||
begin
|
||||
var liveFileBaseName := Format('%s_%d_%02d.tab-live', [symbolName, yearValue, monthValue]);
|
||||
var potentialLivePath := TPath.Combine(pathName, liveFileBaseName);
|
||||
if TFile.Exists(potentialLivePath) then
|
||||
liveFilePath := potentialLivePath;
|
||||
end;
|
||||
end;
|
||||
|
||||
var loadedFileList := TList<TFuture<TArray<TDataPoint<T>>>>.Create;
|
||||
var loadStates := TList<TState>.Create;
|
||||
try
|
||||
for currentFile in tabFiles do
|
||||
begin
|
||||
var data := LoadDataFile(currentFile);
|
||||
loadedFileList.Add(data);
|
||||
loadStates.Add(data.Done);
|
||||
end;
|
||||
|
||||
liveData := TFuture<TArray<TDataPoint<T>>>.Null;
|
||||
if liveFilePath <> '' then
|
||||
begin
|
||||
liveData := LoadDataFile(liveFilePath);
|
||||
loadedFileList.Add(liveData);
|
||||
loadStates.Add(liveData.Done);
|
||||
end;
|
||||
|
||||
loadedState := TState.All(loadStates.ToArray);
|
||||
loadedFiles := loadedFileList.ToArray;
|
||||
finally
|
||||
loadStates.Free;
|
||||
loadedFileList.Free;
|
||||
end;
|
||||
finally
|
||||
tabFiles.Free;
|
||||
end;
|
||||
|
||||
Result :=
|
||||
TFuture<TArray<TDataPoint<T>>>.Construct(
|
||||
loadedState,
|
||||
function: TArray<TDataPoint<T>>
|
||||
begin
|
||||
var tickList := TList<TDataPoint<T>>.Create;
|
||||
try
|
||||
var overallLastTabTickTime: TDateTime := 0;
|
||||
var cnt := 0;
|
||||
for var P in loadedFiles do
|
||||
inc(cnt, Length(P.Value));
|
||||
tickList.Capacity := cnt;
|
||||
|
||||
for var P in loadedFiles do
|
||||
for var iterTickData in P.Value do
|
||||
if overallLastTabTickTime < iterTickData.Time then
|
||||
begin
|
||||
tickList.Add(iterTickData);
|
||||
overallLastTabTickTime := iterTickData.Time;
|
||||
end;
|
||||
Result := tickList.ToArray;
|
||||
finally
|
||||
tickList.Free;
|
||||
end;
|
||||
end
|
||||
);
|
||||
Result := FCachedFiles.GetOrAdd(Filename);
|
||||
end;
|
||||
|
||||
class function TAuraDataServer<T>.ReadCompressedData(const InputStream: TStream): TArray<TDataPoint<T>>;
|
||||
@@ -583,14 +439,13 @@ end;
|
||||
|
||||
{ TAuraFileStream<T> }
|
||||
|
||||
constructor TAuraFileStream<T>.Create(ADataServer: TAuraDataServer<T>; const AFilename: String);
|
||||
constructor TAuraFileStream<T>.Create(ADataServer: TAuraDataServer<T>; const ASymbol: String);
|
||||
begin
|
||||
inherited Create;
|
||||
Assert(Assigned(ADataServer));
|
||||
FDataServer := ADataServer;
|
||||
FCurrentFileName := AFilename;
|
||||
// Create an event
|
||||
FHasData := TEvent.CreateEvent; // interface helper benutzen!
|
||||
FSymbol := ASymbol;
|
||||
FHasData := TEvent.CreateEvent;
|
||||
end;
|
||||
|
||||
destructor TAuraFileStream<T>.Destroy;
|
||||
@@ -599,56 +454,69 @@ begin
|
||||
end;
|
||||
|
||||
procedure TAuraFileStream<T>.AfterConstruction;
|
||||
var
|
||||
firstFile: TFuture<TAuraDataServer<T>.TDataFile>;
|
||||
begin
|
||||
inherited;
|
||||
|
||||
firstFile := TFuture<TAuraDataServer<T>.TDataFile>.Construct(function: TAuraDataServer<T>.TDataFile begin Result := FDataServer.FindFirstDataFile(FSymbol); end);
|
||||
|
||||
FCurrent :=
|
||||
firstFile.Chain<TDataStream>(
|
||||
function(const FileInfo: TAuraDataServer<T>.TDataFile): TDataStream
|
||||
begin
|
||||
if FileInfo.IsValid then
|
||||
begin
|
||||
// Changed: Use renamed field "FileInfo".
|
||||
Result.FileInfo := FileInfo;
|
||||
Result.Data := FDataServer.LoadDataFile(FileInfo.GetFullFileName);
|
||||
Result.Data.Done.Subscribe(FHasData);
|
||||
end;
|
||||
end
|
||||
);
|
||||
|
||||
FLastTimeStamp := 0;
|
||||
FCurrPosInFile := 0;
|
||||
FCurrentData := FDataServer.LoadDataFile(FCurrentFileName);
|
||||
// Forward all future done events, because this means there is new data.
|
||||
FCurrentData.Done.Subscribe(FHasData);
|
||||
|
||||
FNextFileName := FDataServer.FindNextDataFile(FCurrentFileName);
|
||||
if FNextFileName <> '' then
|
||||
begin
|
||||
FNextData := FDataServer.LoadDataFile(FNextFileName);
|
||||
FNextData.Done.Subscribe(FHasData);
|
||||
end;
|
||||
PreloadNext;
|
||||
end;
|
||||
|
||||
procedure TAuraFileStream<T>.BeforeDestruction;
|
||||
begin
|
||||
FCurrent.WaitFor;
|
||||
FCurrent.Value.Data.WaitFor;
|
||||
FNext.WaitFor;
|
||||
FNext.Value.Data.WaitFor;
|
||||
inherited;
|
||||
end;
|
||||
|
||||
function TAuraFileStream<T>.GetChunk(var Data: array of TDataPoint<T>): Integer;
|
||||
var
|
||||
item: TDataPoint<T>;
|
||||
currData: TArray<TDataPoint<T>>;
|
||||
label
|
||||
loop;
|
||||
begin
|
||||
Result := 0;
|
||||
|
||||
// This is an asynchronous operation! We don't wait for data. It's totally valid to result nothing, if there is nothing.
|
||||
if not FCurrentData.Done.IsSet then
|
||||
loop:
|
||||
if not FCurrent.Done.IsSet then
|
||||
exit;
|
||||
if not FCurrent.Value.Data.Done.IsSet then
|
||||
exit;
|
||||
|
||||
if FCurrPosInFile >= Length(FCurrentData.Value) then
|
||||
// Changed: Use renamed field "FileInfo".
|
||||
if not FCurrent.Value.FileInfo.IsValid then
|
||||
exit;
|
||||
|
||||
if FCurrPosInFile >= Length(FCurrent.Value.Data.Value) then
|
||||
begin
|
||||
FCurrentData := FNextData;
|
||||
FCurrentFileName := FNextFileName;
|
||||
FCurrPosInFile := 0;
|
||||
FNextData := TFuture<TArray<TDataPoint<T>>>.Null;
|
||||
|
||||
if FCurrentFileName = '' then
|
||||
exit;
|
||||
|
||||
FNextFileName := FDataServer.FindNextDataFile(FCurrentFileName);
|
||||
if FNextFileName <> '' then
|
||||
begin
|
||||
FNextData := FDataServer.LoadDataFile(FNextFileName);
|
||||
FNextData.Done.Subscribe(FHasData);
|
||||
end;
|
||||
|
||||
if not FCurrentData.Done.IsSet then
|
||||
exit;
|
||||
FCurrent := FNext;
|
||||
PreloadNext;
|
||||
goto loop;
|
||||
end;
|
||||
|
||||
currData := FCurrentData.Value;
|
||||
|
||||
var currData := FCurrent.Value.Data.Value;
|
||||
var maxLen := Length(Data);
|
||||
while (Result < maxLen) and (FCurrPosInFile < Length(currData)) do
|
||||
begin
|
||||
@@ -662,8 +530,6 @@ begin
|
||||
Inc(FCurrPosInFile);
|
||||
end;
|
||||
|
||||
// If there is data left in the current file, signal new data. If it is finished, we do nothing, because
|
||||
// the next signal will come from the next future being done.
|
||||
if FCurrPosInFile < Length(currData) then
|
||||
begin
|
||||
FHasData.Notify;
|
||||
@@ -675,92 +541,191 @@ begin
|
||||
Result := FHasData.Signal;
|
||||
end;
|
||||
|
||||
function TAuraFileStream<T>.GetSymbol: String;
|
||||
begin
|
||||
Result := FSymbol;
|
||||
end;
|
||||
|
||||
function TAuraFileStream<T>.IsHistory: Boolean;
|
||||
begin
|
||||
Result := True;
|
||||
end;
|
||||
|
||||
procedure TAuraFileStream<T>.PreloadNext;
|
||||
begin
|
||||
FNext :=
|
||||
FCurrent.Chain<TDataStream>(
|
||||
function(const Prev: TDataStream): TDataStream
|
||||
var
|
||||
nextFileInfo: TAuraDataServer<T>.TDataFile;
|
||||
begin
|
||||
// Changed: Use renamed field "FileInfo".
|
||||
if Prev.FileInfo.IsValid then
|
||||
begin
|
||||
nextFileInfo := TAuraDataServer<T>.FindNextDataFile(Prev.FileInfo);
|
||||
if nextFileInfo.IsValid then
|
||||
begin
|
||||
Result.FileInfo := nextFileInfo;
|
||||
Result.Data := FDataServer.LoadDataFile(nextFileInfo.GetFullFileName);
|
||||
Result.Data.Done.Subscribe(FHasData);
|
||||
end;
|
||||
end;
|
||||
end
|
||||
);
|
||||
end;
|
||||
|
||||
{ TAuraTABFileServer }
|
||||
|
||||
function TAuraTABFileServer.CreateStream(const Symbol: String): IDataStream<TAskBidItem>;
|
||||
var
|
||||
fileName: string;
|
||||
begin
|
||||
// FindFirstDataFile uses the server's path and the given symbol to locate the initial data file.
|
||||
fileName := FindFirstDataFile(FPath, Symbol);
|
||||
|
||||
// Exit if no file is found or if it is not a .tab file, preventing stream creation for invalid sources.
|
||||
if (fileName = '') or (not TPath.GetExtension(fileName).StartsWith('.tab')) then
|
||||
exit(nil);
|
||||
|
||||
// Creates a new stream instance, passing itself as the data server and the found filename.
|
||||
Result := TAuraFileStream<TAskBidItem>.Create(Self, fileName);
|
||||
Result := TAuraFileStream<TAskBidItem>.Create(Self, Symbol);
|
||||
end;
|
||||
|
||||
class function TAuraTABFileServer.TryParseFileName(
|
||||
const FileName: string;
|
||||
out PathValue, SymbolValue: string;
|
||||
out YearValue, MonthValue: Integer
|
||||
): Boolean;
|
||||
function TAuraTABFileServer.LoadDataSeries(const FileName: string): TFuture<TArray<TDataPoint<TAskBidItem>>>;
|
||||
var
|
||||
fileNameNoPath: string;
|
||||
nameForParsing: string;
|
||||
loadedState: TState;
|
||||
loadedFiles: TArray<TFuture<TArray<TDataPoint<TAskBidItem>>>>;
|
||||
liveData: TFuture<TArray<TDataPoint<TAskBidItem>>>;
|
||||
tabFiles: TList<string>;
|
||||
liveFilePath: string;
|
||||
currentFileInfo: TDataFile;
|
||||
begin
|
||||
tabFiles := TList<string>.Create;
|
||||
try
|
||||
if TryParseFileName(FileName, currentFileInfo) then
|
||||
begin
|
||||
while currentFileInfo.IsValid do
|
||||
begin
|
||||
tabFiles.Add(currentFileInfo.GetFullFileName);
|
||||
currentFileInfo := FindNextDataFile(currentFileInfo);
|
||||
end;
|
||||
end;
|
||||
|
||||
liveFilePath := '';
|
||||
if tabFiles.Count > 0 then
|
||||
begin
|
||||
var lastTabFile := tabFiles.Last;
|
||||
if TryParseFileName(lastTabFile, currentFileInfo) then
|
||||
begin
|
||||
var liveFileBaseName := Format('%s_%d_%02d.tab-live', [currentFileInfo.Symbol, currentFileInfo.Year, currentFileInfo.Month]);
|
||||
var potentialLivePath := TPath.Combine(currentFileInfo.Path, liveFileBaseName);
|
||||
if TFile.Exists(potentialLivePath) then
|
||||
liveFilePath := potentialLivePath;
|
||||
end;
|
||||
end;
|
||||
|
||||
var loadedFileList := TList<TFuture<TArray<TDataPoint<TAskBidItem>>>>.Create;
|
||||
var loadStates := TList<TState>.Create;
|
||||
try
|
||||
for var fileStr in tabFiles do
|
||||
begin
|
||||
var data := LoadDataFile(fileStr);
|
||||
loadedFileList.Add(data);
|
||||
loadStates.Add(data.Done);
|
||||
end;
|
||||
|
||||
liveData := TFuture<TArray<TDataPoint<TAskBidItem>>>.Null;
|
||||
if liveFilePath <> '' then
|
||||
begin
|
||||
liveData := LoadDataFile(liveFilePath);
|
||||
loadedFileList.Add(liveData);
|
||||
loadStates.Add(liveData.Done);
|
||||
end;
|
||||
|
||||
loadedState := TState.All(loadStates.ToArray);
|
||||
loadedFiles := loadedFileList.ToArray;
|
||||
finally
|
||||
loadStates.Free;
|
||||
loadedFileList.Free;
|
||||
end;
|
||||
finally
|
||||
tabFiles.Free;
|
||||
end;
|
||||
|
||||
Result :=
|
||||
TFuture<TArray<TDataPoint<TAskBidItem>>>.Construct(
|
||||
loadedState,
|
||||
function: TArray<TDataPoint<TAskBidItem>>
|
||||
begin
|
||||
var tickList := TList<TDataPoint<TAskBidItem>>.Create;
|
||||
try
|
||||
var overallLastTabTickTime: TDateTime := 0;
|
||||
var cnt := 0;
|
||||
for var P in loadedFiles do
|
||||
inc(cnt, Length(P.Value));
|
||||
tickList.Capacity := cnt;
|
||||
|
||||
for var P in loadedFiles do
|
||||
for var iterTickData in P.Value do
|
||||
if overallLastTabTickTime < iterTickData.Time then
|
||||
begin
|
||||
tickList.Add(iterTickData);
|
||||
overallLastTabTickTime := iterTickData.Time;
|
||||
end;
|
||||
Result := tickList.ToArray;
|
||||
finally
|
||||
tickList.Free;
|
||||
end;
|
||||
end
|
||||
);
|
||||
end;
|
||||
|
||||
class function TAuraTABFileServer.TryParseFileName(const FileName: string; out DataFile: TAuraDataServer<TAskBidItem>.TDataFile): Boolean;
|
||||
var
|
||||
fileNameNoPath, nameForParsing, baseName, ext: string;
|
||||
parts: TArray<string>;
|
||||
baseName: string;
|
||||
begin
|
||||
Result := False;
|
||||
PathValue := '';
|
||||
SymbolValue := '';
|
||||
YearValue := 0;
|
||||
MonthValue := 0;
|
||||
PathValue := TPath.GetDirectoryName(FileName);
|
||||
DataFile := Default(TDataFile);
|
||||
|
||||
DataFile.Path := TPath.GetDirectoryName(FileName);
|
||||
fileNameNoPath := TPath.GetFileName(FileName);
|
||||
nameForParsing := fileNameNoPath;
|
||||
|
||||
ext := '';
|
||||
if nameForParsing.EndsWith('_zip', True) then
|
||||
begin
|
||||
baseName := nameForParsing.Substring(0, nameForParsing.Length - 4);
|
||||
if TPath.GetExtension(baseName) <> '' then
|
||||
begin
|
||||
nameForParsing := baseName;
|
||||
end;
|
||||
ext := '_zip';
|
||||
nameForParsing := nameForParsing.Substring(0, nameForParsing.Length - 4);
|
||||
end;
|
||||
|
||||
nameForParsing := TPath.GetFileNameWithoutExtension(nameForParsing);
|
||||
parts := nameForParsing.Split(['_']);
|
||||
DataFile.Extension := TPath.GetExtension(nameForParsing) + ext;
|
||||
baseName := TPath.GetFileNameWithoutExtension(nameForParsing);
|
||||
|
||||
parts := baseName.Split(['_']);
|
||||
|
||||
if Length(parts) < 3 then
|
||||
exit;
|
||||
|
||||
if not TryStrToInt(parts[High(parts)], MonthValue) then
|
||||
if not TryStrToInt(parts[High(parts)], DataFile.Month) then
|
||||
exit;
|
||||
|
||||
if (MonthValue < 1) or (MonthValue > 12) then
|
||||
if (DataFile.Month < 1) or (DataFile.Month > 12) then
|
||||
begin
|
||||
MonthValue := 0;
|
||||
DataFile.Month := 0;
|
||||
exit;
|
||||
end;
|
||||
|
||||
if not TryStrToInt(parts[High(parts) - 1], YearValue) then
|
||||
if not TryStrToInt(parts[High(parts) - 1], DataFile.Year) then
|
||||
begin
|
||||
MonthValue := 0;
|
||||
DataFile.Month := 0;
|
||||
exit;
|
||||
end;
|
||||
|
||||
if YearValue <= 0 then
|
||||
if DataFile.Year <= 0 then
|
||||
begin
|
||||
YearValue := 0;
|
||||
MonthValue := 0;
|
||||
DataFile.Year := 0;
|
||||
DataFile.Month := 0;
|
||||
exit;
|
||||
end;
|
||||
|
||||
SymbolValue := string.Join('_', Copy(parts, 0, Length(parts) - 2));
|
||||
DataFile.Symbol := string.Join('_', Copy(parts, 0, Length(parts) - 2));
|
||||
|
||||
if SymbolValue = '' then
|
||||
if DataFile.Symbol = '' then
|
||||
begin
|
||||
PathValue := '';
|
||||
YearValue := 0;
|
||||
MonthValue := 0;
|
||||
DataFile.Path := '';
|
||||
DataFile.Year := 0;
|
||||
DataFile.Month := 0;
|
||||
exit;
|
||||
end;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user