Data file handling
This commit is contained in:
@@ -44,7 +44,7 @@
|
|||||||
* Befolge die gängigen Delphi-Formatierungsstandards mit folgenden Ausnahmen:
|
* Befolge die gängigen Delphi-Formatierungsstandards mit folgenden Ausnahmen:
|
||||||
|
|
||||||
- Einrückung mit 4 Leerzeichen anstelle von 2. Auch bei Kommentaren.
|
- Einrückung mit 4 Leerzeichen anstelle von 2. Auch bei Kommentaren.
|
||||||
- Das Code-Format ist UTF-8.
|
- Das Code-Format ist UTF-8. Nur ASCII, keine Sonderzeichen erlaubt (insb. kein No-Break-Space!)
|
||||||
- Folgende Schlüsselwörter müssen klein geschrieben werden:
|
- Folgende Schlüsselwörter müssen klein geschrieben werden:
|
||||||
and, or, not, mod, div, in, as, is, array of, sizeof(), inc(), dec()
|
and, or, not, mod, div, in, as, is, array of, sizeof(), inc(), dec()
|
||||||
|
|
||||||
|
|||||||
+27
-18
@@ -100,26 +100,35 @@ begin
|
|||||||
|
|
||||||
FLock.Enter;
|
FLock.Enter;
|
||||||
try
|
try
|
||||||
// Caching is active. First, try to free memory if needed.
|
if IsMemoryLow() then
|
||||||
fileList := FCachedFiles.Values.ToArray;
|
|
||||||
TArray.Sort<TCachedFile>(
|
|
||||||
fileList,
|
|
||||||
TComparer<TCachedFile>.Construct(
|
|
||||||
function(const Left, Right: TCachedFile): Integer
|
|
||||||
begin
|
|
||||||
// Sort by LastUsed timestamp, oldest first.
|
|
||||||
Result := Sign(Left.LastUsed - Right.LastUsed);
|
|
||||||
end
|
|
||||||
)
|
|
||||||
);
|
|
||||||
|
|
||||||
for i := 0 to High(fileList) do
|
|
||||||
begin
|
begin
|
||||||
if not IsMemoryLow() then
|
// Caching is active. First, try to free memory if needed.
|
||||||
break; // Stop evicting if memory pressure is relieved.
|
fileList := FCachedFiles.Values.ToArray;
|
||||||
|
TArray.Sort<TCachedFile>(
|
||||||
|
fileList,
|
||||||
|
TComparer<TCachedFile>.Construct(
|
||||||
|
function(const Left, Right: TCachedFile): Integer
|
||||||
|
begin
|
||||||
|
// Sort by LastUsed timestamp, oldest first.
|
||||||
|
Result := Sign(Left.LastUsed - Right.LastUsed);
|
||||||
|
end
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
if fileList[i].Name <> AFileName then
|
for i := 0 to High(fileList) do
|
||||||
FCachedFiles.Remove(fileList[i].Name);
|
begin
|
||||||
|
if not IsMemoryLow() then
|
||||||
|
break; // Stop evicting if memory pressure is relieved.
|
||||||
|
|
||||||
|
if fileList[i].Name <> AFileName then
|
||||||
|
begin
|
||||||
|
if fileList[i].Data.Done.IsSet then
|
||||||
|
begin
|
||||||
|
FCachedFiles.Remove(fileList[i].Name);
|
||||||
|
fileList[i].Data := TFuture<T>.Null;
|
||||||
|
end;
|
||||||
|
end;
|
||||||
|
end;
|
||||||
end;
|
end;
|
||||||
|
|
||||||
// Check if a valid entry exists in the cache.
|
// Check if a valid entry exists in the cache.
|
||||||
|
|||||||
+240
-227
@@ -5,7 +5,7 @@ unit Myc.Trade.DataStream;
|
|||||||
Provides a data server for loading time-series data and streaming it.
|
Provides a data server for loading time-series data and streaming it.
|
||||||
|
|
||||||
This unit contains the core components for handling historical data:
|
This unit contains the core components for handling historical data:
|
||||||
- TAuraFileDataRecord: A generic record for a single time-stamped data entry.
|
- TFileRecord: A generic record for a single time-stamped data entry.
|
||||||
- IDataServer/TAuraDataServer: A server component responsible for loading and caching
|
- IDataServer/TAuraDataServer: A server component responsible for loading and caching
|
||||||
series of data files from disk. Each server instance manages its own cache,
|
series of data files from disk. Each server instance manages its own cache,
|
||||||
but all instances share a central load gate.
|
but all instances share a central load gate.
|
||||||
@@ -30,6 +30,34 @@ uses
|
|||||||
Myc.Core.FileCache;
|
Myc.Core.FileCache;
|
||||||
|
|
||||||
type
|
type
|
||||||
|
// Represents metadata for a single data file.
|
||||||
|
TDataFile = record
|
||||||
|
private
|
||||||
|
FExtension: String;
|
||||||
|
FPath: String;
|
||||||
|
FSymbol: String;
|
||||||
|
FYear: Integer;
|
||||||
|
FMonth: Integer;
|
||||||
|
function GetIsValid: Boolean;
|
||||||
|
public
|
||||||
|
constructor Create(const APath, ASymbol, AExtension: String; AYear, AMonth: Integer);
|
||||||
|
function GetBaseFileName: string;
|
||||||
|
function GetFullFileName: string;
|
||||||
|
// Gets the next consecutive data file, if it exists on disk.
|
||||||
|
function GetNextFile: TDataFile;
|
||||||
|
class function ParseFileName(const FileName: string): TDataFile; static;
|
||||||
|
// Scans a directory to find the oldest file for a specific symbol.
|
||||||
|
class function FindFirst(const DirectoryPath, Symbol: string): TDataFile; static;
|
||||||
|
// Scans a directory and returns the oldest file found for each symbol.
|
||||||
|
class function FindOldestPerSymbol(const DirectoryPath: string): TArray<TDataFile>; static;
|
||||||
|
property IsValid: Boolean read GetIsValid;
|
||||||
|
property Extension: String read FExtension;
|
||||||
|
property Path: String read FPath;
|
||||||
|
property Symbol: String read FSymbol;
|
||||||
|
property Year: Integer read FYear;
|
||||||
|
property Month: Integer read FMonth;
|
||||||
|
end;
|
||||||
|
|
||||||
// Represents a generic data stream capable of providing sequential data chunks.
|
// Represents a generic data stream capable of providing sequential data chunks.
|
||||||
// IsHistory:
|
// IsHistory:
|
||||||
// - true, if this stream is a history stream. Once HasData becomes false, it reached it's end and will not provide more data.
|
// - true, if this stream is a history stream. Once HasData becomes false, it reached it's end and will not provide more data.
|
||||||
@@ -71,35 +99,13 @@ type
|
|||||||
['{1F8E5A9D-E92A-44C1-9F3F-C4B82A6E94B3}']
|
['{1F8E5A9D-E92A-44C1-9F3F-C4B82A6E94B3}']
|
||||||
function CreateStream(const Symbol: String): IDataStream<T>;
|
function CreateStream(const Symbol: String): IDataStream<T>;
|
||||||
procedure ClearCache;
|
procedure ClearCache;
|
||||||
end;
|
function EnumerateSymbols: TArray<String>;
|
||||||
|
|
||||||
// A generic record for a single time-stamped data entry.
|
|
||||||
TAuraFileDataRecord<T: record> = packed record
|
|
||||||
TimeStamp: TDateTime;
|
|
||||||
Data: T;
|
|
||||||
end;
|
|
||||||
|
|
||||||
IAuraDataServer<T: record> = interface(IDataServer<T>)
|
|
||||||
['{481E27DE-DC58-49AF-B8A8-B6316980C423}']
|
|
||||||
function LoadDataFile(const FileName: string): TFuture<TArray<TDataPoint<T>>>;
|
|
||||||
function EnumerateAssets: TArray<String>;
|
|
||||||
end;
|
end;
|
||||||
|
|
||||||
// Aura files
|
// Aura files
|
||||||
|
|
||||||
// Generic server for loading and managing sequential time-series data from files.
|
// Generic server for loading and managing sequential time-series data from files.
|
||||||
TAuraDataServer<T: record> = class(TInterfacedObject, IDataServer<T>, IAuraDataServer<T>)
|
TAuraDataServer<T: record> = class(TInterfacedObject, IDataServer<T>)
|
||||||
public type
|
|
||||||
TDataFile = record
|
|
||||||
Extension: String;
|
|
||||||
Path, Symbol: String;
|
|
||||||
Year, Month: Integer;
|
|
||||||
function GetBaseFileName: string;
|
|
||||||
function GetFullFileName: string;
|
|
||||||
function GetIsValid: Boolean;
|
|
||||||
property IsValid: Boolean read GetIsValid;
|
|
||||||
end;
|
|
||||||
|
|
||||||
private
|
private
|
||||||
FCachedFiles: TDataFileCache<TArray<TDataPoint<T>>>;
|
FCachedFiles: TDataFileCache<TArray<TDataPoint<T>>>;
|
||||||
FPath: String;
|
FPath: String;
|
||||||
@@ -117,17 +123,10 @@ type
|
|||||||
public
|
public
|
||||||
constructor Create(const APath: String);
|
constructor Create(const APath: String);
|
||||||
destructor Destroy; override;
|
destructor Destroy; override;
|
||||||
|
|
||||||
class function TryParseFileName(const FileName: string; out DataFile: TDataFile): Boolean; virtual; abstract;
|
|
||||||
|
|
||||||
function FindFirstDataFile(const Symbol: string): TDataFile;
|
|
||||||
class function FindNextDataFile(const CurrentFile: TDataFile): TDataFile;
|
|
||||||
class function FindOldestFilesPerSymbol(const DirectoryPath: string): TArray<TDataFile>;
|
|
||||||
|
|
||||||
function CreateStream(const Symbol: String): IDataStream<T>; virtual; abstract;
|
function CreateStream(const Symbol: String): IDataStream<T>; virtual; abstract;
|
||||||
procedure ClearCache;
|
procedure ClearCache;
|
||||||
function EnumerateAssets: TArray<String>;
|
function EnumerateSymbols: TArray<String>;
|
||||||
function LoadDataFile(const FileName: string): TFuture<TArray<TDataPoint<T>>>;
|
function LoadDataFile(const DataFile: TDataFile): TFuture<TArray<TDataPoint<T>>>;
|
||||||
property Path: String read GetPath;
|
property Path: String read GetPath;
|
||||||
end;
|
end;
|
||||||
|
|
||||||
@@ -135,9 +134,8 @@ type
|
|||||||
TAuraFileStream<T: record> = class(TDataStream<T>, IDataStream<T>)
|
TAuraFileStream<T: record> = class(TDataStream<T>, IDataStream<T>)
|
||||||
private
|
private
|
||||||
type
|
type
|
||||||
// Changed: Renamed "File" to "FileInfo" for clarity.
|
|
||||||
TDataStream = record
|
TDataStream = record
|
||||||
FileInfo: TAuraDataServer<T>.TDataFile;
|
FileInfo: TDataFile;
|
||||||
Data: TFuture<TArray<TDataPoint<T>>>;
|
Data: TFuture<TArray<TDataPoint<T>>>;
|
||||||
end;
|
end;
|
||||||
private
|
private
|
||||||
@@ -166,8 +164,7 @@ type
|
|||||||
TAuraTABFileServer = class(TAuraDataServer<TAskBidItem>)
|
TAuraTABFileServer = class(TAuraDataServer<TAskBidItem>)
|
||||||
public
|
public
|
||||||
function CreateStream(const Symbol: String): IDataStream<TAskBidItem>; override;
|
function CreateStream(const Symbol: String): IDataStream<TAskBidItem>; override;
|
||||||
function LoadDataSeries(const FileName: string): TFuture<TArray<TDataPoint<TAskBidItem>>>;
|
function LoadDataSeries(const InitialFile: TDataFile): TFuture<TArray<TDataPoint<TAskBidItem>>>;
|
||||||
class function TryParseFileName(const FileName: string; out DataFile: TAuraDataServer<TAskBidItem>.TDataFile): Boolean; override;
|
|
||||||
end;
|
end;
|
||||||
|
|
||||||
implementation
|
implementation
|
||||||
@@ -178,23 +175,180 @@ uses
|
|||||||
System.StrUtils,
|
System.StrUtils,
|
||||||
Myc.TaskManager;
|
Myc.TaskManager;
|
||||||
|
|
||||||
{ TAuraDataServer<T>.TDataFile }
|
{ TDataFile }
|
||||||
|
|
||||||
function TAuraDataServer<T>.TDataFile.GetBaseFileName: string;
|
constructor TDataFile.Create(const APath, ASymbol, AExtension: String; AYear, AMonth: Integer);
|
||||||
begin
|
begin
|
||||||
Result := Format('%s_%.4d_%.2d', [Symbol, Year, Month]);
|
FPath := APath;
|
||||||
|
FSymbol := ASymbol;
|
||||||
|
FExtension := AExtension;
|
||||||
|
FYear := AYear;
|
||||||
|
FMonth := AMonth;
|
||||||
end;
|
end;
|
||||||
|
|
||||||
function TAuraDataServer<T>.TDataFile.GetFullFileName: string;
|
function TDataFile.GetBaseFileName: string;
|
||||||
begin
|
begin
|
||||||
Result := TPath.Combine(Path, GetBaseFileName + Extension);
|
Result := Format('%s_%.4d_%.2d', [FSymbol, FYear, FMonth]);
|
||||||
end;
|
end;
|
||||||
|
|
||||||
function TAuraDataServer<T>.TDataFile.GetIsValid: Boolean;
|
function TDataFile.GetFullFileName: string;
|
||||||
begin
|
begin
|
||||||
Result := (Symbol <> '') and (Year > 0);
|
Result := TPath.Combine(FPath, GetBaseFileName + FExtension);
|
||||||
end;
|
end;
|
||||||
|
|
||||||
|
function TDataFile.GetIsValid: Boolean;
|
||||||
|
begin
|
||||||
|
Result := (FSymbol <> '') and (FYear > 0);
|
||||||
|
end;
|
||||||
|
|
||||||
|
function TDataFile.GetNextFile: TDataFile;
|
||||||
|
var
|
||||||
|
nextMonth, nextYear: Integer;
|
||||||
|
nextFile: TDataFile;
|
||||||
|
begin
|
||||||
|
if not IsValid then
|
||||||
|
exit(Default(TDataFile));
|
||||||
|
|
||||||
|
nextMonth := FMonth + 1;
|
||||||
|
nextYear := FYear;
|
||||||
|
if (nextMonth > 12) then
|
||||||
|
begin
|
||||||
|
nextMonth := 1;
|
||||||
|
Inc(nextYear);
|
||||||
|
end;
|
||||||
|
|
||||||
|
// Probe for zipped file first
|
||||||
|
nextFile := TDataFile.Create(FPath, FSymbol, '.tab_zip', nextYear, nextMonth);
|
||||||
|
if TFile.Exists(nextFile.GetFullFileName) then
|
||||||
|
exit(nextFile);
|
||||||
|
|
||||||
|
// Probe for uncompressed file
|
||||||
|
nextFile := TDataFile.Create(FPath, FSymbol, '.tab', nextYear, nextMonth);
|
||||||
|
if TFile.Exists(nextFile.GetFullFileName) then
|
||||||
|
exit(nextFile);
|
||||||
|
|
||||||
|
// No next file found
|
||||||
|
Result := Default(TDataFile);
|
||||||
|
end;
|
||||||
|
|
||||||
|
class function TDataFile.ParseFileName(const FileName: string): TDataFile;
|
||||||
|
var
|
||||||
|
fileNameNoPath, nameForParsing, baseName, ext, path, symbol: string;
|
||||||
|
year, month: Integer;
|
||||||
|
parts: TArray<string>;
|
||||||
|
begin
|
||||||
|
Result := Default(TDataFile);
|
||||||
|
|
||||||
|
path := TPath.GetDirectoryName(FileName);
|
||||||
|
fileNameNoPath := TPath.GetFileName(FileName);
|
||||||
|
nameForParsing := fileNameNoPath;
|
||||||
|
|
||||||
|
ext := '';
|
||||||
|
if nameForParsing.EndsWith('_zip', True) then
|
||||||
|
begin
|
||||||
|
ext := '_zip';
|
||||||
|
nameForParsing := nameForParsing.Substring(0, nameForParsing.Length - 4);
|
||||||
|
end;
|
||||||
|
|
||||||
|
var fileExt := TPath.GetExtension(nameForParsing) + ext;
|
||||||
|
baseName := TPath.GetFileNameWithoutExtension(nameForParsing);
|
||||||
|
|
||||||
|
parts := baseName.Split(['_']);
|
||||||
|
|
||||||
|
if Length(parts) < 3 then
|
||||||
|
exit;
|
||||||
|
|
||||||
|
if not TryStrToInt(parts[High(parts)], month) then
|
||||||
|
exit;
|
||||||
|
|
||||||
|
if (month < 1) or (month > 12) then
|
||||||
|
exit;
|
||||||
|
|
||||||
|
if not TryStrToInt(parts[High(parts) - 1], year) then
|
||||||
|
exit;
|
||||||
|
|
||||||
|
if year <= 0 then
|
||||||
|
exit;
|
||||||
|
|
||||||
|
symbol := string.Join('_', Copy(parts, 0, Length(parts) - 2));
|
||||||
|
|
||||||
|
if symbol = '' then
|
||||||
|
exit;
|
||||||
|
|
||||||
|
Result := TDataFile.Create(path, symbol, fileExt, year, month);
|
||||||
|
end;
|
||||||
|
|
||||||
|
class function TDataFile.FindFirst(const DirectoryPath, Symbol: string): TDataFile;
|
||||||
|
var
|
||||||
|
fileNames: TArray<string>;
|
||||||
|
currentFileName: string;
|
||||||
|
parsedFile: TDataFile;
|
||||||
|
begin
|
||||||
|
Result := Default(TDataFile);
|
||||||
|
|
||||||
|
if not TDirectory.Exists(DirectoryPath) then
|
||||||
|
exit;
|
||||||
|
|
||||||
|
fileNames := TDirectory.GetFiles(DirectoryPath);
|
||||||
|
for currentFileName in fileNames do
|
||||||
|
begin
|
||||||
|
parsedFile := TDataFile.ParseFileName(currentFileName);
|
||||||
|
if parsedFile.IsValid then
|
||||||
|
begin
|
||||||
|
if SameText(parsedFile.Symbol, Symbol) then
|
||||||
|
begin
|
||||||
|
// If it's the first match or older than the current result, update.
|
||||||
|
if (not Result.IsValid) or
|
||||||
|
(parsedFile.Year < Result.Year) or
|
||||||
|
((parsedFile.Year = Result.Year) and (parsedFile.Month < Result.Month)) then
|
||||||
|
begin
|
||||||
|
Result := parsedFile;
|
||||||
|
end;
|
||||||
|
end;
|
||||||
|
end;
|
||||||
|
end;
|
||||||
|
end;
|
||||||
|
|
||||||
|
class function TDataFile.FindOldestPerSymbol(const DirectoryPath: string): TArray<TDataFile>;
|
||||||
|
var
|
||||||
|
oldestFilesPerSymbol: TDictionary<string, TDataFile>;
|
||||||
|
fileNames: TArray<string>;
|
||||||
|
currentFile: string;
|
||||||
|
dataFile: TDataFile;
|
||||||
|
trackedInfo: TDataFile;
|
||||||
|
begin
|
||||||
|
oldestFilesPerSymbol := TDictionary<string, TDataFile>.Create;
|
||||||
|
try
|
||||||
|
if not TDirectory.Exists(DirectoryPath) then
|
||||||
|
exit(nil);
|
||||||
|
|
||||||
|
fileNames := TDirectory.GetFiles(DirectoryPath);
|
||||||
|
for currentFile in fileNames do
|
||||||
|
begin
|
||||||
|
dataFile := TDataFile.ParseFileName(currentFile);
|
||||||
|
if dataFile.IsValid then
|
||||||
|
begin
|
||||||
|
if oldestFilesPerSymbol.TryGetValue(dataFile.Symbol, trackedInfo) then
|
||||||
|
begin
|
||||||
|
if (dataFile.Year < trackedInfo.Year) or ((dataFile.Year = trackedInfo.Year) and (dataFile.Month < trackedInfo.Month)) then
|
||||||
|
begin
|
||||||
|
oldestFilesPerSymbol.AddOrSetValue(dataFile.Symbol, dataFile);
|
||||||
|
end;
|
||||||
|
end
|
||||||
|
else
|
||||||
|
begin
|
||||||
|
oldestFilesPerSymbol.Add(dataFile.Symbol, dataFile);
|
||||||
|
end;
|
||||||
|
end;
|
||||||
|
end;
|
||||||
|
|
||||||
|
Result := oldestFilesPerSymbol.Values.ToArray;
|
||||||
|
finally
|
||||||
|
oldestFilesPerSymbol.Free;
|
||||||
|
end;
|
||||||
|
end;
|
||||||
|
|
||||||
|
|
||||||
{ TAuraDataServer<T> }
|
{ TAuraDataServer<T> }
|
||||||
|
|
||||||
constructor TAuraDataServer<T>.Create(const APath: String);
|
constructor TAuraDataServer<T>.Create(const APath: String);
|
||||||
@@ -219,91 +373,12 @@ begin
|
|||||||
FCachedFiles.Clear;
|
FCachedFiles.Clear;
|
||||||
end;
|
end;
|
||||||
|
|
||||||
function TAuraDataServer<T>.FindFirstDataFile(const Symbol: string): TDataFile;
|
function TAuraDataServer<T>.EnumerateSymbols: TArray<String>;
|
||||||
var
|
|
||||||
oldestFiles: TArray<TDataFile>;
|
|
||||||
fileInfo: TDataFile;
|
|
||||||
begin
|
|
||||||
oldestFiles := FindOldestFilesPerSymbol(FPath);
|
|
||||||
for fileInfo in oldestFiles do
|
|
||||||
begin
|
|
||||||
if SameText(fileInfo.Symbol, Symbol) then
|
|
||||||
exit(fileInfo);
|
|
||||||
end;
|
|
||||||
Result := Default(TDataFile);
|
|
||||||
end;
|
|
||||||
|
|
||||||
class function TAuraDataServer<T>.FindNextDataFile(const CurrentFile: TDataFile): TDataFile;
|
|
||||||
var
|
|
||||||
dataFileToProbe: TDataFile;
|
|
||||||
begin
|
|
||||||
Result := Default(TDataFile);
|
|
||||||
|
|
||||||
if not CurrentFile.IsValid then
|
|
||||||
exit;
|
|
||||||
|
|
||||||
// Changed: Simplified logic, no reparsing needed.
|
|
||||||
dataFileToProbe := CurrentFile;
|
|
||||||
Inc(dataFileToProbe.Month);
|
|
||||||
if (dataFileToProbe.Month > 12) then
|
|
||||||
begin
|
|
||||||
dataFileToProbe.Month := 1;
|
|
||||||
Inc(dataFileToProbe.Year);
|
|
||||||
end;
|
|
||||||
|
|
||||||
dataFileToProbe.Extension := '.tab_zip';
|
|
||||||
if TFile.Exists(dataFileToProbe.GetFullFileName) then
|
|
||||||
exit(dataFileToProbe);
|
|
||||||
|
|
||||||
dataFileToProbe.Extension := '.tab';
|
|
||||||
if TFile.Exists(dataFileToProbe.GetFullFileName) then
|
|
||||||
exit(dataFileToProbe);
|
|
||||||
end;
|
|
||||||
|
|
||||||
class function TAuraDataServer<T>.FindOldestFilesPerSymbol(const DirectoryPath: string): TArray<TDataFile>;
|
|
||||||
var
|
|
||||||
oldestFilesPerSymbol: TDictionary<string, TDataFile>;
|
|
||||||
fileNames: TArray<string>;
|
|
||||||
currentFile: string;
|
|
||||||
dataFile: TDataFile;
|
|
||||||
trackedInfo: TDataFile;
|
|
||||||
begin
|
|
||||||
oldestFilesPerSymbol := TDictionary<string, TDataFile>.Create;
|
|
||||||
try
|
|
||||||
if not TDirectory.Exists(DirectoryPath) then
|
|
||||||
exit(nil);
|
|
||||||
|
|
||||||
fileNames := TDirectory.GetFiles(DirectoryPath);
|
|
||||||
for currentFile in fileNames do
|
|
||||||
begin
|
|
||||||
if TryParseFileName(currentFile, dataFile) then
|
|
||||||
begin
|
|
||||||
if oldestFilesPerSymbol.TryGetValue(dataFile.Symbol, trackedInfo) then
|
|
||||||
begin
|
|
||||||
if (dataFile.Year < trackedInfo.Year) or ((dataFile.Year = trackedInfo.Year) and (dataFile.Month < trackedInfo.Month)) then
|
|
||||||
begin
|
|
||||||
oldestFilesPerSymbol.AddOrSetValue(dataFile.Symbol, dataFile);
|
|
||||||
end;
|
|
||||||
end
|
|
||||||
else
|
|
||||||
begin
|
|
||||||
oldestFilesPerSymbol.Add(dataFile.Symbol, dataFile);
|
|
||||||
end;
|
|
||||||
end;
|
|
||||||
end;
|
|
||||||
|
|
||||||
Result := oldestFilesPerSymbol.Values.ToArray;
|
|
||||||
finally
|
|
||||||
oldestFilesPerSymbol.Free;
|
|
||||||
end;
|
|
||||||
end;
|
|
||||||
|
|
||||||
function TAuraDataServer<T>.EnumerateAssets: TArray<String>;
|
|
||||||
var
|
var
|
||||||
dataFiles: TArray<TDataFile>;
|
dataFiles: TArray<TDataFile>;
|
||||||
i: Integer;
|
i: Integer;
|
||||||
begin
|
begin
|
||||||
dataFiles := FindOldestFilesPerSymbol(FPath);
|
dataFiles := TDataFile.FindOldestPerSymbol(FPath);
|
||||||
SetLength(Result, Length(dataFiles));
|
SetLength(Result, Length(dataFiles));
|
||||||
for i := 0 to High(dataFiles) do
|
for i := 0 to High(dataFiles) do
|
||||||
Result[i] := dataFiles[i].Symbol;
|
Result[i] := dataFiles[i].Symbol;
|
||||||
@@ -367,9 +442,9 @@ begin
|
|||||||
end;
|
end;
|
||||||
end;
|
end;
|
||||||
|
|
||||||
function TAuraDataServer<T>.LoadDataFile(const FileName: string): TFuture<TArray<TDataPoint<T>>>;
|
function TAuraDataServer<T>.LoadDataFile(const DataFile: TDataFile): TFuture<TArray<TDataPoint<T>>>;
|
||||||
begin
|
begin
|
||||||
Result := FCachedFiles.GetOrAdd(Filename);
|
Result := FCachedFiles.GetOrAdd(DataFile.GetFullFileName);
|
||||||
end;
|
end;
|
||||||
|
|
||||||
class function TAuraDataServer<T>.ReadCompressedData(const InputStream: TStream): TArray<TDataPoint<T>>;
|
class function TAuraDataServer<T>.ReadCompressedData(const InputStream: TStream): TArray<TDataPoint<T>>;
|
||||||
@@ -412,25 +487,30 @@ begin
|
|||||||
end;
|
end;
|
||||||
|
|
||||||
class function TAuraDataServer<T>.ReadUncompressedData(const InputStream: TStream): TArray<TDataPoint<T>>;
|
class function TAuraDataServer<T>.ReadUncompressedData(const InputStream: TStream): TArray<TDataPoint<T>>;
|
||||||
|
type
|
||||||
|
TFileRecord = packed record
|
||||||
|
TimeStamp: TDateTime;
|
||||||
|
Data: T;
|
||||||
|
end;
|
||||||
var
|
var
|
||||||
fileSize: Int64;
|
fileSize: Int64;
|
||||||
recordCount, bytesRead: Integer;
|
recordCount, bytesRead: Integer;
|
||||||
rec: TAuraFileDataRecord<T>;
|
rec: TFileRecord;
|
||||||
begin
|
begin
|
||||||
SetLength(Result, 0);
|
SetLength(Result, 0);
|
||||||
InputStream.Position := 0;
|
InputStream.Position := 0;
|
||||||
fileSize := InputStream.Size;
|
fileSize := InputStream.Size;
|
||||||
if (fileSize = 0) or ((fileSize mod SizeOf(TAuraFileDataRecord<T>)) <> 0) then
|
if (fileSize = 0) or ((fileSize mod SizeOf(TFileRecord)) <> 0) then
|
||||||
exit;
|
exit;
|
||||||
|
|
||||||
recordCount := fileSize div SizeOf(TAuraFileDataRecord<T>);
|
recordCount := fileSize div SizeOf(TFileRecord);
|
||||||
if recordCount > 0 then
|
if recordCount > 0 then
|
||||||
begin
|
begin
|
||||||
SetLength(Result, recordCount);
|
SetLength(Result, recordCount);
|
||||||
for var i := 0 to High(Result) do
|
for var i := 0 to High(Result) do
|
||||||
begin
|
begin
|
||||||
bytesRead := InputStream.Read(rec, SizeOf(TAuraFileDataRecord<T>));
|
bytesRead := InputStream.Read(rec, SizeOf(TFileRecord));
|
||||||
if bytesRead <> SizeOf(TAuraFileDataRecord<T>) then
|
if bytesRead <> SizeOf(TFileRecord) then
|
||||||
raise EReadError.CreateFmt('Read error. Expected %d bytes, read %d.', [fileSize, bytesRead]);
|
raise EReadError.CreateFmt('Read error. Expected %d bytes, read %d.', [fileSize, bytesRead]);
|
||||||
Result[i].Create(rec.TimeStamp, rec.Data);
|
Result[i].Create(rec.TimeStamp, rec.Data);
|
||||||
end;
|
end;
|
||||||
@@ -455,21 +535,23 @@ end;
|
|||||||
|
|
||||||
procedure TAuraFileStream<T>.AfterConstruction;
|
procedure TAuraFileStream<T>.AfterConstruction;
|
||||||
var
|
var
|
||||||
firstFile: TFuture<TAuraDataServer<T>.TDataFile>;
|
firstFile: TFuture<TDataFile>;
|
||||||
|
serverPath: string;
|
||||||
begin
|
begin
|
||||||
inherited;
|
inherited;
|
||||||
|
|
||||||
firstFile := TFuture<TAuraDataServer<T>.TDataFile>.Construct(function: TAuraDataServer<T>.TDataFile begin Result := FDataServer.FindFirstDataFile(FSymbol); end);
|
// We need the path from the server instance to find the file.
|
||||||
|
serverPath := (FDataServer as TAuraDataServer<T>).Path;
|
||||||
|
firstFile := TFuture<TDataFile>.Construct(function: TDataFile begin Result := TDataFile.FindFirst(serverPath, FSymbol); end);
|
||||||
|
|
||||||
FCurrent :=
|
FCurrent :=
|
||||||
firstFile.Chain<TDataStream>(
|
firstFile.Chain<TDataStream>(
|
||||||
function(const FileInfo: TAuraDataServer<T>.TDataFile): TDataStream
|
function(const FileInfo: TDataFile): TDataStream
|
||||||
begin
|
begin
|
||||||
if FileInfo.IsValid then
|
if FileInfo.IsValid then
|
||||||
begin
|
begin
|
||||||
// Changed: Use renamed field "FileInfo".
|
|
||||||
Result.FileInfo := FileInfo;
|
Result.FileInfo := FileInfo;
|
||||||
Result.Data := FDataServer.LoadDataFile(FileInfo.GetFullFileName);
|
Result.Data := FDataServer.LoadDataFile(FileInfo);
|
||||||
Result.Data.Done.Subscribe(FHasData);
|
Result.Data.Done.Subscribe(FHasData);
|
||||||
end;
|
end;
|
||||||
end
|
end
|
||||||
@@ -504,7 +586,6 @@ begin
|
|||||||
if not FCurrent.Value.Data.Done.IsSet then
|
if not FCurrent.Value.Data.Done.IsSet then
|
||||||
exit;
|
exit;
|
||||||
|
|
||||||
// Changed: Use renamed field "FileInfo".
|
|
||||||
if not FCurrent.Value.FileInfo.IsValid then
|
if not FCurrent.Value.FileInfo.IsValid then
|
||||||
exit;
|
exit;
|
||||||
|
|
||||||
@@ -557,16 +638,15 @@ begin
|
|||||||
FCurrent.Chain<TDataStream>(
|
FCurrent.Chain<TDataStream>(
|
||||||
function(const Prev: TDataStream): TDataStream
|
function(const Prev: TDataStream): TDataStream
|
||||||
var
|
var
|
||||||
nextFileInfo: TAuraDataServer<T>.TDataFile;
|
nextFileInfo: TDataFile;
|
||||||
begin
|
begin
|
||||||
// Changed: Use renamed field "FileInfo".
|
|
||||||
if Prev.FileInfo.IsValid then
|
if Prev.FileInfo.IsValid then
|
||||||
begin
|
begin
|
||||||
nextFileInfo := TAuraDataServer<T>.FindNextDataFile(Prev.FileInfo);
|
nextFileInfo := Prev.FileInfo.GetNextFile;
|
||||||
if nextFileInfo.IsValid then
|
if nextFileInfo.IsValid then
|
||||||
begin
|
begin
|
||||||
Result.FileInfo := nextFileInfo;
|
Result.FileInfo := nextFileInfo;
|
||||||
Result.Data := FDataServer.LoadDataFile(nextFileInfo.GetFullFileName);
|
Result.Data := FDataServer.LoadDataFile(nextFileInfo);
|
||||||
Result.Data.Done.Subscribe(FHasData);
|
Result.Data.Done.Subscribe(FHasData);
|
||||||
end;
|
end;
|
||||||
end;
|
end;
|
||||||
@@ -581,53 +661,48 @@ begin
|
|||||||
Result := TAuraFileStream<TAskBidItem>.Create(Self, Symbol);
|
Result := TAuraFileStream<TAskBidItem>.Create(Self, Symbol);
|
||||||
end;
|
end;
|
||||||
|
|
||||||
function TAuraTABFileServer.LoadDataSeries(const FileName: string): TFuture<TArray<TDataPoint<TAskBidItem>>>;
|
function TAuraTABFileServer.LoadDataSeries(const InitialFile: TDataFile): TFuture<TArray<TDataPoint<TAskBidItem>>>;
|
||||||
var
|
var
|
||||||
loadedState: TState;
|
loadedState: TState;
|
||||||
loadedFiles: TArray<TFuture<TArray<TDataPoint<TAskBidItem>>>>;
|
loadedFiles: TArray<TFuture<TArray<TDataPoint<TAskBidItem>>>>;
|
||||||
liveData: TFuture<TArray<TDataPoint<TAskBidItem>>>;
|
liveData: TFuture<TArray<TDataPoint<TAskBidItem>>>;
|
||||||
tabFiles: TList<string>;
|
tabFiles: TList<TDataFile>;
|
||||||
liveFilePath: string;
|
liveFile: TDataFile;
|
||||||
currentFileInfo: TDataFile;
|
currentFileInfo: TDataFile;
|
||||||
begin
|
begin
|
||||||
tabFiles := TList<string>.Create;
|
tabFiles := TList<TDataFile>.Create;
|
||||||
try
|
try
|
||||||
if TryParseFileName(FileName, currentFileInfo) then
|
currentFileInfo := InitialFile;
|
||||||
|
while currentFileInfo.IsValid do
|
||||||
begin
|
begin
|
||||||
while currentFileInfo.IsValid do
|
tabFiles.Add(currentFileInfo);
|
||||||
begin
|
currentFileInfo := currentFileInfo.GetNextFile;
|
||||||
tabFiles.Add(currentFileInfo.GetFullFileName);
|
|
||||||
currentFileInfo := FindNextDataFile(currentFileInfo);
|
|
||||||
end;
|
|
||||||
end;
|
end;
|
||||||
|
|
||||||
liveFilePath := '';
|
liveFile := Default(TDataFile);
|
||||||
if tabFiles.Count > 0 then
|
if tabFiles.Count > 0 then
|
||||||
begin
|
begin
|
||||||
var lastTabFile := tabFiles.Last;
|
var lastTabFile := tabFiles.Last;
|
||||||
if TryParseFileName(lastTabFile, currentFileInfo) then
|
var liveFileBaseName := Format('%s_%d_%02d.tab-live', [lastTabFile.Symbol, lastTabFile.Year, lastTabFile.Month]);
|
||||||
begin
|
var potentialLivePath := TPath.Combine(lastTabFile.Path, liveFileBaseName);
|
||||||
var liveFileBaseName := Format('%s_%d_%02d.tab-live', [currentFileInfo.Symbol, currentFileInfo.Year, currentFileInfo.Month]);
|
if TFile.Exists(potentialLivePath) then
|
||||||
var potentialLivePath := TPath.Combine(currentFileInfo.Path, liveFileBaseName);
|
liveFile := TDataFile.ParseFileName(potentialLivePath);
|
||||||
if TFile.Exists(potentialLivePath) then
|
|
||||||
liveFilePath := potentialLivePath;
|
|
||||||
end;
|
|
||||||
end;
|
end;
|
||||||
|
|
||||||
var loadedFileList := TList<TFuture<TArray<TDataPoint<TAskBidItem>>>>.Create;
|
var loadedFileList := TList<TFuture<TArray<TDataPoint<TAskBidItem>>>>.Create;
|
||||||
var loadStates := TList<TState>.Create;
|
var loadStates := TList<TState>.Create;
|
||||||
try
|
try
|
||||||
for var fileStr in tabFiles do
|
for var fileInfo in tabFiles do
|
||||||
begin
|
begin
|
||||||
var data := LoadDataFile(fileStr);
|
var data := LoadDataFile(fileInfo);
|
||||||
loadedFileList.Add(data);
|
loadedFileList.Add(data);
|
||||||
loadStates.Add(data.Done);
|
loadStates.Add(data.Done);
|
||||||
end;
|
end;
|
||||||
|
|
||||||
liveData := TFuture<TArray<TDataPoint<TAskBidItem>>>.Null;
|
liveData := TFuture<TArray<TDataPoint<TAskBidItem>>>.Null;
|
||||||
if liveFilePath <> '' then
|
if liveFile.IsValid then
|
||||||
begin
|
begin
|
||||||
liveData := LoadDataFile(liveFilePath);
|
liveData := LoadDataFile(liveFile);
|
||||||
loadedFileList.Add(liveData);
|
loadedFileList.Add(liveData);
|
||||||
loadStates.Add(liveData.Done);
|
loadStates.Add(liveData.Done);
|
||||||
end;
|
end;
|
||||||
@@ -670,66 +745,4 @@ begin
|
|||||||
);
|
);
|
||||||
end;
|
end;
|
||||||
|
|
||||||
class function TAuraTABFileServer.TryParseFileName(const FileName: string; out DataFile: TAuraDataServer<TAskBidItem>.TDataFile): Boolean;
|
|
||||||
var
|
|
||||||
fileNameNoPath, nameForParsing, baseName, ext: string;
|
|
||||||
parts: TArray<string>;
|
|
||||||
begin
|
|
||||||
Result := False;
|
|
||||||
DataFile := Default(TDataFile);
|
|
||||||
|
|
||||||
DataFile.Path := TPath.GetDirectoryName(FileName);
|
|
||||||
fileNameNoPath := TPath.GetFileName(FileName);
|
|
||||||
nameForParsing := fileNameNoPath;
|
|
||||||
|
|
||||||
ext := '';
|
|
||||||
if nameForParsing.EndsWith('_zip', True) then
|
|
||||||
begin
|
|
||||||
ext := '_zip';
|
|
||||||
nameForParsing := nameForParsing.Substring(0, nameForParsing.Length - 4);
|
|
||||||
end;
|
|
||||||
|
|
||||||
DataFile.Extension := TPath.GetExtension(nameForParsing) + ext;
|
|
||||||
baseName := TPath.GetFileNameWithoutExtension(nameForParsing);
|
|
||||||
|
|
||||||
parts := baseName.Split(['_']);
|
|
||||||
|
|
||||||
if Length(parts) < 3 then
|
|
||||||
exit;
|
|
||||||
|
|
||||||
if not TryStrToInt(parts[High(parts)], DataFile.Month) then
|
|
||||||
exit;
|
|
||||||
|
|
||||||
if (DataFile.Month < 1) or (DataFile.Month > 12) then
|
|
||||||
begin
|
|
||||||
DataFile.Month := 0;
|
|
||||||
exit;
|
|
||||||
end;
|
|
||||||
|
|
||||||
if not TryStrToInt(parts[High(parts) - 1], DataFile.Year) then
|
|
||||||
begin
|
|
||||||
DataFile.Month := 0;
|
|
||||||
exit;
|
|
||||||
end;
|
|
||||||
|
|
||||||
if DataFile.Year <= 0 then
|
|
||||||
begin
|
|
||||||
DataFile.Year := 0;
|
|
||||||
DataFile.Month := 0;
|
|
||||||
exit;
|
|
||||||
end;
|
|
||||||
|
|
||||||
DataFile.Symbol := string.Join('_', Copy(parts, 0, Length(parts) - 2));
|
|
||||||
|
|
||||||
if DataFile.Symbol = '' then
|
|
||||||
begin
|
|
||||||
DataFile.Path := '';
|
|
||||||
DataFile.Year := 0;
|
|
||||||
DataFile.Month := 0;
|
|
||||||
exit;
|
|
||||||
end;
|
|
||||||
|
|
||||||
Result := True;
|
|
||||||
end;
|
|
||||||
|
|
||||||
end.
|
end.
|
||||||
|
|||||||
Reference in New Issue
Block a user