Brummel/MycLib
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

TDataSeries<T>

Browse Source
This commit is contained in:
Michael Schimmel
2025-06-05 14:36:05 +02:00
parent 6bed68748d
commit f8c3ffceb8
14 changed files with 1051 additions and 517 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Src/Myc.Core.Futures.pas
+16 -16
View File
@@ -59,22 +59,22 @@ begin
// Subscribe the job execution to AGate.
// The job will run when AGate notifies the subscriber returned by Run.
FInit :=
ATaskManager.CreateTask(
AGate,
procedure
begin
try
try
Self.FResult := AProc();
except
Self.FResult := Default(T); // Set result to Default(T) on error
raise; // Re-raise for TaskFactory to handle
end;
finally
Self.FDone.Notify; // Signal that this future is done (successfully or with error)
end;
end
);
ATaskManager.CreateTask(
AGate,
procedure
begin
try
try
Self.FResult := AProc();
except
Self.FResult := Default(T); // Set result to Default(T) on error
raise; // Re-raise for TaskFactory to handle
end;
finally
Self.FDone.Notify; // Signal that this future is done (successfully or with error)
end;
end
);
end;
destructor TMycGateFuncFuture<T>.Destroy;
Src/Myc.Core.Notifier.pas
+1 -1
View File
@@ -43,7 +43,7 @@ type
procedure Release; inline; // Releases the previously acquired exclusive lock.
function IsLocked: Boolean; inline; // Checks if the list is currently locked by any thread.
procedure Notify(
Func: TPredicate<T>
Func: TPredicate<T>
); // Iterates through registered receivers and invokes the predicate; removes receiver if predicate returns false.
end;
Src/Myc.Core.Tasks.pas
+11 -11
View File
@@ -211,17 +211,17 @@ begin
capturedProc := Proc; // Capture Proc for the anonymous method
CreateAnonymousThread(
'Thread',
procedure
begin
try
capturedProc(); // Execute the provided procedure
finally
capturedProc := nil; // Clear the captured proc
res.Notify; // Signal completion via the latch's Notify method
end;
end)
.Start;
'Thread',
procedure
begin
try
capturedProc(); // Execute the provided procedure
finally
capturedProc := nil; // Clear the captured proc
res.Notify; // Signal completion via the latch's Notify method
end;
end)
.Start;
// Return the state interface of the latch
exit(res.State);
Src/Myc.Test.Core.Atomic.pas
+3 -3
View File
@@ -279,9 +279,9 @@ begin
entryData := AllocEntryData(123);
// This should not raise an assertion error from TSList.Push.
Assert.WillNotRaise(
procedure begin FList.Push(@entryData.Entry); end,
nil,
'Pushing an aligned entry should not raise an exception/assertion.'
procedure begin FList.Push(@entryData.Entry); end,
nil,
'Pushing an aligned entry should not raise an exception/assertion.'
);
Assert.AreEqual(1, FList.QueryDepth, 'QueryDepth should be 1 after pushing an aligned entry.');
end;
Src/Myc.Test.Core.Lazy.pas
+62 -62
View File
@@ -187,14 +187,14 @@ begin
procExecuted := False;
expectedValue := 50;
funcLazy :=
TMycFuncLazy<Integer>.Create(
sourceDirty.State,
function: Integer
begin
procExecuted := True;
Result := expectedValue;
end
);
TMycFuncLazy<Integer>.Create(
sourceDirty.State,
function: Integer
begin
procExecuted := True;
Result := expectedValue;
end
);
Assert.IsNotNull(funcLazy, 'TMycFuncLazy<Integer> instance should not be nil');
Assert.IsTrue(funcLazy.GetChanged.IsSet, 'Changed.IsSet should be true before the first Pop by design');
value := 0;
@@ -223,9 +223,9 @@ begin
result := funcLazy.Pop(value);
Assert.IsTrue(result, 'First Pop should return true by design, even if FProc is nil');
Assert.AreEqual(
preCallValue,
value,
'Value should be unchanged as FProc was nil and current Pop implementation does not assign Default(T)'
preCallValue,
value,
'Value should be unchanged as FProc was nil and current Pop implementation does not assign Default(T)'
);
Assert.IsFalse(funcLazy.GetChanged.IsSet, 'Changed state should be false after Pop');
end;
@@ -260,14 +260,14 @@ begin
initialProcValue := 44;
procExecutedCount := 0;
funcLazy :=
TMycFuncLazy<Integer>.Create(
sourceDirty.State,
function: Integer
begin
Inc(procExecutedCount);
Result := initialProcValue;
end
);
TMycFuncLazy<Integer>.Create(
sourceDirty.State,
function: Integer
begin
Inc(procExecutedCount);
Result := initialProcValue;
end
);
Helper_ConsumeInitialPop(funcLazy, initialProcValue);
Assert.AreEqual(1, procExecutedCount, 'Proc should have executed once for initial pop');
result := funcLazy.Pop(value);
@@ -305,17 +305,17 @@ begin
sourceDirty.Reset;
procCallCount := 0;
funcLazy :=
TMycFuncLazy<Integer>.Create(
sourceDirty.State,
function: Integer
begin
Inc(procCallCount);
if procCallCount = 1 then
Result := 30
else
Result := 300 + procCallCount;
end
);
TMycFuncLazy<Integer>.Create(
sourceDirty.State,
function: Integer
begin
Inc(procCallCount);
if procCallCount = 1 then
Result := 30
else
Result := 300 + procCallCount;
end
);
currentExpectedValue := 30;
Helper_ConsumeInitialPop(funcLazy, currentExpectedValue);
Assert.AreEqual(1, procCallCount, 'Proc executed for initial Pop');
@@ -342,14 +342,14 @@ begin
sourceDirty.Reset;
procCallCount := 0;
funcLazy :=
TMycFuncLazy<Integer>.Create(
sourceDirty.State,
function: Integer
begin
Inc(procCallCount);
Result := 40;
end
);
TMycFuncLazy<Integer>.Create(
sourceDirty.State,
function: Integer
begin
Inc(procCallCount);
Result := 40;
end
);
resultPop1 := funcLazy.Pop(value);
Assert.IsTrue(resultPop1, 'First Pop should return true by design');
Assert.AreEqual(40, value, 'Value from first Pop');
@@ -378,19 +378,19 @@ begin
firstSourceChangeValue := 52;
funcLazy :=
TMycFuncLazy<Integer>.Create(
sourceDirty.State,
function: Integer
begin
Inc(procCallCount);
if procCallCount = 1 then
Result := initialValue // For initial pop
else if procCallCount = 2 then
Result := firstSourceChangeValue // For pop after first effective source change
else
Result := 999; // Should not be reached in this specific test logic
end
);
TMycFuncLazy<Integer>.Create(
sourceDirty.State,
function: Integer
begin
Inc(procCallCount);
if procCallCount = 1 then
Result := initialValue // For initial pop
else if procCallCount = 2 then
Result := firstSourceChangeValue // For pop after first effective source change
else
Result := 999; // Should not be reached in this specific test logic
end
);
// 1. Initial Pop (consumes "by design" changed state)
Assert.IsTrue(funcLazy.GetChanged.IsSet, 'Changed state should be true before first Pop (by design)');
@@ -436,9 +436,9 @@ begin
Assert.IsTrue(funcLazyObj.Pop(tempVal), 'Initial Pop should succeed');
funcLazyObj.Destroy;
Assert.WillNotRaise(
procedure begin sourceDirty.Notify; end,
nil,
'Destroy test assumes TMycSubscription.Unsubscribe works. Verified by no crash on source notify post-destroy.'
procedure begin sourceDirty.Notify; end,
nil,
'Destroy test assumes TMycSubscription.Unsubscribe works. Verified by no crash on source notify post-destroy.'
);
sourceDirty := nil;
end;
@@ -457,14 +457,14 @@ begin
expectedValue := 70;
procExecuted := False;
funcLazy :=
TMycFuncLazy<Integer>.Create(
sourceDirty.State,
function: Integer
begin
procExecuted := True;
Result := expectedValue;
end
);
TMycFuncLazy<Integer>.Create(
sourceDirty.State,
function: Integer
begin
procExecuted := True;
Result := expectedValue;
end
);
Assert.IsNotNull(funcLazy, 'TMycFuncLazy<Integer> instance should not be nil');
Assert.IsTrue(funcLazy.GetChanged.IsSet, 'funcLazy.GetChanged.IsSet should be true after creation (by design)');
value := 0;
Src/Myc.Test.Lazy.pas
+54 -54
View File
@@ -146,14 +146,14 @@ begin
procExecuted := False;
// FChangingSignal is reset in Setup
lazyIntf :=
TLazy<Integer>.Construct(
FChangingSignal.State,
function: Integer
begin
procExecuted := True;
Result := 10;
end
);
TLazy<Integer>.Construct(
FChangingSignal.State,
function: Integer
begin
procExecuted := True;
Result := 10;
end
);
Assert.IsNotNull(lazyIntf, 'TLazy.Construct should return a valid interface');
lazyRec := lazyIntf; // Implicit conversion
@@ -171,14 +171,14 @@ begin
procExecuted := False;
expectedValue := 20;
lazyIntf :=
TLazy<Integer>.Construct(
FChangingSignal.State,
function: Integer
begin
procExecuted := True;
Result := expectedValue;
end
);
TLazy<Integer>.Construct(
FChangingSignal.State,
function: Integer
begin
procExecuted := True;
Result := expectedValue;
end
);
lazyRec := lazyIntf;
ConsumeInitialPop(lazyRec, expectedValue, 'TestConstruct_FirstPop');
@@ -236,14 +236,14 @@ var
begin
procCallCount := 0;
lazyIntf :=
TLazy<Integer>.Construct(
FChangingSignal.State,
function: Integer
begin
Inc(procCallCount);
Result := 100 + procCallCount; // Value changes per call
end
);
TLazy<Integer>.Construct(
FChangingSignal.State,
function: Integer
begin
Inc(procCallCount);
Result := 100 + procCallCount; // Value changes per call
end
);
lazyRec := lazyIntf;
ConsumeInitialPop(lazyRec, 101, 'TestConstruct_StateInteraction_PopResetsChangedAfterSignal (Initial)'); // procCallCount = 1
@@ -267,14 +267,14 @@ var
begin
procCallCount := 0;
lazyIntf :=
TLazy<Integer>.Construct(
FChangingSignal.State,
function: Integer
begin
Inc(procCallCount);
Result := 200 + procCallCount;
end
);
TLazy<Integer>.Construct(
FChangingSignal.State,
function: Integer
begin
Inc(procCallCount);
Result := 200 + procCallCount;
end
);
lazyRec := lazyIntf;
// 1. Initial Pop
@@ -322,12 +322,12 @@ begin
// This should trigger its destructor, which should unsubscribe from localChangingSignal.
Assert.WillNotRaise(
procedure
begin
localChangingSignal.Notify; // Notify the source AFTER the lazy object is supposed to be gone.
end,
nil, // Default: any exception is a failure
'Notifying source after lazy object is freed should not crash, indicating unsubscription.'
procedure
begin
localChangingSignal.Notify; // Notify the source AFTER the lazy object is supposed to be gone.
end,
nil, // Default: any exception is a failure
'Notifying source after lazy object is freed should not crash, indicating unsubscription.'
);
localChangingSignal := nil; // Clean up the local signal itself.
@@ -368,14 +368,14 @@ var
begin
procCallCount := 0;
originalLazyIntf :=
TLazy<Integer>.Construct(
FChangingSignal.State,
function: Integer
begin
Inc(procCallCount);
Result := 70 + procCallCount;
end
);
TLazy<Integer>.Construct(
FChangingSignal.State,
function: Integer
begin
Inc(procCallCount);
Result := 70 + procCallCount;
end
);
wrappedLazyRec := TLazy<Integer>.Create(originalLazyIntf);
// First Pop via wrapper (initial pop)
@@ -450,14 +450,14 @@ var
begin
procExecuted := False;
lazyIntf :=
TLazy<Integer>.Construct(
FChangingSignal.State,
function: Integer
begin
procExecuted := True;
Result := 100;
end
);
TLazy<Integer>.Construct(
FChangingSignal.State,
function: Integer
begin
procExecuted := True;
Result := 100;
end
);
lazyRec := lazyIntf;
ConsumeInitialPop(lazyRec, 100, 'TestPop_AfterInitialAndNoSignal (Initial)');
Src/Myc.Trade.DataSeries.pas
+542
View File
@@ -0,0 +1,542 @@
unit Myc.Trade.DataSeries;
(* ------------------------------------------------------------------------------
This unit provides functions to load tick data (TTickData) from
binary files. The data originates from a C# application
and has the structure OADateTime (Double), Ask (Single), and Bid (Single).
The files follow the naming convention:
For .tab files: Symbol_Year_MM.tab OR Symbol_Year_MM.tab_zip
For .tab-live files: Symbol_Year_MM.tab-live (NEVER compressed)
(MM represents the two-digit month, e.g., 01 for January, 12 for December)
If both compressed (.tab_zip) and uncompressed (.tab) versions of a .tab
file exist, the compressed version is preferred. Files ending with _zip
are expected to be ZIP archives containing the actual data file
(e.g., Symbol_Year_MM.tab).
Loading occurs in two phases:
1. All .tab files (regular or _zip) are processed chronologically by
month and year. Unique ticks are loaded, and the timestamp of the
latest tick across all .tab files is determined (overallLastTabTickTime).
Assumes timestamps are unique across all .tab files.
2. All .tab-live files (which are never compressed), for months/years
corresponding to found .tab files, are processed. Ticks from a
.tab-live file are only integrated if their timestamp is strictly later
than overallLastTabTickTime. Assumes such ticks are globally unique.
If a .tab-live file results in no new data being added under these rules,
it is deleted after processing.
The loading function automatically detects subsequent monthly/yearly files.
Main Function:
LoadTickDataSeries - Loads a series of tick data files.
------------------------------------------------------------------------------ *)
interface
uses
System.SysUtils,
System.Classes,
System.Generics.Collections,
System.IOUtils,
Myc.Futures,
Myc.Signals;
type
TDataSeries<T: record> = class
type
TDataPoint = packed record
OADateTime: Double;
Data: T;
end;
protected
class function ReadCompressedData(const InputStream: TStream): TArray<TDataPoint>; static;
class function ReadUncompressedData(const InputStream: TStream): TArray<TDataPoint>; static;
public
class function LoadDataFile(var LoadGate: TLatch; const FileName: string): TFuture<TArray<TDataPoint>>; static;
class function LoadDataSeries(const FileName: string): TFuture<TArray<TDataPoint>>; static;
end;
TAskBidItem = packed record
Ask: Single;
Bid: Single;
end;
TAskBid = class(TDataSeries<TAskBidItem>)
type
TTick = TDataSeries<TAskBidItem>.TDataPoint;
end;
function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue, MonthValue: Integer): Boolean;
// Finds the full filename of the oldest (=first) file for each symbol in the path.
function FindOldestFilesPerSymbol(const DirectoryPath: string): TArray<String>;
implementation
uses
System.Zip,
Myc.TaskManager;
// TryParseFileName parses filenames like "Symbol_Year_MM.Extension" or "Symbol_Year_MM.Extension_zip"
// to extract Path, Symbol, Year, and Month.
function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue, MonthValue: Integer): Boolean;
var
fileNameNoPath: string;
nameForParsing: string;
parts: TArray<string>;
baseName: string;
begin
Result := False; // Default to failure
PathValue := ''; // Initialize all out parameters
SymbolValue := '';
YearValue := 0;
MonthValue := 0;
PathValue := TPath.GetDirectoryName(FileName);
fileNameNoPath := TPath.GetFileName(FileName);
nameForParsing := fileNameNoPath;
// Handle cases like "Symbol_Year_MM.Extension_zip" by removing the trailing "_zip" before extension removal
if nameForParsing.EndsWith('_zip', True) then
begin
// Get the part before "_zip"
baseName := nameForParsing.Substring(0, nameForParsing.Length - 4);
// Get the extension of the original name (e.g. ".csv_zip")
// and check if baseName still contains an extension that needs to be removed by GetFileNameWithoutExtension
if TPath.GetExtension(baseName) <> '' then
begin
nameForParsing := baseName;
end
else
begin
// If baseName (e.g. "EURUSD_2023_01") has no extension, it is the correct parsing base
// However, TPath.GetFileNameWithoutExtension would also work correctly here,
// but this handles a potential case where a file might be named "Symbol_Year_MM_zip" (no dot before zip)
end;
end;
// Remove the actual extension (e.g., ".csv", ".dat") from "Symbol_Year_MM.Extension" or the modified "Symbol_Year_MM.Extension" part of "_zip"
nameForParsing := TPath.GetFileNameWithoutExtension(nameForParsing); // e.g., "EURUSD_2023_01"
parts := nameForParsing.Split(['_']);
if Length(parts) < 3 then // Need at least Symbol_Year_Month
exit;
// Month is the last part.
if not TryStrToInt(parts[High(parts)], MonthValue) then
begin
exit;
end;
if (MonthValue < 1) or (MonthValue > 12) then // Validate month range
begin
MonthValue := 0; // Set to invalid if out of range
exit;
end;
// Year is the second to last part.
if not TryStrToInt(parts[High(parts) - 1], YearValue) then
begin
MonthValue := 0; // Also invalidate month if year parsing fails
exit;
end;
// Ensure YearValue is plausible, e.g. not 0 or negative, adjust as needed for your data's year range
if YearValue <= 0 then
begin
YearValue := 0;
MonthValue := 0;
exit;
end;
// Symbol consists of all parts before Year and Month.
SymbolValue := string.Join('_', Copy(parts, 0, Length(parts) - 2));
if SymbolValue = '' then // Ensure symbol is not empty
begin
// Reset all out parameters on failure before exit
PathValue := '';
YearValue := 0;
MonthValue := 0;
exit;
end;
Result := True;
end;
// Finds the oldest file for each symbol in the given directory.
// The file extension is ignored for the purpose of identifying the symbol, year, and month.
function FindOldestFilesPerSymbol(const DirectoryPath: string): TArray<String>;
type
TTrackedFileInfo = record
FullFileName: string;
Year: Integer;
Month: Integer;
end;
var
oldestFilesPerSymbol: TDictionary<string, TTrackedFileInfo>;
fileNames: TArray<string>;
currentFile: string;
parsedPath: string;
parsedSymbol: string;
parsedYear: Integer;
parsedMonth: Integer;
trackedInfo: TTrackedFileInfo;
begin
oldestFilesPerSymbol := TDictionary<string, TTrackedFileInfo>.Create; // Dictionary to track the oldest file per symbol
try
if not TDirectory.Exists(DirectoryPath) then
begin
// Or raise an exception, depending on desired error handling
exit;
end;
fileNames := TDirectory.GetFiles(DirectoryPath); // Get all files in the directory
for currentFile in fileNames do
begin
// Attempt to parse the file name
if TryParseFileName(currentFile, parsedPath, parsedSymbol, parsedYear, parsedMonth) then
begin
// Check if this symbol is already tracked or if this file is older
if oldestFilesPerSymbol.TryGetValue(parsedSymbol, trackedInfo) then
begin
// If current file's year is less, or year is same and month is less, it's older
if (parsedYear < trackedInfo.Year) or ((parsedYear = trackedInfo.Year) and (parsedMonth < trackedInfo.Month)) then
begin
trackedInfo.FullFileName := currentFile; // Store full path to the file
trackedInfo.Year := parsedYear;
trackedInfo.Month := parsedMonth;
oldestFilesPerSymbol.AddOrSetValue(parsedSymbol, trackedInfo); // Update the entry for this symbol
end;
end
else
begin
// First time we see this symbol, so it's the oldest so far
trackedInfo.FullFileName := currentFile; // Store full path to the file
trackedInfo.Year := parsedYear;
trackedInfo.Month := parsedMonth;
oldestFilesPerSymbol.Add(parsedSymbol, trackedInfo); // Add new entry for this symbol
end;
end;
end;
// Populate the result array with the full names of the oldest files
SetLength(Result, oldestFilesPerSymbol.Count);
var n := 0;
for trackedInfo in oldestFilesPerSymbol.Values do
begin
Result[n] := trackedInfo.FullFileName;
inc(n);
end;
finally
oldestFilesPerSymbol.Free; // Free the dictionary
end;
end;
// Asynchronously reads tick data from a single specified file.
// Returns a TFuture that will provide an array of TAskBidSeries.
class function TDataSeries<T>.LoadDataFile(var LoadGate: TLatch; const FileName: string): TFuture<TArray<TDataPoint>>;
var
parsedPath, parsedSymbol: string;
parsedYear, parsedMonth: Integer;
begin
Result := TFuture<TArray<TDataPoint>>.Null;
// Attempt to parse year and month from the filename
if not TryParseFileName(FileName, parsedPath, parsedSymbol, parsedYear, parsedMonth) then
exit;
if FileName.EndsWith('_zip', True) then
begin
var zipFileBytes := TFile.ReadAllBytes(FileName); // Read all bytes of the .zip file
if Length(zipFileBytes) = 0 then
exit;
var capFileName := FileName;
Result :=
TFuture<TBytes>
.Construct(
TLatch.Enqueue(LoadGate),
function: TBytes
begin
Result := TFile.ReadAllBytes(capFileName); // Read all bytes of the .zip file
end)
.Chain<TArray<TDataPoint>>(
function(bytes: TBytes): TArray<TDataPoint>
begin
var zipMemoryStream := TBytesStream.Create(zipFileBytes); // Create a memory stream to hold the zip file content
try
zipMemoryStream.Position := 0;
Result := ReadCompressedData(zipMemoryStream);
finally
zipMemoryStream.Free;
end;
end);
end
else if TFile.Exists(FileName) then
begin
var capFileName := FileName;
Result :=
TFuture<TArray<TDataPoint>>.Construct(
function: TArray<TDataPoint>
begin
if TFile.Exists(capFileName) then
begin
var stream := TFileStream.Create(capFileName, fmOpenRead or fmShareDenyWrite);
try
try
Result := ReadUncompressedData(stream);
except
on E: EReadError do
begin
raise EReadError.CreateFmt('File %s: %s', [capFileName, E.Message]);
end;
end;
finally
stream.Free;
end;
end;
end
);
end;
end;
class function TDataSeries<T>.LoadDataSeries(const FileName: string): TFuture<TArray<TDataPoint>>;
// Local type declaration for storing file information for phase 1 processing
type
TPhase1FileEntry = record
Path: string;
Year: Integer;
Month: Integer;
end;
var
initialYear, initialMonth: Integer;
currentTabYear, currentTabMonth: Integer;
pathName, symbolName: string;
tabFileEntryRecord: TPhase1FileEntry; // Variable to construct record instances
loadedState: TState;
loadedFiles: TArray<TFuture<TArray<TDataPoint>>>;
liveData: TFuture<TArray<TDataPoint>>;
begin // Start of LoadDataSeries
if not TryParseFileName(FileName, pathName, symbolName, initialYear, initialMonth) then
begin
raise EArgumentException.CreateFmt(
'Invalid initial file name format: %s. Expected Symbol_Year_MM.Extension or Symbol_Year_MM.Extension_zip',
[FileName]);
end;
currentTabYear := initialYear;
currentTabMonth := initialMonth;
var tabBaseNameFormat: string;
var compressedTabPath, uncompressedTabPath: string;
var actualTabFileToLoad: string;
var maxTabYearFound := -1;
var maxTabMonthFound := -1;
var filesToLoadPhase1 := TList<TPhase1FileEntry>.Create; // Create list with the named record type
try
while True do
begin
tabBaseNameFormat := Format('%s_%.4d_%.2d.tab', [symbolName, currentTabYear, currentTabMonth]);
compressedTabPath := TPath.Combine(pathName, tabBaseNameFormat + '_zip');
uncompressedTabPath := TPath.Combine(pathName, tabBaseNameFormat);
actualTabFileToLoad := '';
if TFile.Exists(compressedTabPath) then
begin
actualTabFileToLoad := compressedTabPath;
end
else if TFile.Exists(uncompressedTabPath) then
begin
actualTabFileToLoad := uncompressedTabPath;
end;
if actualTabFileToLoad = '' then
break; // No more sequential .tab files
// Construct the record and add it to the list
tabFileEntryRecord.Path := actualTabFileToLoad;
tabFileEntryRecord.Year := currentTabYear;
tabFileEntryRecord.Month := currentTabMonth;
filesToLoadPhase1.Add(tabFileEntryRecord);
// Advance to the next month
Inc(currentTabMonth);
if currentTabMonth > 12 then
begin
currentTabMonth := 1;
Inc(currentTabYear);
end;
end;
// Now load and process identified .tab files
var loadedFileList := TList<TFuture<TArray<TDataPoint>>>.Create;
var loadStates := TList<TState>.Create;
try
var LoadGate: TLatch;
for var tabFileEntry: TPhase1FileEntry in filesToLoadPhase1 do // Iterate with the correct type
begin
var data := LoadDataFile(LoadGate, tabFileEntry.Path);
loadedFileList.Add(TFuture<TArray<TDataPoint>>.Create(data));
loadStates.Add(data.Done);
if tabFileEntry.Year > maxTabYearFound then
begin
maxTabYearFound := tabFileEntry.Year;
maxTabMonthFound := tabFileEntry.Month;
end
else if (tabFileEntry.Year = maxTabYearFound) and (tabFileEntry.Month > maxTabMonthFound) then
begin
maxTabMonthFound := tabFileEntry.Month;
end;
end;
var liveFileBaseName := Format('%s_%d_%02d.tab-live', [symbolName, maxTabYearFound, maxTabMonthFound]);
liveData := LoadDataFile(LoadGate, TPath.Combine(pathName, liveFileBaseName));
loadStates.Add(liveData.Done);
loadedFileList.Add(liveData);
loadedState := TState.All(loadStates.ToArray);
loadedFiles := loadedFileList.ToArray;
finally
loadStates.Free;
loadedFileList.Free;
end;
finally
filesToLoadPhase1.Free;
end;
Result :=
TFuture<TArray<TDataPoint>>.Construct(
loadedState,
function: TArray<TDataPoint>
begin
var tickList := TList<TDataPoint>.Create;
try
var totalTicks := Length(liveData.Result);
var overallLastTabTickTime := 0.0;
for var P in loadedFiles do
Inc(totalTicks, Length(P.Result));
tickList.Capacity := totalTicks;
for var P in loadedFiles do
begin
if Length(P.Result) > 0 then
begin
for var iterTickData in P.Result do
if iterTickData.OADateTime > overallLastTabTickTime then
begin
tickList.Add(iterTickData);
overallLastTabTickTime := iterTickData.OADateTime;
end;
end;
end;
Result := tickList.ToArray;
finally
tickList.Free;
end;
end
);
end;
class function TDataSeries<T>.ReadCompressedData(const InputStream: TStream): TArray<TDataPoint>;
var
decompressionStream: TStream; // Holds uncompressed data of a single zip entry
localHeader: TZipHeader;
entryIndex: Integer;
i: Integer;
zipFileInstance: TZipFile;
begin
// Initialize Result record
SetLength(Result, 0);
decompressionStream := nil;
zipFileInstance := nil;
try
InputStream.Position := 0; // Reset stream position to the beginning for TZipFile
// Open TZipFile from this memory stream
zipFileInstance := TZipFile.Create;
zipFileInstance.Open(InputStream, TZipMode.zmRead); // Open the zip from the memory stream
if zipFileInstance.FileCount = 0 then
begin
// If TZipFile opens an empty or invalid stream successfully but finds no files
exit; // FileSuccessfullyLoaded remains False
end;
entryIndex := -1;
for i := 0 to zipFileInstance.FileCount - 1 do
begin
if zipFileInstance.FileNames[i].EndsWith('.tab', True) then
begin
entryIndex := i;
break;
end;
end;
if entryIndex = -1 then // If specific name not found, default to the first entry
begin
if zipFileInstance.FileCount > 0 then
entryIndex := 0
else
exit; // Should not happen if FileCount > 0 check passed
end;
// Decompress the specific entry from the in-memory zip file.
// TZipFile.Read creates and populates decompressionStream with uncompressed data.
zipFileInstance.Read(entryIndex, decompressionStream, localHeader);
if not Assigned(decompressionStream) then
exit; // Exit if no valid stream could be prepared
Result := ReadUncompressedData(decompressionStream);
finally
// Ensure all potentially created objects are freed
if Assigned(decompressionStream) then // Holds uncompressed data from a zip entry
decompressionStream.Free;
if Assigned(zipFileInstance) then
zipFileInstance.Free;
end;
end;
class function TDataSeries<T>.ReadUncompressedData(const InputStream: TStream): TArray<TDataPoint>;
var
fileSize: Int64;
recordCount: Integer;
bytesRead: Integer;
begin
// Initialize Result record
SetLength(Result, 0);
InputStream.Position := 0;
fileSize := InputStream.Size;
if fileSize = 0 then
exit;
if (fileSize mod SizeOf(TDataPoint)) <> 0 then
raise EReadError.CreateFmt('File corrupted. Size %d is not a multiple of record size %d.', [fileSize, SizeOf(TDataPoint)]);
recordCount := fileSize div SizeOf(TDataPoint);
if recordCount > 0 then
begin
SetLength(Result, recordCount);
bytesRead := InputStream.Read(Result[0], fileSize);
if bytesRead <> fileSize then
raise EReadError.CreateFmt('Read error. Expected to read %d bytes, but actually read %d bytes.', [fileSize, bytesRead]);
end;
end;
end.