Brummel/MycLib
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

DataServer 1st Version

Browse Source
This commit is contained in:
Michael Schimmel
2025-06-06 12:43:51 +02:00
parent f70cfe0ec1
commit 98de7176c3
9 changed files with 271 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Src/Myc.Trade.DataSeries.pas
+7 -45
View File
@@ -1,39 +1,5 @@
unit Myc.Trade.DataSeries;
(* ------------------------------------------------------------------------------
This unit provides functions to load tick data (TTickData) from
binary files. The data originates from a C# application
and has the structure OADateTime (Double), Ask (Single), and Bid (Single).
The files follow the naming convention:
For .tab files: Symbol_Year_MM.tab OR Symbol_Year_MM.tab_zip
For .tab-live files: Symbol_Year_MM.tab-live (NEVER compressed)
(MM represents the two-digit month, e.g., 01 for January, 12 for December)
If both compressed (.tab_zip) and uncompressed (.tab) versions of a .tab
file exist, the compressed version is preferred. Files ending with _zip
are expected to be ZIP archives containing the actual data file
(e.g., Symbol_Year_MM.tab).
Loading occurs in two phases:
1. All .tab files (regular or _zip) are processed chronologically by
month and year. Unique ticks are loaded, and the timestamp of the
latest tick across all .tab files is determined (overallLastTabTickTime).
Assumes timestamps are unique across all .tab files.
2. All .tab-live files (which are never compressed), for months/years
corresponding to found .tab files, are processed. Ticks from a
.tab-live file are only integrated if their timestamp is strictly later
than overallLastTabTickTime. Assumes such ticks are globally unique.
If a .tab-live file results in no new data being added under these rules,
it is deleted after processing.
The loading function automatically detects subsequent monthly/yearly files.
Main Function:
LoadTickDataSeries - Loads a series of tick data files.
------------------------------------------------------------------------------ *)
interface
uses
@@ -293,9 +259,9 @@ begin
for var i := 0 to High(fL) do
begin
FCachedFiles.Remove(fL[i].Name);
if not IsMemoryLow then
break;
FCachedFiles.Remove(fL[i].Name);
end;
end;
@@ -319,10 +285,6 @@ begin
if FileName.EndsWith('_zip', True) then
begin
var zipFileBytes := TFile.ReadAllBytes(FileName); // Read all bytes of the .zip file
if Length(zipFileBytes) = 0 then
exit;
var capFileName := FileName;
Result :=
TFuture<TBytes>
@@ -335,7 +297,7 @@ begin
.Chain<TArray<TDataPoint>>(
function(bytes: TBytes): TArray<TDataPoint>
begin
var zipMemoryStream := TBytesStream.Create(zipFileBytes); // Create a memory stream to hold the zip file content
var zipMemoryStream := TBytesStream.Create(bytes); // Create a memory stream to hold the zip file content
try
zipMemoryStream.Position := 0;
Result := ReadCompressedData(zipMemoryStream);
@@ -349,6 +311,7 @@ begin
var capFileName := FileName;
Result :=
TFuture<TArray<TDataPoint>>.Construct(
TLatch.Enqueue(LoadGate),
function: TArray<TDataPoint>
begin
if TFile.Exists(capFileName) then
@@ -375,7 +338,6 @@ begin
begin
var cachedFile: TCachedFile;
cachedFile.Name := Filename;
var age: TDateTime;
if FileAge(Filename, cachedFile.Age, true) then
begin
cachedFile.Data := Result;
@@ -418,7 +380,7 @@ begin // Start of LoadDataSeries
var maxTabYearFound := -1;
var maxTabMonthFound := -1;
var filesToLoadPhase1 := TList<TPhase1FileEntry>.Create; // Create list with the named record type
var filesToLoad := TList<TPhase1FileEntry>.Create; // Create list with the named record type
try
while True do
begin
@@ -443,7 +405,7 @@ begin // Start of LoadDataSeries
tabFileEntryRecord.Path := actualTabFileToLoad;
tabFileEntryRecord.Year := currentTabYear;
tabFileEntryRecord.Month := currentTabMonth;
filesToLoadPhase1.Add(tabFileEntryRecord);
filesToLoad.Add(tabFileEntryRecord);
// Advance to the next month
Inc(currentTabMonth);
@@ -460,7 +422,7 @@ begin // Start of LoadDataSeries
try
var LoadGate: TLatch;
for var tabFileEntry: TPhase1FileEntry in filesToLoadPhase1 do // Iterate with the correct type
for var tabFileEntry: TPhase1FileEntry in filesToLoad do // Iterate with the correct type
begin
var data := LoadDataFile(LoadGate, tabFileEntry.Path);
@@ -490,7 +452,7 @@ begin // Start of LoadDataSeries
loadedFileList.Free;
end;
finally
filesToLoadPhase1.Free;
filesToLoad.Free;
end;
Result :=