Brummel/MycLib
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

TSeries optimized + Unit tests

Browse Source
This commit is contained in:
Michael Schimmel
2025-07-23 14:46:49 +02:00
parent d1d3393392
commit b623be13fa
7 changed files with 560 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Src/Myc.Trade.DataArray.pas
+337 -68
View File
@@ -3,10 +3,7 @@ unit Myc.Trade.DataArray;
interface
type
// A series is an array of values with the newest ite at index=0. Each series counts the total of added items since creation, but
// it actually may contain less items, because the array size is limited by the lookback parameter, when adding items.
// Series are immutable.
TSeries<T> = record
TChunkArray<T> = record
private
const
ChunkSize = 1024;
@@ -14,118 +11,390 @@ type
TChunk = TArray<T>;
private
FChunks: TArray<TChunk>;
FCount: Int64;
FTotalCount: Int64;
FOffset: Integer;
FCount: Integer;
function GetItems(Idx: Integer): T; inline;
function LogicalToPhysicalIndex(LogicalIndex: Int64): Int64; inline;
function GetItems(Idx: Int64): T; inline;
public
constructor Create(const AChunks: TArray<TChunk>; ACount, ATotalCount: Int64);
constructor Create(const AChunks: TArray<TChunk>; ACount: Integer; AOffset: Integer);
class operator Initialize(out Dest: TChunkArray<T>);
// Add a single item without creating a temporary array.
procedure Add(const Data: T; MaxCount: Integer); overload;
// Add items, but ensure that the result has MaxCount items.
procedure Add(const Data: array of T; MaxCount: Integer); overload;
class function CreateFromArray(const AData: TArray<T>; MaxCount: Integer): TChunkArray<T>; static;
// Creates a deep copy of the array, optionally truncating it to the last MaxCount items.
function Copy(MaxCount: Integer = -1): TChunkArray<T>;
property Count: Integer read FCount;
property Items[Idx: Integer]: T read GetItems; default;
end;
// A series where the last added item has index 0.
TSeries<T> = record
private
FArray: TChunkArray<T>;
FTotalCount: Int64;
function GetCount: Integer;
function GetItems(Idx: Integer): T;
public
constructor Create(const AArray: TChunkArray<T>; ATotalCount: Int64);
class operator Initialize(out Dest: TSeries<T>);
// Add a singe item
function Add(const Data: T; Lookback: Int64 = -1): TSeries<T>; overload;
// Add a ranmge of items
function Add(const Data: array of T; First, Count, Lookback: Int64): TSeries<T>; overload;
// Helper to create a data array from a raw TArray.
class function CreateFromArray(const AData: TArray<T>; First, Count: Integer): TSeries<T>; static;
property Count: Int64 read FCount;
// Creates a deep copy of the series, optionally truncating it to the last Lookback items.
function Copy(Lookback: Integer = -1): TSeries<T>;
procedure Add(const Data: T; Lookback: Int64 = -1); overload;
procedure Add(const Data: array of T; Lookback: Integer); overload;
class function CreateFromArray(const AData: TArray<T>; Lookback: Integer): TSeries<T>; static;
property Count: Integer read GetCount;
property TotalCount: Int64 read FTotalCount;
property Items[Idx: Int64]: T read GetItems; default;
property Items[Idx: Integer]: T read GetItems; default;
end;
implementation
uses
System.Generics.Collections,
System.Math;
{ TSeries<T> }
{ TChunkArray<T> }
constructor TSeries<T>.Create(const AChunks: TArray<TChunk>; ACount, ATotalCount: Int64);
constructor TChunkArray<T>.Create(const AChunks: TArray<TChunk>; ACount, AOffset: Integer);
begin
FChunks := AChunks;
FCount := ACount;
FTotalCount := ATotalCount;
FOffset := AOffset;
Assert(FOffset < ChunkSize);
end;
class function TSeries<T>.CreateFromArray(const AData: TArray<T>; First, Count: Integer): TSeries<T>;
begin
// Use the Add method on an empty array to perform the chunking logic.
Result.Add(AData, First, Count, Count);
end;
function TSeries<T>.Add(const Data: array of T; First, Count, Lookback: Int64): TSeries<T>;
procedure TChunkArray<T>.Add(const Data: T; MaxCount: Integer);
var
destPhysicalIdx, sourcePhysicalIdx: Int64;
itemsToSkip: Int64;
numNewChunks: Integer;
newChunks: TArray<TChunk>;
destChunkIdx, destSubIdx: Integer;
sumCount, newCount: Int64;
newCount, totalNewOffset: Integer;
oldLogicalEnd, newLogicalEnd, oldStartChunk, newStartChunk, oldEndChunk, newEndChunk, chunksToRemove, chunksToAdd: Integer;
begin
if Count < 0 then
Count := Length(Data) - First;
if Count = 0 then
exit(Self);
if MaxCount < 0 then
MaxCount := MaxInt;
Assert(Count <= (Length(Data) - First), 'Count cannot be larger than the source array');
if MaxCount <= 0 then
exit;
sumCount := FCount + Count;
newCount := sumCount;
if (Lookback >= 0) and (newCount > Lookback) then
newCount := Lookback;
itemsToSkip := sumCount - newCount;
// 1. Calculate the new logical state (numToAdd is always 1)
newCount := Min(FCount + 1, MaxCount);
totalNewOffset := FOffset + FCount + 1 - newCount;
numNewChunks := 0;
// 2. Determine if the underlying FChunks array needs restructuring
oldLogicalEnd := FOffset + FCount - 1;
newLogicalEnd := totalNewOffset + newCount - 1;
oldStartChunk := FOffset div ChunkSize;
newStartChunk := totalNewOffset div ChunkSize;
oldEndChunk := -1;
if FCount > 0 then
oldEndChunk := oldLogicalEnd div ChunkSize;
newEndChunk := -1;
if newCount > 0 then
numNewChunks := (newCount - 1) div ChunkSize + 1;
SetLength(newChunks, numNewChunks);
newEndChunk := newLogicalEnd div ChunkSize;
for destPhysicalIdx := 0 to newCount - 1 do
chunksToRemove := newStartChunk - oldStartChunk;
chunksToAdd := newEndChunk - oldEndChunk;
// 3. Perform the add operation
if (chunksToRemove > 0) or (chunksToAdd > 0) then
begin
destChunkIdx := destPhysicalIdx div ChunkSize;
destSubIdx := destPhysicalIdx mod ChunkSize;
// --- Rebuild Path: Chunks must be added or removed ---
var newChunks: TArray<TChunk>;
var oldChunkCount := Length(FChunks);
var newChunkCount := oldChunkCount - chunksToRemove + chunksToAdd;
SetLength(newChunks, newChunkCount);
if destSubIdx = 0 then
SetLength(newChunks[destChunkIdx], ChunkSize);
var numChunksToKeep := oldChunkCount - chunksToRemove;
if numChunksToKeep > 0 then
TArray.Copy<TChunk>(FChunks, newChunks, chunksToRemove, 0, numChunksToKeep);
sourcePhysicalIdx := itemsToSkip + destPhysicalIdx;
for var i := numChunksToKeep to newChunkCount - 1 do
SetLength(newChunks[i], ChunkSize);
if sourcePhysicalIdx < FCount then
FChunks := newChunks;
// Copy the new data item into the rebuilt chunks
var writeStartLogicalIdx := FOffset + FCount;
var physicalWriteStartIdx := writeStartLogicalIdx - (chunksToRemove * ChunkSize);
var chunkIdx := physicalWriteStartIdx div ChunkSize;
var idxInChunk := physicalWriteStartIdx mod ChunkSize;
Assert(chunkIdx < Length(FChunks), 'Chunk index out of bounds during rebuild');
FChunks[chunkIdx][idxInChunk] := Data;
end
else
begin
// --- Fast Path: Window slides within existing chunk allocation ---
var writeStartLogicalIdx := FOffset + FCount;
var chunkIdx := writeStartLogicalIdx div ChunkSize;
var idxInChunk := writeStartLogicalIdx mod ChunkSize;
Assert(chunkIdx < Length(FChunks), 'Chunk index out of bounds on fast path');
if Length(FChunks[chunkIdx]) = 0 then // Safeguard
SetLength(FChunks[chunkIdx], ChunkSize);
FChunks[chunkIdx][idxInChunk] := Data;
end;
// 4. Finalize the new state
FOffset := totalNewOffset mod ChunkSize;
FCount := newCount;
end;
class function TChunkArray<T>.CreateFromArray(const AData: TArray<T>; MaxCount: Integer): TChunkArray<T>;
begin
Result.Add(AData, MaxCount);
end;
procedure TChunkArray<T>.Add(const Data: array of T; MaxCount: Integer);
var
dataLen, dataReadOffset, numToAdd, newCount, totalNewOffset: Integer;
oldLogicalEnd, newLogicalEnd, oldStartChunk, newStartChunk, oldEndChunk, newEndChunk, chunksToRemove, chunksToAdd: Integer;
begin
if MaxCount < 0 then
MaxCount := MaxInt;
dataLen := Length(Data);
if (dataLen = 0) or (MaxCount <= 0) then
exit;
// 1. Trim input data if it's larger than MaxCount
dataReadOffset := 0;
if dataLen > MaxCount then
dataReadOffset := dataLen - MaxCount;
numToAdd := dataLen - dataReadOffset;
// 2. Calculate the new logical state (total offset and count)
newCount := Min(FCount + numToAdd, MaxCount);
// Total logical offset of the new window's start
totalNewOffset := FOffset + FCount + numToAdd - newCount;
// 3. Determine if the underlying FChunks array needs restructuring
oldLogicalEnd := FOffset + FCount - 1;
newLogicalEnd := totalNewOffset + newCount - 1;
oldStartChunk := FOffset div ChunkSize;
newStartChunk := totalNewOffset div ChunkSize;
oldEndChunk := -1;
if FCount > 0 then
oldEndChunk := oldLogicalEnd div ChunkSize;
newEndChunk := -1;
if newCount > 0 then
newEndChunk := newLogicalEnd div ChunkSize;
chunksToRemove := newStartChunk - oldStartChunk;
chunksToAdd := newEndChunk - oldEndChunk;
// 4. Perform the add operation
if (chunksToRemove > 0) or (chunksToAdd > 0) then
begin
// --- Rebuild Path: Chunks must be added or removed ---
var newChunks: TArray<TChunk>;
var oldChunkCount := Length(FChunks);
var newChunkCount := oldChunkCount - chunksToRemove + chunksToAdd;
SetLength(newChunks, newChunkCount);
// Copy references to the chunks that are kept
var numChunksToKeep := oldChunkCount - chunksToRemove;
if numChunksToKeep > 0 then
TArray.Copy<TChunk>(FChunks, newChunks, chunksToRemove, 0, numChunksToKeep);
// Allocate new chunks at the end
for var i := numChunksToKeep to newChunkCount - 1 do
SetLength(newChunks[i], ChunkSize);
FChunks := newChunks;
// Copy the new data into the rebuilt chunks
var writeStartLogicalIdx := FOffset + FCount;
var physicalWriteStartIdx := writeStartLogicalIdx - (chunksToRemove * ChunkSize);
var remainingToAdd := numToAdd;
var currentDataOffset := dataReadOffset;
while remainingToAdd > 0 do
begin
newChunks[destChunkIdx][destSubIdx] := FChunks[sourcePhysicalIdx div ChunkSize][sourcePhysicalIdx mod ChunkSize];
end
else
var chunkIdx := physicalWriteStartIdx div ChunkSize;
var idxInChunk := physicalWriteStartIdx mod ChunkSize;
var spaceInChunk := ChunkSize - idxInChunk;
var countToCopy := Min(remainingToAdd, spaceInChunk);
Assert(chunkIdx < Length(FChunks), 'Chunk index out of bounds during rebuild');
TArray.Copy<T>(Data, FChunks[chunkIdx], currentDataOffset, idxInChunk, countToCopy);
inc(physicalWriteStartIdx, countToCopy);
inc(currentDataOffset, countToCopy);
dec(remainingToAdd, countToCopy);
end;
end
else
begin
// --- Fast Path: Window slides within existing chunk allocation ---
var writeStartLogicalIdx := FOffset + FCount;
var remainingToAdd := numToAdd;
var currentDataOffset := dataReadOffset;
while remainingToAdd > 0 do
begin
newChunks[destChunkIdx][destSubIdx] := Data[First + (sourcePhysicalIdx - FCount)];
var chunkIdx := writeStartLogicalIdx div ChunkSize;
var idxInChunk := writeStartLogicalIdx mod ChunkSize;
var spaceInChunk := ChunkSize - idxInChunk;
var countToCopy := Min(remainingToAdd, spaceInChunk);
Assert(chunkIdx < Length(FChunks), 'Chunk index out of bounds on fast path');
if Length(FChunks[chunkIdx]) = 0 then // Should not happen on fast path, but as a safeguard
SetLength(FChunks[chunkIdx], ChunkSize);
TArray.Copy<T>(Data, FChunks[chunkIdx], currentDataOffset, idxInChunk, countToCopy);
inc(writeStartLogicalIdx, countToCopy);
inc(currentDataOffset, countToCopy);
dec(remainingToAdd, countToCopy);
end;
end;
Result := TSeries<T>.Create(newChunks, newCount, FTotalCount + Count);
// 5. Finalize the new state
FOffset := totalNewOffset mod ChunkSize;
FCount := newCount;
end;
function TSeries<T>.Add(const Data: T; Lookback: Int64): TSeries<T>;
function TChunkArray<T>.GetItems(Idx: Integer): T;
begin
Result := Add([Data], 0, 1, Lookback);
// Convert logical index to physical index inside chunks
var physicalIdx := Idx + FOffset;
Result := FChunks[physicalIdx div ChunkSize][physicalIdx mod ChunkSize];
end;
function TSeries<T>.GetItems(Idx: Int64): T;
function TChunkArray<T>.Copy(MaxCount: Integer): TChunkArray<T>;
var
physicalIndex: Int64;
effectiveCount: Integer;
logicalStartIndex: Integer;
physicalStartIndex, physicalEndIndex: Integer;
startChunkIdx, endChunkIdx: Integer;
newOffset: Integer;
newChunkCount: Integer;
i: Integer;
begin
Assert((Idx >= 0) and (Idx < FCount), 'Logical index is out of bounds.');
physicalIndex := LogicalToPhysicalIndex(Idx);
Result := FChunks[physicalIndex div ChunkSize][physicalIndex mod ChunkSize];
// 1. Determine the number of items to copy.
effectiveCount := FCount;
if (MaxCount >= 0) and (MaxCount < FCount) then
effectiveCount := MaxCount;
if effectiveCount = 0 then
begin
// Return an empty array.
Result := Default(TChunkArray<T>);
exit;
end;
// 2. Calculate physical location of the data to be copied.
logicalStartIndex := FCount - effectiveCount;
physicalStartIndex := FOffset + logicalStartIndex;
physicalEndIndex := physicalStartIndex + effectiveCount - 1;
startChunkIdx := physicalStartIndex div ChunkSize;
endChunkIdx := physicalEndIndex div ChunkSize;
newOffset := physicalStartIndex mod ChunkSize;
// 3. Create the new chunk array for the result.
newChunkCount := endChunkIdx - startChunkIdx + 1;
SetLength(Result.FChunks, newChunkCount);
// 4. Copy chunks. Boundary chunks are deep-copied, internal chunks are shared.
if newChunkCount = 1 then
begin
// All data is in a single chunk, which is both a start and end boundary.
// It must always be a deep copy.
Result.FChunks[0] := System.Copy(FChunks[startChunkIdx]);
end
else
begin
// First chunk (start boundary) must be a deep copy.
Result.FChunks[0] := System.Copy(FChunks[startChunkIdx]);
// Middle, immutable chunks can be shared by reference.
// This loop only runs if there are 3 or more chunks in the copy.
for i := 1 to newChunkCount - 2 do
begin
Result.FChunks[i] := FChunks[startChunkIdx + i];
end;
// Last chunk (end boundary) must be a deep copy.
Result.FChunks[newChunkCount - 1] := System.Copy(FChunks[endChunkIdx]);
end;
// 5. Finalize the new TChunkArray state.
Result.FOffset := newOffset;
Result.FCount := effectiveCount;
end;
function TSeries<T>.LogicalToPhysicalIndex(LogicalIndex: Int64): Int64;
class operator TChunkArray<T>.Initialize(out Dest: TChunkArray<T>);
begin
Result := FCount - LogicalIndex - 1;
Dest.FOffset := 0;
Dest.FCount := 0;
end;
{ TSeries<T> }
procedure TSeries<T>.Add(const Data: T; Lookback: Int64);
begin
FArray.Add(Data, Lookback);
inc(FTotalCount);
end;
procedure TSeries<T>.Add(const Data: array of T; Lookback: Integer);
begin
FArray.Add(Data, Lookback);
inc(FTotalCount, Length(Data));
end;
constructor TSeries<T>.Create(const AArray: TChunkArray<T>; ATotalCount: Int64);
begin
FArray := AArray;
FTotalCount := ATotalCount;
end;
function TSeries<T>.Copy(Lookback: Integer): TSeries<T>;
begin
// Create a copy of the underlying chunk array, optionally truncating it.
Result.FArray := FArray.Copy(Lookback);
// The new series' total count is the number of items it actually contains.
Result.FTotalCount := Result.FArray.Count;
end;
class function TSeries<T>.CreateFromArray(const AData: TArray<T>; Lookback: Integer): TSeries<T>;
begin
Result.FArray.Add(AData, Lookback);
Result.FTotalCount := Length(AData);
end;
function TSeries<T>.GetCount: Integer;
begin
Result := FArray.Count;
end;
function TSeries<T>.GetItems(Idx: Integer): T;
begin
// Access is reversed: series index 0 is the last element in the underlying array
Assert((Idx >= 0) and (Idx < FArray.Count), 'Index out of bounds');
Result := FArray.Items[FArray.Count - 1 - Idx];
end;
class operator TSeries<T>.Initialize(out Dest: TSeries<T>);
begin
Dest.FCount := 0;
Dest.FTotalCount := 0;
end;
Src/Myc.Trade.DataPoint.Impl.pas
+1 -1
View File
@@ -554,7 +554,7 @@ begin
end;
Assert(FCount = 0);
Value := Value.Add(Arr, 0, Length(Arr), FLookback);
Value.Add(Arr, FLookback);
end;
end;
finally
Src/Myc.Trade.Indicators.pas
+8 -8
View File
@@ -216,7 +216,7 @@ begin
var
stdDev: Double;
begin
sourceData := sourceData.Add(Value, Period);
sourceData.Add(Value, Period);
Result.MiddleBand := Double.NaN;
Result.UpperBand := Double.NaN;
Result.LowerBand := Double.NaN;
@@ -240,7 +240,7 @@ begin
Result :=
function(const Value: Double): Double
begin
sourceData := sourceData.Add(Value, Period);
sourceData.Add(Value, Period);
if (sourceData.Count < Period) then
begin
@@ -281,7 +281,7 @@ begin
Result := Double.NaN;
// Add new price to the source data array, respecting the lookback period.
sourceData := sourceData.Add(price, Period);
sourceData.Add(price, Period);
// Check if there is enough data to start the first stage of calculation.
if (sourceData.Count >= Period) then
@@ -292,7 +292,7 @@ begin
// Calculate the difference and add to the intermediate series.
diff := 2 * wmaHalf - wmaFull;
diffSeries := diffSeries.Add(diff, periodSqrt);
diffSeries.Add(diff, periodSqrt);
// Check if there is enough intermediate data for the final calculation.
if (diffSeries.Count >= periodSqrt) then
@@ -352,7 +352,7 @@ begin
gainSum, lossSum: Double;
i: Integer;
begin
sourceData := sourceData.Add(Value, Period + 1);
sourceData.Add(Value, Period + 1);
Result := Double.NaN;
if (sourceData.Count <= Period) then
@@ -404,7 +404,7 @@ begin
Result :=
function(const Value: Double): Double
begin
sourceData := sourceData.Add(Value, Period);
sourceData.Add(Value, Period);
if (sourceData.Count >= Period) then
Result := CalculateSMA(sourceData, Period)
else
@@ -432,7 +432,7 @@ begin
i: Integer;
highestHigh, lowestLow: Double;
begin
sourceData := sourceData.Add(Value, KPeriod);
sourceData.Add(Value, KPeriod);
Result.K := Double.NaN;
Result.D := Double.NaN;
@@ -477,7 +477,7 @@ begin
tr: Double;
begin
// We only need the previous bar to calculate true range.
sourceData := sourceData.Add(Value, 2);
sourceData.Add(Value, 2);
if (sourceData.Count < 2) then
begin