MycLib/Src/Myc.Trade.Core.DataPoint.pas

unit Myc.Trade.Core.DataPoint;

interface

uses
    System.Generics.Collections,
    System.TimeSpan,
    Myc.Trade.DataPoint;

type
    TMycDataArray<T> = record
    private
        const
            ChunkSize = 1024;
        type
            TChunk = TArray<TDataPoint<T>>;
    private
        FChunks: TArray<TChunk>;
        FCount: Int64;

        function LogicalToPhysicalIndex(LogicalIndex: Int64): Int64; inline;
        function GetItems(Idx: Int64): TDataPoint<T>; inline;
    public
        constructor Create(const AChunks: TArray<TChunk>; ACount: Int64);
        function Add(const Data: array of TDataPoint<T>; First, Count, Lookback: Int64): TMycDataArray<T>;
        class function CreateEmpty: TMycDataArray<T>; static;
        // Helper to create a data array from a raw TArray.
        class function CreateFromArray(const AData: TArray<TDataPoint<T>>; First, Count: Integer): TMycDataArray<T>; static;
        property Count: Int64 read FCount;
        property Items[Idx: Int64]: TDataPoint<T> read GetItems; default;
    end;

    // The implementation class for IDataSeries<T>.
    TMycDataSeries<T> = class(TInterfacedObject, IDataSeries<T>)
    private
        FData: TMycDataArray<T>;
        FLookback: Int64;
        FTotalCount: Int64;
        function GetCount: Int64;
        function GetItems(Idx: Int64): TDataPoint<T>;
        function GetLookback: Int64;
        function GetTotalCount: Int64;
    public
        constructor Create(ALookback: Int64; const AData: TMycDataArray<T>; ATotalCount: Int64);
        destructor Destroy; override;
        function Add(const Data: TArray<TDataPoint<T>>; First, Count: Integer): IDataSeries<T>;
        class function CreateDataSeries(Lookback: Int64; const AData: TMycDataArray<T>; ATotalCount: Int64): IDataSeries<T>; static;
    end;

    // Null object implementation for IDataSeries<T>
    TNullDataSeries<T> = class(TInterfacedObject, IDataSeries<T>)
    strict private
        class var
            FNull: IDataSeries<T>;
    private
        FTotalCount: Int64;
        function GetCount: Int64;
        function GetItems(Idx: Int64): TDataPoint<T>;
        function GetTotalCount: Int64;
        function GetLookback: Int64;
        class constructor CreateClass;
    public
        constructor Create(ATotalCount: Int64);
        function Add(const Data: TArray<TDataPoint<T>>; First, Count: Integer): IDataSeries<T>;
        class property Null: IDataSeries<T> read FNull;
    end;

    // A virtual series that combines a base series and an array of new data without copying.
    TCompositeDataSeries<T> = class(TInterfacedObject, IDataSeries<T>)
    private
        FBaseSeries: IDataSeries<T>;
        FAddedData: TMycDataArray<T>;
        FLookback: Int64;
        FCount: Int64;
        function GetCount: Int64;
        function GetItems(Idx: Int64): TDataPoint<T>;
        function GetLookback: Int64;
        function GetTotalCount: Int64;
    public
        constructor Create(const ABaseSeries: IDataSeries<T>; const AAddedData: TMycDataArray<T>);
        function Add(const Data: TArray<TDataPoint<T>>; First, Count: Integer): IDataSeries<T>;
        class function CreateComposite(
            const BaseSeries: IDataSeries<T>;
            const Data: TArray<TDataPoint<T>>;
            First, Count: Integer
        ): IDataSeries<T>; static;
    end;

    TConvertSeries<T, S> = class(TInterfacedObject, IDataSeries<S>)
    private
        FSource: IDataSeries<T>;
        FConvertFunc: TDataSeries<T>.TConvertFunc<S>;
        function GetCount: Int64;
        function GetItems(Idx: Int64): TDataPoint<S>;
        function GetLookback: Int64;
        function GetTotalCount: Int64;
    public
        constructor Create(const ASource: IDataSeries<T>; const AConvertFunc: TDataSeries<T>.TConvertFunc<S>);
        function Add(const Data: TArray<TDataPoint<S>>; First, Count: Integer): IDataSeries<S>;
    end;

    TAggregateDataSeries<T, S> = class(TInterfacedObject, IDataSeries<S>)
    public
        // Defines the function signature for aggregating a set of source data points into a single target value.
        type
            TAggregateFunc = reference to function(const ASourcePoints: TArray<TDataPoint<T>>): S;
    private
        FSource: IDataSeries<T>;
        FTimeFrame: TDateTime;
        FAggregateFunc: TAggregateFunc;
        FCachedItems: TDictionary<Int64, TDataPoint<S>>;
        FBaseTime: TDateTime;
        FAggregatedCount: Int64;
        function GetCount: Int64;
        function GetItems(Idx: Int64): TDataPoint<S>;
        function GetLookback: Int64;
        function GetTotalCount: Int64;
        procedure CalculateAggregatedCount;
    public
        constructor Create(const ASource: IDataSeries<T>; ATimeFrame: TDateTime; const AAggregateFunc: TAggregateFunc);
        destructor Destroy; override;
        function Add(const Data: TArray<TDataPoint<S>>; First, Count: Integer): IDataSeries<S>;
    end;

implementation

uses
    System.SysUtils,
    System.Math;

{ TMycDataArray<T> }

constructor TMycDataArray<T>.Create(const AChunks: TArray<TChunk>; ACount: Int64);
begin
    FChunks := AChunks;
    FCount := ACount;
end;

class function TMycDataArray<T>.CreateEmpty: TMycDataArray<T>;
begin
    Result.FChunks := nil;
    Result.FCount := 0;
end;

class function TMycDataArray<T>.CreateFromArray(const AData: TArray<TDataPoint<T>>; First, Count: Integer): TMycDataArray<T>;
begin
    // Use the Add method on an empty array to perform the chunking logic.
    Result := CreateEmpty.Add(AData, First, Count, Count);
end;

function TMycDataArray<T>.Add(const Data: array of TDataPoint<T>; First, Count, Lookback: Int64): TMycDataArray<T>;
var
    destPhysicalIdx, sourcePhysicalIdx: Int64;
    itemsToSkip: Int64;
    numNewChunks: Integer;
    newChunks: TArray<TChunk>;
    destChunkIdx, destSubIdx: Integer;
    sumCount, newCount: Int64;
begin
    if Count < 0 then
        Count := Length(Data) - First;
    if (Lookback <= 0) or (Count = 0) then
        exit(Self);

    Assert(Count <= (Length(Data) - First), 'Count cannot be larger than the source array');
    for var i := First + 1 to First + Count - 1 do
        Assert(Data[i].Time >= Data[i - 1].Time, 'Input array for Add is not chronologically sorted');
    if FCount > 0 then
        Assert(Data[First].Time >= Self.Items[0].Time, 'First new item is older than last existing item');

    sumCount := FCount + Count;
    newCount := sumCount;
    if (Lookback > 0) and (newCount > Lookback) then
        newCount := Lookback;
    itemsToSkip := sumCount - newCount;

    numNewChunks := 0;
    if newCount > 0 then
        numNewChunks := (newCount - 1) div ChunkSize + 1;
    SetLength(newChunks, numNewChunks);

    for destPhysicalIdx := 0 to newCount - 1 do
    begin
        destChunkIdx := destPhysicalIdx div ChunkSize;
        destSubIdx := destPhysicalIdx mod ChunkSize;

        if destSubIdx = 0 then
            SetLength(newChunks[destChunkIdx], ChunkSize);

        sourcePhysicalIdx := itemsToSkip + destPhysicalIdx;

        if sourcePhysicalIdx < FCount then
        begin
            newChunks[destChunkIdx][destSubIdx] := FChunks[sourcePhysicalIdx div ChunkSize][sourcePhysicalIdx mod ChunkSize];
        end
        else
        begin
            newChunks[destChunkIdx][destSubIdx] := Data[First + (sourcePhysicalIdx - FCount)];
        end;
    end;

    Result := TMycDataArray<T>.Create(newChunks, newCount);
end;

function TMycDataArray<T>.GetItems(Idx: Int64): TDataPoint<T>;
var
    physicalIndex: Int64;
begin
    Assert((Idx >= 0) and (Idx < FCount), 'Logical index is out of bounds.');
    physicalIndex := LogicalToPhysicalIndex(Idx);
    Result := FChunks[physicalIndex div ChunkSize][physicalIndex mod ChunkSize];
end;

function TMycDataArray<T>.LogicalToPhysicalIndex(LogicalIndex: Int64): Int64;
begin
    Result := FCount - LogicalIndex - 1;
end;

{ TMycDataSeries<T> }

constructor TMycDataSeries<T>.Create(ALookback: Int64; const AData: TMycDataArray<T>; ATotalCount: Int64);
begin
    inherited Create;
    FLookback := ALookback;
    FData := AData;
    FTotalCount := ATotalCount;
end;

destructor TMycDataSeries<T>.Destroy;
begin
    inherited;
end;

class function TMycDataSeries<T>.CreateDataSeries(Lookback: Int64; const AData: TMycDataArray<T>; ATotalCount: Int64): IDataSeries<T>;
begin
    if Lookback > 0 then
        Result := TMycDataSeries<T>.Create(Lookback, AData, ATotalCount)
    else
        Result := TNullDataSeries<T>.Null;
end;

function TMycDataSeries<T>.Add(const Data: TArray<TDataPoint<T>>; First, Count: Integer): IDataSeries<T>;
var
    newData: TMycDataArray<T>;
    newTotalCount: Int64;
begin
    if Count < 0 then
        Count := Length(Data) - First;
    newData := FData.Add(Data, First, Count, FLookback);
    newTotalCount := FTotalCount + Count;
    Result := TMycDataSeries<T>.Create(FLookback, newData, newTotalCount);
end;

function TMycDataSeries<T>.GetCount: Int64;
begin
    Result := FData.Count;
end;

function TMycDataSeries<T>.GetItems(Idx: Int64): TDataPoint<T>;
begin
    Assert((Idx >= 0) and (Idx < FData.Count), 'Index is out of bounds.');
    Result := FData.Items[Idx];
end;

function TMycDataSeries<T>.GetLookback: Int64;
begin
    Result := FLookback;
end;

function TMycDataSeries<T>.GetTotalCount: Int64;
begin
    Result := FTotalCount;
end;

{ TNullDataSeries<T> }

class constructor TNullDataSeries<T>.CreateClass;
begin
    FNull := TNullDataSeries<T>.Create(0);
end;

constructor TNullDataSeries<T>.Create(ATotalCount: Int64);
begin
    inherited Create;
    FTotalCount := ATotalCount;
end;

function TNullDataSeries<T>.Add(const Data: TArray<TDataPoint<T>>; First, Count: Integer): IDataSeries<T>;
begin
    if Count < 0 then
        Count := Length(Data) - First;

    if Count > 0 then
        Result := TNullDataSeries<T>.Create(FTotalCount + Count)
    else
        Result := Self;
end;

function TNullDataSeries<T>.GetCount: Int64;
begin
    Result := 0;
end;

function TNullDataSeries<T>.GetItems(Idx: Int64): TDataPoint<T>;
begin
    Assert(false, 'Data series is empty.');
    Result := Default(TDataPoint<T>);
end;

function TNullDataSeries<T>.GetLookback: Int64;
begin
    Result := 0;
end;

function TNullDataSeries<T>.GetTotalCount: Int64;
begin
    Result := FTotalCount;
end;

{ TCompositeDataSeries<T> }

constructor TCompositeDataSeries<T>.Create(const ABaseSeries: IDataSeries<T>; const AAddedData: TMycDataArray<T>);
begin
    inherited Create;
    FBaseSeries := ABaseSeries;
    FAddedData := AAddedData;
    FLookback := FBaseSeries.Lookback;
    Assert(FLookback > 0);

    FCount := FBaseSeries.Count + FAddedData.Count;
    if FCount > FLookback then
        FCount := FLookback;
end;

function TCompositeDataSeries<T>.Add(const Data: TArray<TDataPoint<T>>; First, Count: Integer): IDataSeries<T>;
var
    newAddedData: TMycDataArray<T>;
    itemsInBase: Int64;
    lookbackForAdd: Int64;
begin
    if Count < 0 then
        Count := Length(Data) - First;
    if Count = 0 then
        exit(Self);

    itemsInBase := FBaseSeries.Count;
    lookbackForAdd := FLookback - itemsInBase;
    if lookbackForAdd < 0 then
        lookbackForAdd := 0;

    newAddedData := FAddedData.Add(Data, First, Count, lookbackForAdd);

    // Optimization: If the added data fills the entire lookback window,
    // the base series is no longer relevant. We can return a simpler TMycDataSeries.
    if newAddedData.Count >= FLookback then
    begin
        Result := TMycDataSeries<T>.Create(FLookback, newAddedData, GetTotalCount + Count);
    end
    else
    begin
        Result := TCompositeDataSeries<T>.Create(FBaseSeries, newAddedData);
    end;
end;

class function TCompositeDataSeries<T>.CreateComposite(
    const BaseSeries: IDataSeries<T>;
    const Data: TArray<TDataPoint<T>>;
    First, Count: Integer
): IDataSeries<T>;
begin
    if Count < 0 then
        Count := Length(Data) - First;
    if Count = 0 then
        exit(BaseSeries);

    Result := TCompositeDataSeries<T>.Create(BaseSeries, TMycDataArray<T>.CreateFromArray(Data, First, Count));
end;

function TCompositeDataSeries<T>.GetCount: Int64;
begin
    Result := FCount;
end;

function TCompositeDataSeries<T>.GetItems(Idx: Int64): TDataPoint<T>;
var
    addedCount: Int64;
begin
    Assert((Idx >= 0) and (Idx < FCount), 'Logical index is out of bounds.');
    addedCount := FAddedData.Count;
    if Idx < addedCount then
    begin
        Result := FAddedData.Items[Idx];
    end
    else
    begin
        Result := FBaseSeries.Items[Idx - addedCount];
    end;
end;

function TCompositeDataSeries<T>.GetLookback: Int64;
begin
    Result := FLookback;
end;

function TCompositeDataSeries<T>.GetTotalCount: Int64;
begin
    Result := FBaseSeries.TotalCount + FAddedData.Count;
end;

{ TConvertSeries<T, S> }

constructor TConvertSeries<T, S>.Create(const ASource: IDataSeries<T>; const AConvertFunc: TDataSeries<T>.TConvertFunc<S>);
begin
    inherited Create;
    FSource := ASource;
    FConvertFunc := AConvertFunc;
end;

function TConvertSeries<T, S>.Add(const Data: TArray<TDataPoint<S>>; First, Count: Integer): IDataSeries<S>;
begin
    Result := TCompositeDataSeries<S>.CreateComposite(Self, Data, First, Count);
end;

function TConvertSeries<T, S>.GetCount: Int64;
begin
    Result := FSource.Count;
end;

function TConvertSeries<T, S>.GetItems(Idx: Int64): TDataPoint<S>;
var
    P: TDataPoint<T>;
begin
    P := FSource[Idx];
    Result.Create(P.Time, FConvertFunc(P));
end;

function TConvertSeries<T, S>.GetLookback: Int64;
begin
    Result := FSource.Lookback;
end;

function TConvertSeries<T, S>.GetTotalCount: Int64;
begin
    Result := FSource.TotalCount;
end;

{ TAggregateDataSeries<T, S> }

constructor TAggregateDataSeries<T, S>.Create(const ASource: IDataSeries<T>; ATimeFrame: TDateTime; const AAggregateFunc: TAggregateFunc);
begin
    inherited Create;
    FSource := ASource;
    FTimeFrame := ATimeFrame;
    FAggregateFunc := AAggregateFunc;
    FCachedItems := TDictionary<Int64, TDataPoint<S>>.Create;
    FAggregatedCount := -1; // -1 indicates that it has not been calculated yet
end;

destructor TAggregateDataSeries<T, S>.Destroy;
begin
    FCachedItems.Free;
    inherited;
end;

procedure TAggregateDataSeries<T, S>.CalculateAggregatedCount;
var
    totalTimeSpan: Double;
begin
    if FSource.Count = 0 then
    begin
        FBaseTime := 0;
        FAggregatedCount := 0;
    end
    else
    begin
        // The timestamp of the oldest element serves as the anchor for our time grid.
        FBaseTime := FSource.Items[FSource.Count - 1].Time;
        // Total duration covered by the source series.
        totalTimeSpan := FSource.Items[0].Time - FBaseTime;
        if totalTimeSpan >= 0 then
            FAggregatedCount := Trunc(totalTimeSpan / FTimeFrame) + 1
        else
            FAggregatedCount := 0;
    end;
end;

function TAggregateDataSeries<T, S>.Add(const Data: TArray<TDataPoint<S>>; First, Count: Integer): IDataSeries<S>;
begin
    // Adding to an aggregated series is complex.
    // The most straightforward approach is to create a composite series.
    Result := TCompositeDataSeries<S>.CreateComposite(Self, Data, First, Count);
end;

function TAggregateDataSeries<T, S>.GetCount: Int64;
begin
    if FAggregatedCount = -1 then
        CalculateAggregatedCount;
    Result := FAggregatedCount;
end;

function TAggregateDataSeries<T, S>.GetItems(Idx: Int64): TDataPoint<S>;
var
    startTime, endTime: TDateTime;
    sourcePoints: TList<TDataPoint<T>>;
    sourceIdx: Int64;
begin
    Assert((Idx >= 0) and (Idx < GetCount), 'Index is out of bounds.');

    if FCachedItems.TryGetValue(Idx, Result) then
        exit;

    // Calculate the time window for the requested aggregated data point.
    // Index 0 is the newest, so we calculate backwards from the total count.
    endTime := FBaseTime + (FAggregatedCount - Idx) * FTimeFrame;
    startTime := endTime - FTimeFrame;

    sourcePoints := TList<TDataPoint<T>>.Create;
    try
        // Find all source data points that fall into this time window.
        // We can optimize the start of the search using the IndexOf function.
        sourceIdx := TDataSeries<T>(FSource).IndexOf(endTime);
        if sourceIdx = -1 then
            sourceIdx := 0; // If endTime is after the last element, start at the newest.

        while (sourceIdx < FSource.Count) do
        begin
            var P := FSource.Items[sourceIdx];
            if P.Time < startTime then
                break; // We have moved past our time window

            if P.Time < endTime then // Time is within [startTime, endTime)
            begin
                sourcePoints.Add(P);
            end;
            Inc(sourceIdx);
        end;

        // The aggregation function expects the data in chronological order (oldest first),
        // but we collected it in reverse. So we reverse the list.
        sourcePoints.Reverse;
        Result.Create(endTime, FAggregateFunc(sourcePoints.ToArray));
    finally
        sourcePoints.Free;
    end;

    // Cache the result for future calls
    FCachedItems.Add(Idx, Result);
end;

function TAggregateDataSeries<T, S>.GetLookback: Int64;
begin
    Result := FSource.Lookback; // The lookback is defined by the source series.
end;

function TAggregateDataSeries<T, S>.GetTotalCount: Int64;
begin
    // The total count of aggregated items is simply its current count, as it's a view.
    Result := GetCount;
end;

end.