Concurrent data processing

Src/Myc.Core.Notifier.pas

@@ -37,6 +37,9 @@ type
         // Frees memory previously allocated for a TItem.
         class procedure FreeItem(Item: PItem); static; inline;
 
+    private
+        function GetFirst: PItem;
+
     public
         // Initializes the list in an unlocked state.
         class operator Initialize(out Dest: TMycNotifyList<T>);
@@ -58,6 +61,7 @@ type
         // If the predicate returns false, the receiver is detached from further notifications
         // by setting its interface reference to nil. The list item itself is not freed here.
         procedure Notify(const Func: TNotifyProc);
+        property First: PItem read GetFirst;
     end;
 
 implementation
@@ -112,6 +116,12 @@ begin
     FreeMem(Item, sizeof(TItem));
 end;
 
+function TMycNotifyList<T>.GetFirst: PItem;
+begin
+    Assert(IsLocked);
+    Result := FList;
+end;
+
 function TMycNotifyList<T>.IsLocked: Boolean;
 begin
     Result := NativeUInt(FList) and 1 = 0;

Src/Myc.Fmx.Chart.Series.pas

@@ -18,7 +18,7 @@ type
     private
         FCount: Int64;
     protected
-        function ProcessData(const Value: T): Boolean; override;
+        function ProcessData(const Value: T): TState; override;
     public
         constructor Create;
     end;
@@ -30,7 +30,7 @@ type
     private
         FData: TWriteable<TMycDataArray<T>>;
     protected
-        function ProcessData(const Value: T): Boolean; override;
+        function ProcessData(const Value: T): TState; override;
     public
         constructor Create(const ALookback: TMutable<Int64>);
         property Data: TWriteable<TMycDataArray<T>> read FData;
@@ -116,10 +116,9 @@ begin
     FCount := 0;
 end;
 
-function TChartSeriesCounter<T>.ProcessData(const Value: T): Boolean;
+function TChartSeriesCounter<T>.ProcessData(const Value: T): TState;
 begin
-    Result := true;
-    Broadcast(FCount);
+    Result := Broadcast(FCount);
     inc(FCount);
 end;
 
@@ -133,9 +132,9 @@ begin
     FData := TWriteable<TMycDataArray<T>>.CreateWriteable(FCurrData).Protect;
 end;
 
-function TChartSeriesReceiver<T>.ProcessData(const Value: T): Boolean;
+function TChartSeriesReceiver<T>.ProcessData(const Value: T): TState;
 begin
-    Result := true;
+    Result := TState.Null;
     FCurrData := FCurrData.Add(Value, FLookback.Value);
     FData.Value := FCurrData;
 end;

Src/Myc.Fmx.Chart.pas

@@ -210,7 +210,7 @@ begin
     end;
 
     // finally, check the repaint signal
-    var repaintSignal := FNeedRepaint.Reset;
+    var repaintSignal := false; // FNeedRepaint.Reset;
 
     if repaintSignal or (seriesRepaint and seriesChanged) then
         Repaint;

Src/Myc.Signals.pas

@@ -79,7 +79,7 @@ type
         class operator Implicit(const A: TState): IState; overload;
 
         class function All(const States: TArray<TState>): TState; static;
-        class function Any(const States: TArray<TState>): TState; static;
+        class function Any(const States: TArray<TState>; Count: Integer = 1): TState; static;
 
         class property Null: IState read FNull;
 
@@ -138,6 +138,7 @@ type
     strict private
         class var
             FNull: ILatch;
+            [volatile]
             FQueueLock: Integer;
         class constructor ClassCreate;
 
@@ -243,32 +244,23 @@ begin
 end;
 
 class function TState.All(const States: TArray<TState>): TState;
+begin
+    Result := Any(States, Length(States));
+end;
+
+class function TState.Any(const States: TArray<TState>; Count: Integer = 1): TState;
 var
     Latch: TLatch.ILatch;
 begin
-    Latch := TLatch.CreateLatch(Length(States));
+    if (States = nil) or (Count <= 0) then
+        exit(TState.Null);
+
+    Latch := TLatch.CreateLatch(Count);
     for var state in States do
         state.Signal.Subscribe(Latch);
     Result := Latch.State;
 end;
 
-class function TState.Any(const States: TArray<TState>): TState;
-var
-    Latch: TLatch.ILatch;
-begin
-    if Length(States) = 0 then
-    begin
-        Result := TState.Null;
-    end
-    else
-    begin
-        Latch := TLatch.CreateLatch(1);
-        for var state in States do
-            state.Signal.Subscribe(Latch);
-        Result := Latch.State;
-    end;
-end;
-
 function TState.GetSignal: TSignal;
 begin
     Result := FState.Signal;

Src/Myc.TaskManager.pas

@@ -4,7 +4,8 @@ interface
 
 uses
     System.SysUtils,
-    Myc.Signals;
+    Myc.Signals,
+    System.Contnrs;
 
 type
     TTaskManager = record
@@ -38,6 +39,16 @@ type
         procedure WaitFor(const State: TState); inline;
     end;
 
+    TQueue = record
+    strict private
+        [volatile]
+        FQueueLock: Integer;
+        FState: TState;
+    public
+        function Enqueue(const Func: TFunc<TState>): TState;
+        class operator Initialize(out Dest: TQueue);
+    end;
+
 var
     TaskManager: TTaskManager;
 
@@ -174,4 +185,29 @@ begin
     Result := A.FTaskManager;
 end;
 
+function TQueue.Enqueue(const Func: TFunc<TState>): TState;
+begin
+    var state := TaskManager.RunTask(FState, Func);
+
+    repeat
+        if FQueueLock > 0 then
+        begin
+            YieldProcessor;
+            continue;
+        end;
+    until AtomicCmpExchange(FQueueLock, 1, 0) = 0;
+
+    try
+        Result := FState;
+        FState := state;
+    finally
+        AtomicDecrement(FQueueLock);
+    end;
+end;
+
+class operator TQueue.Initialize(out Dest: TQueue);
+begin
+    Dest.FQueueLock := 0;
+end;
+
 end.

Src/Myc.Trade.DataPoint.pas

@@ -4,6 +4,7 @@ interface
 
 uses
     System.TimeSpan,
+    Myc.Signals,
     Myc.Core.Notifier;
 
 type
@@ -31,12 +32,12 @@ type
     end;
 
     IMycProcessor<T> = interface
-        function ProcessData(const Value: T): Boolean;
+        function ProcessData(const Value: T): TState;
     end;
 
     TMycProcessor<T> = class abstract(TInterfacedObject, IMycProcessor<T>)
     protected
-        function ProcessData(const Value: T): Boolean; virtual; abstract;
+        function ProcessData(const Value: T): TState; virtual; abstract;
     end;
 
     TTag = Pointer;
@@ -53,6 +54,7 @@ type
         constructor Create(const Controller: IInterface);
         destructor Destroy; override;
         // Notifies all linked processors.
+        function Broadcast(const Value: T): TState;
         procedure Notify(const Func: TMycNotifyList<IMycProcessor<T>>.TNotifyProc);
         // Link a Processor
         function Link(const Processor: IMycProcessor<T>): TTag;
@@ -62,21 +64,21 @@ type
 
     TMycGenericProcessor<T> = class(TMycProcessor<T>)
         type
-            TProc = reference to function(const Value: T): Boolean;
+            TProc = reference to function(const Value: T): TState;
     private
         FProc: TProc;
     protected
-        function ProcessData(const Value: T): Boolean; override; final;
+        function ProcessData(const Value: T): TState; override; final;
     public
         constructor Create(const AProc: TProc);
     end;
 
     TMycContainedProcessor<T> = class(TContainedObject, IMycProcessor<T>)
         type
-            TProc = function(const Value: T): Boolean of object;
+            TProc = function(const Value: T): TState of object;
     private
         FProc: TProc;
-        function ProcessData(const Value: T): Boolean;
+        function ProcessData(const Value: T): TState;
     public
         constructor Create(const Controller: IInterface; const AProc: TProc);
     end;
@@ -91,10 +93,10 @@ type
         FSender: TMycDataProvider<T>;
         function GetSender: IMycDataProvider<T>;
     protected
-        function ProcessData(const Value: S): Boolean; override; abstract;
+        function ProcessData(const Value: S): TState; override; abstract;
 
         // Broadcasts the given data to all linked processors.
-        procedure Broadcast(const Value: T);
+        function Broadcast(const Value: T): TState;
 
     public
         constructor Create;
@@ -469,7 +471,7 @@ begin
     FProc := AProc;
 end;
 
-function TMycGenericProcessor<T>.ProcessData(const Value: T): Boolean;
+function TMycGenericProcessor<T>.ProcessData(const Value: T): TState;
 begin
     Result := FProc(Value);
 end;
@@ -480,7 +482,7 @@ begin
     FProc := AProc;
 end;
 
-function TMycContainedProcessor<T>.ProcessData(const Value: T): Boolean;
+function TMycContainedProcessor<T>.ProcessData(const Value: T): TState;
 begin
     Result := FProc(Value);
 end;
@@ -498,6 +500,34 @@ begin
     inherited Destroy;
 end;
 
+function TMycDataProvider<T>.Broadcast(const Value: T): TState;
+begin
+    FListeners.Lock;
+    try
+        var item := FListeners.First;
+        if item = nil then
+            exit(TState.Null);
+
+        var i := 1;
+        while item.Next <> nil do
+        begin
+            inc(i);
+            item := item.Next;
+        end;
+
+        var done := TLatch.CreateLatch(i);
+        while item <> nil do
+        begin
+            item.Receiver.ProcessData(Value).Signal.Subscribe(done);
+            item := item.Prev;
+        end;
+
+        Result := done.State;
+    finally
+        FListeners.Release;
+    end;
+end;
+
 procedure TMycDataProvider<T>.Notify(const Func: TMycNotifyList<IMycProcessor<T>>.TNotifyProc);
 begin
     FListeners.Lock;
@@ -543,10 +573,9 @@ begin
     inherited Destroy;
 end;
 
-procedure TMycConverter<S, T>.Broadcast(const Value: T);
+function TMycConverter<S, T>.Broadcast(const Value: T): TState;
 begin
-    var cValue := Value;
-    FSender.Notify(function(const Processor: IMycProcessor<T>): Boolean begin Result := Processor.ProcessData(cValue) end);
+    Result := FSender.Broadcast(Value);
 end;
 
 function TMycConverter<S, T>.GetSender: IMycDataProvider<T>;

Src/Myc.Trade.DataStream.pas

@@ -445,20 +445,20 @@ begin
         DataFile.Chain(
             function(const DataChunks: TArray<TArray<TDataPoint<T>>>): TState
             begin
-                if Assigned(DataChunks) then
-                begin
-                    // Process each chunk, checking for termination between chunks.
-                    for var chunk in DataChunks do
-                    begin
-                        if cTerminated.IsSet then
-                            exit(TState.Null);
+                // Process each chunk, checking for termination between chunks.
+                var done := TLatch.CreateLatch(Length(DataChunks));
+                for var i := 0 to High(DataChunks) do
+                    if not cTerminated.IsSet then
+                        Processor.ProcessData(DataChunks[i]).Signal.Subscribe(done)
+                    else
+                        done.Notify;
 
-                        if not Processor.ProcessData(chunk) then
-                            exit(TState.Null);
-                    end;
-                end;
-
-                Result := ProcessFile(nextFileInfo, nextFile, cTerminated, Processor);
+                // Move to next file (which is currently preloading) once all Processors are done
+                Result :=
+                    TaskManager.RunTask(
+                        done.State,
+                        function: TState begin Result := ProcessFile(nextFileInfo, nextFile, cTerminated, Processor); end
+                    )
             end
         );
 end;