Concurrent data processing

Src/Myc.Trade.DataStream.pas

@@ -445,20 +445,20 @@ begin
         DataFile.Chain(
             function(const DataChunks: TArray<TArray<TDataPoint<T>>>): TState
             begin
-                if Assigned(DataChunks) then
-                begin
-                    // Process each chunk, checking for termination between chunks.
-                    for var chunk in DataChunks do
-                    begin
-                        if cTerminated.IsSet then
-                            exit(TState.Null);
+                // Process each chunk, checking for termination between chunks.
+                var done := TLatch.CreateLatch(Length(DataChunks));
+                for var i := 0 to High(DataChunks) do
+                    if not cTerminated.IsSet then
+                        Processor.ProcessData(DataChunks[i]).Signal.Subscribe(done)
+                    else
+                        done.Notify;
 
-                        if not Processor.ProcessData(chunk) then
-                            exit(TState.Null);
-                    end;
-                end;
-
-                Result := ProcessFile(nextFileInfo, nextFile, cTerminated, Processor);
+                // Move to next file (which is currently preloading) once all Processors are done
+                Result :=
+                    TaskManager.RunTask(
+                        done.State,
+                        function: TState begin Result := ProcessFile(nextFileInfo, nextFile, cTerminated, Processor); end
+                    )
             end
         );
 end;