Brummel/cTrader-Algo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
Michael Schimmel
2026-01-28 09:10:52 +01:00
commit a1d2e96f8a
513 changed files with 19503 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Sources/Robots/MS_IncrementalExport/.vscode/tasks.json
Vendored
+1
View File
@@ -0,0 +1 @@
{"version":"2.0.0","tasks":[{"label":"build","command":"dotnet","type":"process","args":["build","${workspaceFolder}","/property:GenerateFullPaths=true","/consoleLoggerParameters:NoSummary"],"problemMatcher":"$msCompile"}]}
Sources/Robots/MS_IncrementalExport/MS_IncrementalExport.sln
+22
View File
@@ -0,0 +1,22 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.30011.22
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MS_IncrementalExport", "MS_IncrementalExport\MS_IncrementalExport.csproj", "{c3b052a5-8e47-4bd1-99ba-f114dbdcfa8f}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{c3b052a5-8e47-4bd1-99ba-f114dbdcfa8f}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{c3b052a5-8e47-4bd1-99ba-f114dbdcfa8f}.Debug|Any CPU.Build.0 = Debug|Any CPU
{c3b052a5-8e47-4bd1-99ba-f114dbdcfa8f}.Release|Any CPU.ActiveCfg = Release|Any CPU
{c3b052a5-8e47-4bd1-99ba-f114dbdcfa8f}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
EndGlobal
Sources/Robots/MS_IncrementalExport/MS_IncrementalExport/MS_IncrementalExport.cs
+186
View File
@@ -0,0 +1,186 @@
/* * MS_IncrementalExport
* High-performance M1 and Tick data exporter.
* Naming: {Symbol}_{Start-YYYY-MM}_{End-YYYY-MM}.{ext}
*/
using System;
using System.IO;
using System.IO.Compression;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using cAlgo.API;
namespace cAlgo.Robots
{
public enum ExportMode { M1, Tick }
[Robot(AccessRights = AccessRights.FullAccess, AddIndicators = true)]
public class MS_IncrementalExport : Robot
{
[StructLayout(LayoutKind.Sequential, Pack = 1, Size = 56)]
private struct M1Record
{
public double OADateTime; // Must be at Offset 0
public long Open, High, Low, Close, Spread;
public int TickVolume, Digits;
}
[StructLayout(LayoutKind.Sequential, Pack = 1, Size = 16)]
private struct TickDataRecord
{
public double OADateTime; // Must be at Offset 0
public float Ask;
public float Bid;
}
[Parameter("Export Mode", DefaultValue = ExportMode.M1)]
public ExportMode Mode { get; set; }
[Parameter("Output Path", DefaultValue = "/mnt/Data")]
public string DirectoryName { get; set; }
[Parameter("Existing File Path", DefaultValue = "")]
public string ExistingFilePath { get; set; }
[Parameter("Max File Size (kB)", DefaultValue = 10000)]
public int MaxFileSizeKb { get; set; }
private List<M1Record> _m1Buffer = new List<M1Record>();
private List<TickDataRecord> _tickBuffer = new List<TickDataRecord>();
private double _lastTimestamp = double.MinValue;
private double _priceFactor;
protected override void OnStart()
{
_priceFactor = Math.Pow(10, Symbol.Digits);
if (!string.IsNullOrWhiteSpace(ExistingFilePath) && File.Exists(ExistingFilePath))
{
LoadExistingData(ExistingFilePath);
}
}
protected override void OnTick()
{
if (Mode != ExportMode.Tick) return;
var time = Server.TimeInUtc.ToOADate();
if (time <= _lastTimestamp) return;
_tickBuffer.Add(new TickDataRecord
{
OADateTime = time,
Ask = (float)Symbol.Ask,
Bid = (float)Symbol.Bid
});
}
protected override void OnBar()
{
if (Mode != ExportMode.M1) return;
var barIndex = Bars.Count - 2;
if (barIndex < 0) return;
var bar = Bars[barIndex];
var barTime = bar.OpenTime.ToOADate();
if (barTime <= _lastTimestamp) return;
_m1Buffer.Add(new M1Record
{
OADateTime = barTime,
Open = (long)(bar.Open * _priceFactor),
High = (long)(bar.High * _priceFactor),
Low = (long)(bar.Low * _priceFactor),
Close = (long)(bar.Close * _priceFactor),
Spread = (long)(Symbol.Spread * _priceFactor),
TickVolume = (int)bar.TickVolume,
Digits = Symbol.Digits
});
}
protected override void OnStop()
{
if (Mode == ExportMode.M1)
ProcessExport(_m1Buffer, "m1");
else
ProcessExport(_tickBuffer, "tick");
}
private void ProcessExport<T>(List<T> buffer, string extension) where T : unmanaged
{
if (buffer.Count == 0) return;
int recordSize = Marshal.SizeOf<T>();
long maxBytes = (long)MaxFileSizeKb * 1024;
int batchSize = (int)(maxBytes / recordSize);
for (int i = 0; i < buffer.Count; i += batchSize)
{
var count = Math.Min(batchSize, buffer.Count - i);
var batch = buffer.GetRange(i, count);
WriteBatch(batch, extension);
}
}
private void WriteBatch<T>(List<T> records, string ext) where T : unmanaged
{
ReadOnlySpan<T> span = CollectionsMarshal.AsSpan(records);
// Extract timestamps via Cast to interpret the first 8 bytes as double
double start = MemoryMarshal.Cast<T, double>(span)[0];
double end = MemoryMarshal.Cast<T, double>(span.Slice(records.Count - 1))[0];
string startStr = DateTime.FromOADate(start).ToString("yyyy-MM");
string endStr = DateTime.FromOADate(end).ToString("yyyy-MM");
string fileName = $"{Symbol.Name}_{startStr}_{endStr}.{ext}";
string archivePath = Path.Combine(DirectoryName, fileName);
ReadOnlySpan<byte> byteSpan = MemoryMarshal.Cast<T, byte>(span);
using var fileStream = new FileStream(archivePath, FileMode.Create);
using var archive = new ZipArchive(fileStream, ZipArchiveMode.Create);
var entry = archive.CreateEntry($"{Symbol.Name}.bin", CompressionLevel.Fastest);
using var entryStream = entry.Open();
entryStream.Write(byteSpan);
Print("Saved: {0}", archivePath);
}
private void LoadExistingData(string path)
{
try
{
using var archive = ZipFile.OpenRead(path);
var entry = archive.Entries.FirstOrDefault();
if (entry == null) return;
using var stream = entry.Open();
using var ms = new MemoryStream();
stream.CopyTo(ms);
byte[] data = ms.ToArray();
if (Mode == ExportMode.M1)
{
var records = MemoryMarshal.Cast<byte, M1Record>(data);
foreach (var r in records) if (r.OADateTime > _lastTimestamp) _lastTimestamp = r.OADateTime;
}
else
{
var records = MemoryMarshal.Cast<byte, TickDataRecord>(data);
foreach (var r in records) if (r.OADateTime > _lastTimestamp) _lastTimestamp = r.OADateTime;
}
Print("Loaded existing data. Last timestamp: {0}", DateTime.FromOADate(_lastTimestamp));
}
catch (Exception ex)
{
Print("Error loading existing data: {0}", ex.Message);
}
}
}
}
Sources/Robots/MS_IncrementalExport/MS_IncrementalExport/MS_IncrementalExport.csproj
+9
View File
@@ -0,0 +1,9 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="cTrader.Automate" Version="*" />
</ItemGroup>
</Project>
Sources/Robots/MS_IncrementalExport/MS_IncrementalExport/obj/Debug/net6.0/.NETCoreApp,Version=v6.0.AssemblyAttributes.cs
+4
View File
@@ -0,0 +1,4 @@
// <autogenerated />
using System;
using System.Reflection;
[assembly: global::System.Runtime.Versioning.TargetFrameworkAttribute(".NETCoreApp,Version=v6.0", FrameworkDisplayName = "")]
Sources/Robots/MS_IncrementalExport/MS_IncrementalExport/obj/Debug/net6.0/MS_IncrementalExport.AssemblyInfo.cs
+22
View File
@@ -0,0 +1,22 @@
//------------------------------------------------------------------------------
// <auto-generated>
// This code was generated by a tool.
//
// Changes to this file may cause incorrect behavior and will be lost if
// the code is regenerated.
// </auto-generated>
//------------------------------------------------------------------------------
using System;
using System.Reflection;
[assembly: System.Reflection.AssemblyCompanyAttribute("MS_IncrementalExport")]
[assembly: System.Reflection.AssemblyConfigurationAttribute("Debug")]
[assembly: System.Reflection.AssemblyFileVersionAttribute("1.0.0.0")]
[assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.0")]
[assembly: System.Reflection.AssemblyProductAttribute("MS_IncrementalExport")]
[assembly: System.Reflection.AssemblyTitleAttribute("MS_IncrementalExport")]
[assembly: System.Reflection.AssemblyVersionAttribute("1.0.0.0")]
// Generated by the MSBuild WriteCodeFragment class.
Sources/Robots/MS_IncrementalExport/MS_IncrementalExport/obj/Debug/net6.0/MS_IncrementalExport.AssemblyInfoInputs.cache
+1
View File
@@ -0,0 +1 @@
5773c38d4319cb7f76ca5b658c23774f3fba8ad5
Sources/Robots/MS_IncrementalExport/MS_IncrementalExport/obj/Debug/net6.0/MS_IncrementalExport.GeneratedMSBuildEditorConfig.editorconfig
+10
View File
@@ -0,0 +1,10 @@
is_global = true
build_property.TargetFramework = net6.0
build_property.TargetPlatformMinVersion =
build_property.UsingMicrosoftNETSdkWeb =
build_property.ProjectTypeGuids =
build_property.InvariantGlobalization =
build_property.PlatformNeutralAssembly =
build_property._SupportedPlatformList = Linux,macOS,Windows
build_property.RootNamespace = MS_IncrementalExport
build_property.ProjectDir = C:\Users\Brummel\Documents\cAlgo\Sources\Robots\MS_IncrementalExport\MS_IncrementalExport\
Sources/Robots/MS_IncrementalExport/MS_IncrementalExport/obj/Debug/net6.0/MS_IncrementalExport.assets.cache
BIN
View File
Binary file not shown.
Sources/Robots/MS_IncrementalExport/MS_IncrementalExport/obj/Debug/net6.0/MS_IncrementalExport.csproj.AssemblyReference.cache
BIN
View File
Binary file not shown.
Sources/Robots/MS_IncrementalExport/MS_IncrementalExport/obj/MS_IncrementalExport.csproj.nuget.dgspec.json
+66
View File
@@ -0,0 +1,66 @@
{
"format": 1,
"restore": {
"C:\\Users\\Brummel\\Documents\\cAlgo\\Sources\\Robots\\MS_IncrementalExport\\MS_IncrementalExport\\MS_IncrementalExport.csproj": {}
},
"projects": {
"C:\\Users\\Brummel\\Documents\\cAlgo\\Sources\\Robots\\MS_IncrementalExport\\MS_IncrementalExport\\MS_IncrementalExport.csproj": {
"version": "1.0.0",
"restore": {
"projectUniqueName": "C:\\Users\\Brummel\\Documents\\cAlgo\\Sources\\Robots\\MS_IncrementalExport\\MS_IncrementalExport\\MS_IncrementalExport.csproj",
"projectName": "MS_IncrementalExport",
"projectPath": "C:\\Users\\Brummel\\Documents\\cAlgo\\Sources\\Robots\\MS_IncrementalExport\\MS_IncrementalExport\\MS_IncrementalExport.csproj",
"packagesPath": "C:\\Users\\Brummel\\.nuget\\packages\\",
"outputPath": "C:\\Users\\Brummel\\Documents\\cAlgo\\Sources\\Robots\\MS_IncrementalExport\\MS_IncrementalExport\\obj\\",
"projectStyle": "PackageReference",
"configFilePaths": [
"C:\\Users\\Brummel\\AppData\\Roaming\\NuGet\\NuGet.Config"
],
"originalTargetFrameworks": [
"net6.0"
],
"sources": {
"https://api.nuget.org/v3/index.json": {}
},
"frameworks": {
"net6.0": {
"targetAlias": "net6.0",
"projectReferences": {}
}
},
"warningProperties": {
"warnAsError": [
"NU1605"
]
}
},
"frameworks": {
"net6.0": {
"targetAlias": "net6.0",
"dependencies": {
"cTrader.Automate": {
"target": "Package",
"version": "[*, )"
}
},
"imports": [
"net461",
"net462",
"net47",
"net471",
"net472",
"net48"
],
"assetTargetFallback": true,
"warn": true,
"frameworkReferences": {
"Microsoft.NETCore.App": {
"privateAssets": "all"
}
},
"runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\6.0.200\\RuntimeIdentifierGraph.json"
}
}
}
}
}
Sources/Robots/MS_IncrementalExport/MS_IncrementalExport/obj/MS_IncrementalExport.csproj.nuget.g.props
+21
View File
@@ -0,0 +1,21 @@
<?xml version="1.0" encoding="utf-8" standalone="no"?>
<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
<RestoreSuccess Condition=" '$(RestoreSuccess)' == '' ">True</RestoreSuccess>
<RestoreTool Condition=" '$(RestoreTool)' == '' ">NuGet</RestoreTool>
<ProjectAssetsFile Condition=" '$(ProjectAssetsFile)' == '' ">$(MSBuildThisFileDirectory)project.assets.json</ProjectAssetsFile>
<NuGetPackageRoot Condition=" '$(NuGetPackageRoot)' == '' ">$(UserProfile)\.nuget\packages\</NuGetPackageRoot>
<NuGetPackageFolders Condition=" '$(NuGetPackageFolders)' == '' ">C:\Users\Brummel\.nuget\packages\</NuGetPackageFolders>
<NuGetProjectStyle Condition=" '$(NuGetProjectStyle)' == '' ">PackageReference</NuGetProjectStyle>
<NuGetToolVersion Condition=" '$(NuGetToolVersion)' == '' ">6.1.0</NuGetToolVersion>
</PropertyGroup>
<ItemGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
<SourceRoot Include="C:\Users\Brummel\.nuget\packages\" />
</ItemGroup>
<ImportGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
<Import Project="$(NuGetPackageRoot)ctrader.automate\1.0.14\build\cTrader.Automate.props" Condition="Exists('$(NuGetPackageRoot)ctrader.automate\1.0.14\build\cTrader.Automate.props')" />
</ImportGroup>
<PropertyGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
<PkgcTrader_Automate Condition=" '$(PkgcTrader_Automate)' == '' ">C:\Users\Brummel\.nuget\packages\ctrader.automate\1.0.14</PkgcTrader_Automate>
</PropertyGroup>
</Project>
Sources/Robots/MS_IncrementalExport/MS_IncrementalExport/obj/MS_IncrementalExport.csproj.nuget.g.targets
+6
View File
@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8" standalone="no"?>
<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ImportGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
<Import Project="$(NuGetPackageRoot)ctrader.automate\1.0.14\build\cTrader.Automate.targets" Condition="Exists('$(NuGetPackageRoot)ctrader.automate\1.0.14\build\cTrader.Automate.targets')" />
</ImportGroup>
</Project>
Sources/Robots/MS_IncrementalExport/MS_IncrementalExport/obj/project.assets.json
+139
View File
@@ -0,0 +1,139 @@
{
"version": 3,
"targets": {
"net6.0": {
"cTrader.Automate/1.0.14": {
"type": "package",
"compile": {
"lib/net6.0/cAlgo.API.dll": {}
},
"runtime": {
"lib/net6.0/cAlgo.API.dll": {}
},
"build": {
"build/cTrader.Automate.props": {},
"build/cTrader.Automate.targets": {}
}
}
}
},
"libraries": {
"cTrader.Automate/1.0.14": {
"sha512": "eNwE7WL90MGBKb5MuLAtZLdQy0vxkI5EVhLWAQ9S83EAdYkGAzdccvGFLk2oqmtSGBtD+gEpyrJUW/ej4dI4jw==",
"type": "package",
"path": "ctrader.automate/1.0.14",
"hasTools": true,
"files": [
".nupkg.metadata",
".signature.p7s",
"build/cTrader.Automate.props",
"build/cTrader.Automate.targets",
"ctrader.automate.1.0.14.nupkg.sha512",
"ctrader.automate.nuspec",
"eula.md",
"icon.png",
"lib/net40/cAlgo.API.dll",
"lib/net40/cAlgo.API.xml",
"lib/net6.0/cAlgo.API.dll",
"lib/net6.0/cAlgo.API.xml",
"tools/net472/Core.AlgoFormat.Compose.Reflection.dll",
"tools/net472/Core.AlgoFormat.Writer.dll",
"tools/net472/Core.AlgoFormat.dll",
"tools/net472/Core.Domain.Primitives.dll",
"tools/net472/Newtonsoft.Json.dll",
"tools/net472/System.Buffers.dll",
"tools/net472/System.Collections.Immutable.dll",
"tools/net472/System.Memory.dll",
"tools/net472/System.Numerics.Vectors.dll",
"tools/net472/System.Reflection.Metadata.dll",
"tools/net472/System.Reflection.MetadataLoadContext.dll",
"tools/net472/System.Runtime.CompilerServices.Unsafe.dll",
"tools/net472/cTrader.Automate.Sdk.Tasks.dll",
"tools/net6.0/Core.AlgoFormat.Compose.Reflection.dll",
"tools/net6.0/Core.AlgoFormat.Writer.dll",
"tools/net6.0/Core.AlgoFormat.dll",
"tools/net6.0/Core.Connection.Protobuf.Common.dll",
"tools/net6.0/Core.Domain.Primitives.dll",
"tools/net6.0/Microsoft.Win32.SystemEvents.dll",
"tools/net6.0/Newtonsoft.Json.dll",
"tools/net6.0/System.Drawing.Common.dll",
"tools/net6.0/System.Reflection.MetadataLoadContext.dll",
"tools/net6.0/System.Security.Permissions.dll",
"tools/net6.0/System.Windows.Extensions.dll",
"tools/net6.0/cTrader.Automate.Sdk.Tasks.dll",
"tools/net6.0/protobuf-net.Core.dll",
"tools/net6.0/protobuf-net.dll",
"tools/net6.0/runtimes/unix/lib/net6.0/System.Drawing.Common.dll",
"tools/net6.0/runtimes/win/lib/net6.0/Microsoft.Win32.SystemEvents.dll",
"tools/net6.0/runtimes/win/lib/net6.0/System.Drawing.Common.dll",
"tools/net6.0/runtimes/win/lib/net6.0/System.Windows.Extensions.dll"
]
}
},
"projectFileDependencyGroups": {
"net6.0": [
"cTrader.Automate >= *"
]
},
"packageFolders": {
"C:\\Users\\Brummel\\.nuget\\packages\\": {}
},
"project": {
"version": "1.0.0",
"restore": {
"projectUniqueName": "C:\\Users\\Brummel\\Documents\\cAlgo\\Sources\\Robots\\MS_IncrementalExport\\MS_IncrementalExport\\MS_IncrementalExport.csproj",
"projectName": "MS_IncrementalExport",
"projectPath": "C:\\Users\\Brummel\\Documents\\cAlgo\\Sources\\Robots\\MS_IncrementalExport\\MS_IncrementalExport\\MS_IncrementalExport.csproj",
"packagesPath": "C:\\Users\\Brummel\\.nuget\\packages\\",
"outputPath": "C:\\Users\\Brummel\\Documents\\cAlgo\\Sources\\Robots\\MS_IncrementalExport\\MS_IncrementalExport\\obj\\",
"projectStyle": "PackageReference",
"configFilePaths": [
"C:\\Users\\Brummel\\AppData\\Roaming\\NuGet\\NuGet.Config"
],
"originalTargetFrameworks": [
"net6.0"
],
"sources": {
"https://api.nuget.org/v3/index.json": {}
},
"frameworks": {
"net6.0": {
"targetAlias": "net6.0",
"projectReferences": {}
}
},
"warningProperties": {
"warnAsError": [
"NU1605"
]
}
},
"frameworks": {
"net6.0": {
"targetAlias": "net6.0",
"dependencies": {
"cTrader.Automate": {
"target": "Package",
"version": "[*, )"
}
},
"imports": [
"net461",
"net462",
"net47",
"net471",
"net472",
"net48"
],
"assetTargetFallback": true,
"warn": true,
"frameworkReferences": {
"Microsoft.NETCore.App": {
"privateAssets": "all"
}
},
"runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\6.0.200\\RuntimeIdentifierGraph.json"
}
}
}
}
Sources/Robots/MS_IncrementalExport/MS_IncrementalExport/obj/project.nuget.cache
+10
View File
@@ -0,0 +1,10 @@
{
"version": 2,
"dgSpecHash": "yjqqJ1FuNh4unJj0bFrEA5zuzdz+mayZe5I9PxwxfOVpWAKWRwhTHWIw6G7wF7IgsOktZDLcmYJEDZ4N/mWgGw==",
"success": true,
"projectFilePath": "C:\\Users\\Brummel\\Documents\\cAlgo\\Sources\\Robots\\MS_IncrementalExport\\MS_IncrementalExport\\MS_IncrementalExport.csproj",
"expectedPackageFiles": [
"C:\\Users\\Brummel\\.nuget\\packages\\ctrader.automate\\1.0.14\\ctrader.automate.1.0.14.nupkg.sha512"
],
"logs": []
}