Add MS_SimpleExport bot + hardened Linux export-runner

MS_SimpleExport is a market-data exporter (M1/Tick -> zipped binary), driven
headless via the official cTrader-console Docker image. This adds the bot and a
hardened bash driver that runs the scrape on a Linux host, spawning one ephemeral
console container per backtest window.

- src/MS_SimpleExport: the cBot; csproj aligned to the repo's Linux template
  (AlgoPublish=false, cTrader.Automate 1.*-*).
- scripts/export-runner.sh: hardened port of the Unraid driver -- retry-then-abort
  on transient failure, gap-aware resume, current-month end-date clamp, 3-way run
  classification with a per-run timeout, and ZIP validation before rename.
- scripts/export.env.example: deployment config template (real values live in the
  gitignored runtime/export.env).
- docs/export-runner-quirks.md: the multi-lens audit findings and hardening decisions.
- .gitignore: ignore runtime/ (secrets, the placed .algo, the symbol list).
This commit is contained in:
2026-06-24 13:31:41 +02:00
parent 9c39a743d0
commit cc3f0cbd46
6 changed files with 547 additions and 0 deletions
+108
View File
@@ -0,0 +1,108 @@
using System;
using System.IO;
using System.IO.Compression;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using cAlgo.API;
namespace cAlgo.Robots
{
public enum ExportMode { M1, Tick }
[Robot(AccessRights = AccessRights.FullAccess)]
public class MS_SimpleExport : Robot
{
[StructLayout(LayoutKind.Sequential, Pack = 1)]
private struct M1Record {
public double Time;
public double O, H, L, C;
public float S;
public int V;
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
private struct TickRecord {
public double Time;
public double Ask, Bid;
}
[Parameter("Export Mode", DefaultValue = ExportMode.M1)]
public ExportMode Mode { get; set; }
[Parameter("Output File (ZIP)", DefaultValue = "")]
public string OutputFile { get; set; }
[Parameter("Is Probing", DefaultValue = false)]
public bool IsProbing { get; set; }
private List<M1Record> _m1Buffer = new List<M1Record>();
private List<TickRecord> _tickBuffer = new List<TickRecord>();
protected override void OnStart() {
// No price factor needed for double/float export
}
protected override void OnTick() {
if (Mode != ExportMode.Tick) return;
if (IsProbing) {
Print("DATA_FOUND");
Stop();
return;
}
_tickBuffer.Add(new TickRecord {
Time = Server.TimeInUtc.ToOADate(),
Ask = Symbol.Ask,
Bid = Symbol.Bid
});
}
protected override void OnBar() {
if (Mode != ExportMode.M1) return;
if (IsProbing) {
Print("DATA_FOUND");
Stop();
return;
}
var bar = Bars.Last(1);
_m1Buffer.Add(new M1Record {
Time = bar.OpenTime.ToOADate(),
O = bar.Open,
H = bar.High,
L = bar.Low,
C = bar.Close,
S = (float)Symbol.Spread,
V = (int)bar.TickVolume
});
}
protected override void OnStop() {
if (IsProbing || string.IsNullOrEmpty(OutputFile)) return;
try {
using var fileStream = new FileStream(OutputFile, FileMode.Create);
using var archive = new ZipArchive(fileStream, ZipArchiveMode.Create);
var entry = archive.CreateEntry($"{Symbol.Name}.bin", CompressionLevel.Fastest);
using var entryStream = entry.Open();
if (Mode == ExportMode.M1) {
ReadOnlySpan<M1Record> span = CollectionsMarshal.AsSpan(_m1Buffer);
ReadOnlySpan<byte> byteSpan = MemoryMarshal.Cast<M1Record, byte>(span);
if (!byteSpan.IsEmpty) entryStream.Write(byteSpan);
} else {
ReadOnlySpan<TickRecord> span = CollectionsMarshal.AsSpan(_tickBuffer);
ReadOnlySpan<byte> byteSpan = MemoryMarshal.Cast<TickRecord, byte>(span);
if (!byteSpan.IsEmpty) entryStream.Write(byteSpan);
}
Print("EXPORT_SUCCESS");
} catch (Exception ex) {
Print("ERROR: {0}", ex.Message);
}
}
}
}
+23
View File
@@ -0,0 +1,23 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<!-- net6.0 is the runtime cTrader's algo host expects. -->
<TargetFramework>net6.0</TargetFramework>
<LangVersion>latest</LangVersion>
<Nullable>disable</Nullable>
<ImplicitUsings>disable</ImplicitUsings>
<!--
Turn off the cTrader.Automate publish step that copies the .algo into the
Windows-only ~/Documents/cAlgo/Sources path. The .algo is still emitted into
bin/<config>/<tfm>/; scripts/build.sh collects it into dist/.
-->
<AlgoPublish>false</AlgoPublish>
</PropertyGroup>
<ItemGroup>
<!-- Official Spotware SDK (owner "spotware", not the third-party "cAlgo.API"). -->
<PackageReference Include="cTrader.Automate" Version="1.*-*" />
</ItemGroup>
</Project>