Add MS_SimpleExport bot + hardened Linux export-runner
MS_SimpleExport is a market-data exporter (M1/Tick -> zipped binary), driven headless via the official cTrader-console Docker image. This adds the bot and a hardened bash driver that runs the scrape on a Linux host, spawning one ephemeral console container per backtest window. - src/MS_SimpleExport: the cBot; csproj aligned to the repo's Linux template (AlgoPublish=false, cTrader.Automate 1.*-*). - scripts/export-runner.sh: hardened port of the Unraid driver -- retry-then-abort on transient failure, gap-aware resume, current-month end-date clamp, 3-way run classification with a per-run timeout, and ZIP validation before rename. - scripts/export.env.example: deployment config template (real values live in the gitignored runtime/export.env). - docs/export-runner-quirks.md: the multi-lens audit findings and hardening decisions. - .gitignore: ignore runtime/ (secrets, the placed .algo, the symbol list).
This commit is contained in:
Executable
+108
@@ -0,0 +1,108 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.InteropServices;
|
||||
using cAlgo.API;
|
||||
|
||||
namespace cAlgo.Robots
|
||||
{
|
||||
public enum ExportMode { M1, Tick }
|
||||
|
||||
[Robot(AccessRights = AccessRights.FullAccess)]
|
||||
public class MS_SimpleExport : Robot
|
||||
{
|
||||
[StructLayout(LayoutKind.Sequential, Pack = 1)]
|
||||
private struct M1Record {
|
||||
public double Time;
|
||||
public double O, H, L, C;
|
||||
public float S;
|
||||
public int V;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential, Pack = 1)]
|
||||
private struct TickRecord {
|
||||
public double Time;
|
||||
public double Ask, Bid;
|
||||
}
|
||||
|
||||
[Parameter("Export Mode", DefaultValue = ExportMode.M1)]
|
||||
public ExportMode Mode { get; set; }
|
||||
|
||||
[Parameter("Output File (ZIP)", DefaultValue = "")]
|
||||
public string OutputFile { get; set; }
|
||||
|
||||
[Parameter("Is Probing", DefaultValue = false)]
|
||||
public bool IsProbing { get; set; }
|
||||
|
||||
private List<M1Record> _m1Buffer = new List<M1Record>();
|
||||
private List<TickRecord> _tickBuffer = new List<TickRecord>();
|
||||
|
||||
protected override void OnStart() {
|
||||
// No price factor needed for double/float export
|
||||
}
|
||||
|
||||
protected override void OnTick() {
|
||||
if (Mode != ExportMode.Tick) return;
|
||||
|
||||
if (IsProbing) {
|
||||
Print("DATA_FOUND");
|
||||
Stop();
|
||||
return;
|
||||
}
|
||||
|
||||
_tickBuffer.Add(new TickRecord {
|
||||
Time = Server.TimeInUtc.ToOADate(),
|
||||
Ask = Symbol.Ask,
|
||||
Bid = Symbol.Bid
|
||||
});
|
||||
}
|
||||
|
||||
protected override void OnBar() {
|
||||
if (Mode != ExportMode.M1) return;
|
||||
|
||||
if (IsProbing) {
|
||||
Print("DATA_FOUND");
|
||||
Stop();
|
||||
return;
|
||||
}
|
||||
|
||||
var bar = Bars.Last(1);
|
||||
_m1Buffer.Add(new M1Record {
|
||||
Time = bar.OpenTime.ToOADate(),
|
||||
O = bar.Open,
|
||||
H = bar.High,
|
||||
L = bar.Low,
|
||||
C = bar.Close,
|
||||
S = (float)Symbol.Spread,
|
||||
V = (int)bar.TickVolume
|
||||
});
|
||||
}
|
||||
|
||||
protected override void OnStop() {
|
||||
if (IsProbing || string.IsNullOrEmpty(OutputFile)) return;
|
||||
|
||||
try {
|
||||
using var fileStream = new FileStream(OutputFile, FileMode.Create);
|
||||
using var archive = new ZipArchive(fileStream, ZipArchiveMode.Create);
|
||||
var entry = archive.CreateEntry($"{Symbol.Name}.bin", CompressionLevel.Fastest);
|
||||
|
||||
using var entryStream = entry.Open();
|
||||
|
||||
if (Mode == ExportMode.M1) {
|
||||
ReadOnlySpan<M1Record> span = CollectionsMarshal.AsSpan(_m1Buffer);
|
||||
ReadOnlySpan<byte> byteSpan = MemoryMarshal.Cast<M1Record, byte>(span);
|
||||
if (!byteSpan.IsEmpty) entryStream.Write(byteSpan);
|
||||
} else {
|
||||
ReadOnlySpan<TickRecord> span = CollectionsMarshal.AsSpan(_tickBuffer);
|
||||
ReadOnlySpan<byte> byteSpan = MemoryMarshal.Cast<TickRecord, byte>(span);
|
||||
if (!byteSpan.IsEmpty) entryStream.Write(byteSpan);
|
||||
}
|
||||
|
||||
Print("EXPORT_SUCCESS");
|
||||
} catch (Exception ex) {
|
||||
Print("ERROR: {0}", ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Executable
+23
@@ -0,0 +1,23 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<!-- net6.0 is the runtime cTrader's algo host expects. -->
|
||||
<TargetFramework>net6.0</TargetFramework>
|
||||
<LangVersion>latest</LangVersion>
|
||||
<Nullable>disable</Nullable>
|
||||
<ImplicitUsings>disable</ImplicitUsings>
|
||||
|
||||
<!--
|
||||
Turn off the cTrader.Automate publish step that copies the .algo into the
|
||||
Windows-only ~/Documents/cAlgo/Sources path. The .algo is still emitted into
|
||||
bin/<config>/<tfm>/; scripts/build.sh collects it into dist/.
|
||||
-->
|
||||
<AlgoPublish>false</AlgoPublish>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<!-- Official Spotware SDK (owner "spotware", not the third-party "cAlgo.API"). -->
|
||||
<PackageReference Include="cTrader.Automate" Version="1.*-*" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
Reference in New Issue
Block a user