diff --git a/.gitignore b/.gitignore index dcb195b..566fa0f 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,7 @@ dist/ # Local tool installs .dotnet/ + +# Deployment runtime: secrets (password.txt), the placed .algo, the symbol list, +# and the credentials env file. Never commit any of this. +runtime/ diff --git a/docs/export-runner-quirks.md b/docs/export-runner-quirks.md new file mode 100644 index 0000000..56aad3a --- /dev/null +++ b/docs/export-runner-quirks.md @@ -0,0 +1,73 @@ +# export-runner.sh — quirk audit & hardening decisions + +`scripts/export-runner.sh` is the Linux-host port of the original Unraid data-export +driver. Before going live it was put through a multi-lens adversarial audit (5 review +lenses over the script **and** the `MS_SimpleExport` cBot, each finding independently +verified). 49 findings survived verification, deduplicating to ~18 distinct quirks. + +This document records what was fixed, what was a deliberate behaviour choice, and what +is intentionally left alone — so the rationale is not lost. + +## Behaviour decisions (the load-bearing ones) + +| Topic | Decision | Why | +|---|---|---| +| **Transient failure** | Retry ×3, then **hard-abort** the whole run | A login stall / crash / timeout must never be silently treated as "empty month". Aborting loudly means a gap is noticed, not buried. | +| **Resume** | **Gap-aware**: walk every month from the symbol's first data month to now; scrape only **missing** months + the **trailing edge** (latest existing + current). | Fixes the silent middle-gap (old `ls\|sort\|tail-1` resume never backfilled) **and** stops overwriting known-good middle files. | +| **Empty month** | Don't write empty files. A clean `No historical data` is classified **empty → skip**. | For continuously-active instruments a true in-range empty month effectively never occurs; the empty path is purely defensive and must not trip the hard-abort. | +| **Current month** | Clamp the end date to **yesterday** | cTrader rejects a backtest whose end date is in the future (proven), so the whole current month was being dropped. Clamping captures the partial month now; next month's edge re-scrape completes it. | + +## Classification — the core of the hardening + +`run_one` reads the console log stream (with a per-run `timeout`) and classifies the +outcome, instead of trusting a single stdout sentinel: + +- `DATA_FOUND` / `EXPORT_SUCCESS` → **success** (rc 0) +- `No historical data for the specified period` → **clean-empty** (rc 3, skipped) +- anything else — timeout, `ERROR:`, a result block with no verdict, a dead stream → **transient failure** (rc 2, retried then aborted) + +A produced ZIP is validated (`testzip` + non-empty entry) **before** the `mv`, so a +truncated artifact from a cut-off run is caught instead of being promoted to a final file. + +## Quirks fixed (no behaviour choice needed) + +| Quirk | Fix | +|---|---| +| No per-run timeout → a hung console blocks the whole export forever | `timeout` around the log drain; a hang is a transient failure | +| `--rm` + `docker logs -f` race could lose markers on a fast-exiting container | dropped `--rm`; container kept until logs drained, then removed; trap reaps any in-flight container on Ctrl-C (no orphans) | +| `*ERROR*` matched as an unanchored substring | matched against the bot's `ERROR:` form / the explicit verdict markers | +| Resume parser `cut -f2/-f3` corrupts year/month for symbols containing `_` | parsed with a bash regex anchored on the symbol | +| Resume `sort\|tail` is lexical | month walk is purely numeric (`year*12 + month-1` index) | +| Empty / corrupt ZIP promoted to a final file | `validate_export` (PK + CRC + non-empty entry) gates the `mv` | +| Monthly start-probe only tested days 01–10 | discovery probes the **full** month; yearly probe uses December (catches same-year-later starts) | +| Completion banner always printed "success" | banner reports saved / empty counts | + +## Intentionally NOT changed — frozen by the existing corpus + +The binary record layout (`M1Record` = 48 B: raw `double` OHLC, `float` spread, `int` +volume; `TickRecord` = 24 B) is **frozen by the ~4380 existing `.m1`/`.tick` files** the +bot already produced. Any layout change makes new files incompatible with the corpus, so +the following audit findings are **WONTFIX** unless the whole dataset is re-exported: + +- `TickVolume` stored as `int` (would need `long`) — harmless in practice: an M1 bar's + tick count never approaches `int` range. +- `Spread` as raw `float` price units, and no `Digits` field — a downstream reader must + know the contract out-of-band. (Also: `Symbol.Spread` is usually 0 in M1 bar replay, so + that column is largely meaningless for M1 mode — a data-source limitation, not fixable + without tick data.) +- ZIP entry named after `Symbol.Name` while the file is named after the requested symbol — + cosmetic: the file is renamed by the driver and the reader opens the single entry by + position, not name. + +If the format is ever migrated, that is a separate project that also re-exports the corpus. + +## Verification + +The hardened script was validated end-to-end against a scratch directory (the real NFS +data dir untouched): + +- **Gap-fill / edge / clamp / trusted-middle**: seeded `2026_03` + `2026_05` (gap at `04`); + the run filled `04`, re-scraped the `05` edge, captured the current month `06` clamped to + yesterday, and left the trusted middle `03` byte-identical. +- **Clean-empty safety**: `run_one` on an empty future window returns rc 3 (skip), never the + rc 2 that would abort the run. diff --git a/scripts/export-runner.sh b/scripts/export-runner.sh new file mode 100755 index 0000000..8d00a50 --- /dev/null +++ b/scripts/export-runner.sh @@ -0,0 +1,322 @@ +#!/usr/bin/env bash +# +# Headless market-data export driver for the MS_SimpleExport cBot (hardened). +# +# Runs on THIS host and spawns one cTrader-console container per backtest window. +# Deployment-specific config lives in runtime/export.env (gitignored). +# +# Hardening over the original Unraid script (see docs/export-runner-quirks.md): +# - Per-run timeout + 3-way classification (success / clean-empty / transient-fail) +# read from the log stream; a transient failure is retried, then HARD-ABORTS the +# whole run (never silently skipped) so gaps cannot form unnoticed. +# - Gap-aware resume: every expected month from the symbol's first data month to now +# is checked; only MISSING months and the trailing edge (latest existing + current) +# are (re-)scraped. Known-good middle files are never overwritten. +# - Current (incomplete) month end-date is clamped to yesterday, so partial current +# month data is captured instead of being rejected for a future end date. +# - Clean "No historical data" is treated as legitimately-empty -> skipped, no file +# written (no empty-file pollution). Empty/corrupt ZIPs are caught before mv. +# - No --rm: the container is kept until logs are drained, then removed explicitly; +# a trap removes any in-flight container on interrupt (no orphans). +# +# Required: runtime/export.env defining at least HOST_DATA_PATH, CTID, ACCOUNT. + +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +# --- DEFAULTS (overridable via runtime/export.env or the environment) --- +CONTAINER_DATA_PATH="/mnt/Data" +BOT_FILE="MS_SimpleExport.algo" +PWD_FILE="/mnt/Robots/password.txt" # container-internal path (console reads it) +IMAGE="ghcr.io/spotware/ctrader-console:latest" +HOST_ROBOTS_PATH="$REPO_ROOT/runtime/Robots" # holds MS_SimpleExport.algo + password.txt +SYMBOL_FILE="$REPO_ROOT/runtime/ScrapeSymbols.txt" +RETRIES="${RETRIES:-3}" # attempts before a transient failure aborts +RUN_TIMEOUT="${RUN_TIMEOUT:-600}" # seconds per backtest before it counts as a hang +DISCOVER_MIN_YEAR="${DISCOVER_MIN_YEAR:-2005}" # earliest year the first-data probe scans + +# --- DEPLOYMENT CONFIG (secrets/IDs/paths) --- +ENV_FILE="${ENV_FILE:-$REPO_ROOT/runtime/export.env}" +if [[ ! -f "$ENV_FILE" ]]; then + echo "❌ Missing $ENV_FILE — copy scripts/export.env.example and fill it in." >&2 + exit 1 +fi +# shellcheck disable=SC1090 +source "$ENV_FILE" + +: "${HOST_DATA_PATH:?HOST_DATA_PATH must be set in runtime/export.env}" +: "${CTID:?CTID must be set in runtime/export.env}" +: "${ACCOUNT:?ACCOUNT must be set in runtime/export.env}" + +CURRENT_YEAR=$(date +%Y) +CURRENT_MONTH=$((10#$(date +%m))) + +# Track an in-flight container so the trap can reap it on Ctrl-C / kill (no orphans). +CURRENT_CONTAINER="" +cleanup() { [[ -n "$CURRENT_CONTAINER" ]] && docker rm -f "$CURRENT_CONTAINER" >/dev/null 2>&1; } +trap cleanup EXIT INT TERM + +# Month arithmetic as a single 0-based index (year*12 + month-1) for easy comparison. +midx() { echo $(( $1 * 12 + $2 - 1 )); } +idx_year() { echo $(( $1 / 12 )); } +idx_month() { echo $(( $1 % 12 + 1 )); } + +# --------------------------------------------------------------------------- +# run_one: execute ONE backtest window and classify the outcome from the logs. +# returns 0 = success (DATA_FOUND / EXPORT_SUCCESS) +# 3 = clean empty ("No historical data for the specified period") +# 2 = transient failure (timeout, crash, login error, no terminator) +# args: S_DATE E_DATE SYMBOL MODE PROBING OUT_FILE +# --------------------------------------------------------------------------- +run_one() { + local S_DATE=$1 E_DATE=$2 SYMBOL=$3 MODE=$4 PROBING=$5 OUT_FILE=$6 + local C_NAME="ctrader-${SYMBOL//[^A-Za-z0-9]/-}-$(date +%s%N)" + local DATA_MODE="m1"; [[ "$MODE" == "Tick" ]] && DATA_MODE="ticks" + + CURRENT_CONTAINER="$C_NAME" + # NOTE: no --rm — the console lingers after the backtest; we drain logs then remove it. + docker run -d --name "$C_NAME" --init \ + -v "$HOST_ROBOTS_PATH":"/mnt/Robots" \ + -v "$HOST_DATA_PATH":"$CONTAINER_DATA_PATH" \ + -e TZ="Europe/Berlin" \ + "$IMAGE" \ + backtest "/mnt/Robots/$BOT_FILE" \ + --start="$S_DATE" --end="$E_DATE" --symbol="$SYMBOL" \ + --ctid="$CTID" --pwd-file="$PWD_FILE" --account="$ACCOUNT" \ + --period="M1" --data-mode="$DATA_MODE" \ + --Mode="$MODE" --IsProbing="$PROBING" \ + --OutputFile="${CONTAINER_DATA_PATH}/${OUT_FILE}" --full-access > /dev/null 2>&1 || { + echo " ⚠️ docker run failed to start container"; CURRENT_CONTAINER=""; return 2; } + + local saw_success=0 saw_empty=0 saw_error=0 saw_term=0 + local last_progress=0 now + while IFS= read -r line; do + if [[ "$line" == *"Progress"* ]]; then + now=$(date +%s) + if (( now - last_progress >= 5 )); then echo " [BOT] $line"; last_progress=$now; fi + else + echo " [BOT] $line" + fi + case "$line" in + *DATA_FOUND*|*EXPORT_SUCCESS*) saw_success=1; saw_term=1; break ;; + *"No historical data for the specified period"*) saw_empty=1 ;; + *"ERROR:"*) saw_error=1 ;; # the bot prints Print("ERROR: ...") on an OnStop exception + esac + [[ "$line" == "}" ]] && { saw_term=1; break; } # end-of-result terminator (always printed) + done < <(timeout "$RUN_TIMEOUT" docker logs -f "$C_NAME" 2>&1) + + docker rm -f "$C_NAME" >/dev/null 2>&1 + CURRENT_CONTAINER="" + + if (( saw_success )); then return 0 + elif (( saw_empty )); then return 3 + else return 2 # saw_error, no terminator (timeout/stream died), or `}` without a verdict + fi +} + +# --------------------------------------------------------------------------- +# run_bot: run_one with retry. Clean-empty is NOT retried. A transient failure +# that survives all retries returns 2 (the caller decides to abort). +# --------------------------------------------------------------------------- +run_bot() { + local attempt rc + for ((attempt=1; attempt<=RETRIES; attempt++)); do + run_one "$@"; rc=$? + case $rc in + 0) return 0 ;; + 3) return 3 ;; + 2) echo " ⚠️ attempt $attempt/$RETRIES failed (transient) for [$3 $1..$2]" + (( attempt < RETRIES )) && sleep $(( attempt * 5 )) ;; + esac + done + return 2 +} + +# validate_export FILE -> echoes "ok" | "empty" | "bad" +# "bad" catches a truncated/corrupt ZIP (e.g. a crash mid-write); "empty" is a valid +# ZIP whose entry has zero bytes (a ran-but-no-bars month we do not want to keep). +validate_export() { + python3 - "$1" <<'PY' +import sys, zipfile +try: + z = zipfile.ZipFile(sys.argv[1]) + if z.testzip() is not None: print("bad"); sys.exit() + infos = z.infolist() + if not infos: print("bad"); sys.exit() + print("empty" if sum(i.file_size for i in infos) == 0 else "ok") +except Exception: + print("bad") +PY +} + +# month_end_or_clamp Y M -> echoes dd/mm/yyyy ; the current (incomplete) month is +# clamped to yesterday. Echoes empty string if the current month has no complete day +# yet (today is the 1st) so the caller can skip it. +month_end_or_clamp() { + local y=$1 m=$2 MM; MM=$(printf "%02d" "$m") + if (( y == CURRENT_YEAR && m == CURRENT_MONTH )); then + local yest first; yest="$(date -d 'yesterday' +%Y-%m-%d)"; first="$y-$MM-01" + [[ "$yest" < "$first" ]] && { echo ""; return; } # today is the 1st: nothing complete yet + date -d "$yest" +%d/%m/%Y + else + date -d "$y-$MM-01 +1 month -1 day" +%d/%m/%Y + fi +} + +# scrape_month SYMBOL MODE EXT YEAR MONTH +# Scrapes one month into a .tmp, validates, and renames to the final file. Aborts the +# whole run on a transient failure (retries exhausted) or a corrupt artifact. +scrape_month() { + local S=$1 MODE=$2 EXT=$3 y=$4 m=$5 + local MM; MM=$(printf "%02d" "$m") + local FINAL="${S}_${y}_${MM}.${EXT}" + local TEMP="${FINAL}.tmp" + local S_DATE="01/${MM}/${y}" E_DATE + E_DATE="$(month_end_or_clamp "$y" "$m")" + [[ -z "$E_DATE" ]] && { echo " ↪︎ $y-$MM just started — nothing complete yet, skipping"; return 0; } + + echo "📦 Scraping $y-$MM ($S_DATE → $E_DATE)..." + run_bot "$S_DATE" "$E_DATE" "$S" "$MODE" "false" "$TEMP" + local rc=$? + case $rc in + 0) if [[ ! -f "$HOST_DATA_PATH/$TEMP" ]]; then + echo "❌ ABORT: bot reported success but $TEMP not found ($S $y-$MM)"; exit 1 + fi + case "$(validate_export "$HOST_DATA_PATH/$TEMP")" in + ok) if mv "$HOST_DATA_PATH/$TEMP" "$HOST_DATA_PATH/$FINAL"; then + echo " ✅ Saved: $FINAL"; SAVED=$((SAVED+1)) + else echo "❌ ABORT: rename $TEMP → $FINAL failed"; exit 1; fi ;; + empty) echo " ◦ empty result for $y-$MM — not saved"; rm -f "$HOST_DATA_PATH/$TEMP"; EMPTY=$((EMPTY+1)) ;; + bad) echo "❌ ABORT: corrupt/truncated ZIP for $S $y-$MM (likely a cut-off run)"; exit 1 ;; + esac ;; + 3) echo " ◦ no data (clean) for $y-$MM — skipped"; rm -f "$HOST_DATA_PATH/$TEMP" 2>/dev/null; EMPTY=$((EMPTY+1)) ;; + 2) echo "❌ ABORT: $S $y-$MM failed after $RETRIES retries (transient). Stopping so the gap is not lost."; exit 1 ;; + esac +} + +# probe_run SYMBOL MODE S_DATE E_DATE -> 0 if data present, 1 if clean-empty (aborts on transient) +probe_run() { + run_bot "$3" "$4" "$1" "$2" "true" "probe.tmp" + case $? in + 0) return 0 ;; + 3) return 1 ;; + 2) echo "❌ ABORT: probe failed after $RETRIES retries for $1 ($3..$4)"; exit 1 ;; + esac +} + +# discover_start SYMBOL MODE -> echoes "YEAR MONTH" of the first month with data, or +# nothing if the symbol has no data at all. Yearly probe uses December (catches data +# that started any time that year); monthly probe uses the FULL month (catches data +# starting after the 10th — the old probe only tested days 01–10). +discover_start() { + local S=$1 MODE=$2 y mm MM edate found_year="" + echo "🌑 Discovery: locating first data month for $S ($MODE)..." >&2 + for ((y=DISCOVER_MIN_YEAR; y&2 + if probe_run "$S" "$MODE" "01/12/$y" "31/12/$y"; then found_year=$y; break; fi + done + if [[ -z "$found_year" ]]; then found_year=$CURRENT_YEAR; fi # brand-new this year → check months below + for ((mm=1; mm<=12; mm++)); do + (( found_year == CURRENT_YEAR && mm > CURRENT_MONTH )) && break + MM=$(printf "%02d" "$mm") + edate="$(month_end_or_clamp "$found_year" "$mm")" + [[ -z "$edate" ]] && break + echo " 🧪 month $found_year-$MM" >&2 + if probe_run "$S" "$MODE" "01/$MM/$found_year" "$edate"; then echo "$found_year $mm"; return 0; fi + done + return 1 # no data anywhere +} + +# scan_existing SYMBOL EXT -> echoes "MIN_IDX MAX_IDX" of existing month files, or nothing. +scan_existing() { + local S=$1 EXT=$2 f base y m i min="" max="" + shopt -s nullglob + for f in "$HOST_DATA_PATH/${S}"_*_*."$EXT"; do + base=$(basename "$f") + if [[ "$base" =~ ^${S}_([0-9]{4})_([0-9]{2})\.${EXT}$ ]]; then + y=${BASH_REMATCH[1]}; m=$((10#${BASH_REMATCH[2]})) + i=$(midx "$y" "$m") + if [[ -z "$min" ]] || (( i < min )); then min=$i; fi + if [[ -z "$max" ]] || (( i > max )); then max=$i; fi + fi + done + shopt -u nullglob + [[ -n "$min" ]] && echo "$min $max" +} + +# --- SYMBOL PROCESSOR (gap-aware) --- +process_symbol() { + local S=$1 MODE=$2 + local EXT; EXT=$(echo "$MODE" | tr '[:upper:]' '[:lower:]') + local now_idx; now_idx=$(midx "$CURRENT_YEAR" "$CURRENT_MONTH") + + local start_idx latest_idx + local existing; existing=$(scan_existing "$S" "$EXT") + if [[ -n "$existing" ]]; then + start_idx=${existing% *}; latest_idx=${existing#* } + echo "🔄 Resume: $S has data $(idx_year "$start_idx")-$(printf '%02d' "$(idx_month "$start_idx")") .. $(idx_year "$latest_idx")-$(printf '%02d' "$(idx_month "$latest_idx")")" + else + local sd; sd=$(discover_start "$S" "$MODE") || { echo " ❌ No data at all for $S — skipping symbol."; return; } + start_idx=$(midx ${sd% *} ${sd#* }); latest_idx=-1 # nothing on disk yet + echo " ✨ First data: $(echo "$sd" | awk '{printf "%d-%02d", $1, $2}')" + fi + + # Walk every expected month from the first data month to the current month. + # Scrape a month iff: it is missing, OR it is the trailing edge (latest existing + # month — may be a partial current-month from a previous run) OR the current month. + # Known-good middle months are trusted and never overwritten. + local i y m MM FINAL + for ((i=start_idx; i<=now_idx; i++)); do + y=$(idx_year "$i"); m=$(idx_month "$i"); MM=$(printf "%02d" "$m") + FINAL="${S}_${y}_${MM}.${EXT}" + if [[ -f "$HOST_DATA_PATH/$FINAL" ]] && (( i != latest_idx && i != now_idx )); then + continue # trusted middle file + fi + scrape_month "$S" "$MODE" "$EXT" "$y" "$m" + done +} + +# --- EXECUTION (skipped when the script is sourced, e.g. for unit tests) --- +main() { +SAVED=0; EMPTY=0 + +# Clean up stale temp files at start (only *.tmp — never touches real month files). +[[ -d "$HOST_DATA_PATH" ]] && rm -f "$HOST_DATA_PATH"/*.tmp 2>/dev/null + +echo "🏁 Starting data export task: $(date)" + +if [[ ! -f "$SYMBOL_FILE" ]]; then + echo "❌ Error: symbol file $SYMBOL_FILE not found!"; exit 1 +fi + +SYMBOLS_DONE=0 +while IFS= read -r line || [[ -n "$line" ]]; do + line="${line//$'\r'/}" # strip CR (Windows line endings) + line="$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" # trim + [[ -z "$line" || "$line" =~ ^# ]] && continue + + read -r S MODES <<< "$line" + [[ -z "$MODES" ]] && MODES="M1" + + for MODE in $MODES; do + if [[ "$MODE" == "M1" || "$MODE" == "Tick" ]]; then + echo "------------------------------------------" + echo "🔍 Processing: $S (mode: $MODE)" + process_symbol "$S" "$MODE" + SYMBOLS_DONE=$((SYMBOLS_DONE+1)) + else + echo "⚠️ Skipping unknown mode: '$MODE' for symbol '$S'" + fi + done +done < "$SYMBOL_FILE" + +echo "------------------------------------------" +echo "🏁 Done: $(date) — $SYMBOLS_DONE jobs, $SAVED month(s) saved, $EMPTY empty/skipped." +} + +# Run main only when executed directly; `source`-ing the script (for tests) defines the +# functions without running the scrape. +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/scripts/export.env.example b/scripts/export.env.example new file mode 100644 index 0000000..4bc4640 --- /dev/null +++ b/scripts/export.env.example @@ -0,0 +1,17 @@ +# Deployment config for scripts/export-runner.sh. +# Copy this to runtime/export.env (gitignored) and fill in. Never commit the real file. + +# Output data directory on THIS host — the cBot writes here via the container mount. +# On this machine the Unraid "TickData" share is NFS-mounted at /mnt/tickdata, so the +# Pepperstone subfolder is: +HOST_DATA_PATH="/mnt/tickdata/Pepperstone" + +# cTrader ID login (email) and trading account number the console uses to pull data. +CTID="you@example.com" +ACCOUNT="0000000" + +# --- Optional overrides (the script derives sensible defaults from the repo) --- +# Must contain MS_SimpleExport.algo + password.txt: +#HOST_ROBOTS_PATH="$REPO_ROOT/runtime/Robots" +#SYMBOL_FILE="$REPO_ROOT/runtime/ScrapeSymbols.txt" +#IMAGE="ghcr.io/spotware/ctrader-console:latest" diff --git a/src/MS_SimpleExport/MS_SimpleExport.cs b/src/MS_SimpleExport/MS_SimpleExport.cs new file mode 100755 index 0000000..675d067 --- /dev/null +++ b/src/MS_SimpleExport/MS_SimpleExport.cs @@ -0,0 +1,108 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using cAlgo.API; + +namespace cAlgo.Robots +{ + public enum ExportMode { M1, Tick } + + [Robot(AccessRights = AccessRights.FullAccess)] + public class MS_SimpleExport : Robot + { + [StructLayout(LayoutKind.Sequential, Pack = 1)] + private struct M1Record { + public double Time; + public double O, H, L, C; + public float S; + public int V; + } + + [StructLayout(LayoutKind.Sequential, Pack = 1)] + private struct TickRecord { + public double Time; + public double Ask, Bid; + } + + [Parameter("Export Mode", DefaultValue = ExportMode.M1)] + public ExportMode Mode { get; set; } + + [Parameter("Output File (ZIP)", DefaultValue = "")] + public string OutputFile { get; set; } + + [Parameter("Is Probing", DefaultValue = false)] + public bool IsProbing { get; set; } + + private List _m1Buffer = new List(); + private List _tickBuffer = new List(); + + protected override void OnStart() { + // No price factor needed for double/float export + } + + protected override void OnTick() { + if (Mode != ExportMode.Tick) return; + + if (IsProbing) { + Print("DATA_FOUND"); + Stop(); + return; + } + + _tickBuffer.Add(new TickRecord { + Time = Server.TimeInUtc.ToOADate(), + Ask = Symbol.Ask, + Bid = Symbol.Bid + }); + } + + protected override void OnBar() { + if (Mode != ExportMode.M1) return; + + if (IsProbing) { + Print("DATA_FOUND"); + Stop(); + return; + } + + var bar = Bars.Last(1); + _m1Buffer.Add(new M1Record { + Time = bar.OpenTime.ToOADate(), + O = bar.Open, + H = bar.High, + L = bar.Low, + C = bar.Close, + S = (float)Symbol.Spread, + V = (int)bar.TickVolume + }); + } + + protected override void OnStop() { + if (IsProbing || string.IsNullOrEmpty(OutputFile)) return; + + try { + using var fileStream = new FileStream(OutputFile, FileMode.Create); + using var archive = new ZipArchive(fileStream, ZipArchiveMode.Create); + var entry = archive.CreateEntry($"{Symbol.Name}.bin", CompressionLevel.Fastest); + + using var entryStream = entry.Open(); + + if (Mode == ExportMode.M1) { + ReadOnlySpan span = CollectionsMarshal.AsSpan(_m1Buffer); + ReadOnlySpan byteSpan = MemoryMarshal.Cast(span); + if (!byteSpan.IsEmpty) entryStream.Write(byteSpan); + } else { + ReadOnlySpan span = CollectionsMarshal.AsSpan(_tickBuffer); + ReadOnlySpan byteSpan = MemoryMarshal.Cast(span); + if (!byteSpan.IsEmpty) entryStream.Write(byteSpan); + } + + Print("EXPORT_SUCCESS"); + } catch (Exception ex) { + Print("ERROR: {0}", ex.Message); + } + } + } +} \ No newline at end of file diff --git a/src/MS_SimpleExport/MS_SimpleExport.csproj b/src/MS_SimpleExport/MS_SimpleExport.csproj new file mode 100755 index 0000000..cc976cb --- /dev/null +++ b/src/MS_SimpleExport/MS_SimpleExport.csproj @@ -0,0 +1,23 @@ + + + + + net6.0 + latest + disable + disable + + + false + + + + + + + +