Brummel
cc3f0cbd46
MS_SimpleExport is a market-data exporter (M1/Tick -> zipped binary), driven headless via the official cTrader-console Docker image. This adds the bot and a hardened bash driver that runs the scrape on a Linux host, spawning one ephemeral console container per backtest window. - src/MS_SimpleExport: the cBot; csproj aligned to the repo's Linux template (AlgoPublish=false, cTrader.Automate 1.*-*). - scripts/export-runner.sh: hardened port of the Unraid driver -- retry-then-abort on transient failure, gap-aware resume, current-month end-date clamp, 3-way run classification with a per-run timeout, and ZIP validation before rename. - scripts/export.env.example: deployment config template (real values live in the gitignored runtime/export.env). - docs/export-runner-quirks.md: the multi-lens audit findings and hardening decisions. - .gitignore: ignore runtime/ (secrets, the placed .algo, the symbol list).
323 lines
14 KiB
Bash
Executable File
323 lines
14 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
#
|
||
# Headless market-data export driver for the MS_SimpleExport cBot (hardened).
|
||
#
|
||
# Runs on THIS host and spawns one cTrader-console container per backtest window.
|
||
# Deployment-specific config lives in runtime/export.env (gitignored).
|
||
#
|
||
# Hardening over the original Unraid script (see docs/export-runner-quirks.md):
|
||
# - Per-run timeout + 3-way classification (success / clean-empty / transient-fail)
|
||
# read from the log stream; a transient failure is retried, then HARD-ABORTS the
|
||
# whole run (never silently skipped) so gaps cannot form unnoticed.
|
||
# - Gap-aware resume: every expected month from the symbol's first data month to now
|
||
# is checked; only MISSING months and the trailing edge (latest existing + current)
|
||
# are (re-)scraped. Known-good middle files are never overwritten.
|
||
# - Current (incomplete) month end-date is clamped to yesterday, so partial current
|
||
# month data is captured instead of being rejected for a future end date.
|
||
# - Clean "No historical data" is treated as legitimately-empty -> skipped, no file
|
||
# written (no empty-file pollution). Empty/corrupt ZIPs are caught before mv.
|
||
# - No --rm: the container is kept until logs are drained, then removed explicitly;
|
||
# a trap removes any in-flight container on interrupt (no orphans).
|
||
#
|
||
# Required: runtime/export.env defining at least HOST_DATA_PATH, CTID, ACCOUNT.
|
||
|
||
set -uo pipefail
|
||
|
||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||
|
||
# --- DEFAULTS (overridable via runtime/export.env or the environment) ---
|
||
CONTAINER_DATA_PATH="/mnt/Data"
|
||
BOT_FILE="MS_SimpleExport.algo"
|
||
PWD_FILE="/mnt/Robots/password.txt" # container-internal path (console reads it)
|
||
IMAGE="ghcr.io/spotware/ctrader-console:latest"
|
||
HOST_ROBOTS_PATH="$REPO_ROOT/runtime/Robots" # holds MS_SimpleExport.algo + password.txt
|
||
SYMBOL_FILE="$REPO_ROOT/runtime/ScrapeSymbols.txt"
|
||
RETRIES="${RETRIES:-3}" # attempts before a transient failure aborts
|
||
RUN_TIMEOUT="${RUN_TIMEOUT:-600}" # seconds per backtest before it counts as a hang
|
||
DISCOVER_MIN_YEAR="${DISCOVER_MIN_YEAR:-2005}" # earliest year the first-data probe scans
|
||
|
||
# --- DEPLOYMENT CONFIG (secrets/IDs/paths) ---
|
||
ENV_FILE="${ENV_FILE:-$REPO_ROOT/runtime/export.env}"
|
||
if [[ ! -f "$ENV_FILE" ]]; then
|
||
echo "❌ Missing $ENV_FILE — copy scripts/export.env.example and fill it in." >&2
|
||
exit 1
|
||
fi
|
||
# shellcheck disable=SC1090
|
||
source "$ENV_FILE"
|
||
|
||
: "${HOST_DATA_PATH:?HOST_DATA_PATH must be set in runtime/export.env}"
|
||
: "${CTID:?CTID must be set in runtime/export.env}"
|
||
: "${ACCOUNT:?ACCOUNT must be set in runtime/export.env}"
|
||
|
||
CURRENT_YEAR=$(date +%Y)
|
||
CURRENT_MONTH=$((10#$(date +%m)))
|
||
|
||
# Track an in-flight container so the trap can reap it on Ctrl-C / kill (no orphans).
|
||
CURRENT_CONTAINER=""
|
||
cleanup() { [[ -n "$CURRENT_CONTAINER" ]] && docker rm -f "$CURRENT_CONTAINER" >/dev/null 2>&1; }
|
||
trap cleanup EXIT INT TERM
|
||
|
||
# Month arithmetic as a single 0-based index (year*12 + month-1) for easy comparison.
|
||
midx() { echo $(( $1 * 12 + $2 - 1 )); }
|
||
idx_year() { echo $(( $1 / 12 )); }
|
||
idx_month() { echo $(( $1 % 12 + 1 )); }
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# run_one: execute ONE backtest window and classify the outcome from the logs.
|
||
# returns 0 = success (DATA_FOUND / EXPORT_SUCCESS)
|
||
# 3 = clean empty ("No historical data for the specified period")
|
||
# 2 = transient failure (timeout, crash, login error, no terminator)
|
||
# args: S_DATE E_DATE SYMBOL MODE PROBING OUT_FILE
|
||
# ---------------------------------------------------------------------------
|
||
run_one() {
|
||
local S_DATE=$1 E_DATE=$2 SYMBOL=$3 MODE=$4 PROBING=$5 OUT_FILE=$6
|
||
local C_NAME="ctrader-${SYMBOL//[^A-Za-z0-9]/-}-$(date +%s%N)"
|
||
local DATA_MODE="m1"; [[ "$MODE" == "Tick" ]] && DATA_MODE="ticks"
|
||
|
||
CURRENT_CONTAINER="$C_NAME"
|
||
# NOTE: no --rm — the console lingers after the backtest; we drain logs then remove it.
|
||
docker run -d --name "$C_NAME" --init \
|
||
-v "$HOST_ROBOTS_PATH":"/mnt/Robots" \
|
||
-v "$HOST_DATA_PATH":"$CONTAINER_DATA_PATH" \
|
||
-e TZ="Europe/Berlin" \
|
||
"$IMAGE" \
|
||
backtest "/mnt/Robots/$BOT_FILE" \
|
||
--start="$S_DATE" --end="$E_DATE" --symbol="$SYMBOL" \
|
||
--ctid="$CTID" --pwd-file="$PWD_FILE" --account="$ACCOUNT" \
|
||
--period="M1" --data-mode="$DATA_MODE" \
|
||
--Mode="$MODE" --IsProbing="$PROBING" \
|
||
--OutputFile="${CONTAINER_DATA_PATH}/${OUT_FILE}" --full-access > /dev/null 2>&1 || {
|
||
echo " ⚠️ docker run failed to start container"; CURRENT_CONTAINER=""; return 2; }
|
||
|
||
local saw_success=0 saw_empty=0 saw_error=0 saw_term=0
|
||
local last_progress=0 now
|
||
while IFS= read -r line; do
|
||
if [[ "$line" == *"Progress"* ]]; then
|
||
now=$(date +%s)
|
||
if (( now - last_progress >= 5 )); then echo " [BOT] $line"; last_progress=$now; fi
|
||
else
|
||
echo " [BOT] $line"
|
||
fi
|
||
case "$line" in
|
||
*DATA_FOUND*|*EXPORT_SUCCESS*) saw_success=1; saw_term=1; break ;;
|
||
*"No historical data for the specified period"*) saw_empty=1 ;;
|
||
*"ERROR:"*) saw_error=1 ;; # the bot prints Print("ERROR: ...") on an OnStop exception
|
||
esac
|
||
[[ "$line" == "}" ]] && { saw_term=1; break; } # end-of-result terminator (always printed)
|
||
done < <(timeout "$RUN_TIMEOUT" docker logs -f "$C_NAME" 2>&1)
|
||
|
||
docker rm -f "$C_NAME" >/dev/null 2>&1
|
||
CURRENT_CONTAINER=""
|
||
|
||
if (( saw_success )); then return 0
|
||
elif (( saw_empty )); then return 3
|
||
else return 2 # saw_error, no terminator (timeout/stream died), or `}` without a verdict
|
||
fi
|
||
}
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# run_bot: run_one with retry. Clean-empty is NOT retried. A transient failure
|
||
# that survives all retries returns 2 (the caller decides to abort).
|
||
# ---------------------------------------------------------------------------
|
||
run_bot() {
|
||
local attempt rc
|
||
for ((attempt=1; attempt<=RETRIES; attempt++)); do
|
||
run_one "$@"; rc=$?
|
||
case $rc in
|
||
0) return 0 ;;
|
||
3) return 3 ;;
|
||
2) echo " ⚠️ attempt $attempt/$RETRIES failed (transient) for [$3 $1..$2]"
|
||
(( attempt < RETRIES )) && sleep $(( attempt * 5 )) ;;
|
||
esac
|
||
done
|
||
return 2
|
||
}
|
||
|
||
# validate_export FILE -> echoes "ok" | "empty" | "bad"
|
||
# "bad" catches a truncated/corrupt ZIP (e.g. a crash mid-write); "empty" is a valid
|
||
# ZIP whose entry has zero bytes (a ran-but-no-bars month we do not want to keep).
|
||
validate_export() {
|
||
python3 - "$1" <<'PY'
|
||
import sys, zipfile
|
||
try:
|
||
z = zipfile.ZipFile(sys.argv[1])
|
||
if z.testzip() is not None: print("bad"); sys.exit()
|
||
infos = z.infolist()
|
||
if not infos: print("bad"); sys.exit()
|
||
print("empty" if sum(i.file_size for i in infos) == 0 else "ok")
|
||
except Exception:
|
||
print("bad")
|
||
PY
|
||
}
|
||
|
||
# month_end_or_clamp Y M -> echoes dd/mm/yyyy ; the current (incomplete) month is
|
||
# clamped to yesterday. Echoes empty string if the current month has no complete day
|
||
# yet (today is the 1st) so the caller can skip it.
|
||
month_end_or_clamp() {
|
||
local y=$1 m=$2 MM; MM=$(printf "%02d" "$m")
|
||
if (( y == CURRENT_YEAR && m == CURRENT_MONTH )); then
|
||
local yest first; yest="$(date -d 'yesterday' +%Y-%m-%d)"; first="$y-$MM-01"
|
||
[[ "$yest" < "$first" ]] && { echo ""; return; } # today is the 1st: nothing complete yet
|
||
date -d "$yest" +%d/%m/%Y
|
||
else
|
||
date -d "$y-$MM-01 +1 month -1 day" +%d/%m/%Y
|
||
fi
|
||
}
|
||
|
||
# scrape_month SYMBOL MODE EXT YEAR MONTH
|
||
# Scrapes one month into a .tmp, validates, and renames to the final file. Aborts the
|
||
# whole run on a transient failure (retries exhausted) or a corrupt artifact.
|
||
scrape_month() {
|
||
local S=$1 MODE=$2 EXT=$3 y=$4 m=$5
|
||
local MM; MM=$(printf "%02d" "$m")
|
||
local FINAL="${S}_${y}_${MM}.${EXT}"
|
||
local TEMP="${FINAL}.tmp"
|
||
local S_DATE="01/${MM}/${y}" E_DATE
|
||
E_DATE="$(month_end_or_clamp "$y" "$m")"
|
||
[[ -z "$E_DATE" ]] && { echo " ↪︎ $y-$MM just started — nothing complete yet, skipping"; return 0; }
|
||
|
||
echo "📦 Scraping $y-$MM ($S_DATE → $E_DATE)..."
|
||
run_bot "$S_DATE" "$E_DATE" "$S" "$MODE" "false" "$TEMP"
|
||
local rc=$?
|
||
case $rc in
|
||
0) if [[ ! -f "$HOST_DATA_PATH/$TEMP" ]]; then
|
||
echo "❌ ABORT: bot reported success but $TEMP not found ($S $y-$MM)"; exit 1
|
||
fi
|
||
case "$(validate_export "$HOST_DATA_PATH/$TEMP")" in
|
||
ok) if mv "$HOST_DATA_PATH/$TEMP" "$HOST_DATA_PATH/$FINAL"; then
|
||
echo " ✅ Saved: $FINAL"; SAVED=$((SAVED+1))
|
||
else echo "❌ ABORT: rename $TEMP → $FINAL failed"; exit 1; fi ;;
|
||
empty) echo " ◦ empty result for $y-$MM — not saved"; rm -f "$HOST_DATA_PATH/$TEMP"; EMPTY=$((EMPTY+1)) ;;
|
||
bad) echo "❌ ABORT: corrupt/truncated ZIP for $S $y-$MM (likely a cut-off run)"; exit 1 ;;
|
||
esac ;;
|
||
3) echo " ◦ no data (clean) for $y-$MM — skipped"; rm -f "$HOST_DATA_PATH/$TEMP" 2>/dev/null; EMPTY=$((EMPTY+1)) ;;
|
||
2) echo "❌ ABORT: $S $y-$MM failed after $RETRIES retries (transient). Stopping so the gap is not lost."; exit 1 ;;
|
||
esac
|
||
}
|
||
|
||
# probe_run SYMBOL MODE S_DATE E_DATE -> 0 if data present, 1 if clean-empty (aborts on transient)
|
||
probe_run() {
|
||
run_bot "$3" "$4" "$1" "$2" "true" "probe.tmp"
|
||
case $? in
|
||
0) return 0 ;;
|
||
3) return 1 ;;
|
||
2) echo "❌ ABORT: probe failed after $RETRIES retries for $1 ($3..$4)"; exit 1 ;;
|
||
esac
|
||
}
|
||
|
||
# discover_start SYMBOL MODE -> echoes "YEAR MONTH" of the first month with data, or
|
||
# nothing if the symbol has no data at all. Yearly probe uses December (catches data
|
||
# that started any time that year); monthly probe uses the FULL month (catches data
|
||
# starting after the 10th — the old probe only tested days 01–10).
|
||
discover_start() {
|
||
local S=$1 MODE=$2 y mm MM edate found_year=""
|
||
echo "🌑 Discovery: locating first data month for $S ($MODE)..." >&2
|
||
for ((y=DISCOVER_MIN_YEAR; y<CURRENT_YEAR; y++)); do
|
||
echo " 🧪 year $y" >&2
|
||
if probe_run "$S" "$MODE" "01/12/$y" "31/12/$y"; then found_year=$y; break; fi
|
||
done
|
||
if [[ -z "$found_year" ]]; then found_year=$CURRENT_YEAR; fi # brand-new this year → check months below
|
||
for ((mm=1; mm<=12; mm++)); do
|
||
(( found_year == CURRENT_YEAR && mm > CURRENT_MONTH )) && break
|
||
MM=$(printf "%02d" "$mm")
|
||
edate="$(month_end_or_clamp "$found_year" "$mm")"
|
||
[[ -z "$edate" ]] && break
|
||
echo " 🧪 month $found_year-$MM" >&2
|
||
if probe_run "$S" "$MODE" "01/$MM/$found_year" "$edate"; then echo "$found_year $mm"; return 0; fi
|
||
done
|
||
return 1 # no data anywhere
|
||
}
|
||
|
||
# scan_existing SYMBOL EXT -> echoes "MIN_IDX MAX_IDX" of existing month files, or nothing.
|
||
scan_existing() {
|
||
local S=$1 EXT=$2 f base y m i min="" max=""
|
||
shopt -s nullglob
|
||
for f in "$HOST_DATA_PATH/${S}"_*_*."$EXT"; do
|
||
base=$(basename "$f")
|
||
if [[ "$base" =~ ^${S}_([0-9]{4})_([0-9]{2})\.${EXT}$ ]]; then
|
||
y=${BASH_REMATCH[1]}; m=$((10#${BASH_REMATCH[2]}))
|
||
i=$(midx "$y" "$m")
|
||
if [[ -z "$min" ]] || (( i < min )); then min=$i; fi
|
||
if [[ -z "$max" ]] || (( i > max )); then max=$i; fi
|
||
fi
|
||
done
|
||
shopt -u nullglob
|
||
[[ -n "$min" ]] && echo "$min $max"
|
||
}
|
||
|
||
# --- SYMBOL PROCESSOR (gap-aware) ---
|
||
process_symbol() {
|
||
local S=$1 MODE=$2
|
||
local EXT; EXT=$(echo "$MODE" | tr '[:upper:]' '[:lower:]')
|
||
local now_idx; now_idx=$(midx "$CURRENT_YEAR" "$CURRENT_MONTH")
|
||
|
||
local start_idx latest_idx
|
||
local existing; existing=$(scan_existing "$S" "$EXT")
|
||
if [[ -n "$existing" ]]; then
|
||
start_idx=${existing% *}; latest_idx=${existing#* }
|
||
echo "🔄 Resume: $S has data $(idx_year "$start_idx")-$(printf '%02d' "$(idx_month "$start_idx")") .. $(idx_year "$latest_idx")-$(printf '%02d' "$(idx_month "$latest_idx")")"
|
||
else
|
||
local sd; sd=$(discover_start "$S" "$MODE") || { echo " ❌ No data at all for $S — skipping symbol."; return; }
|
||
start_idx=$(midx ${sd% *} ${sd#* }); latest_idx=-1 # nothing on disk yet
|
||
echo " ✨ First data: $(echo "$sd" | awk '{printf "%d-%02d", $1, $2}')"
|
||
fi
|
||
|
||
# Walk every expected month from the first data month to the current month.
|
||
# Scrape a month iff: it is missing, OR it is the trailing edge (latest existing
|
||
# month — may be a partial current-month from a previous run) OR the current month.
|
||
# Known-good middle months are trusted and never overwritten.
|
||
local i y m MM FINAL
|
||
for ((i=start_idx; i<=now_idx; i++)); do
|
||
y=$(idx_year "$i"); m=$(idx_month "$i"); MM=$(printf "%02d" "$m")
|
||
FINAL="${S}_${y}_${MM}.${EXT}"
|
||
if [[ -f "$HOST_DATA_PATH/$FINAL" ]] && (( i != latest_idx && i != now_idx )); then
|
||
continue # trusted middle file
|
||
fi
|
||
scrape_month "$S" "$MODE" "$EXT" "$y" "$m"
|
||
done
|
||
}
|
||
|
||
# --- EXECUTION (skipped when the script is sourced, e.g. for unit tests) ---
|
||
main() {
|
||
SAVED=0; EMPTY=0
|
||
|
||
# Clean up stale temp files at start (only *.tmp — never touches real month files).
|
||
[[ -d "$HOST_DATA_PATH" ]] && rm -f "$HOST_DATA_PATH"/*.tmp 2>/dev/null
|
||
|
||
echo "🏁 Starting data export task: $(date)"
|
||
|
||
if [[ ! -f "$SYMBOL_FILE" ]]; then
|
||
echo "❌ Error: symbol file $SYMBOL_FILE not found!"; exit 1
|
||
fi
|
||
|
||
SYMBOLS_DONE=0
|
||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||
line="${line//$'\r'/}" # strip CR (Windows line endings)
|
||
line="$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" # trim
|
||
[[ -z "$line" || "$line" =~ ^# ]] && continue
|
||
|
||
read -r S MODES <<< "$line"
|
||
[[ -z "$MODES" ]] && MODES="M1"
|
||
|
||
for MODE in $MODES; do
|
||
if [[ "$MODE" == "M1" || "$MODE" == "Tick" ]]; then
|
||
echo "------------------------------------------"
|
||
echo "🔍 Processing: $S (mode: $MODE)"
|
||
process_symbol "$S" "$MODE"
|
||
SYMBOLS_DONE=$((SYMBOLS_DONE+1))
|
||
else
|
||
echo "⚠️ Skipping unknown mode: '$MODE' for symbol '$S'"
|
||
fi
|
||
done
|
||
done < "$SYMBOL_FILE"
|
||
|
||
echo "------------------------------------------"
|
||
echo "🏁 Done: $(date) — $SYMBOLS_DONE jobs, $SAVED month(s) saved, $EMPTY empty/skipped."
|
||
}
|
||
|
||
# Run main only when executed directly; `source`-ing the script (for tests) defines the
|
||
# functions without running the scrape.
|
||
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
||
main "$@"
|
||
fi
|