diff --git a/docs/export-runner-quirks.md b/docs/export-runner-quirks.md index 302007b..bd5fd13 100644 --- a/docs/export-runner-quirks.md +++ b/docs/export-runner-quirks.md @@ -79,3 +79,24 @@ data dir untouched): yesterday, and left the trusted middle `03` byte-identical. - **Clean-empty safety**: `run_one` on an empty future window returns rc 3 (skip), never the rc 2 that would abort the run. + +## Post-deploy fix — discovery stdout pollution (year-0000 abort) + +The first **new** symbol added after the rewrite (AUDUSD) aborted on the scheduled +2026-06-25 run with `--start=01/01/0 --end=31/01/0000` ("Value for parameter start can't +be parsed"). Root cause: a function-output contract violation, not bad data. + +`process_symbol` resolves a brand-new symbol's first month via `sd=$(discover_start …)` — +command substitution, so `discover_start`'s **stdout is its result channel**. But the +probes inside it (`probe_run` → `run_one`) stream the cBot's `[BOT] …` progress on stdout +too. Those lines leaked into `sd`; `start_idx=$(midx ${sd% *} ${sd#* })` then parsed +garbage and collapsed to `0` → the walk started at year 0, month 1 → an unparseable +`01/01/0` backtest. (The abort was correct behaviour: it stopped a ~24 000-iteration walk +from year 0 to now, and nothing was written to the data dir — the scrape failed before any +file op.) Only the no-existing-files branch calls `discover_start`, so only a *new* symbol +could trigger it — invisible until AUDUSD. + +**Fix**: every probe call inside `discover_start` is redirected `1>&2`, so its stdout +carries only the result line. A documented CONTRACT comment now guards the function. +Covered by `scripts/test-export-runner.sh` (source-level; stubs `probe_run` to emit the +same stdout noise and asserts a clean `YEAR MONTH` result — RED without the redirect). diff --git a/scripts/export-runner.sh b/scripts/export-runner.sh index ec84968..dc14e47 100755 --- a/scripts/export-runner.sh +++ b/scripts/export-runner.sh @@ -31,7 +31,7 @@ BOT_FILE="MS_SimpleExport.algo" PWD_FILE="/mnt/Robots/password.txt" # container-internal path (console reads it) IMAGE="ghcr.io/spotware/ctrader-console:latest" HOST_ROBOTS_PATH="$REPO_ROOT/runtime/Robots" # holds MS_SimpleExport.algo + password.txt -SYMBOL_FILE="$REPO_ROOT/runtime/ScrapeSymbols.txt" +SYMBOL_FILE="${SYMBOL_FILE:-$REPO_ROOT/runtime/ScrapeSymbols.txt}" # overridable for a scoped run/test RETRIES="${RETRIES:-3}" # attempts before a transient failure aborts RUN_TIMEOUT="${RUN_TIMEOUT:-600}" # seconds per backtest before it counts as a hang DISCOVER_MIN_YEAR="${DISCOVER_MIN_YEAR:-2005}" # earliest year the first-data probe scans @@ -211,12 +211,18 @@ probe_run() { # nothing if the symbol has no data at all. Yearly probe uses December (catches data # that started any time that year); monthly probe uses the FULL month (catches data # starting after the 10th — the old probe only tested days 01–10). +# +# CONTRACT: this function's STDOUT carries ONLY the result line; the caller reads it via +# command substitution. probe_run -> run_one stream the cBot's "[BOT] …" lines on stdout, +# so every probe call here MUST be redirected to stderr (`1>&2`) — otherwise that progress +# noise pollutes the captured value and start_idx is parsed from garbage (see the AUDUSD +# year-0000 abort, docs/export-runner-quirks.md). All diagnostics here go to stderr too. discover_start() { local S=$1 MODE=$2 y mm MM edate found_year="" echo "🌑 Discovery: locating first data month for $S ($MODE)..." >&2 for ((y=DISCOVER_MIN_YEAR; y&2 - if probe_run "$S" "$MODE" "01/12/$y" "31/12/$y"; then found_year=$y; break; fi + if probe_run "$S" "$MODE" "01/12/$y" "31/12/$y" 1>&2; then found_year=$y; break; fi done if [[ -z "$found_year" ]]; then found_year=$CURRENT_YEAR; fi # brand-new this year → check months below for ((mm=1; mm<=12; mm++)); do @@ -225,7 +231,7 @@ discover_start() { edate="$(month_end_or_clamp "$found_year" "$mm")" [[ -z "$edate" ]] && break echo " 🧪 month $found_year-$MM" >&2 - if probe_run "$S" "$MODE" "01/$MM/$found_year" "$edate"; then echo "$found_year $mm"; return 0; fi + if probe_run "$S" "$MODE" "01/$MM/$found_year" "$edate" 1>&2; then echo "$found_year $mm"; return 0; fi done return 1 # no data anywhere } diff --git a/scripts/test-export-runner.sh b/scripts/test-export-runner.sh new file mode 100755 index 0000000..2c5ea99 --- /dev/null +++ b/scripts/test-export-runner.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# +# Source-level unit tests for export-runner.sh. The runner is written to be sourceable +# (main runs only when executed directly), so we can load its functions, stub the parts +# that would spawn containers, and assert pure logic. No Docker, no broker, no data dir. +# +# Run: scripts/test-export-runner.sh (exit 0 = all pass) +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUNNER="$HERE/export-runner.sh" + +# The runner sources runtime/export.env and hard-requires HOST_DATA_PATH/CTID/ACCOUNT. +# Provide throwaway values so sourcing succeeds without the real deployment config. +export ENV_FILE; ENV_FILE="$(mktemp)" +cat > "$ENV_FILE" <<'ENV' +HOST_DATA_PATH=/nonexistent +CTID=test@example.com +ACCOUNT=0 +ENV +trap 'rm -f "$ENV_FILE"' EXIT + +# shellcheck disable=SC1090 +source "$RUNNER" + +PASS=0 FAIL=0 +check() { # check NAME EXPECTED ACTUAL + if [[ "$2" == "$3" ]]; then echo " ✓ $1"; PASS=$((PASS+1)) + else echo " ✗ $1 — expected [$2], got [$3]"; FAIL=$((FAIL+1)); fi +} + +# --- discover_start contamination regression ------------------------------------------- +# run_one streams "[BOT] …" progress on STDOUT; discover_start's STDOUT is its result +# channel (read via command substitution). A stub that mimics that noise must NOT leak +# into the result. Pre-fix this returned garbage like "0-000-…" -> start_idx 0 -> the +# year-0000 "--start=01/01/0" abort. CURRENT_YEAR/MONTH come from the host clock; pick a +# data-start strictly before today so the assertion is stable regardless of the date. +emit_noise() { echo " [BOT] Progress 50% | bla"; echo " [BOT] | Info | noise"; } + +# Scenario 1: data starts in a past December (yearly probe hits), Jan of that year empty, +# Feb has data -> expect " 2". +TARGET_YEAR=$(( CURRENT_YEAR - 1 )) +probe_run() { emit_noise + case "$3" in + "01/12/$TARGET_YEAR") return 0 ;; # yearly probe: this December has data + "01/02/$TARGET_YEAR") return 0 ;; # monthly probe: February is the first month + *) return 1 ;; + esac +} +got="$(discover_start FOO M1)" +check "discover_start: result is clean (no [BOT] leakage)" "$TARGET_YEAR 2" "$got" + +# Scenario 2: no data in any past year, first data in an early month of the current year. +probe_run() { emit_noise + [[ "$3" == "01/0$(( CURRENT_MONTH > 1 ? CURRENT_MONTH - 1 : CURRENT_MONTH ))/$CURRENT_YEAR" ]] && return 0 + return 1 +} +got="$(discover_start FOO M1)" +# first month of the current year that the stub reports data for: +want_m=$(( CURRENT_MONTH > 1 ? CURRENT_MONTH - 1 : CURRENT_MONTH )) +check "discover_start: current-year fallback, clean result" "$CURRENT_YEAR $want_m" "$got" + +# Scenario 3: genuinely no data anywhere -> empty stdout, rc 1. +probe_run() { emit_noise; return 1; } +got="$(discover_start FOO M1)"; rc=$? +check "discover_start: no data -> empty result" "" "$got" +check "discover_start: no data -> rc 1" "1" "$rc" + +# --- month index round-trip (sanity on the arithmetic the resume loop depends on) ------ +i=$(midx 2026 5) +check "midx/idx_year round-trip" "2026" "$(idx_year "$i")" +check "midx/idx_month round-trip" "5" "$(idx_month "$i")" + +echo "-----" +echo "$PASS passed, $FAIL failed" +[[ $FAIL -eq 0 ]]