export-runner: fix new-symbol discovery aborting with a year-0000 backtest
discover_start's stdout is its result channel (read via command substitution in process_symbol), but the probes inside it stream the cBot's "[BOT] …" progress on stdout too. That noise leaked into the captured value, so start_idx parsed to 0 and the resume walk began at year 0 -> an unparseable "--start=01/01/0" backtest that retried out and hard-aborted the whole run. Only a brand-new symbol (no files on disk yet) hits this path, so it stayed invisible until AUDUSD was added. Redirect every probe call in discover_start to stderr (1>&2) so its stdout carries only the "YEAR MONTH" result, and document the contract. Add scripts/test-export-runner.sh: source-level tests that stub probe_run to emit the same stdout noise and assert a clean result (RED without the redirect). Make SYMBOL_FILE overridable for scoped runs/tests.
This commit is contained in:
@@ -31,7 +31,7 @@ BOT_FILE="MS_SimpleExport.algo"
|
||||
PWD_FILE="/mnt/Robots/password.txt" # container-internal path (console reads it)
|
||||
IMAGE="ghcr.io/spotware/ctrader-console:latest"
|
||||
HOST_ROBOTS_PATH="$REPO_ROOT/runtime/Robots" # holds MS_SimpleExport.algo + password.txt
|
||||
SYMBOL_FILE="$REPO_ROOT/runtime/ScrapeSymbols.txt"
|
||||
SYMBOL_FILE="${SYMBOL_FILE:-$REPO_ROOT/runtime/ScrapeSymbols.txt}" # overridable for a scoped run/test
|
||||
RETRIES="${RETRIES:-3}" # attempts before a transient failure aborts
|
||||
RUN_TIMEOUT="${RUN_TIMEOUT:-600}" # seconds per backtest before it counts as a hang
|
||||
DISCOVER_MIN_YEAR="${DISCOVER_MIN_YEAR:-2005}" # earliest year the first-data probe scans
|
||||
@@ -211,12 +211,18 @@ probe_run() {
|
||||
# nothing if the symbol has no data at all. Yearly probe uses December (catches data
|
||||
# that started any time that year); monthly probe uses the FULL month (catches data
|
||||
# starting after the 10th — the old probe only tested days 01–10).
|
||||
#
|
||||
# CONTRACT: this function's STDOUT carries ONLY the result line; the caller reads it via
|
||||
# command substitution. probe_run -> run_one stream the cBot's "[BOT] …" lines on stdout,
|
||||
# so every probe call here MUST be redirected to stderr (`1>&2`) — otherwise that progress
|
||||
# noise pollutes the captured value and start_idx is parsed from garbage (see the AUDUSD
|
||||
# year-0000 abort, docs/export-runner-quirks.md). All diagnostics here go to stderr too.
|
||||
discover_start() {
|
||||
local S=$1 MODE=$2 y mm MM edate found_year=""
|
||||
echo "🌑 Discovery: locating first data month for $S ($MODE)..." >&2
|
||||
for ((y=DISCOVER_MIN_YEAR; y<CURRENT_YEAR; y++)); do
|
||||
echo " 🧪 year $y" >&2
|
||||
if probe_run "$S" "$MODE" "01/12/$y" "31/12/$y"; then found_year=$y; break; fi
|
||||
if probe_run "$S" "$MODE" "01/12/$y" "31/12/$y" 1>&2; then found_year=$y; break; fi
|
||||
done
|
||||
if [[ -z "$found_year" ]]; then found_year=$CURRENT_YEAR; fi # brand-new this year → check months below
|
||||
for ((mm=1; mm<=12; mm++)); do
|
||||
@@ -225,7 +231,7 @@ discover_start() {
|
||||
edate="$(month_end_or_clamp "$found_year" "$mm")"
|
||||
[[ -z "$edate" ]] && break
|
||||
echo " 🧪 month $found_year-$MM" >&2
|
||||
if probe_run "$S" "$MODE" "01/$MM/$found_year" "$edate"; then echo "$found_year $mm"; return 0; fi
|
||||
if probe_run "$S" "$MODE" "01/$MM/$found_year" "$edate" 1>&2; then echo "$found_year $mm"; return 0; fi
|
||||
done
|
||||
return 1 # no data anywhere
|
||||
}
|
||||
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Source-level unit tests for export-runner.sh. The runner is written to be sourceable
|
||||
# (main runs only when executed directly), so we can load its functions, stub the parts
|
||||
# that would spawn containers, and assert pure logic. No Docker, no broker, no data dir.
|
||||
#
|
||||
# Run: scripts/test-export-runner.sh (exit 0 = all pass)
|
||||
set -uo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
RUNNER="$HERE/export-runner.sh"
|
||||
|
||||
# The runner sources runtime/export.env and hard-requires HOST_DATA_PATH/CTID/ACCOUNT.
|
||||
# Provide throwaway values so sourcing succeeds without the real deployment config.
|
||||
export ENV_FILE; ENV_FILE="$(mktemp)"
|
||||
cat > "$ENV_FILE" <<'ENV'
|
||||
HOST_DATA_PATH=/nonexistent
|
||||
CTID=test@example.com
|
||||
ACCOUNT=0
|
||||
ENV
|
||||
trap 'rm -f "$ENV_FILE"' EXIT
|
||||
|
||||
# shellcheck disable=SC1090
|
||||
source "$RUNNER"
|
||||
|
||||
PASS=0 FAIL=0
|
||||
check() { # check NAME EXPECTED ACTUAL
|
||||
if [[ "$2" == "$3" ]]; then echo " ✓ $1"; PASS=$((PASS+1))
|
||||
else echo " ✗ $1 — expected [$2], got [$3]"; FAIL=$((FAIL+1)); fi
|
||||
}
|
||||
|
||||
# --- discover_start contamination regression -------------------------------------------
|
||||
# run_one streams "[BOT] …" progress on STDOUT; discover_start's STDOUT is its result
|
||||
# channel (read via command substitution). A stub that mimics that noise must NOT leak
|
||||
# into the result. Pre-fix this returned garbage like "0-000-…" -> start_idx 0 -> the
|
||||
# year-0000 "--start=01/01/0" abort. CURRENT_YEAR/MONTH come from the host clock; pick a
|
||||
# data-start strictly before today so the assertion is stable regardless of the date.
|
||||
emit_noise() { echo " [BOT] Progress 50% | bla"; echo " [BOT] | Info | noise"; }
|
||||
|
||||
# Scenario 1: data starts in a past December (yearly probe hits), Jan of that year empty,
|
||||
# Feb has data -> expect "<year> 2".
|
||||
TARGET_YEAR=$(( CURRENT_YEAR - 1 ))
|
||||
probe_run() { emit_noise
|
||||
case "$3" in
|
||||
"01/12/$TARGET_YEAR") return 0 ;; # yearly probe: this December has data
|
||||
"01/02/$TARGET_YEAR") return 0 ;; # monthly probe: February is the first month
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
got="$(discover_start FOO M1)"
|
||||
check "discover_start: result is clean (no [BOT] leakage)" "$TARGET_YEAR 2" "$got"
|
||||
|
||||
# Scenario 2: no data in any past year, first data in an early month of the current year.
|
||||
probe_run() { emit_noise
|
||||
[[ "$3" == "01/0$(( CURRENT_MONTH > 1 ? CURRENT_MONTH - 1 : CURRENT_MONTH ))/$CURRENT_YEAR" ]] && return 0
|
||||
return 1
|
||||
}
|
||||
got="$(discover_start FOO M1)"
|
||||
# first month of the current year that the stub reports data for:
|
||||
want_m=$(( CURRENT_MONTH > 1 ? CURRENT_MONTH - 1 : CURRENT_MONTH ))
|
||||
check "discover_start: current-year fallback, clean result" "$CURRENT_YEAR $want_m" "$got"
|
||||
|
||||
# Scenario 3: genuinely no data anywhere -> empty stdout, rc 1.
|
||||
probe_run() { emit_noise; return 1; }
|
||||
got="$(discover_start FOO M1)"; rc=$?
|
||||
check "discover_start: no data -> empty result" "" "$got"
|
||||
check "discover_start: no data -> rc 1" "1" "$rc"
|
||||
|
||||
# --- month index round-trip (sanity on the arithmetic the resume loop depends on) ------
|
||||
i=$(midx 2026 5)
|
||||
check "midx/idx_year round-trip" "2026" "$(idx_year "$i")"
|
||||
check "midx/idx_month round-trip" "5" "$(idx_month "$i")"
|
||||
|
||||
echo "-----"
|
||||
echo "$PASS passed, $FAIL failed"
|
||||
[[ $FAIL -eq 0 ]]
|
||||
Reference in New Issue
Block a user