#!/usr/bin/env bash
#
# Headless market-data export driver for the MS_SimpleExport cBot (hardened).
#
# Runs on THIS host and spawns one cTrader-console container per backtest window.
# Deployment-specific config lives in runtime/export.env (gitignored).
#
# Hardening over the original Unraid script (see docs/export-runner-quirks.md):
#   - Per-run timeout + 3-way classification (success / clean-empty / transient-fail)
#     read from the log stream; a transient failure is retried, then HARD-ABORTS the
#     whole run (never silently skipped) so gaps cannot form unnoticed.
#   - Gap-aware resume: every expected month from the symbol's first data month to now
#     is checked; only MISSING months and the trailing edge (latest existing + current)
#     are (re-)scraped. Known-good middle files are never overwritten.
#   - Current (incomplete) month end-date is clamped to yesterday, so partial current
#     month data is captured instead of being rejected for a future end date.
#   - Clean "No historical data" is treated as legitimately-empty -> skipped, no file
#     written (no empty-file pollution). Empty/corrupt ZIPs are caught before mv.
#   - No --rm: the container is kept until logs are drained, then removed explicitly;
#     a trap removes any in-flight container on interrupt (no orphans).
#
# Required: runtime/export.env defining at least HOST_DATA_PATH, CTID, ACCOUNT.

set -uo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"

# --- DEFAULTS (overridable via runtime/export.env or the environment) ---
CONTAINER_DATA_PATH="/mnt/Data"
BOT_FILE="MS_SimpleExport.algo"
PWD_FILE="/mnt/Robots/password.txt"          # container-internal path (console reads it)
IMAGE="ghcr.io/spotware/ctrader-console:latest"
HOST_ROBOTS_PATH="$REPO_ROOT/runtime/Robots" # holds MS_SimpleExport.algo + password.txt
SYMBOL_FILE="${SYMBOL_FILE:-$REPO_ROOT/runtime/ScrapeSymbols.txt}"  # overridable for a scoped run/test
RETRIES="${RETRIES:-3}"                       # attempts before a transient failure aborts
RUN_TIMEOUT="${RUN_TIMEOUT:-600}"             # seconds per backtest before it counts as a hang
DISCOVER_MIN_YEAR="${DISCOVER_MIN_YEAR:-2005}" # earliest year the first-data probe scans

# --- DEPLOYMENT CONFIG (secrets/IDs/paths) ---
ENV_FILE="${ENV_FILE:-$REPO_ROOT/runtime/export.env}"
if [[ ! -f "$ENV_FILE" ]]; then
    echo "❌ Missing $ENV_FILE — copy scripts/export.env.example and fill it in." >&2
    exit 1
fi
# shellcheck disable=SC1090
source "$ENV_FILE"

: "${HOST_DATA_PATH:?HOST_DATA_PATH must be set in runtime/export.env}"
: "${CTID:?CTID must be set in runtime/export.env}"
: "${ACCOUNT:?ACCOUNT must be set in runtime/export.env}"

CURRENT_YEAR=$(date +%Y)
CURRENT_MONTH=$((10#$(date +%m)))

# Track an in-flight container so the trap can reap it on Ctrl-C / kill (no orphans).
CURRENT_CONTAINER=""
cleanup() { [[ -n "$CURRENT_CONTAINER" ]] && docker rm -f "$CURRENT_CONTAINER" >/dev/null 2>&1; }
trap cleanup EXIT INT TERM

# Month arithmetic as a single 0-based index (year*12 + month-1) for easy comparison.
midx() { echo $(( $1 * 12 + $2 - 1 )); }
idx_year()  { echo $(( $1 / 12 )); }
idx_month() { echo $(( $1 % 12 + 1 )); }

# ---------------------------------------------------------------------------
# run_one: execute ONE backtest window and classify the outcome from the logs.
#   returns 0 = success (DATA_FOUND / EXPORT_SUCCESS)
#           3 = clean empty ("No historical data for the specified period")
#           2 = transient failure (timeout, crash, login error, no terminator)
# args: S_DATE E_DATE SYMBOL MODE PROBING OUT_FILE
# ---------------------------------------------------------------------------
run_one() {
    local S_DATE=$1 E_DATE=$2 SYMBOL=$3 MODE=$4 PROBING=$5 OUT_FILE=$6
    local C_NAME="ctrader-${SYMBOL//[^A-Za-z0-9]/-}-$(date +%s%N)"
    local DATA_MODE="m1"; [[ "$MODE" == "Tick" ]] && DATA_MODE="ticks"

    CURRENT_CONTAINER="$C_NAME"
    # NOTE: no --rm — the console lingers after the backtest; we drain logs then remove it.
    docker run -d --name "$C_NAME" --init \
        -v "$HOST_ROBOTS_PATH":"/mnt/Robots" \
        -v "$HOST_DATA_PATH":"$CONTAINER_DATA_PATH" \
        -e TZ="Europe/Berlin" \
        "$IMAGE" \
        backtest "/mnt/Robots/$BOT_FILE" \
        --start="$S_DATE" --end="$E_DATE" --symbol="$SYMBOL" \
        --ctid="$CTID" --pwd-file="$PWD_FILE" --account="$ACCOUNT" \
        --period="M1" --data-mode="$DATA_MODE" \
        --Mode="$MODE" --IsProbing="$PROBING" \
        --OutputFile="${CONTAINER_DATA_PATH}/${OUT_FILE}" --full-access > /dev/null 2>&1 || {
            echo "  ⚠️ docker run failed to start container"; CURRENT_CONTAINER=""; return 2; }

    local saw_success=0 saw_empty=0 saw_error=0 saw_term=0
    local last_progress=0 now
    while IFS= read -r line; do
        if [[ "$line" == *"Progress"* ]]; then
            now=$(date +%s)
            if (( now - last_progress >= 5 )); then echo "  [BOT] $line"; last_progress=$now; fi
        else
            echo "  [BOT] $line"
        fi
        case "$line" in
            *DATA_FOUND*|*EXPORT_SUCCESS*) saw_success=1; saw_term=1; break ;;
            *"No historical data for the specified period"*) saw_empty=1 ;;
            *"ERROR:"*) saw_error=1 ;;   # the bot prints Print("ERROR: ...") on an OnStop exception
        esac
        [[ "$line" == "}" ]] && { saw_term=1; break; }   # end-of-result terminator (always printed)
    done < <(timeout "$RUN_TIMEOUT" docker logs -f "$C_NAME" 2>&1)

    docker rm -f "$C_NAME" >/dev/null 2>&1
    CURRENT_CONTAINER=""

    if   (( saw_success )); then return 0   # DATA_FOUND / EXPORT_SUCCESS
    elif (( saw_error ));   then return 2   # bot raised an exception (e.g. write failure) -> retry/abort
    elif (( saw_empty ));   then return 3   # console: "No historical data for the specified period"
    elif (( saw_term ));    then return 3   # backtest reached its `}` summary but exported nothing -> data hole / empty month
    else                         return 2   # no terminator: timeout / crash / dead stream -> transient failure
    fi
}

# ---------------------------------------------------------------------------
# run_bot: run_one with retry. Clean-empty is NOT retried. A transient failure
# that survives all retries returns 2 (the caller decides to abort).
# ---------------------------------------------------------------------------
run_bot() {
    local attempt rc
    for ((attempt=1; attempt<=RETRIES; attempt++)); do
        run_one "$@"; rc=$?
        case $rc in
            0) return 0 ;;
            3) return 3 ;;
            2) echo "  ⚠️ attempt $attempt/$RETRIES failed (transient) for [$3 $1..$2]"
               (( attempt < RETRIES )) && sleep $(( attempt * 5 )) ;;
        esac
    done
    return 2
}

# validate_export FILE -> echoes "ok" | "empty" | "bad"
# "bad" catches a truncated/corrupt ZIP (e.g. a crash mid-write); "empty" is a valid
# ZIP whose entry has zero bytes (a ran-but-no-bars month we do not want to keep).
validate_export() {
    python3 - "$1" <<'PY'
import sys, zipfile
try:
    z = zipfile.ZipFile(sys.argv[1])
    if z.testzip() is not None: print("bad"); sys.exit()
    infos = z.infolist()
    if not infos: print("bad"); sys.exit()
    print("empty" if sum(i.file_size for i in infos) == 0 else "ok")
except Exception:
    print("bad")
PY
}

# month_end_or_clamp Y M -> echoes dd/mm/yyyy ; the current (incomplete) month is
# clamped to yesterday. Echoes empty string if the current month has no complete day
# yet (today is the 1st) so the caller can skip it.
month_end_or_clamp() {
    local y=$1 m=$2 MM; MM=$(printf "%02d" "$m")
    if (( y == CURRENT_YEAR && m == CURRENT_MONTH )); then
        local yest first; yest="$(date -d 'yesterday' +%Y-%m-%d)"; first="$y-$MM-01"
        [[ "$yest" < "$first" ]] && { echo ""; return; }   # today is the 1st: nothing complete yet
        date -d "$yest" +%d/%m/%Y
    else
        date -d "$y-$MM-01 +1 month -1 day" +%d/%m/%Y
    fi
}

# scrape_month SYMBOL MODE EXT YEAR MONTH
# Scrapes one month into a .tmp, validates, and renames to the final file. Aborts the
# whole run on a transient failure (retries exhausted) or a corrupt artifact.
scrape_month() {
    local S=$1 MODE=$2 EXT=$3 y=$4 m=$5
    local MM; MM=$(printf "%02d" "$m")
    local FINAL="${S}_${y}_${MM}.${EXT}"
    local TEMP="${FINAL}.tmp"
    local S_DATE="01/${MM}/${y}" E_DATE
    E_DATE="$(month_end_or_clamp "$y" "$m")"
    [[ -z "$E_DATE" ]] && { echo "  ↪︎ $y-$MM just started — nothing complete yet, skipping"; return 0; }

    echo "📦 Scraping $y-$MM ($S_DATE → $E_DATE)..."
    run_bot "$S_DATE" "$E_DATE" "$S" "$MODE" "false" "$TEMP"
    local rc=$?
    case $rc in
        0)  if [[ ! -f "$HOST_DATA_PATH/$TEMP" ]]; then
                echo "❌ ABORT: bot reported success but $TEMP not found ($S $y-$MM)"; exit 1
            fi
            case "$(validate_export "$HOST_DATA_PATH/$TEMP")" in
                ok)    if mv "$HOST_DATA_PATH/$TEMP" "$HOST_DATA_PATH/$FINAL"; then
                           echo "  ✅ Saved: $FINAL"; SAVED=$((SAVED+1))
                       else echo "❌ ABORT: rename $TEMP → $FINAL failed"; exit 1; fi ;;
                empty) echo "  ◦ empty result for $y-$MM — not saved"; rm -f "$HOST_DATA_PATH/$TEMP"; EMPTY=$((EMPTY+1)) ;;
                bad)   echo "❌ ABORT: corrupt/truncated ZIP for $S $y-$MM (likely a cut-off run)"; exit 1 ;;
            esac ;;
        3)  echo "  ◦ no data (clean) for $y-$MM — skipped"; rm -f "$HOST_DATA_PATH/$TEMP" 2>/dev/null; EMPTY=$((EMPTY+1)) ;;
        2)  echo "❌ ABORT: $S $y-$MM failed after $RETRIES retries (transient). Stopping so the gap is not lost."; exit 1 ;;
    esac
}

# probe_run SYMBOL MODE S_DATE E_DATE -> 0 if data present, 1 if clean-empty (aborts on transient)
probe_run() {
    run_bot "$3" "$4" "$1" "$2" "true" "probe.tmp"
    case $? in
        0) return 0 ;;
        3) return 1 ;;
        2) echo "❌ ABORT: probe failed after $RETRIES retries for $1 ($3..$4)"; exit 1 ;;
    esac
}

# discover_start SYMBOL MODE -> echoes "YEAR MONTH" of the first month with data, or
# nothing if the symbol has no data at all. Yearly probe uses December (catches data
# that started any time that year); monthly probe uses the FULL month (catches data
# starting after the 10th — the old probe only tested days 01–10).
#
# CONTRACT: this function's STDOUT carries ONLY the result line; the caller reads it via
# command substitution. probe_run -> run_one stream the cBot's "[BOT] …" lines on stdout,
# so every probe call here MUST be redirected to stderr (`1>&2`) — otherwise that progress
# noise pollutes the captured value and start_idx is parsed from garbage (see the AUDUSD
# year-0000 abort, docs/export-runner-quirks.md). All diagnostics here go to stderr too.
discover_start() {
    local S=$1 MODE=$2 y mm MM edate found_year=""
    echo "🌑 Discovery: locating first data month for $S ($MODE)..." >&2
    for ((y=DISCOVER_MIN_YEAR; y<CURRENT_YEAR; y++)); do
        echo "  🧪 year $y" >&2
        if probe_run "$S" "$MODE" "01/12/$y" "31/12/$y" 1>&2; then found_year=$y; break; fi
    done
    if [[ -z "$found_year" ]]; then found_year=$CURRENT_YEAR; fi   # brand-new this year → check months below
    for ((mm=1; mm<=12; mm++)); do
        (( found_year == CURRENT_YEAR && mm > CURRENT_MONTH )) && break
        MM=$(printf "%02d" "$mm")
        edate="$(month_end_or_clamp "$found_year" "$mm")"
        [[ -z "$edate" ]] && break
        echo "  🧪 month $found_year-$MM" >&2
        if probe_run "$S" "$MODE" "01/$MM/$found_year" "$edate" 1>&2; then echo "$found_year $mm"; return 0; fi
    done
    return 1   # no data anywhere
}

# scan_existing SYMBOL EXT -> echoes "MIN_IDX MAX_IDX" of existing month files, or nothing.
scan_existing() {
    local S=$1 EXT=$2 f base y m i min="" max=""
    shopt -s nullglob
    for f in "$HOST_DATA_PATH/${S}"_*_*."$EXT"; do
        base=$(basename "$f")
        if [[ "$base" =~ ^${S}_([0-9]{4})_([0-9]{2})\.${EXT}$ ]]; then
            y=${BASH_REMATCH[1]}; m=$((10#${BASH_REMATCH[2]}))
            i=$(midx "$y" "$m")
            if [[ -z "$min" ]] || (( i < min )); then min=$i; fi
            if [[ -z "$max" ]] || (( i > max )); then max=$i; fi
        fi
    done
    shopt -u nullglob
    [[ -n "$min" ]] && echo "$min $max"
}

# --- SYMBOL PROCESSOR (gap-aware) ---
process_symbol() {
    local S=$1 MODE=$2
    local EXT; EXT=$(echo "$MODE" | tr '[:upper:]' '[:lower:]')
    local now_idx; now_idx=$(midx "$CURRENT_YEAR" "$CURRENT_MONTH")

    local start_idx latest_idx
    local existing; existing=$(scan_existing "$S" "$EXT")
    if [[ -n "$existing" ]]; then
        start_idx=${existing% *}; latest_idx=${existing#* }
        echo "🔄 Resume: $S has data $(idx_year "$start_idx")-$(printf '%02d' "$(idx_month "$start_idx")") .. $(idx_year "$latest_idx")-$(printf '%02d' "$(idx_month "$latest_idx")")"
    else
        local sd; sd=$(discover_start "$S" "$MODE") || { echo "  ❌ No data at all for $S — skipping symbol."; return; }
        start_idx=$(midx ${sd% *} ${sd#* }); latest_idx=-1   # nothing on disk yet
        echo "  ✨ First data: $(echo "$sd" | awk '{printf "%d-%02d", $1, $2}')"
    fi

    # Walk every expected month from the first data month to the current month.
    # Scrape a month iff: it is missing, OR it is the trailing edge (latest existing
    # month — may be a partial current-month from a previous run) OR the current month.
    # Known-good middle months are trusted and never overwritten.
    local i y m MM FINAL
    for ((i=start_idx; i<=now_idx; i++)); do
        y=$(idx_year "$i"); m=$(idx_month "$i"); MM=$(printf "%02d" "$m")
        FINAL="${S}_${y}_${MM}.${EXT}"
        if [[ -f "$HOST_DATA_PATH/$FINAL" ]] && (( i != latest_idx && i != now_idx )); then
            continue   # trusted middle file
        fi
        scrape_month "$S" "$MODE" "$EXT" "$y" "$m"
    done
}

# --- EXECUTION (skipped when the script is sourced, e.g. for unit tests) ---
main() {
SAVED=0; EMPTY=0

# Clean up stale temp files at start (only *.tmp — never touches real month files).
[[ -d "$HOST_DATA_PATH" ]] && rm -f "$HOST_DATA_PATH"/*.tmp 2>/dev/null

echo "🏁 Starting data export task: $(date)"

if [[ ! -f "$SYMBOL_FILE" ]]; then
    echo "❌ Error: symbol file $SYMBOL_FILE not found!"; exit 1
fi

SYMBOLS_DONE=0
while IFS= read -r line || [[ -n "$line" ]]; do
    line="${line//$'\r'/}"                                   # strip CR (Windows line endings)
    line="$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"  # trim
    [[ -z "$line" || "$line" =~ ^# ]] && continue

    read -r S MODES <<< "$line"
    [[ -z "$MODES" ]] && MODES="M1"

    for MODE in $MODES; do
        if [[ "$MODE" == "M1" || "$MODE" == "Tick" ]]; then
            echo "------------------------------------------"
            echo "🔍 Processing: $S (mode: $MODE)"
            process_symbol "$S" "$MODE"
            SYMBOLS_DONE=$((SYMBOLS_DONE+1))
        else
            echo "⚠️  Skipping unknown mode: '$MODE' for symbol '$S'"
        fi
    done
done < "$SYMBOL_FILE"

echo "------------------------------------------"
echo "🏁 Done: $(date) — $SYMBOLS_DONE jobs, $SAVED month(s) saved, $EMPTY empty/skipped."
}

# Run main only when executed directly; `source`-ing the script (for tests) defines the
# functions without running the scrape.
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
    main "$@"
fi