110 lines
3.4 KiB
Bash
Executable File
110 lines
3.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Regression test for the whisper service: runs every fixture under
|
|
# tests/fixtures/dictations/ through /asr twice (with and without hotwords)
|
|
# and reports word error rate vs the golden `.expected.txt`.
|
|
#
|
|
# Usage:
|
|
# ./scripts/regress_whisper.sh
|
|
# WHISPER_URL=http://minerva.lan:9001 ./scripts/regress_whisper.sh
|
|
# ./scripts/regress_whisper.sh --verbose # also print each hypothesis
|
|
#
|
|
# Exit code is the sum of mismatches across all hotwords runs (0 = perfect).
|
|
set -euo pipefail
|
|
|
|
WHISPER_URL="${WHISPER_URL:-http://minerva.lan:9001}"
|
|
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
|
DIR="$REPO_ROOT/tests/fixtures/dictations"
|
|
VERBOSE=0
|
|
[[ "${1:-}" == "--verbose" || "${1:-}" == "-v" ]] && VERBOSE=1
|
|
|
|
die() { echo "error: $*" >&2; exit 1; }
|
|
[ -d "$DIR" ] || die "fixtures dir missing: $DIR"
|
|
command -v python3 >/dev/null || die "python3 not found"
|
|
|
|
# POST a single fixture. If hotwords arg is non-empty, forward it as a form
|
|
# field so the whisper service can feed it into faster_whisper.
|
|
run_once() {
|
|
local m4a="$1" hotwords="$2"
|
|
if [ -n "$hotwords" ]; then
|
|
curl -sS --max-time 120 -X POST "$WHISPER_URL/asr?output=txt&language=de" \
|
|
-F "audio_file=@$m4a" \
|
|
-F "hotwords=$hotwords"
|
|
else
|
|
curl -sS --max-time 120 -X POST "$WHISPER_URL/asr?output=txt&language=de" \
|
|
-F "audio_file=@$m4a"
|
|
fi
|
|
}
|
|
|
|
# Word error rate via Levenshtein. Prints "errors/ref_len".
|
|
wer() {
|
|
python3 - "$1" "$2" <<'PY'
|
|
import re, sys, unicodedata
|
|
|
|
def norm(path):
|
|
with open(path, encoding="utf-8") as f:
|
|
s = f.read()
|
|
s = unicodedata.normalize("NFC", s).lower()
|
|
s = re.sub(r"[^\w\s]", " ", s, flags=re.UNICODE)
|
|
return s.split()
|
|
|
|
def lev(a, b):
|
|
m, n = len(a), len(b)
|
|
if m == 0 or n == 0: return max(m, n)
|
|
prev = list(range(n + 1))
|
|
for i, ca in enumerate(a, 1):
|
|
cur = [i] + [0] * n
|
|
for j, cb in enumerate(b, 1):
|
|
cost = 0 if ca == cb else 1
|
|
cur[j] = min(cur[j - 1] + 1, prev[j] + 1, prev[j - 1] + cost)
|
|
prev = cur
|
|
return prev[n]
|
|
|
|
ref, hyp = norm(sys.argv[1]), norm(sys.argv[2])
|
|
print(f"{lev(ref, hyp)}/{len(ref)}")
|
|
PY
|
|
}
|
|
|
|
printf "%-38s %-13s %-13s\n" "fixture" "no_hotwords" "with_hotwords"
|
|
printf "%-38s %-13s %-13s\n" "--------------------------------------" "-------------" "-------------"
|
|
|
|
total_hw_errs=0
|
|
for m4a in "$DIR"/*.m4a; do
|
|
name="$(basename "$m4a" .m4a)"
|
|
exp="$DIR/$name.expected.txt"
|
|
hw_file="$DIR/$name.hotwords.txt"
|
|
[ -f "$exp" ] || { echo "skip $name (no .expected.txt)"; continue; }
|
|
|
|
tmp_no="$(mktemp)"
|
|
tmp_hw="$(mktemp)"
|
|
trap 'rm -f "$tmp_no" "$tmp_hw"' EXIT
|
|
|
|
run_once "$m4a" "" > "$tmp_no"
|
|
wer_no="$(wer "$exp" "$tmp_no")"
|
|
|
|
if [ -f "$hw_file" ]; then
|
|
hw="$(tr -d '\n' < "$hw_file")"
|
|
run_once "$m4a" "$hw" > "$tmp_hw"
|
|
wer_hw="$(wer "$exp" "$tmp_hw")"
|
|
total_hw_errs=$(( total_hw_errs + ${wer_hw%/*} ))
|
|
else
|
|
wer_hw="—"
|
|
total_hw_errs=$(( total_hw_errs + ${wer_no%/*} ))
|
|
fi
|
|
|
|
printf "%-38s %-13s %-13s\n" "$name" "$wer_no" "$wer_hw"
|
|
|
|
if [ "$VERBOSE" -eq 1 ]; then
|
|
echo " expected: $(tr -d '\n' < "$exp")"
|
|
echo " no_hotwords: $(tr -d '\n' < "$tmp_no")"
|
|
if [ -f "$hw_file" ]; then
|
|
echo " with_hotwords: $(tr -d '\n' < "$tmp_hw")"
|
|
fi
|
|
fi
|
|
|
|
rm -f "$tmp_no" "$tmp_hw"
|
|
done
|
|
|
|
echo
|
|
echo "Total hotwords-run word errors: $total_hw_errs"
|
|
exit "$total_hw_errs"
|