feat: Add Canary ASR model service and sweep experiments
This commit introduces the `doctate-canary` service and associated sweep experiments. The service provides access to the `nvidia/canary-1b-v2` ASR model via a native FastAPI API. It includes deployment scripts, Docker configurations, and detailed documentation on installation, API usage, and environment variables. The sweep experiments aim to thoroughly evaluate the Canary model's performance under various configurations. This includes testing different inference pipelines (single-shot vs. buffered), decoding strategies (greedy vs. beam search), and parameter tuning (chunk length, overlap, batch size, precision). The goal is to reproduce previous findings and identify optimal settings. The commit also includes: - Utility scripts for audio transcoding and manipulation. - Comprehensive logging and result collection mechanisms for the sweep runs. - Detailed analysis of `dur=0` occurrences and word confidence, concluding they are not reliable indicators of hallucination. - Documentation on the interaction between decoding parameters, especially `return_hypotheses`, and the availability of confidence scores.
This commit is contained in:
@@ -0,0 +1,178 @@
|
||||
"""Pull cross-attention scores and word confidence on the dosing audio.
|
||||
|
||||
Goal: see whether the AED's cross-attention distribution and per-word
|
||||
confidence flag the hallucinated "5 milligramm 101" cluster that we
|
||||
already identified via zero-duration word offsets.
|
||||
|
||||
Two new signals beyond what probe_canary_ts.py captured:
|
||||
1. word_confidence — per-word probability proxy. Hallucinations *should*
|
||||
have lower confidence than real words, though not guaranteed because
|
||||
AEDs hallucinate with high confidence too.
|
||||
2. xatt_scores — cross-attention from decoder tokens to encoder frames.
|
||||
For each output token we summarize where it "looked" in the audio:
|
||||
- peak_frame (argmax) and peak_time_s
|
||||
- entropy of the distribution (flat = no clear acoustic anchor)
|
||||
Hallucinated tokens should show high entropy (no clear acoustic peak).
|
||||
|
||||
Output:
|
||||
/sweep/probe_attn_conf.json — text + timestamp + word_confidence + per-token
|
||||
xatt summary (argmax frame, entropy)
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
import torch
|
||||
from omegaconf import OmegaConf
|
||||
|
||||
from nemo.collections.asr.models import EncDecMultiTaskModel
|
||||
from nemo.collections.asr.models.aed_multitask_models import (
|
||||
MultiTaskTranscriptionConfig,
|
||||
parse_multitask_prompt,
|
||||
)
|
||||
|
||||
|
||||
WAV_SRC = "/sweep/case_c414cf52_15-01-39.m4a"
|
||||
OUT_PATH = "/sweep/probe_attn_conf_kwargs_returnhyp.json"
|
||||
|
||||
|
||||
def transcode(src: str) -> str:
|
||||
dst = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
|
||||
dst.close()
|
||||
subprocess.run(["ffmpeg", "-y", "-loglevel", "error", "-i", src,
|
||||
"-ac", "1", "-ar", "16000", "-f", "wav", dst.name],
|
||||
check=True)
|
||||
return dst.name
|
||||
|
||||
|
||||
def to_jsonable(o):
|
||||
if isinstance(o, torch.Tensor):
|
||||
return o.detach().cpu().tolist()
|
||||
if hasattr(o, "tolist"):
|
||||
return o.tolist()
|
||||
if hasattr(o, "item"):
|
||||
return o.item()
|
||||
return str(o)
|
||||
|
||||
|
||||
def entropy_of(probs: torch.Tensor) -> float:
|
||||
p = probs.clamp_min(1e-12)
|
||||
return float(-(p * p.log()).sum().item())
|
||||
|
||||
|
||||
def summarize_xatt(xatt_scores, frame_stride_s: float):
|
||||
"""xatt_scores: List[Tensor] over decoder layers.
|
||||
|
||||
Multi-head attention output shape is typically [heads, tokens, frames]
|
||||
or [tokens, frames] depending on the decoder. We mean-pool away any
|
||||
leading dimensions until we get a 2D [tokens, frames] matrix.
|
||||
"""
|
||||
if not xatt_scores:
|
||||
return None
|
||||
layers = [x.float() for x in xatt_scores]
|
||||
print(f" xatt layer shapes: {[tuple(l.shape) for l in layers]}", flush=True)
|
||||
avg = torch.stack(layers, dim=0).mean(dim=0)
|
||||
# Collapse all leading dims (batch, heads, ...) until 2D
|
||||
while avg.dim() > 2:
|
||||
avg = avg.mean(dim=0)
|
||||
n_tokens, n_frames = avg.shape
|
||||
summaries = []
|
||||
for t in range(n_tokens):
|
||||
row = avg[t]
|
||||
norm = row / row.sum().clamp_min(1e-12)
|
||||
peak = int(norm.argmax().item())
|
||||
summaries.append({
|
||||
"token_idx": t,
|
||||
"peak_frame": peak,
|
||||
"peak_time_s": round(peak * frame_stride_s, 3),
|
||||
"peak_prob": round(float(norm[peak].item()), 4),
|
||||
"entropy": round(entropy_of(norm), 4),
|
||||
"max_log_entropy": round(math.log(n_frames), 4),
|
||||
})
|
||||
return {"n_tokens": n_tokens, "n_frames": n_frames,
|
||||
"frame_stride_s": frame_stride_s,
|
||||
"n_layers": len(layers), "per_token": summaries}
|
||||
|
||||
|
||||
def main():
|
||||
print("loading Canary 1B v2", flush=True)
|
||||
model = EncDecMultiTaskModel.from_pretrained("nvidia/canary-1b-v2")
|
||||
model = model.to("cuda").to(torch.bfloat16)
|
||||
model.eval()
|
||||
|
||||
# Patch the live decoding config:
|
||||
# - greedy strategy (confidence is only filled in greedy path, not beam)
|
||||
# - return_xattn_scores=True
|
||||
# - confidence_cfg with all preserve flags
|
||||
# - enable_chunking=False is set via override_config below (NeMo gates xatt
|
||||
# on chunking; chunking also blocks word-level confidence aggregation).
|
||||
cfg = OmegaConf.to_container(model.cfg.decoding, resolve=True)
|
||||
cfg["strategy"] = "greedy"
|
||||
cfg["return_xattn_scores"] = True
|
||||
cfg.setdefault("confidence_cfg", {})
|
||||
cfg["confidence_cfg"]["preserve_word_confidence"] = True
|
||||
cfg["confidence_cfg"]["preserve_token_confidence"] = True
|
||||
cfg["confidence_cfg"]["preserve_frame_confidence"] = True
|
||||
cfg["confidence_cfg"]["aggregation"] = "min" # most signal-preserving
|
||||
cfg.setdefault("greedy", {})
|
||||
cfg["greedy"]["preserve_token_confidence"] = True
|
||||
cfg["greedy"]["preserve_alignments"] = True
|
||||
print("applying decoding override:", json.dumps(cfg, indent=2), flush=True)
|
||||
model.change_decoding_strategy(OmegaConf.create(cfg))
|
||||
|
||||
# Encoder frame stride: feature_stride * subsampling
|
||||
feature_stride = float(model.cfg.preprocessor["window_stride"])
|
||||
subsampling = 8 # FastConformer default; verified empirically against
|
||||
# 80ms-per-frame in probe_tdt.md
|
||||
frame_stride_s = feature_stride * subsampling
|
||||
print(f"frame stride: {frame_stride_s} s", flush=True)
|
||||
|
||||
wav = transcode(WAV_SRC)
|
||||
# Clean single-knob ablation: same override_config for both runs,
|
||||
# only `enable_chunking` differs. All other fields (batch_size, prompt,
|
||||
# timestamps, return_hypotheses) held constant.
|
||||
print(f"running transcribe(KWARGS path, return_hypotheses=True, timestamps=True) on {WAV_SRC}",
|
||||
flush=True)
|
||||
# Final clean test: kwargs path (like the original Mode A run that
|
||||
# showed None confidence), but with return_hypotheses=True forced.
|
||||
# If confidence appears, then `return_hypotheses` was the real
|
||||
# confounder all along — not timestamps, not the call form.
|
||||
with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16):
|
||||
with torch.no_grad():
|
||||
hyps = model.transcribe([wav], source_lang="de", target_lang="de",
|
||||
pnc="yes", timestamps=True, batch_size=1,
|
||||
return_hypotheses=True)
|
||||
h = hyps[0]
|
||||
|
||||
out = {
|
||||
"text": getattr(h, "text", None),
|
||||
"score": getattr(h, "score", None),
|
||||
"timestamp": getattr(h, "timestamp", None),
|
||||
"tokens": getattr(h, "tokens", None),
|
||||
"word_confidence": getattr(h, "word_confidence", None),
|
||||
"token_confidence": getattr(h, "token_confidence", None),
|
||||
"xatt_summary": summarize_xatt(getattr(h, "xatt_scores", None),
|
||||
frame_stride_s),
|
||||
}
|
||||
|
||||
with open(OUT_PATH, "w") as f:
|
||||
json.dump(out, f, default=to_jsonable, ensure_ascii=False, indent=2)
|
||||
|
||||
# Quick sanity print
|
||||
print(f"\ntext: {h.text}", flush=True)
|
||||
print(f"score: {h.score}", flush=True)
|
||||
wc = getattr(h, "word_confidence", None)
|
||||
print(f"word_confidence: {len(wc) if wc is not None else 'None'} entries", flush=True)
|
||||
tc = getattr(h, "token_confidence", None)
|
||||
print(f"token_confidence: {len(tc) if tc is not None else 'None'} entries", flush=True)
|
||||
xs = getattr(h, "xatt_scores", None)
|
||||
if xs is not None:
|
||||
print(f"xatt_scores: {len(xs)} layer(s); shapes={[tuple(x.shape) for x in xs]}", flush=True)
|
||||
else:
|
||||
print("xatt_scores: None", flush=True)
|
||||
print(f"\nfull dump -> {OUT_PATH}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Show forced-alignment timestamps from Canary's main transcribe(timestamps=True).
|
||||
|
||||
Demonstrates the full Hypothesis structure produced by:
|
||||
AED text + CTC submodel acoustic alignment via Viterbi.
|
||||
|
||||
Output:
|
||||
/sweep/canary_timestamps_demo.json — full pretty-printed dict
|
||||
stdout — short summary
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
import torch
|
||||
|
||||
from nemo.collections.asr.models import EncDecMultiTaskModel
|
||||
|
||||
|
||||
WAV_SRC = "/sweep/case_c414cf52_15-01-39.m4a"
|
||||
OUT_PATH = "/sweep/canary_timestamps_dosing.json"
|
||||
|
||||
|
||||
def transcode(src: str) -> str:
|
||||
dst = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
|
||||
dst.close()
|
||||
subprocess.run(["ffmpeg", "-y", "-loglevel", "error", "-i", src,
|
||||
"-ac", "1", "-ar", "16000",
|
||||
"-f", "wav", dst.name], check=True)
|
||||
return dst.name
|
||||
|
||||
|
||||
def to_jsonable(o):
|
||||
if hasattr(o, "tolist"):
|
||||
return o.tolist()
|
||||
if hasattr(o, "item"):
|
||||
return o.item()
|
||||
return str(o)
|
||||
|
||||
|
||||
def main():
|
||||
print("loading Canary 1B v2", flush=True)
|
||||
model = EncDecMultiTaskModel.from_pretrained("nvidia/canary-1b-v2")
|
||||
model = model.to("cuda").to(torch.bfloat16)
|
||||
model.eval()
|
||||
|
||||
wav = transcode(WAV_SRC)
|
||||
print(f"running model.transcribe(timestamps=True) on {WAV_SRC}", flush=True)
|
||||
with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16):
|
||||
with torch.no_grad():
|
||||
hyps = model.transcribe([wav], source_lang="de", target_lang="de",
|
||||
pnc="yes", timestamps=True, batch_size=4)
|
||||
h = hyps[0]
|
||||
|
||||
out = {"text": getattr(h, "text", None),
|
||||
"timestamp": getattr(h, "timestamp", None)}
|
||||
with open(OUT_PATH, "w") as f:
|
||||
json.dump(out, f, default=to_jsonable, ensure_ascii=False, indent=2)
|
||||
|
||||
print(f"\ntext: {h.text}", flush=True)
|
||||
ts = h.timestamp if isinstance(h.timestamp, dict) else {}
|
||||
for k, v in ts.items():
|
||||
n = len(v) if hasattr(v, "__len__") else "?"
|
||||
print(f"timestamp[{k}]: {n} entries", flush=True)
|
||||
print(f"\nfull dump -> {OUT_PATH}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Standalone CTC-submodel transcription on the dosing audio.
|
||||
|
||||
Loads Canary, accesses model.timestamps_asr_model directly, calls its own
|
||||
transcribe() — bypassing the AED decoder entirely. Goal: see whether the CTC
|
||||
submodel renders the spoken digit sequence "eins null eins" as separate
|
||||
tokens or also collapses it.
|
||||
"""
|
||||
import json
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
import torch
|
||||
|
||||
from nemo.collections.asr.models import EncDecMultiTaskModel
|
||||
|
||||
|
||||
WAV_SRC = "/sweep/case_c414cf52_15-01-39.m4a"
|
||||
OUT_PATH = "/sweep/ctc_standalone_dosing.json"
|
||||
|
||||
|
||||
def transcode(src: str) -> str:
|
||||
dst = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
|
||||
dst.close()
|
||||
subprocess.run(["ffmpeg", "-y", "-loglevel", "error", "-i", src,
|
||||
"-ac", "1", "-ar", "16000", "-f", "wav", dst.name],
|
||||
check=True)
|
||||
return dst.name
|
||||
|
||||
|
||||
def to_jsonable(o):
|
||||
if hasattr(o, "tolist"):
|
||||
return o.tolist()
|
||||
if hasattr(o, "item"):
|
||||
return o.item()
|
||||
return str(o)
|
||||
|
||||
|
||||
def main():
|
||||
print("loading Canary 1B v2 (just for the CTC submodel)", flush=True)
|
||||
model = EncDecMultiTaskModel.from_pretrained("nvidia/canary-1b-v2")
|
||||
model = model.to("cuda").to(torch.bfloat16)
|
||||
model.eval()
|
||||
|
||||
tdt = model.timestamps_asr_model
|
||||
print(f"submodel: {type(tdt).__name__}", flush=True)
|
||||
|
||||
wav = transcode(WAV_SRC)
|
||||
print(f"running tdt.transcribe(timestamps=True) on {WAV_SRC}", flush=True)
|
||||
with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16):
|
||||
with torch.no_grad():
|
||||
hyps = tdt.transcribe([wav], timestamps=True)
|
||||
h = hyps[0]
|
||||
|
||||
out = {"text": getattr(h, "text", None),
|
||||
"timestamp": getattr(h, "timestamp", None)}
|
||||
with open(OUT_PATH, "w") as f:
|
||||
json.dump(out, f, default=to_jsonable, ensure_ascii=False, indent=2)
|
||||
|
||||
print(f"\nstandalone CTC text:\n{h.text}\n", flush=True)
|
||||
ts = h.timestamp if isinstance(h.timestamp, dict) else {}
|
||||
for k, v in ts.items():
|
||||
n = len(v) if hasattr(v, "__len__") else "?"
|
||||
print(f"timestamp[{k}]: {n} entries", flush=True)
|
||||
print(f"\nfull dump -> {OUT_PATH}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,215 @@
|
||||
"""TDT probe — can timestamps_asr_model handle audio standalone, including >40 s?
|
||||
|
||||
Background:
|
||||
EncDecMultiTaskModel.timestamps_asr_model is a Token-Duration-Transducer
|
||||
(RNN-T variant) with hybrid CTC head, FastConformer encoder enc_hidden=1024.
|
||||
Bundled inside Canary 1B v2 specifically to provide forced-alignment
|
||||
timestamps for the AED head, but it is itself a complete ASR model.
|
||||
|
||||
Open question: does the relative positional encoding lift the 40 s wall that
|
||||
AED hits, or does TDT inherit the same training-time max_duration limit?
|
||||
|
||||
Test set:
|
||||
- 20 s : sanity check, well within both models
|
||||
- 80 s : full source — Canary path1 truncates here, baseline expectation
|
||||
- 160 s : concat(80, 80), no re-encode — pushes 4x past TDT training horizon
|
||||
|
||||
Per case:
|
||||
- TDT.transcribe(timestamps=False)
|
||||
- TDT.transcribe(timestamps=True) — also verify timestamp shape
|
||||
- Canary path1 — reference, will truncate >40 s
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import wave
|
||||
|
||||
import torch
|
||||
|
||||
from nemo.collections.asr.models import EncDecMultiTaskModel
|
||||
|
||||
|
||||
AUDIO_DIR = "/sweep/threshold_audios"
|
||||
RESULTS_PATH = "/sweep/probe_tdt.jsonl"
|
||||
LOG_PATH = "/sweep/probe_tdt.log"
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
line = f"{time.strftime('%H:%M:%S')} {msg}"
|
||||
print(line, flush=True)
|
||||
with open(LOG_PATH, "a") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
|
||||
def transcode(src: str, duration_secs: float | None = None) -> str:
|
||||
dst = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
|
||||
dst.close()
|
||||
cmd = ["ffmpeg", "-y", "-loglevel", "error", "-i", src,
|
||||
"-ac", "1", "-ar", "16000"]
|
||||
if duration_secs is not None:
|
||||
cmd += ["-t", str(duration_secs)]
|
||||
cmd += ["-f", "wav", dst.name]
|
||||
subprocess.run(cmd, check=True)
|
||||
return dst.name
|
||||
|
||||
|
||||
def concat_wavs(wavs: list[str], repeats: int = 1) -> str:
|
||||
list_file = tempfile.NamedTemporaryFile(suffix=".txt", mode="w", delete=False)
|
||||
for w in wavs * repeats:
|
||||
list_file.write(f"file '{w}'\n")
|
||||
list_file.close()
|
||||
dst = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
|
||||
dst.close()
|
||||
subprocess.run(["ffmpeg", "-y", "-loglevel", "error",
|
||||
"-f", "concat", "-safe", "0", "-i", list_file.name,
|
||||
"-c", "copy", dst.name], check=True)
|
||||
os.unlink(list_file.name)
|
||||
return dst.name
|
||||
|
||||
|
||||
def wav_duration(p: str) -> float:
|
||||
with wave.open(p, "rb") as w:
|
||||
return w.getnframes() / float(w.getframerate())
|
||||
|
||||
|
||||
def vram_mb() -> float:
|
||||
return torch.cuda.max_memory_allocated() / (1024 * 1024) if torch.cuda.is_available() else 0.0
|
||||
|
||||
|
||||
def run_tdt(tdt, wav: str, timestamps: bool) -> dict:
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16):
|
||||
with torch.no_grad():
|
||||
hyps = tdt.transcribe([wav], timestamps=timestamps)
|
||||
elapsed = time.monotonic() - t0
|
||||
h = hyps[0]
|
||||
text = (getattr(h, "text", None) or "").strip()
|
||||
ts = getattr(h, "timestamp", None)
|
||||
ts_summary = None
|
||||
if isinstance(ts, dict):
|
||||
ts_summary = {k: (len(v) if hasattr(v, "__len__") else str(type(v).__name__))
|
||||
for k, v in ts.items()}
|
||||
return {"ok": True, "text": text, "chars": len(text),
|
||||
"duration_infer": round(elapsed, 2),
|
||||
"vram_peak_mb": round(vram_mb(), 0),
|
||||
"timestamp_summary": ts_summary}
|
||||
except Exception as e:
|
||||
return {"ok": False, "error": f"{type(e).__name__}: {e}",
|
||||
"duration_infer": round(time.monotonic() - t0, 2)}
|
||||
|
||||
|
||||
def run_canary(model, wav: str) -> dict:
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16):
|
||||
with torch.no_grad():
|
||||
hyps = model.transcribe([wav], source_lang="de", target_lang="de",
|
||||
pnc="yes", batch_size=4)
|
||||
elapsed = time.monotonic() - t0
|
||||
text = (getattr(hyps[0], "text", None) or str(hyps[0])).strip()
|
||||
return {"ok": True, "text": text, "chars": len(text),
|
||||
"duration_infer": round(elapsed, 2),
|
||||
"vram_peak_mb": round(vram_mb(), 0)}
|
||||
except Exception as e:
|
||||
return {"ok": False, "error": f"{type(e).__name__}: {e}",
|
||||
"duration_infer": round(time.monotonic() - t0, 2)}
|
||||
|
||||
|
||||
def find_source() -> str | None:
|
||||
for c in ("trim_full.m4a", "full.m4a", "source.m4a"):
|
||||
p = os.path.join(AUDIO_DIR, c)
|
||||
if os.path.exists(p):
|
||||
return p
|
||||
if not os.path.isdir(AUDIO_DIR):
|
||||
return None
|
||||
m4as = [f for f in os.listdir(AUDIO_DIR) if f.endswith(".m4a")]
|
||||
if not m4as:
|
||||
return None
|
||||
m4as.sort(key=lambda f: os.path.getsize(os.path.join(AUDIO_DIR, f)), reverse=True)
|
||||
return os.path.join(AUDIO_DIR, m4as[0])
|
||||
|
||||
|
||||
def main():
|
||||
open(RESULTS_PATH, "w").close()
|
||||
open(LOG_PATH, "w").close()
|
||||
|
||||
log("loading EncDecMultiTaskModel (Canary 1B v2)")
|
||||
model = EncDecMultiTaskModel.from_pretrained("nvidia/canary-1b-v2")
|
||||
model = model.to("cuda").to(torch.bfloat16)
|
||||
model.eval()
|
||||
log(f" vram after load: {vram_mb():.0f} MB")
|
||||
|
||||
tdt = getattr(model, "timestamps_asr_model", None)
|
||||
if tdt is None:
|
||||
log("FATAL: model has no timestamps_asr_model attribute")
|
||||
return
|
||||
log(f" tdt class: {type(tdt).__name__}")
|
||||
log(f" tdt device: {next(tdt.parameters()).device}, dtype: {next(tdt.parameters()).dtype}")
|
||||
|
||||
src = find_source()
|
||||
if src is None:
|
||||
log(f"FATAL: no source audio found in {AUDIO_DIR}")
|
||||
return
|
||||
log(f"source audio: {src}")
|
||||
|
||||
wav_full = transcode(src)
|
||||
dur_full = wav_duration(wav_full)
|
||||
log(f" full wav: {dur_full:.1f} s")
|
||||
|
||||
wav_20 = transcode(src, duration_secs=20.0)
|
||||
log(f" 20 s wav: {wav_duration(wav_20):.1f} s")
|
||||
|
||||
wav_80 = transcode(src, duration_secs=80.0) if dur_full >= 80 else wav_full
|
||||
log(f" 80 s wav: {wav_duration(wav_80):.1f} s")
|
||||
|
||||
wav_160 = concat_wavs([wav_80], repeats=2)
|
||||
log(f" 160 s concat wav: {wav_duration(wav_160):.1f} s")
|
||||
|
||||
cases = [("20s", wav_20), ("80s", wav_80), ("160s_concat", wav_160)]
|
||||
|
||||
def record(d: dict) -> None:
|
||||
with open(RESULTS_PATH, "a") as f:
|
||||
f.write(json.dumps(d, ensure_ascii=False) + "\n")
|
||||
|
||||
for label, wav in cases:
|
||||
dur = wav_duration(wav)
|
||||
log(f"\n=== case {label} ({dur:.1f} s) ===")
|
||||
|
||||
log(" TDT timestamps=False")
|
||||
r = run_tdt(tdt, wav, timestamps=False)
|
||||
record({"case": label, "duration_audio": dur, "model": "tdt",
|
||||
"timestamps": False, **r})
|
||||
if r["ok"]:
|
||||
log(f" -> {r['chars']} chars in {r['duration_infer']} s, "
|
||||
f"{r['vram_peak_mb']} MB")
|
||||
else:
|
||||
log(f" FAIL: {r['error']}")
|
||||
|
||||
log(" TDT timestamps=True")
|
||||
r = run_tdt(tdt, wav, timestamps=True)
|
||||
record({"case": label, "duration_audio": dur, "model": "tdt",
|
||||
"timestamps": True, **r})
|
||||
if r["ok"]:
|
||||
log(f" -> {r['chars']} chars, ts_summary={r['timestamp_summary']}")
|
||||
else:
|
||||
log(f" FAIL: {r['error']}")
|
||||
|
||||
log(" Canary path1 (reference)")
|
||||
r = run_canary(model, wav)
|
||||
record({"case": label, "duration_audio": dur, "model": "canary_path1",
|
||||
**r})
|
||||
if r["ok"]:
|
||||
log(f" -> {r['chars']} chars in {r['duration_infer']} s")
|
||||
else:
|
||||
log(f" FAIL: {r['error']}")
|
||||
|
||||
log("done")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+425
@@ -0,0 +1,425 @@
|
||||
"""Canary parameter sweep — direct NeMo calls, no HTTP layer.
|
||||
|
||||
Runs many parameter combinations against a fixed set of audios and writes
|
||||
JSONL + a human-readable summary. Designed to run inside the doctate-canary
|
||||
Docker image (or its base) with the model volume mounted at /models.
|
||||
|
||||
Outputs:
|
||||
/sweep/results.jsonl — one JSON object per run
|
||||
/sweep/log.txt — chronological log
|
||||
"""
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import wave
|
||||
from contextlib import contextmanager
|
||||
|
||||
import torch
|
||||
from omegaconf import OmegaConf
|
||||
|
||||
from nemo.collections.asr.models import ASRModel
|
||||
from nemo.collections.asr.parts.utils.streaming_utils import FrameBatchMultiTaskAED
|
||||
from nemo.collections.asr.parts.utils.transcribe_utils import get_buffered_pred_feat_multitaskAED
|
||||
|
||||
|
||||
AUDIO_DIR = "/sweep/audios"
|
||||
OUT_DIR = "/sweep"
|
||||
RESULTS_PATH = f"{OUT_DIR}/results.jsonl"
|
||||
LOG_PATH = f"{OUT_DIR}/log.txt"
|
||||
|
||||
# Subset for most axes — one short, one medium, one long (>40 s each).
|
||||
SUBSET = [
|
||||
"2026-04-27T15-00-48Z.m4a", # 48.4 s
|
||||
"2026-04-27T14-55-41Z.m4a", # 65.9 s
|
||||
"2026-04-27T14-58-17Z.m4a", # 80.9 s
|
||||
]
|
||||
|
||||
# Single-audio subset for speed/VRAM-only axes.
|
||||
SINGLE = ["2026-04-27T14-58-17Z.m4a"] # 80.9 s
|
||||
|
||||
DTYPE_MAP = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
line = f"{time.strftime('%H:%M:%S')} {msg}"
|
||||
print(line, flush=True)
|
||||
with open(LOG_PATH, "a") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
|
||||
def transcode_to_wav16k(src: str) -> str:
|
||||
dst = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
|
||||
dst.close()
|
||||
subprocess.run(
|
||||
["ffmpeg", "-y", "-loglevel", "error", "-i", src,
|
||||
"-ac", "1", "-ar", "16000", "-f", "wav", dst.name],
|
||||
check=True,
|
||||
)
|
||||
return dst.name
|
||||
|
||||
|
||||
def wav_duration(path: str) -> float:
|
||||
with wave.open(path, "rb") as w:
|
||||
return w.getnframes() / float(w.getframerate())
|
||||
|
||||
|
||||
def vram_mb() -> float:
|
||||
if torch.cuda.is_available():
|
||||
return torch.cuda.max_memory_allocated() / (1024 * 1024)
|
||||
return 0.0
|
||||
|
||||
|
||||
@contextmanager
|
||||
def vram_track():
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
pass
|
||||
|
||||
|
||||
def load_model(precision: str):
|
||||
log(f"loading model precision={precision}")
|
||||
t0 = time.monotonic()
|
||||
model = ASRModel.from_pretrained("nvidia/canary-1b-v2")
|
||||
model = model.to("cuda")
|
||||
if precision != "fp32":
|
||||
model = model.to(DTYPE_MAP[precision])
|
||||
model.eval()
|
||||
log(f"model loaded in {time.monotonic() - t0:.1f}s, vram={vram_mb():.0f}MB")
|
||||
return model
|
||||
|
||||
|
||||
def prep_cfg(model):
|
||||
cfg = copy.deepcopy(model._cfg)
|
||||
OmegaConf.set_struct(cfg.preprocessor, False)
|
||||
cfg.preprocessor.dither = 0.0
|
||||
cfg.preprocessor.pad_to = 0
|
||||
OmegaConf.set_struct(cfg.preprocessor, True)
|
||||
return cfg
|
||||
|
||||
|
||||
def write_manifest(wav_path: str, duration: float, pnc: str = "yes",
|
||||
src_lang: str = "de", tgt_lang: str = "de",
|
||||
task: str = "asr") -> str:
|
||||
f = tempfile.NamedTemporaryFile(suffix=".json", mode="w", delete=False)
|
||||
json.dump({
|
||||
"audio_filepath": wav_path,
|
||||
"duration": duration,
|
||||
"taskname": task,
|
||||
"source_lang": src_lang,
|
||||
"target_lang": tgt_lang,
|
||||
"pnc": pnc,
|
||||
}, f)
|
||||
f.write("\n")
|
||||
f.close()
|
||||
return f.name
|
||||
|
||||
|
||||
def run_buffered(model, model_cfg, wav_path: str, duration: float,
|
||||
chunk_len: float, total_buffer: float, batch_size: int,
|
||||
precision: str, pnc: str = "yes", timestamps: bool = False):
|
||||
manifest = write_manifest(wav_path, duration, pnc=pnc)
|
||||
feature_stride = model_cfg.preprocessor["window_stride"]
|
||||
model_stride_secs = feature_stride * 8
|
||||
|
||||
frame_asr = FrameBatchMultiTaskAED(
|
||||
asr_model=model,
|
||||
frame_len=chunk_len,
|
||||
total_buffer=total_buffer,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
amp_dtype = DTYPE_MAP[precision]
|
||||
use_amp = precision != "fp32"
|
||||
t0 = time.monotonic()
|
||||
with vram_track():
|
||||
with torch.amp.autocast("cuda", enabled=use_amp, dtype=amp_dtype):
|
||||
with torch.no_grad():
|
||||
hyps = get_buffered_pred_feat_multitaskAED(
|
||||
frame_asr, model_cfg.preprocessor, model_stride_secs,
|
||||
model.device, manifest=manifest, filepaths=None,
|
||||
timestamps=timestamps,
|
||||
)
|
||||
elapsed = time.monotonic() - t0
|
||||
peak_vram = vram_mb()
|
||||
os.unlink(manifest)
|
||||
text = (getattr(hyps[0], "text", None) or str(hyps[0])).strip()
|
||||
return text, elapsed, peak_vram
|
||||
|
||||
|
||||
def run_transcribe_path1(model, wav_path: str, batch_size: int, pnc: str,
|
||||
timestamps: bool, precision: str):
|
||||
"""Path 1 — direct .transcribe() with batch_size=1 enables auto-chunking
|
||||
in NeMo >= 2.5 (per docstring of speech_to_text_aed_chunked_infer.py)."""
|
||||
amp_dtype = DTYPE_MAP[precision]
|
||||
use_amp = precision != "fp32"
|
||||
t0 = time.monotonic()
|
||||
with vram_track():
|
||||
with torch.amp.autocast("cuda", enabled=use_amp, dtype=amp_dtype):
|
||||
with torch.no_grad():
|
||||
hyps = model.transcribe(
|
||||
[wav_path],
|
||||
source_lang="de", target_lang="de",
|
||||
pnc=pnc, timestamps=timestamps,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
elapsed = time.monotonic() - t0
|
||||
peak_vram = vram_mb()
|
||||
text = (getattr(hyps[0], "text", None) or str(hyps[0])).strip()
|
||||
return text, elapsed, peak_vram
|
||||
|
||||
|
||||
def write_result(record: dict) -> None:
|
||||
with open(RESULTS_PATH, "a") as f:
|
||||
f.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
def sweep():
|
||||
open(LOG_PATH, "w").close()
|
||||
open(RESULTS_PATH, "w").close()
|
||||
|
||||
log("=== preparing audios ===")
|
||||
wavs = {}
|
||||
for name in set(SUBSET + SINGLE):
|
||||
src = os.path.join(AUDIO_DIR, name)
|
||||
wav = transcode_to_wav16k(src)
|
||||
dur = wav_duration(wav)
|
||||
wavs[name] = (wav, dur)
|
||||
log(f" {name} → {wav} ({dur:.1f}s)")
|
||||
|
||||
# ===== precision axis (B.2) — separate model loads =====
|
||||
for precision in ["bf16", "fp16", "fp32"]:
|
||||
try:
|
||||
torch.cuda.empty_cache()
|
||||
model = load_model(precision)
|
||||
cfg = prep_cfg(model)
|
||||
for name in SINGLE:
|
||||
wav, dur = wavs[name]
|
||||
axis = f"B2_precision={precision}"
|
||||
log(f"running {axis} on {name}")
|
||||
try:
|
||||
text, t, vram = run_buffered(
|
||||
model, cfg, wav, dur,
|
||||
chunk_len=40.0, total_buffer=40.0, batch_size=8,
|
||||
precision=precision,
|
||||
)
|
||||
write_result({
|
||||
"axis": "precision", "precision": precision,
|
||||
"chunk_len": 40.0, "overlap": 0, "batch_size": 8,
|
||||
"decoding": "greedy", "pnc": "yes", "timestamps": False,
|
||||
"audio": name, "duration_audio": dur,
|
||||
"duration_infer": round(t, 2), "vram_peak_mb": round(vram, 0),
|
||||
"chars": len(text), "text": text,
|
||||
})
|
||||
except Exception as e:
|
||||
log(f" FAIL {axis} on {name}: {e}")
|
||||
write_result({"axis": "precision", "precision": precision,
|
||||
"audio": name, "error": str(e)})
|
||||
del model
|
||||
torch.cuda.empty_cache()
|
||||
except Exception as e:
|
||||
log(f"FAIL precision={precision}: {e}")
|
||||
|
||||
# All remaining axes use bf16 default model.
|
||||
torch.cuda.empty_cache()
|
||||
model = load_model("bf16")
|
||||
cfg = prep_cfg(model)
|
||||
|
||||
# ===== chunk_len axis (A.1) =====
|
||||
for chunk_len in [10.0, 20.0, 40.0, 60.0]:
|
||||
for name in SUBSET:
|
||||
wav, dur = wavs[name]
|
||||
axis = f"A1_chunk_len={chunk_len}"
|
||||
log(f"running {axis} on {name}")
|
||||
try:
|
||||
text, t, vram = run_buffered(
|
||||
model, cfg, wav, dur,
|
||||
chunk_len=chunk_len, total_buffer=chunk_len, batch_size=8,
|
||||
precision="bf16",
|
||||
)
|
||||
write_result({
|
||||
"axis": "chunk_len", "chunk_len": chunk_len,
|
||||
"overlap": 0, "batch_size": 8, "precision": "bf16",
|
||||
"decoding": "greedy", "pnc": "yes", "timestamps": False,
|
||||
"audio": name, "duration_audio": dur,
|
||||
"duration_infer": round(t, 2), "vram_peak_mb": round(vram, 0),
|
||||
"chars": len(text), "text": text,
|
||||
})
|
||||
except Exception as e:
|
||||
log(f" FAIL: {e}")
|
||||
write_result({"axis": "chunk_len", "chunk_len": chunk_len,
|
||||
"audio": name, "error": str(e)})
|
||||
|
||||
# ===== overlap axis (A.2) — chunk_len=40, total_buffer=chunk_len+overlap =====
|
||||
for overlap in [1.0, 2.0, 5.0, 10.0]:
|
||||
chunk_len = 40.0
|
||||
total_buffer = chunk_len + overlap
|
||||
for name in SUBSET:
|
||||
wav, dur = wavs[name]
|
||||
axis = f"A2_overlap={overlap}"
|
||||
log(f"running {axis} on {name}")
|
||||
try:
|
||||
text, t, vram = run_buffered(
|
||||
model, cfg, wav, dur,
|
||||
chunk_len=chunk_len, total_buffer=total_buffer,
|
||||
batch_size=8, precision="bf16",
|
||||
)
|
||||
write_result({
|
||||
"axis": "overlap", "chunk_len": chunk_len,
|
||||
"overlap": overlap, "batch_size": 8, "precision": "bf16",
|
||||
"decoding": "greedy", "pnc": "yes", "timestamps": False,
|
||||
"audio": name, "duration_audio": dur,
|
||||
"duration_infer": round(t, 2), "vram_peak_mb": round(vram, 0),
|
||||
"chars": len(text), "text": text,
|
||||
})
|
||||
except Exception as e:
|
||||
log(f" FAIL: {e}")
|
||||
write_result({"axis": "overlap", "overlap": overlap,
|
||||
"audio": name, "error": str(e)})
|
||||
|
||||
# ===== batch_size axis (B.1) =====
|
||||
for batch_size in [1, 2, 4, 8, 16]:
|
||||
for name in SINGLE:
|
||||
wav, dur = wavs[name]
|
||||
axis = f"B1_batch={batch_size}"
|
||||
log(f"running {axis} on {name}")
|
||||
try:
|
||||
text, t, vram = run_buffered(
|
||||
model, cfg, wav, dur,
|
||||
chunk_len=40.0, total_buffer=40.0, batch_size=batch_size,
|
||||
precision="bf16",
|
||||
)
|
||||
write_result({
|
||||
"axis": "batch_size", "batch_size": batch_size,
|
||||
"chunk_len": 40.0, "overlap": 0, "precision": "bf16",
|
||||
"decoding": "greedy", "pnc": "yes", "timestamps": False,
|
||||
"audio": name, "duration_audio": dur,
|
||||
"duration_infer": round(t, 2), "vram_peak_mb": round(vram, 0),
|
||||
"chars": len(text), "text": text,
|
||||
})
|
||||
except Exception as e:
|
||||
log(f" FAIL: {e}")
|
||||
write_result({"axis": "batch_size", "batch_size": batch_size,
|
||||
"audio": name, "error": str(e)})
|
||||
|
||||
# ===== pnc axis (C.2) =====
|
||||
for pnc in ["no", "yes"]:
|
||||
for name in SUBSET:
|
||||
wav, dur = wavs[name]
|
||||
axis = f"C2_pnc={pnc}"
|
||||
log(f"running {axis} on {name}")
|
||||
try:
|
||||
text, t, vram = run_buffered(
|
||||
model, cfg, wav, dur,
|
||||
chunk_len=40.0, total_buffer=40.0, batch_size=8,
|
||||
precision="bf16", pnc=pnc,
|
||||
)
|
||||
write_result({
|
||||
"axis": "pnc", "pnc": pnc,
|
||||
"chunk_len": 40.0, "overlap": 0, "batch_size": 8,
|
||||
"precision": "bf16", "decoding": "greedy",
|
||||
"timestamps": False,
|
||||
"audio": name, "duration_audio": dur,
|
||||
"duration_infer": round(t, 2), "vram_peak_mb": round(vram, 0),
|
||||
"chars": len(text), "text": text,
|
||||
})
|
||||
except Exception as e:
|
||||
log(f" FAIL: {e}")
|
||||
write_result({"axis": "pnc", "pnc": pnc,
|
||||
"audio": name, "error": str(e)})
|
||||
|
||||
# ===== timestamps axis (C.3) — when on, NeMo recommends chunk_len=10 =====
|
||||
for ts_flag, chunk_len in [(False, 40.0), (True, 10.0)]:
|
||||
for name in SUBSET:
|
||||
wav, dur = wavs[name]
|
||||
axis = f"C3_timestamps={ts_flag}"
|
||||
log(f"running {axis} (chunk_len={chunk_len}) on {name}")
|
||||
try:
|
||||
text, t, vram = run_buffered(
|
||||
model, cfg, wav, dur,
|
||||
chunk_len=chunk_len, total_buffer=chunk_len, batch_size=8,
|
||||
precision="bf16", timestamps=ts_flag,
|
||||
)
|
||||
write_result({
|
||||
"axis": "timestamps", "timestamps": ts_flag,
|
||||
"chunk_len": chunk_len, "overlap": 0, "batch_size": 8,
|
||||
"precision": "bf16", "decoding": "greedy", "pnc": "yes",
|
||||
"audio": name, "duration_audio": dur,
|
||||
"duration_infer": round(t, 2), "vram_peak_mb": round(vram, 0),
|
||||
"chars": len(text), "text": text,
|
||||
})
|
||||
except Exception as e:
|
||||
log(f" FAIL: {e}")
|
||||
write_result({"axis": "timestamps", "timestamps": ts_flag,
|
||||
"audio": name, "error": str(e)})
|
||||
|
||||
# ===== beam search (C.1) =====
|
||||
try:
|
||||
from nemo.collections.asr.parts.submodules.multitask_decoding import MultiTaskDecodingConfig
|
||||
for strategy, beam_size in [("beam", 4), ("greedy", 1)]:
|
||||
decoding_cfg = MultiTaskDecodingConfig()
|
||||
decoding_cfg.strategy = strategy
|
||||
if hasattr(decoding_cfg, "beam"):
|
||||
decoding_cfg.beam.beam_size = beam_size
|
||||
try:
|
||||
model.change_decoding_strategy(decoding_cfg)
|
||||
except Exception as e:
|
||||
log(f"change_decoding_strategy failed for {strategy}: {e}")
|
||||
continue
|
||||
for name in SUBSET:
|
||||
wav, dur = wavs[name]
|
||||
axis = f"C1_decoding={strategy}_beam={beam_size}"
|
||||
log(f"running {axis} on {name}")
|
||||
try:
|
||||
text, t, vram = run_buffered(
|
||||
model, cfg, wav, dur,
|
||||
chunk_len=40.0, total_buffer=40.0, batch_size=8,
|
||||
precision="bf16",
|
||||
)
|
||||
write_result({
|
||||
"axis": "decoding", "decoding": strategy,
|
||||
"beam_size": beam_size,
|
||||
"chunk_len": 40.0, "overlap": 0, "batch_size": 8,
|
||||
"precision": "bf16", "pnc": "yes", "timestamps": False,
|
||||
"audio": name, "duration_audio": dur,
|
||||
"duration_infer": round(t, 2), "vram_peak_mb": round(vram, 0),
|
||||
"chars": len(text), "text": text,
|
||||
})
|
||||
except Exception as e:
|
||||
log(f" FAIL: {e}")
|
||||
write_result({"axis": "decoding", "decoding": strategy,
|
||||
"audio": name, "error": str(e)})
|
||||
except Exception as e:
|
||||
log(f"beam search axis skipped: {e}")
|
||||
|
||||
# ===== Path-1 (.transcribe() auto-chunking) — D.1 =====
|
||||
log("=== Path 1: model.transcribe() with auto-chunking ===")
|
||||
for name in SUBSET:
|
||||
wav, dur = wavs[name]
|
||||
axis = "D1_path1_transcribe"
|
||||
log(f"running {axis} on {name}")
|
||||
try:
|
||||
text, t, vram = run_transcribe_path1(
|
||||
model, wav, batch_size=1, pnc="yes",
|
||||
timestamps=False, precision="bf16",
|
||||
)
|
||||
write_result({
|
||||
"axis": "path1", "path": "transcribe_auto_chunking",
|
||||
"batch_size": 1, "precision": "bf16",
|
||||
"audio": name, "duration_audio": dur,
|
||||
"duration_infer": round(t, 2), "vram_peak_mb": round(vram, 0),
|
||||
"chars": len(text), "text": text,
|
||||
})
|
||||
except Exception as e:
|
||||
log(f" FAIL: {e}")
|
||||
write_result({"axis": "path1", "audio": name, "error": str(e)})
|
||||
|
||||
log("=== sweep complete ===")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sweep()
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Mini-sweep: does chunk_len == audio_len fix Path 2 for short audios?
|
||||
|
||||
If yes → Path 2 is OK as long as audio fits in exactly one chunk.
|
||||
The bug is in the multi-chunk merging or pad/split logic.
|
||||
If no → Path 2 has a structural problem with single-chunk inputs
|
||||
independent of length matching. Confirms "different inference
|
||||
pipeline" theory.
|
||||
"""
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import wave
|
||||
|
||||
import torch
|
||||
from omegaconf import OmegaConf
|
||||
|
||||
from nemo.collections.asr.models import ASRModel
|
||||
from nemo.collections.asr.parts.utils.streaming_utils import FrameBatchMultiTaskAED
|
||||
from nemo.collections.asr.parts.utils.transcribe_utils import get_buffered_pred_feat_multitaskAED
|
||||
|
||||
|
||||
AUDIO_DIR = "/sweep/threshold_audios"
|
||||
RESULTS_PATH = "/sweep/chunklen_results.jsonl"
|
||||
|
||||
|
||||
def transcode(src):
|
||||
dst = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
|
||||
dst.close()
|
||||
subprocess.run(["ffmpeg", "-y", "-loglevel", "error", "-i", src,
|
||||
"-ac", "1", "-ar", "16000", "-f", "wav", dst.name], check=True)
|
||||
return dst.name
|
||||
|
||||
|
||||
def wav_dur(p):
|
||||
with wave.open(p, "rb") as w:
|
||||
return w.getnframes() / float(w.getframerate())
|
||||
|
||||
|
||||
def write_manifest(wav, dur):
|
||||
f = tempfile.NamedTemporaryFile(suffix=".json", mode="w", delete=False)
|
||||
json.dump({"audio_filepath": wav, "duration": dur, "taskname": "asr",
|
||||
"source_lang": "de", "target_lang": "de", "pnc": "yes"}, f)
|
||||
f.write("\n")
|
||||
f.close()
|
||||
return f.name
|
||||
|
||||
|
||||
def run_path2(model, cfg, wav, dur, chunk_len):
|
||||
manifest = write_manifest(wav, dur)
|
||||
feature_stride = cfg.preprocessor["window_stride"]
|
||||
model_stride = feature_stride * 8
|
||||
fa = FrameBatchMultiTaskAED(asr_model=model, frame_len=chunk_len,
|
||||
total_buffer=chunk_len, batch_size=8)
|
||||
t0 = time.monotonic()
|
||||
with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16):
|
||||
with torch.no_grad():
|
||||
hyps = get_buffered_pred_feat_multitaskAED(
|
||||
fa, cfg.preprocessor, model_stride, model.device,
|
||||
manifest=manifest, filepaths=None, timestamps=False,
|
||||
)
|
||||
elapsed = time.monotonic() - t0
|
||||
os.unlink(manifest)
|
||||
return (getattr(hyps[0], "text", None) or str(hyps[0])).strip(), elapsed
|
||||
|
||||
|
||||
def run_path1(model, wav):
|
||||
t0 = time.monotonic()
|
||||
with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16):
|
||||
with torch.no_grad():
|
||||
hyps = model.transcribe([wav], source_lang="de", target_lang="de",
|
||||
pnc="yes", batch_size=4)
|
||||
elapsed = time.monotonic() - t0
|
||||
return (getattr(hyps[0], "text", None) or str(hyps[0])).strip(), elapsed
|
||||
|
||||
|
||||
def main():
|
||||
open(RESULTS_PATH, "w").close()
|
||||
print("loading model bf16", flush=True)
|
||||
model = ASRModel.from_pretrained("nvidia/canary-1b-v2").to("cuda").to(torch.bfloat16)
|
||||
model.eval()
|
||||
cfg = copy.deepcopy(model._cfg)
|
||||
OmegaConf.set_struct(cfg.preprocessor, False)
|
||||
cfg.preprocessor.dither = 0.0
|
||||
cfg.preprocessor.pad_to = 0
|
||||
OmegaConf.set_struct(cfg.preprocessor, True)
|
||||
print("model loaded", flush=True)
|
||||
|
||||
audios = ["trim_20s.m4a", "trim_30s.m4a"]
|
||||
chunk_lens = {
|
||||
"trim_20s.m4a": [21.0, 25.0, 40.0], # exact, slightly larger, default
|
||||
"trim_30s.m4a": [31.0, 35.0, 40.0],
|
||||
}
|
||||
|
||||
def record(d):
|
||||
with open(RESULTS_PATH, "a") as f:
|
||||
f.write(json.dumps(d, ensure_ascii=False) + "\n")
|
||||
|
||||
for name in audios:
|
||||
src = os.path.join(AUDIO_DIR, name)
|
||||
wav = transcode(src)
|
||||
dur = wav_dur(wav)
|
||||
print(f"\n=== {name} ({dur:.1f}s) ===", flush=True)
|
||||
|
||||
# baseline path1
|
||||
text, t = run_path1(model, wav)
|
||||
print(f"path1: {len(text)} chars in {t:.1f}s", flush=True)
|
||||
record({"audio": name, "duration": dur, "path": "path1",
|
||||
"chunk_len": None, "chars": len(text),
|
||||
"duration_infer": round(t, 2), "text": text})
|
||||
|
||||
for cl in chunk_lens[name]:
|
||||
text, t = run_path2(model, cfg, wav, dur, cl)
|
||||
print(f"path2 chunk_len={cl}: {len(text)} chars in {t:.1f}s", flush=True)
|
||||
record({"audio": name, "duration": dur, "path": "path2",
|
||||
"chunk_len": cl, "chars": len(text),
|
||||
"duration_infer": round(t, 2), "text": text})
|
||||
|
||||
print("done", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,178 @@
|
||||
"""Threshold sweep — varying audio length around the 40 s Encoder-Window.
|
||||
|
||||
Compares three transcription paths on artificially trimmed audios:
|
||||
- path1_default: model.transcribe([wav]) — what main.py uses for ≤ 25 s
|
||||
- path1_bs1: model.transcribe([wav], batch_size=1) — NeMo ≥ 2.5 auto-chunking
|
||||
- path2: FrameBatchMultiTaskAED with chunk_len=40 — what main.py
|
||||
uses for > 25 s
|
||||
|
||||
Audios are 20, 30, 38 and 45 s slices of the 80.9 s source. The 25 s
|
||||
threshold in main.py sits in the middle, so this sweep tells us:
|
||||
- Does Path 1 truncate already at 30/38 s, or only > 40 s?
|
||||
- Does the threshold need to move?
|
||||
"""
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import wave
|
||||
|
||||
import torch
|
||||
from omegaconf import OmegaConf
|
||||
|
||||
from nemo.collections.asr.models import ASRModel
|
||||
from nemo.collections.asr.parts.utils.streaming_utils import FrameBatchMultiTaskAED
|
||||
from nemo.collections.asr.parts.utils.transcribe_utils import get_buffered_pred_feat_multitaskAED
|
||||
|
||||
|
||||
AUDIO_DIR = "/sweep/threshold_audios"
|
||||
OUT_DIR = "/sweep"
|
||||
RESULTS_PATH = f"{OUT_DIR}/threshold_results.jsonl"
|
||||
LOG_PATH = f"{OUT_DIR}/threshold_log.txt"
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
line = f"{time.strftime('%H:%M:%S')} {msg}"
|
||||
print(line, flush=True)
|
||||
with open(LOG_PATH, "a") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
|
||||
def transcode(src: str) -> str:
|
||||
dst = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
|
||||
dst.close()
|
||||
subprocess.run(
|
||||
["ffmpeg", "-y", "-loglevel", "error", "-i", src,
|
||||
"-ac", "1", "-ar", "16000", "-f", "wav", dst.name],
|
||||
check=True,
|
||||
)
|
||||
return dst.name
|
||||
|
||||
|
||||
def wav_duration(p: str) -> float:
|
||||
with wave.open(p, "rb") as w:
|
||||
return w.getnframes() / float(w.getframerate())
|
||||
|
||||
|
||||
def vram_mb() -> float:
|
||||
return torch.cuda.max_memory_allocated() / (1024 * 1024) if torch.cuda.is_available() else 0
|
||||
|
||||
|
||||
def write_manifest(wav_path, duration, pnc="yes"):
|
||||
f = tempfile.NamedTemporaryFile(suffix=".json", mode="w", delete=False)
|
||||
json.dump({"audio_filepath": wav_path, "duration": duration,
|
||||
"taskname": "asr", "source_lang": "de", "target_lang": "de",
|
||||
"pnc": pnc}, f)
|
||||
f.write("\n")
|
||||
f.close()
|
||||
return f.name
|
||||
|
||||
|
||||
def run_path1(model, wav, batch_size):
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
t0 = time.monotonic()
|
||||
with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16):
|
||||
with torch.no_grad():
|
||||
hyps = model.transcribe(
|
||||
[wav], source_lang="de", target_lang="de",
|
||||
pnc="yes", batch_size=batch_size,
|
||||
)
|
||||
return (getattr(hyps[0], "text", None) or str(hyps[0])).strip(), \
|
||||
time.monotonic() - t0, vram_mb()
|
||||
|
||||
|
||||
def run_path2(model, model_cfg, wav, dur, chunk_len=40.0):
|
||||
manifest = write_manifest(wav, dur)
|
||||
feature_stride = model_cfg.preprocessor["window_stride"]
|
||||
model_stride_secs = feature_stride * 8
|
||||
frame_asr = FrameBatchMultiTaskAED(
|
||||
asr_model=model, frame_len=chunk_len,
|
||||
total_buffer=chunk_len, batch_size=8,
|
||||
)
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
t0 = time.monotonic()
|
||||
with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16):
|
||||
with torch.no_grad():
|
||||
hyps = get_buffered_pred_feat_multitaskAED(
|
||||
frame_asr, model_cfg.preprocessor, model_stride_secs,
|
||||
model.device, manifest=manifest, filepaths=None, timestamps=False,
|
||||
)
|
||||
elapsed = time.monotonic() - t0
|
||||
os.unlink(manifest)
|
||||
return (getattr(hyps[0], "text", None) or str(hyps[0])).strip(), \
|
||||
elapsed, vram_mb()
|
||||
|
||||
|
||||
def main():
|
||||
open(LOG_PATH, "w").close()
|
||||
open(RESULTS_PATH, "w").close()
|
||||
|
||||
log("loading model bf16")
|
||||
model = ASRModel.from_pretrained("nvidia/canary-1b-v2")
|
||||
model = model.to("cuda").to(torch.bfloat16)
|
||||
model.eval()
|
||||
cfg = copy.deepcopy(model._cfg)
|
||||
OmegaConf.set_struct(cfg.preprocessor, False)
|
||||
cfg.preprocessor.dither = 0.0
|
||||
cfg.preprocessor.pad_to = 0
|
||||
OmegaConf.set_struct(cfg.preprocessor, True)
|
||||
log(f"model loaded, vram={vram_mb():.0f}MB")
|
||||
|
||||
audios = sorted(os.listdir(AUDIO_DIR))
|
||||
wavs = {}
|
||||
for name in audios:
|
||||
if not name.endswith(".m4a"):
|
||||
continue
|
||||
src = os.path.join(AUDIO_DIR, name)
|
||||
wav = transcode(src)
|
||||
dur = wav_duration(wav)
|
||||
wavs[name] = (wav, dur)
|
||||
log(f" {name} → {dur:.1f}s")
|
||||
|
||||
def record(d):
|
||||
with open(RESULTS_PATH, "a") as f:
|
||||
f.write(json.dumps(d, ensure_ascii=False) + "\n")
|
||||
|
||||
for name, (wav, dur) in wavs.items():
|
||||
# path 1 default (no batch_size override → triggers no auto-chunking
|
||||
# because input is a list, not a single string)
|
||||
log(f"path1_default on {name}")
|
||||
try:
|
||||
text, t, vram = run_path1(model, wav, batch_size=4)
|
||||
record({"audio": name, "duration_audio": dur, "path": "path1_default",
|
||||
"batch_size": 4, "duration_infer": round(t, 2),
|
||||
"vram_peak_mb": round(vram, 0), "chars": len(text), "text": text})
|
||||
except Exception as e:
|
||||
log(f" FAIL: {e}")
|
||||
record({"audio": name, "path": "path1_default", "error": str(e)})
|
||||
|
||||
# path 1 with batch_size=1 (auto-chunking trigger per NeMo docs)
|
||||
log(f"path1_bs1 on {name}")
|
||||
try:
|
||||
text, t, vram = run_path1(model, wav, batch_size=1)
|
||||
record({"audio": name, "duration_audio": dur, "path": "path1_bs1",
|
||||
"batch_size": 1, "duration_infer": round(t, 2),
|
||||
"vram_peak_mb": round(vram, 0), "chars": len(text), "text": text})
|
||||
except Exception as e:
|
||||
log(f" FAIL: {e}")
|
||||
record({"audio": name, "path": "path1_bs1", "error": str(e)})
|
||||
|
||||
# path 2 (always chunked, chunk_len=40)
|
||||
log(f"path2 on {name}")
|
||||
try:
|
||||
text, t, vram = run_path2(model, cfg, wav, dur, chunk_len=40.0)
|
||||
record({"audio": name, "duration_audio": dur, "path": "path2",
|
||||
"chunk_len": 40.0, "batch_size": 8,
|
||||
"duration_infer": round(t, 2),
|
||||
"vram_peak_mb": round(vram, 0), "chars": len(text), "text": text})
|
||||
except Exception as e:
|
||||
log(f" FAIL: {e}")
|
||||
record({"audio": name, "path": "path2", "error": str(e)})
|
||||
|
||||
log("done")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+734
@@ -0,0 +1,734 @@
|
||||
# Canary 1B v2 — Setup, Sweep-Reproduktion und Decoding-Schalter
|
||||
|
||||
Selbsterhaltende Anleitung für **NVIDIA Canary 1B v2** in doctate.
|
||||
Alles, was zur Installation, zur Reproduktion der bisherigen
|
||||
Sweep-Ergebnisse und zur Konfiguration der Decoding-Schalter nötig
|
||||
ist, steht in diesem Dokument. Detail- und Hintergrund-Material ist
|
||||
als Appendix angehängt.
|
||||
|
||||
## 1. Was und warum
|
||||
|
||||
- **Modell:** `nvidia/canary-1b-v2`. FastConformer-Encoder + AED
|
||||
(Attention-Encoder-Decoder), ca. 1 Mrd. Parameter. Vier Sprachen
|
||||
(en/de/fr/es), beste Deutsch-Performance unter den freien Modellen
|
||||
≤ 12 GB VRAM.
|
||||
- **Status in doctate:** zweiter ASR-Service neben Whisper, **nicht**
|
||||
Drop-in-Replacement. Whisper bleibt am `/asr`-Endpoint, Canary läuft
|
||||
separat als nativer FastAPI-Service auf Port 9002 für Vergleichs-
|
||||
und Forschungsläufe.
|
||||
- **Lizenz:** CC-BY-4.0 (kommerzielle Nutzung erlaubt, Attribution
|
||||
nötig).
|
||||
|
||||
## 2. Installation — Containerservice auf Minerva
|
||||
|
||||
Dies ist der Produktionspfad. Der Service läuft 24/7 auf
|
||||
`minerva.lan:9002`. Modellgewichte (~3 GB) werden im Volume
|
||||
`/opt/stacks/doctate-canary/models` gecached, sodass Restarts schnell
|
||||
sind.
|
||||
|
||||
### 2.1 Voraussetzungen auf der Zielmaschine
|
||||
|
||||
- Linux mit Docker und `docker compose`-Plugin
|
||||
- NVIDIA-Treiber + `nvidia-container-toolkit` (für GPU-Zugriff aus
|
||||
Containern)
|
||||
- Mindestens 12 GB GPU-VRAM für bf16-Inferenz
|
||||
- Mindestens 5 GB freier Plattenplatz (3 GB Modell, ca. 4.5 GB
|
||||
Container-Image)
|
||||
- SSH-Zugriff für `deploy.sh`
|
||||
|
||||
### 2.2 Container-Stack
|
||||
|
||||
Der Stack besteht aus fünf Files unter `canary/` im Repo (Inhalte
|
||||
siehe Appendix A):
|
||||
|
||||
| Datei | Zweck |
|
||||
|---|---|
|
||||
| `Dockerfile` | Image-Definition (cuda-devel + Python + ffmpeg + Pip-Deps) |
|
||||
| `requirements.txt` | NeMo, FastAPI, HF-Hub mit Versions-Pins |
|
||||
| `docker-compose.yml` | Service-Definition (Port 9002, GPU-Reservierung, Modell-Volume) |
|
||||
| `main.py` | FastAPI-App mit `/health`, `/info`, `POST /inference` |
|
||||
| `deploy.sh` | rsync + remote build + Health-Polling (10-min-Timeout) |
|
||||
|
||||
### 2.3 Deployment
|
||||
|
||||
Aus dem Dev-Repo:
|
||||
|
||||
```bash
|
||||
cd /home/brummel/dev/doctate/canary
|
||||
./deploy.sh minerva.lan
|
||||
```
|
||||
|
||||
`deploy.sh` führt aus:
|
||||
|
||||
1. `rsync` der Container-Files nach `minerva:/opt/stacks/doctate-canary/`
|
||||
2. `docker compose down` auf der Remote
|
||||
3. `docker pull nvidia/cuda:12.6.1-devel-ubuntu22.04` (umgeht einen
|
||||
TLS-Cert-Bug im Legacy-Builder, der sonst beim Pull innerhalb von
|
||||
`docker build` auftritt)
|
||||
4. `docker build -t doctate-canary .`
|
||||
5. `docker compose up -d`
|
||||
6. Pollt `/health` 120 × 5 s = 10 min lang. Beim Erststart muss das
|
||||
Modell aus HuggingFace geladen werden — das dauert mehrere Minuten.
|
||||
|
||||
Bei Erfolg gibt `deploy.sh` zusätzlich `/info` zurück, sonst die
|
||||
letzten 60 Container-Log-Zeilen.
|
||||
|
||||
### 2.4 API
|
||||
|
||||
Der Service hat eine **native** Canary-API (kein Whisper-Mimik):
|
||||
|
||||
| Methode | Pfad | Zweck |
|
||||
|---|---|---|
|
||||
| `GET` | `/health` | `{"status":"ok","model":...,"device":...,"precision":...}` |
|
||||
| `GET` | `/info` | Erweiterte Metadaten |
|
||||
| `POST` | `/inference` | Multipart-Upload, Inferenz, Antwort als Text oder JSON |
|
||||
|
||||
Form-Felder von `POST /inference`:
|
||||
|
||||
| Feld | Default | Werte |
|
||||
|---|---|---|
|
||||
| `file` | — | Upload (mp3/m4a/opus/wav/...). Intern via ffmpeg → 16 kHz mono WAV |
|
||||
| `language` | `de` | `en`/`de`/`fr`/`es` |
|
||||
| `pnc` | `yes` | `yes`/`no` (Punctuation/Capitalization) |
|
||||
| `timestamps` | `no` | `yes`/`no` (segment-level, nur in `json`-Antwort) |
|
||||
| `response_format` | `text` | `text` (PlainText) / `json` |
|
||||
|
||||
Smoke-Test:
|
||||
|
||||
```bash
|
||||
curl http://minerva.lan:9002/health
|
||||
curl -F file=@dictation.m4a -F language=de \
|
||||
http://minerva.lan:9002/inference
|
||||
```
|
||||
|
||||
### 2.5 Env Vars
|
||||
|
||||
| Variable | Default | Zweck |
|
||||
|---|---|---|
|
||||
| `CANARY_MODEL` | `nvidia/canary-1b-v2` | beliebiges NeMo-ASR-Modell |
|
||||
| `CANARY_DEVICE` | `cuda` | `cuda` oder `cpu` |
|
||||
| `CANARY_PRECISION` | `bf16` | `bf16` / `fp16` / `fp32` |
|
||||
| `CANARY_MODELS_DIR` | `/models` | HF-Cache (volume-persistent) |
|
||||
| `CANARY_CHUNK_LEN_SECS` | `40.0` | Chunk-Fensterlänge für Long-Form |
|
||||
| `CANARY_CHUNKED_THRESHOLD_SECS` | `25.0` | ab welcher Audiolänge gechunkt wird |
|
||||
| `HF_HOME` | wird auf `CANARY_MODELS_DIR` gesetzt | HuggingFace-Cache |
|
||||
|
||||
### 2.6 VRAM-Budget
|
||||
|
||||
Canary 1B v2 in bf16 belegt **ca. 9.7 GB** auf RTX 3060 mit
|
||||
`batch_size=8` für die buffered Pipeline. Das passt **nicht**
|
||||
gleichzeitig mit Whisper + Ollama auf eine 12-GB-Karte. Für
|
||||
Sweep-Läufe (Abschnitt 3) muss der Service kurzzeitig pausiert
|
||||
werden:
|
||||
|
||||
```bash
|
||||
ssh minerva.lan "cd /opt/stacks/doctate-canary && docker compose stop"
|
||||
# ... Sweep-Run hier ...
|
||||
ssh minerva.lan "cd /opt/stacks/doctate-canary && docker compose start"
|
||||
```
|
||||
|
||||
## 3. Reproduktion der Sweep-Ergebnisse vom 2026-04-30
|
||||
|
||||
Der Validation-Sweep hat **101 Einzeldiktate**, **47
|
||||
Per-Case-Concats** und **1 globales 38.7-min-Concat** durch Canary
|
||||
gejagt, um die `dur=0`-Halluzinations-Heuristik gegen ein echtes
|
||||
Korpus zu validieren.
|
||||
|
||||
**Ergebnis** (Disposition siehe Abschnitt 5):
|
||||
|
||||
| Quantity | Value |
|
||||
|---|---:|
|
||||
| dur=0 Vorkommen | 81 |
|
||||
| davon echte Halluzinationen | 2 |
|
||||
| False Positives (echtes Transkript) | 79 |
|
||||
| Precision | **≈ 2.5 %** |
|
||||
|
||||
Diese Schritte regenerieren die Daten von Grund auf. Die Sweep-
|
||||
Skripte liegen in `experiments/canary_sweep/`.
|
||||
|
||||
### 3.1 Cases einsammeln
|
||||
|
||||
```bash
|
||||
mkdir -p experiments/canary_sweep/cases experiments/canary_sweep/concat
|
||||
cd /home/brummel/dev/doctate/tmpdata/dr_mueller
|
||||
find . -name "*.m4a" | while read m4a_rel; do
|
||||
case_uuid=$(echo "$m4a_rel" | cut -d/ -f2)
|
||||
stem=$(basename "$m4a_rel" .m4a)
|
||||
dest="/home/brummel/dev/doctate/experiments/canary_sweep/cases/$case_uuid"
|
||||
mkdir -p "$dest"
|
||||
cp "./$case_uuid/$stem.m4a" "$dest/$stem.m4a"
|
||||
[ -f "./$case_uuid/$stem.json" ] && cp "./$case_uuid/$stem.json" "$dest/$stem.whisper.json"
|
||||
done
|
||||
```
|
||||
|
||||
Resultat: `cases/<uuid>/<stem>.m4a` (101 Files) plus 53 zugehörige
|
||||
`<stem>.whisper.json` aus dem Server-State (Rest sind ältere
|
||||
Aufnahmen, deren Whisper-Output nur in `document.md` zusammengeführt
|
||||
wurde).
|
||||
|
||||
### 3.2 Per-Case- und globalen Concat bauen
|
||||
|
||||
`build_per_case_concat.py` und `build_concat.py` ffmpeg-stitchen
|
||||
alle Audios zu einem WAV pro Case bzw. zu einem 38.7-min-Konglomerat
|
||||
über alle Cases. Beide Skripte schreiben jeweils ein
|
||||
`concat.manifest.json` mit der Reihenfolge und den Time-Offsets der
|
||||
Einzelschnipsel:
|
||||
|
||||
```bash
|
||||
cd experiments/canary_sweep
|
||||
python build_per_case_concat.py # cases/<uuid>/concat.wav
|
||||
python build_concat.py # concat/all.wav (38.7 min)
|
||||
```
|
||||
|
||||
Beide sind idempotent.
|
||||
|
||||
### 3.3 GPU-Konflikt klären, Sweep ausführen
|
||||
|
||||
Der laufende `doctate-canary`-Service belegt fast die ganze GPU. Vor
|
||||
dem Sweep einmal stoppen:
|
||||
|
||||
```bash
|
||||
ssh minerva.lan "cd /opt/stacks/doctate-canary && docker compose stop"
|
||||
```
|
||||
|
||||
Daten + Skripte hochladen:
|
||||
|
||||
```bash
|
||||
rsync -av experiments/canary_sweep/ minerva.lan:/tmp/canary_sweep/
|
||||
```
|
||||
|
||||
**Sweep über die 101 Einzelaudios** — Mode A (Single-Shot, mit
|
||||
Word-Confidence):
|
||||
|
||||
```bash
|
||||
ssh minerva.lan "docker run --rm --name canary-sweep --gpus all \
|
||||
-v /tmp/canary_sweep:/work \
|
||||
-v /opt/stacks/doctate-canary/models:/models \
|
||||
-e HF_HOME=/models \
|
||||
doctate-canary:latest \
|
||||
python3 /work/run_canary.py /work/cases /work/cases"
|
||||
```
|
||||
|
||||
**Per-Case-Concats** — adaptiv (≤ 25 s → Single-Shot, > 25 s →
|
||||
Buffered):
|
||||
|
||||
```bash
|
||||
ssh minerva.lan "docker run --rm --name canary-percase --gpus all \
|
||||
-v /tmp/canary_sweep:/work \
|
||||
-v /opt/stacks/doctate-canary/models:/models \
|
||||
-e HF_HOME=/models \
|
||||
doctate-canary:latest \
|
||||
python3 /work/run_canary_per_case.py /work/cases"
|
||||
```
|
||||
|
||||
**Globaler 38.7-min-Concat** — Buffered ist Pflicht (Single-Shot
|
||||
geht in OOM):
|
||||
|
||||
```bash
|
||||
ssh minerva.lan "docker run --rm --name canary-concat-buffered --gpus all \
|
||||
-v /tmp/canary_sweep:/work \
|
||||
-v /opt/stacks/doctate-canary/models:/models \
|
||||
-e HF_HOME=/models \
|
||||
doctate-canary:latest \
|
||||
python3 /work/run_canary_buffered.py \
|
||||
/work/concat/all.wav /work/concat/all.canary.json"
|
||||
```
|
||||
|
||||
Service wieder hochfahren:
|
||||
|
||||
```bash
|
||||
ssh minerva.lan "cd /opt/stacks/doctate-canary && docker compose start"
|
||||
```
|
||||
|
||||
### 3.4 Outputs zurücksyncen
|
||||
|
||||
```bash
|
||||
rsync -av --include='*/' \
|
||||
--include='*.canary.json' --include='concat.canary.json' \
|
||||
--exclude='*' \
|
||||
minerva.lan:/tmp/canary_sweep/cases/ \
|
||||
experiments/canary_sweep/cases/
|
||||
|
||||
rsync -av minerva.lan:/tmp/canary_sweep/concat/all.canary.json \
|
||||
experiments/canary_sweep/concat/
|
||||
```
|
||||
|
||||
### 3.5 dur=0-Inventar erzeugen
|
||||
|
||||
`dur_zero_inventory.py` walks alle 149 `*.canary.json` Files,
|
||||
extrahiert jedes Wort mit `start_offset == end_offset`, und gibt
|
||||
ein strukturiertes JSON sowie einen Markdown-Report aus:
|
||||
|
||||
```bash
|
||||
cd experiments/canary_sweep
|
||||
python dur_zero_inventory.py
|
||||
```
|
||||
|
||||
Schreibt `dur_zero_inventory.json` (alle 81 Records mit Kontext) und
|
||||
`dur_zero_inventory.md` (gruppiert nach File mit 3-Wörter-Kontext um
|
||||
jeden Treffer).
|
||||
|
||||
## 4. Decoding-Schalter — was, wann, wie
|
||||
|
||||
Canary 1B v2 hat zwei Inferenz-Pipelines mit **unterschiedlichen
|
||||
Output-Schemata**. Die Wahl bestimmt, welche Felder im Hypothesis-
|
||||
Objekt gefüllt werden.
|
||||
|
||||
### 4.1 Pipeline-Wahl
|
||||
|
||||
| Pipeline | Audiolänge | Aufruf | Hypothesis-Felder |
|
||||
|---|---|---|---|
|
||||
| **Single-Shot (Mode A)** | ≤ 25 s | `model.transcribe([wav], …, return_hypotheses=True)` | text, score, timestamp (word+segment), tokens, **word_confidence**, **token_confidence** |
|
||||
| **Buffered** | > 25 s | `FrameBatchMultiTaskAED` + `get_buffered_pred_feat_multitaskAED` | text, score, timestamp (word+segment) — **kein** word_confidence, keine tokens |
|
||||
|
||||
Die Schwelle ist `CHUNKED_THRESHOLD_SECS=25.0` im Service. Hauptlimit
|
||||
der buffered Pipeline: NeMo's `_join_hypotheses` in
|
||||
`nemo/collections/asr/parts/utils/streaming_utils.py` mergt nur
|
||||
`text`, `y_sequence` und `timestamp` über die Chunks — Confidence-
|
||||
Felder werden wegen fehlender `_join_word_confidence`-Methode
|
||||
verworfen. Das ist eine NeMo-Implementations-Lücke, kein
|
||||
Konfigurationsfehler. Die Pro-Chunk-Confidence ginge intern, die
|
||||
Streaming-Aggregation wirft sie nur weg.
|
||||
|
||||
### 4.2 Timestamps an/aus
|
||||
|
||||
**Schalter:** kwarg `timestamps=True` auf `model.transcribe()` bzw.
|
||||
auf `get_buffered_pred_feat_multitaskAED()`.
|
||||
|
||||
**Was sie liefern:**
|
||||
|
||||
```json
|
||||
{
|
||||
"timestamp": {
|
||||
"word": [{"word": "Therapie.", "start_offset": 46, "end_offset": 51,
|
||||
"start": 3.68, "end": 4.08}, …],
|
||||
"segment": [{"segment": "Therapie.", …}, …]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`*_offset` sind Encoder-Frames (FastConformer-Stride 80 ms),
|
||||
`start`/`end` sind Sekunden. Die Word-Timestamps stammen aus dem
|
||||
**CTC-Submodell-Forced-Alignment**, nicht aus dem AED selbst.
|
||||
|
||||
**Kosten:** Vernachlässigbar. Das CTC-Submodell läuft eh, der
|
||||
Schalter erhält nur dessen Output, anstatt ihn zu verwerfen. Im
|
||||
buffered Pfad gilt allerdings die NeMo-Empfehlung
|
||||
`chunk_len_in_secs=10.0` für genauere Word-Timestamps; mit unseren
|
||||
40-s-Chunks sind Word-Offsets weniger präzise (innerhalb eines
|
||||
Chunks ok, an Chunk-Übergängen verlieren sie Genauigkeit).
|
||||
|
||||
**Default im Service:** `timestamps=no` (segment-level, nur im JSON-
|
||||
Response wenn `response_format=json`). Wer Word-Timestamps für
|
||||
Forschungszwecke braucht (z.B. dur=0-Inventar), muss am
|
||||
`transcribe()`-Call selbst flippen — siehe Sweep-Skripte in
|
||||
Abschnitt 3.
|
||||
|
||||
### 4.3 Confidence an/aus
|
||||
|
||||
**Schalter:** vier Stellen, alle vier sind nötig — eine fehlt → Wert
|
||||
ist `None`:
|
||||
|
||||
```python
|
||||
# 1. Decoding-Strategie auf greedy umstellen.
|
||||
# Beam liefert keine Confidence — confidence_cfg wird im
|
||||
# Beam-Pfad ignoriert.
|
||||
cfg = OmegaConf.to_container(model.cfg.decoding, resolve=True)
|
||||
cfg["strategy"] = "greedy"
|
||||
|
||||
# 2. confidence_cfg-Flags setzen
|
||||
cfg.setdefault("confidence_cfg", {})
|
||||
cfg["confidence_cfg"]["preserve_word_confidence"] = True
|
||||
cfg["confidence_cfg"]["preserve_token_confidence"] = True
|
||||
cfg["confidence_cfg"]["preserve_frame_confidence"] = True
|
||||
cfg["confidence_cfg"]["aggregation"] = "min"
|
||||
|
||||
# 3. Greedy-spezifische Flags
|
||||
cfg.setdefault("greedy", {})
|
||||
cfg["greedy"]["preserve_token_confidence"] = True
|
||||
cfg["greedy"]["preserve_alignments"] = True
|
||||
model.change_decoding_strategy(OmegaConf.create(cfg))
|
||||
|
||||
# 4. transcribe() mit return_hypotheses=True (DER stille
|
||||
# Stolperdraht!)
|
||||
hyps = model.transcribe([wav], …, return_hypotheses=True)
|
||||
```
|
||||
|
||||
**Der stille Stolperdraht:** `return_hypotheses` ist standardmäßig
|
||||
`False`. Dann liefert NeMo Strings statt `Hypothesis`-Objekten —
|
||||
ohne dass irgendein Flag oder Log darauf hinweist. Egal welche
|
||||
Confidence-Flags man setzt: ohne `return_hypotheses=True` ist das
|
||||
Feld `None`. Das herauszufinden hat in der Diagnostik-Phase mehrere
|
||||
konfundierte Ablations-Runden gekostet.
|
||||
|
||||
**Wert:** Tsallis-Entropie mit `aggregation=min`, theoretisch im
|
||||
Bereich [0, 1] mit höher = sicherer. In der Praxis Magnituden um
|
||||
3·10⁻⁵ als Median, einzelne Wörter bei float32-Min (1.4·10⁻⁴⁵).
|
||||
Numerisch nutzbar nur über die relative Reihenfolge, nicht über
|
||||
absolute Schwellen.
|
||||
|
||||
**Wichtig — Confidence ist als Halluzinations-Signal nicht
|
||||
brauchbar.** Echtes medizinisches Vokabular ("Bisoprolol",
|
||||
"Ramipril") und Dosierungsdigits ("100", "101") haben dieselben
|
||||
extrem niedrigen Confidence-Werte wie tatsächliche Halluzinationen.
|
||||
Confidence kann "unsicheren echten Inhalt" und "halluzinierten
|
||||
Inhalt" nicht trennen. Detail siehe Appendix C.
|
||||
|
||||
### 4.4 Was zusammen geht — Ablations-Tabelle
|
||||
|
||||
Die kondensierte Wahrheitstabelle für die wichtigen Knöpfe:
|
||||
|
||||
| Strategie | Chunking | timestamps | return_hypotheses | text | confidence | xatt |
|
||||
|---|---|---|---|---|---|---|
|
||||
| beam | ON | True | False (default) | full | None | None |
|
||||
| greedy | ON | True | False | full | None | None |
|
||||
| greedy | ON | True | **True** | full | **gefüllt** | None |
|
||||
| greedy | OFF | False | True | **degradiert** | gefüllt | gefüllt |
|
||||
|
||||
**Lehre:** Confidence + Timestamps + voller Text gehen zusammen,
|
||||
sobald `strategy=greedy` und `return_hypotheses=True`.
|
||||
Cross-Attention (`xatt_scores`) ist die einzige Information, die
|
||||
`enable_chunking=False` erfordert — und damit verliert man bei
|
||||
Audios > 40 s die Textqualität, was Cross-Attention für Production
|
||||
disqualifiziert.
|
||||
|
||||
### 4.5 Cross-Attention (informativ, nicht in Production)
|
||||
|
||||
Falls jemand jemals `xatt_scores` braucht: zusätzlich zu den
|
||||
Confidence-Flags `MultiTaskTranscriptionConfig(enable_chunking=False)`
|
||||
via `override_config` auf `transcribe()`. Liefert pro Output-Token
|
||||
eine Verteilung über Encoder-Frames; Shannon-Entropy pro Token
|
||||
unterscheidet (auf einer einzelnen 76-s-Aufnahme) Halluzination
|
||||
(Entropie ~4.7) von echtem Inhalt (~3.0). Validation auf breiterem
|
||||
Korpus fehlt; nicht weiterverfolgt, weil dur=0 + Confidence schon
|
||||
als Signale gefallen sind.
|
||||
|
||||
## 5. Bekannte Sackgassen
|
||||
|
||||
| Idee | Verworfen weil |
|
||||
|---|---|
|
||||
| **`dur=0` als Halluzinations-Filter oder Advisory-Marker** | 79/81 FP auf dr_mueller — siehe Appendix C |
|
||||
| **Word-Confidence** als Filter oder Marker | medizinisches Vokabular und Digits haben dieselben niedrigen Werte wie Halluzinationen — siehe Appendix C |
|
||||
| Whisper `suppress_tokens=\d+` gegen Digit-Halluzinationen | erzeugt Halluzinations-Loops; `\d` allein lässt Digit-Sequenzen ganz verschwinden |
|
||||
| Whisper-Hotwords aus einzelnen Test-Cases | halluziniert in anderen Cases (Curve-Fitting auf Einzelbeispielen) |
|
||||
| Buffered Pipeline mit Confidence-Output | NeMo `_join_hypotheses` mergt Confidence-Felder nicht (siehe Abschnitt 4.1) |
|
||||
|
||||
Falls Halluzinations-Detektion irgendwann nötig wird: das Signal
|
||||
muss aus *außerhalb* der Per-Word-Telemetrie eines einzelnen ASR-
|
||||
Modells kommen. Die naheliegenden Kandidaten:
|
||||
|
||||
- LLM-Konsolidierung mit medizinischem Domänen-Wissen (geschieht
|
||||
implizit bereits, könnte explizit gemacht werden)
|
||||
- Cross-Modell-Vergleich: AED-Text vs CTC-Standalone, oder Canary
|
||||
vs Whisper, mit Divergenz als Signal
|
||||
- Domänen-Regeln: nicht-existierende Medikamentennamen,
|
||||
unplausible Dosierungswerte etc. — aktuell vom Gazetteer
|
||||
pre-LLM erledigt
|
||||
|
||||
## 6. Domänen-Stolperfallen
|
||||
|
||||
### 6.1 "1-0-1"-Notation
|
||||
|
||||
Deutsche medizinische Diktate sprechen Dosierungsschemata als
|
||||
Digit-Triple ("eins null eins" = morgens 1, mittags 0, abends 1).
|
||||
Varianten: "1-0-0", "½-0-½", "1-0-0-1" (Vier-Slot). Canarys BPE-
|
||||
Tokenizer kollabiert das oft zu einem einzigen Token "101", "100"
|
||||
etc. — derselbe Mechanismus wie der allgemeine Digit-Collapse-
|
||||
Effekt.
|
||||
|
||||
**Konsequenz:** Bare 3-Digit-Tokens im Medikationskontext sind
|
||||
**echter Inhalt**, nicht Halluzination. Eine Pipeline-Stufe, die
|
||||
"100" oder "101" als Halluzination markiert (z.B. weil Confidence
|
||||
niedrig ist), zerstört reale Dosierungsinformation — patient-
|
||||
sicherheitsrelevant.
|
||||
|
||||
**Validierungsbeispiel** (case_c414cf52, Ramipril 5 mg 1-0-1):
|
||||
Der AED produzierte zunächst "100." für gesprochenes "1-0-1"
|
||||
(Digit-Collapse mit falscher Korrektur), dann eine Selbst-Korrektur
|
||||
mit "5 milligramm 101" — beide erscheinen im Transkript. Das "101"
|
||||
ist die korrekte Wiedergabe, das "100." der Fehler. Ein Filter,
|
||||
der nur niedrig-confidente Digits droppt, würde beide vernichten.
|
||||
|
||||
### 6.2 Stille zwischen Schnipsel-Concats
|
||||
|
||||
Beim globalen 38.7-min-Concat (Buffered Pipeline) meldet NeMo für
|
||||
mehrere Chunks `text of utterance with ID: audio_X is empty`. Das
|
||||
sind die Übergangs-Stellen zwischen aneinandergehängten Audios mit
|
||||
Stille — kein Bug, sondern erwartet. Die Output-Hypothese hat
|
||||
trotzdem 1555 Wörter, also keine Information verloren.
|
||||
|
||||
---
|
||||
|
||||
## Appendix A — Container-Files (vollständige Inhalte)
|
||||
|
||||
### Dockerfile
|
||||
|
||||
```dockerfile
|
||||
FROM nvidia/cuda:12.6.1-devel-ubuntu22.04
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
python3 python3-pip python3-dev \
|
||||
ffmpeg \
|
||||
libsndfile1 \
|
||||
git \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --upgrade pip && \
|
||||
pip install -r requirements.txt
|
||||
|
||||
COPY main.py .
|
||||
|
||||
ENV CANARY_MODEL=nvidia/canary-1b-v2 \
|
||||
CANARY_DEVICE=cuda \
|
||||
CANARY_PRECISION=bf16 \
|
||||
CANARY_MODELS_DIR=/models \
|
||||
HF_HOME=/models
|
||||
|
||||
VOLUME ["/models"]
|
||||
EXPOSE 9002
|
||||
|
||||
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "9002", "--workers", "1"]
|
||||
```
|
||||
|
||||
### requirements.txt
|
||||
|
||||
```
|
||||
nemo_toolkit[asr]>=2.7.3,<3.0
|
||||
huggingface_hub>=0.26.0,<1.0
|
||||
fastapi==0.115.0
|
||||
uvicorn[standard]==0.30.6
|
||||
python-multipart==0.0.9
|
||||
```
|
||||
|
||||
**Begründung der Pins:** `nemo_toolkit ≥ 2.7.3` ist Pflicht, weil in
|
||||
NeMo 2.4.x ein Bug verhinderte, dass `FrameBatchMultiTaskAED` mit
|
||||
Canary 1B v2 funktioniert (`CanaryBPETokenizer.vocabulary` fehlte).
|
||||
In 2.5+ ist der Fall `tokenizer.vocab_size`-Fallback eingebaut. Wir
|
||||
pinnen nach unten auf 2.7.3 (in Production verifiziert) und nach
|
||||
oben auf < 3.0 (kein erwartet-stabiler Bruch).
|
||||
|
||||
### docker-compose.yml
|
||||
|
||||
```yaml
|
||||
services:
|
||||
doctate-canary:
|
||||
image: doctate-canary:latest
|
||||
container_name: doctate-canary
|
||||
environment:
|
||||
- CANARY_MODEL=nvidia/canary-1b-v2
|
||||
- CANARY_PRECISION=bf16
|
||||
- NVIDIA_VISIBLE_DEVICES=all
|
||||
- NVIDIA_DRIVER_CAPABILITIES=compute,utility
|
||||
volumes:
|
||||
- /opt/stacks/doctate-canary/models:/models
|
||||
ports:
|
||||
- "9002:9002"
|
||||
shm_size: 2g
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [gpu]
|
||||
restart: unless-stopped
|
||||
```
|
||||
|
||||
### deploy.sh
|
||||
|
||||
```bash
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
HOST="${1:-minerva.lan}"
|
||||
REMOTE_DIR="/opt/stacks/doctate-canary"
|
||||
LOCAL_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
rsync -av --exclude 'deploy.sh' --exclude 'models/' \
|
||||
"$LOCAL_DIR/" "$HOST:$REMOTE_DIR/"
|
||||
|
||||
ssh "$HOST" "cd $REMOTE_DIR && \
|
||||
docker compose down && \
|
||||
docker pull nvidia/cuda:12.6.1-devel-ubuntu22.04 && \
|
||||
docker build -t doctate-canary . && \
|
||||
docker compose up -d"
|
||||
|
||||
for i in $(seq 1 120); do
|
||||
if ssh "$HOST" "curl -sf http://localhost:9002/health" 2>/dev/null; then
|
||||
echo " OK"; exit 0
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
ssh "$HOST" "docker logs doctate-canary --tail 60"
|
||||
exit 1
|
||||
```
|
||||
|
||||
## Appendix B — Service-Schlüssel-Code (`main.py`-Auszug)
|
||||
|
||||
Diese beiden Funktionen aus `canary/main.py` sind die
|
||||
Implementierungs-Quelle für die Pipeline-Wahl in Abschnitt 4.1:
|
||||
|
||||
```python
|
||||
def _transcribe_buffered(wav_path, duration, language, pnc, timestamps):
|
||||
"""Long-audio path: explicit chunked inference via FrameBatchMultiTaskAED.
|
||||
|
||||
Writes a temporary single-line JSONL manifest because language
|
||||
and pnc must be passed via manifest fields when using the
|
||||
buffered API.
|
||||
"""
|
||||
manifest_tmp = tempfile.NamedTemporaryFile(suffix=".json", mode="w", delete=False)
|
||||
json.dump({"audio_filepath": wav_path, "duration": duration,
|
||||
"taskname": "asr", "source_lang": language,
|
||||
"target_lang": language, "pnc": pnc}, manifest_tmp)
|
||||
manifest_tmp.write("\n"); manifest_tmp.close()
|
||||
|
||||
frame_asr = FrameBatchMultiTaskAED(
|
||||
asr_model=model, frame_len=CHUNK_LEN_SECS,
|
||||
total_buffer=CHUNK_LEN_SECS, batch_size=8,
|
||||
)
|
||||
with torch.amp.autocast(model.device.type,
|
||||
enabled=use_amp, dtype=DTYPE_MAP[PRECISION]):
|
||||
with torch.no_grad():
|
||||
hyps = get_buffered_pred_feat_multitaskAED(
|
||||
frame_asr, _model_cfg.preprocessor, MODEL_STRIDE_SECS,
|
||||
model.device, manifest=manifest_tmp.name,
|
||||
filepaths=None, timestamps=timestamps)
|
||||
return hyps
|
||||
|
||||
# Im /inference-Handler:
|
||||
chunked = duration > CHUNKED_THRESHOLD_SECS
|
||||
if chunked:
|
||||
hyps = _transcribe_buffered(wav_path, duration, language, pnc, ts_flag)
|
||||
else:
|
||||
hyps = model.transcribe([wav_path], source_lang=language,
|
||||
target_lang=language, pnc=pnc,
|
||||
timestamps=ts_flag)
|
||||
```
|
||||
|
||||
Pre-Conditions auf das Modell-Config-Objekt, einmal beim Service-
|
||||
Start zu setzen (sonst inkonsistente Feature-Extraktion zwischen
|
||||
Single-Shot- und Buffered-Pfad):
|
||||
|
||||
```python
|
||||
_model_cfg = copy.deepcopy(model._cfg)
|
||||
OmegaConf.set_struct(_model_cfg.preprocessor, False)
|
||||
_model_cfg.preprocessor.dither = 0.0
|
||||
_model_cfg.preprocessor.pad_to = 0
|
||||
OmegaConf.set_struct(_model_cfg.preprocessor, True)
|
||||
assert _model_cfg.preprocessor.normalize == "per_feature"
|
||||
MODEL_STRIDE_SECS = float(_model_cfg.preprocessor["window_stride"]) * 8
|
||||
```
|
||||
|
||||
## Appendix C — `dur=0` Validation auf dr_mueller
|
||||
|
||||
### Methodik
|
||||
|
||||
1. Cohorte: 101 m4a-Aufnahmen aus 48 Case-Verzeichnissen.
|
||||
2. Drei Inferenz-Pässe:
|
||||
- **Single Audios** (101 Files): Mode A — Single-Shot mit
|
||||
`confidence_cfg`, `return_hypotheses=True`, `timestamps=True`.
|
||||
- **Per-Case-Concats** (47 Files): adaptiv (≤ 25 s →
|
||||
Single-Shot, > 25 s → Buffered).
|
||||
- **Globaler Concat** (1 File, 38.7 min): Buffered.
|
||||
3. `dur_zero_inventory.py` walks alle 149 `*.canary.json` und
|
||||
extrahiert jedes Wort mit `start_offset == end_offset`.
|
||||
4. **Ground-Truth-Annotation durch User**: Jedes flagged Vorkommen
|
||||
im 3-Wörter-Kontext gegen die zugehörige Audio-Datei gehört.
|
||||
|
||||
### Resultat
|
||||
|
||||
| Quantity | Value |
|
||||
|---|---:|
|
||||
| dur=0 Vorkommen | 81 |
|
||||
| True Positives (echte Halluzinationen) | 2 |
|
||||
| False Positives (echtes Transkript) | 79 |
|
||||
| Precision | **≈ 2.5 %** |
|
||||
|
||||
Verteilung nach Quelle × Pipeline:
|
||||
|
||||
| Quelle | Pipeline | n_zero_dur |
|
||||
|---|---|---:|
|
||||
| single_audio | single_shot | 29 |
|
||||
| per_case_concat | single_shot | 1 |
|
||||
| per_case_concat | buffered | 21 |
|
||||
| global_concat | buffered | 30 |
|
||||
|
||||
Die zwei True Positives sind beide in case_c414cf52: die Wörter
|
||||
"5" und "milligramm" im Cluster `Bisoprolol 5 mg 100 und Ramipril
|
||||
5 mg 100. [5 milligramm 101] wird vorerst pausiert ...`. Beachte:
|
||||
"101" in diesem Cluster ist **echter Inhalt** (1-0-1-Schema für
|
||||
Ramipril, siehe Abschnitt 6.1).
|
||||
|
||||
Die 79 False Positives sind dominant unbetonte deutsche
|
||||
Funktionswörter. Top-Formen (count ≥ 2):
|
||||
|
||||
```
|
||||
6 und 4 einer 4 sich 3 wie 3 für
|
||||
3 mit 3 einen 2 wird 2 von 2 es
|
||||
2 um 2 pro 2 einmal 2 im
|
||||
```
|
||||
|
||||
Diese Wörter werden im CTC-Viterbi-Forced-Alignment auf 0-Frame-
|
||||
Seams zwischen Nachbarwörter gepresst, weil ihre akustische
|
||||
Realisierung zu kurz oder zu reduziert ist, um eigene Encoder-
|
||||
Frames zu beanspruchen. Strukturell sieht das identisch zu einer
|
||||
Halluzination aus — die Alignment-Mathematik allein kann die
|
||||
beiden nicht trennen.
|
||||
|
||||
### Disposition
|
||||
|
||||
`dur=0` ist als Halluzinations-Signal nicht verwendbar:
|
||||
|
||||
- **Auto-Filter:** Bei 79/81 FP würde das Droppen geflagged Wörter
|
||||
in jedem Diktat echten Transkript zerstören, mit dem Upside, in
|
||||
einer einzigen Aufnahme zwei Halluzinationen zu erwischen.
|
||||
Net-negativ.
|
||||
- **Advisory `[?word]` Marker** für die LLM-Konsolidierung:
|
||||
Gleiches Problem mit niedrigeren Stakes. 79/81 korrekte Wörter
|
||||
als unsicher zu markieren verschmutzt den Prompt und trainiert
|
||||
die LLM, den Marker zu ignorieren — was seinen Nutzen für die
|
||||
echten Halluzinationen kompromittiert.
|
||||
|
||||
Word-Confidence verhält sich auf demselben Korpus analog: das
|
||||
Top-6-niedrigste-Confidence-Wort sind "SC", "Bisoprolol",
|
||||
"Thorazemit", "5", "ausärztliche", "einmal" — eine Mischung aus
|
||||
echtem medizinischen Vokabular, echten Dosierungsdigits und
|
||||
einzelnen tatsächlichen Halluzinationen. Confidence kann "unsicher
|
||||
echt" und "halluziniert" nicht trennen.
|
||||
|
||||
### Implikation für die Pipeline-Architektur
|
||||
|
||||
Halluzinations-Detektion über Per-Word-Telemetrie eines einzelnen
|
||||
ASRs ist ein Sackgasse. Kandidaten für künftige Versuche, falls
|
||||
nötig:
|
||||
|
||||
- Cross-Modell-Agreement (Canary AED vs Canary CTC-Standalone vs
|
||||
Whisper) als Divergenz-Signal
|
||||
- LLM-seitige semantische Plausibilitätsprüfung (passiert implizit,
|
||||
könnte explizit gemacht werden)
|
||||
- Domänenregeln (Medikamenten-Existenz, Dosierungs-Plausibilität)
|
||||
— aktuell vom Gazetteer pre-LLM erledigt
|
||||
|
||||
## Appendix D — Container-Image bauen ohne `deploy.sh`
|
||||
|
||||
Manuell auf einer beliebigen GPU-Maschine:
|
||||
|
||||
```bash
|
||||
cd canary/
|
||||
docker build -t doctate-canary .
|
||||
docker run --rm --gpus all \
|
||||
-v $(pwd)/models:/models \
|
||||
-p 9002:9002 \
|
||||
-e HF_HOME=/models \
|
||||
doctate-canary
|
||||
```
|
||||
|
||||
Beim Erststart lädt der Container ~3 GB aus HuggingFace ins Volume
|
||||
`./models`. Nachfolgende Restarts nutzen den Cache.
|
||||
@@ -1,5 +1,6 @@
|
||||
/target/
|
||||
/_data/
|
||||
/canary_sweep/
|
||||
|
||||
# Fall-spezifische Inputs (Audios, Diktat-Text, Befund-Snapshots,
|
||||
# vocab-Test-Erweiterungen) bleiben strikt lokal — Patientendaten und
|
||||
|
||||
Reference in New Issue
Block a user