diff --git a/canary/probe_attn_conf.py b/canary/probe_attn_conf.py new file mode 100644 index 0000000..1f64ae2 --- /dev/null +++ b/canary/probe_attn_conf.py @@ -0,0 +1,178 @@ +"""Pull cross-attention scores and word confidence on the dosing audio. + +Goal: see whether the AED's cross-attention distribution and per-word +confidence flag the hallucinated "5 milligramm 101" cluster that we +already identified via zero-duration word offsets. + +Two new signals beyond what probe_canary_ts.py captured: + 1. word_confidence — per-word probability proxy. Hallucinations *should* + have lower confidence than real words, though not guaranteed because + AEDs hallucinate with high confidence too. + 2. xatt_scores — cross-attention from decoder tokens to encoder frames. + For each output token we summarize where it "looked" in the audio: + - peak_frame (argmax) and peak_time_s + - entropy of the distribution (flat = no clear acoustic anchor) + Hallucinated tokens should show high entropy (no clear acoustic peak). + +Output: + /sweep/probe_attn_conf.json — text + timestamp + word_confidence + per-token + xatt summary (argmax frame, entropy) +""" +import json +import math +import subprocess +import tempfile + +import torch +from omegaconf import OmegaConf + +from nemo.collections.asr.models import EncDecMultiTaskModel +from nemo.collections.asr.models.aed_multitask_models import ( + MultiTaskTranscriptionConfig, + parse_multitask_prompt, +) + + +WAV_SRC = "/sweep/case_c414cf52_15-01-39.m4a" +OUT_PATH = "/sweep/probe_attn_conf_kwargs_returnhyp.json" + + +def transcode(src: str) -> str: + dst = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) + dst.close() + subprocess.run(["ffmpeg", "-y", "-loglevel", "error", "-i", src, + "-ac", "1", "-ar", "16000", "-f", "wav", dst.name], + check=True) + return dst.name + + +def to_jsonable(o): + if isinstance(o, torch.Tensor): + return o.detach().cpu().tolist() + if hasattr(o, "tolist"): + return o.tolist() + if hasattr(o, "item"): + return o.item() + return str(o) + + +def entropy_of(probs: torch.Tensor) -> float: + p = probs.clamp_min(1e-12) + return float(-(p * p.log()).sum().item()) + + +def summarize_xatt(xatt_scores, frame_stride_s: float): + """xatt_scores: List[Tensor] over decoder layers. + + Multi-head attention output shape is typically [heads, tokens, frames] + or [tokens, frames] depending on the decoder. We mean-pool away any + leading dimensions until we get a 2D [tokens, frames] matrix. + """ + if not xatt_scores: + return None + layers = [x.float() for x in xatt_scores] + print(f" xatt layer shapes: {[tuple(l.shape) for l in layers]}", flush=True) + avg = torch.stack(layers, dim=0).mean(dim=0) + # Collapse all leading dims (batch, heads, ...) until 2D + while avg.dim() > 2: + avg = avg.mean(dim=0) + n_tokens, n_frames = avg.shape + summaries = [] + for t in range(n_tokens): + row = avg[t] + norm = row / row.sum().clamp_min(1e-12) + peak = int(norm.argmax().item()) + summaries.append({ + "token_idx": t, + "peak_frame": peak, + "peak_time_s": round(peak * frame_stride_s, 3), + "peak_prob": round(float(norm[peak].item()), 4), + "entropy": round(entropy_of(norm), 4), + "max_log_entropy": round(math.log(n_frames), 4), + }) + return {"n_tokens": n_tokens, "n_frames": n_frames, + "frame_stride_s": frame_stride_s, + "n_layers": len(layers), "per_token": summaries} + + +def main(): + print("loading Canary 1B v2", flush=True) + model = EncDecMultiTaskModel.from_pretrained("nvidia/canary-1b-v2") + model = model.to("cuda").to(torch.bfloat16) + model.eval() + + # Patch the live decoding config: + # - greedy strategy (confidence is only filled in greedy path, not beam) + # - return_xattn_scores=True + # - confidence_cfg with all preserve flags + # - enable_chunking=False is set via override_config below (NeMo gates xatt + # on chunking; chunking also blocks word-level confidence aggregation). + cfg = OmegaConf.to_container(model.cfg.decoding, resolve=True) + cfg["strategy"] = "greedy" + cfg["return_xattn_scores"] = True + cfg.setdefault("confidence_cfg", {}) + cfg["confidence_cfg"]["preserve_word_confidence"] = True + cfg["confidence_cfg"]["preserve_token_confidence"] = True + cfg["confidence_cfg"]["preserve_frame_confidence"] = True + cfg["confidence_cfg"]["aggregation"] = "min" # most signal-preserving + cfg.setdefault("greedy", {}) + cfg["greedy"]["preserve_token_confidence"] = True + cfg["greedy"]["preserve_alignments"] = True + print("applying decoding override:", json.dumps(cfg, indent=2), flush=True) + model.change_decoding_strategy(OmegaConf.create(cfg)) + + # Encoder frame stride: feature_stride * subsampling + feature_stride = float(model.cfg.preprocessor["window_stride"]) + subsampling = 8 # FastConformer default; verified empirically against + # 80ms-per-frame in probe_tdt.md + frame_stride_s = feature_stride * subsampling + print(f"frame stride: {frame_stride_s} s", flush=True) + + wav = transcode(WAV_SRC) + # Clean single-knob ablation: same override_config for both runs, + # only `enable_chunking` differs. All other fields (batch_size, prompt, + # timestamps, return_hypotheses) held constant. + print(f"running transcribe(KWARGS path, return_hypotheses=True, timestamps=True) on {WAV_SRC}", + flush=True) + # Final clean test: kwargs path (like the original Mode A run that + # showed None confidence), but with return_hypotheses=True forced. + # If confidence appears, then `return_hypotheses` was the real + # confounder all along — not timestamps, not the call form. + with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16): + with torch.no_grad(): + hyps = model.transcribe([wav], source_lang="de", target_lang="de", + pnc="yes", timestamps=True, batch_size=1, + return_hypotheses=True) + h = hyps[0] + + out = { + "text": getattr(h, "text", None), + "score": getattr(h, "score", None), + "timestamp": getattr(h, "timestamp", None), + "tokens": getattr(h, "tokens", None), + "word_confidence": getattr(h, "word_confidence", None), + "token_confidence": getattr(h, "token_confidence", None), + "xatt_summary": summarize_xatt(getattr(h, "xatt_scores", None), + frame_stride_s), + } + + with open(OUT_PATH, "w") as f: + json.dump(out, f, default=to_jsonable, ensure_ascii=False, indent=2) + + # Quick sanity print + print(f"\ntext: {h.text}", flush=True) + print(f"score: {h.score}", flush=True) + wc = getattr(h, "word_confidence", None) + print(f"word_confidence: {len(wc) if wc is not None else 'None'} entries", flush=True) + tc = getattr(h, "token_confidence", None) + print(f"token_confidence: {len(tc) if tc is not None else 'None'} entries", flush=True) + xs = getattr(h, "xatt_scores", None) + if xs is not None: + print(f"xatt_scores: {len(xs)} layer(s); shapes={[tuple(x.shape) for x in xs]}", flush=True) + else: + print("xatt_scores: None", flush=True) + print(f"\nfull dump -> {OUT_PATH}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/canary/probe_canary_ts.py b/canary/probe_canary_ts.py new file mode 100644 index 0000000..9319ec1 --- /dev/null +++ b/canary/probe_canary_ts.py @@ -0,0 +1,69 @@ +"""Show forced-alignment timestamps from Canary's main transcribe(timestamps=True). + +Demonstrates the full Hypothesis structure produced by: + AED text + CTC submodel acoustic alignment via Viterbi. + +Output: + /sweep/canary_timestamps_demo.json — full pretty-printed dict + stdout — short summary +""" +import json +import os +import subprocess +import tempfile + +import torch + +from nemo.collections.asr.models import EncDecMultiTaskModel + + +WAV_SRC = "/sweep/case_c414cf52_15-01-39.m4a" +OUT_PATH = "/sweep/canary_timestamps_dosing.json" + + +def transcode(src: str) -> str: + dst = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) + dst.close() + subprocess.run(["ffmpeg", "-y", "-loglevel", "error", "-i", src, + "-ac", "1", "-ar", "16000", + "-f", "wav", dst.name], check=True) + return dst.name + + +def to_jsonable(o): + if hasattr(o, "tolist"): + return o.tolist() + if hasattr(o, "item"): + return o.item() + return str(o) + + +def main(): + print("loading Canary 1B v2", flush=True) + model = EncDecMultiTaskModel.from_pretrained("nvidia/canary-1b-v2") + model = model.to("cuda").to(torch.bfloat16) + model.eval() + + wav = transcode(WAV_SRC) + print(f"running model.transcribe(timestamps=True) on {WAV_SRC}", flush=True) + with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16): + with torch.no_grad(): + hyps = model.transcribe([wav], source_lang="de", target_lang="de", + pnc="yes", timestamps=True, batch_size=4) + h = hyps[0] + + out = {"text": getattr(h, "text", None), + "timestamp": getattr(h, "timestamp", None)} + with open(OUT_PATH, "w") as f: + json.dump(out, f, default=to_jsonable, ensure_ascii=False, indent=2) + + print(f"\ntext: {h.text}", flush=True) + ts = h.timestamp if isinstance(h.timestamp, dict) else {} + for k, v in ts.items(): + n = len(v) if hasattr(v, "__len__") else "?" + print(f"timestamp[{k}]: {n} entries", flush=True) + print(f"\nfull dump -> {OUT_PATH}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/canary/probe_ctc_standalone.py b/canary/probe_ctc_standalone.py new file mode 100644 index 0000000..ebc2b6e --- /dev/null +++ b/canary/probe_ctc_standalone.py @@ -0,0 +1,68 @@ +"""Standalone CTC-submodel transcription on the dosing audio. + +Loads Canary, accesses model.timestamps_asr_model directly, calls its own +transcribe() — bypassing the AED decoder entirely. Goal: see whether the CTC +submodel renders the spoken digit sequence "eins null eins" as separate +tokens or also collapses it. +""" +import json +import subprocess +import tempfile + +import torch + +from nemo.collections.asr.models import EncDecMultiTaskModel + + +WAV_SRC = "/sweep/case_c414cf52_15-01-39.m4a" +OUT_PATH = "/sweep/ctc_standalone_dosing.json" + + +def transcode(src: str) -> str: + dst = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) + dst.close() + subprocess.run(["ffmpeg", "-y", "-loglevel", "error", "-i", src, + "-ac", "1", "-ar", "16000", "-f", "wav", dst.name], + check=True) + return dst.name + + +def to_jsonable(o): + if hasattr(o, "tolist"): + return o.tolist() + if hasattr(o, "item"): + return o.item() + return str(o) + + +def main(): + print("loading Canary 1B v2 (just for the CTC submodel)", flush=True) + model = EncDecMultiTaskModel.from_pretrained("nvidia/canary-1b-v2") + model = model.to("cuda").to(torch.bfloat16) + model.eval() + + tdt = model.timestamps_asr_model + print(f"submodel: {type(tdt).__name__}", flush=True) + + wav = transcode(WAV_SRC) + print(f"running tdt.transcribe(timestamps=True) on {WAV_SRC}", flush=True) + with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16): + with torch.no_grad(): + hyps = tdt.transcribe([wav], timestamps=True) + h = hyps[0] + + out = {"text": getattr(h, "text", None), + "timestamp": getattr(h, "timestamp", None)} + with open(OUT_PATH, "w") as f: + json.dump(out, f, default=to_jsonable, ensure_ascii=False, indent=2) + + print(f"\nstandalone CTC text:\n{h.text}\n", flush=True) + ts = h.timestamp if isinstance(h.timestamp, dict) else {} + for k, v in ts.items(): + n = len(v) if hasattr(v, "__len__") else "?" + print(f"timestamp[{k}]: {n} entries", flush=True) + print(f"\nfull dump -> {OUT_PATH}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/canary/probe_tdt.py b/canary/probe_tdt.py new file mode 100644 index 0000000..c13676f --- /dev/null +++ b/canary/probe_tdt.py @@ -0,0 +1,215 @@ +"""TDT probe — can timestamps_asr_model handle audio standalone, including >40 s? + +Background: + EncDecMultiTaskModel.timestamps_asr_model is a Token-Duration-Transducer + (RNN-T variant) with hybrid CTC head, FastConformer encoder enc_hidden=1024. + Bundled inside Canary 1B v2 specifically to provide forced-alignment + timestamps for the AED head, but it is itself a complete ASR model. + + Open question: does the relative positional encoding lift the 40 s wall that + AED hits, or does TDT inherit the same training-time max_duration limit? + +Test set: + - 20 s : sanity check, well within both models + - 80 s : full source — Canary path1 truncates here, baseline expectation + - 160 s : concat(80, 80), no re-encode — pushes 4x past TDT training horizon + +Per case: + - TDT.transcribe(timestamps=False) + - TDT.transcribe(timestamps=True) — also verify timestamp shape + - Canary path1 — reference, will truncate >40 s +""" +import json +import os +import subprocess +import tempfile +import time +import wave + +import torch + +from nemo.collections.asr.models import EncDecMultiTaskModel + + +AUDIO_DIR = "/sweep/threshold_audios" +RESULTS_PATH = "/sweep/probe_tdt.jsonl" +LOG_PATH = "/sweep/probe_tdt.log" + + +def log(msg: str) -> None: + line = f"{time.strftime('%H:%M:%S')} {msg}" + print(line, flush=True) + with open(LOG_PATH, "a") as f: + f.write(line + "\n") + + +def transcode(src: str, duration_secs: float | None = None) -> str: + dst = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) + dst.close() + cmd = ["ffmpeg", "-y", "-loglevel", "error", "-i", src, + "-ac", "1", "-ar", "16000"] + if duration_secs is not None: + cmd += ["-t", str(duration_secs)] + cmd += ["-f", "wav", dst.name] + subprocess.run(cmd, check=True) + return dst.name + + +def concat_wavs(wavs: list[str], repeats: int = 1) -> str: + list_file = tempfile.NamedTemporaryFile(suffix=".txt", mode="w", delete=False) + for w in wavs * repeats: + list_file.write(f"file '{w}'\n") + list_file.close() + dst = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) + dst.close() + subprocess.run(["ffmpeg", "-y", "-loglevel", "error", + "-f", "concat", "-safe", "0", "-i", list_file.name, + "-c", "copy", dst.name], check=True) + os.unlink(list_file.name) + return dst.name + + +def wav_duration(p: str) -> float: + with wave.open(p, "rb") as w: + return w.getnframes() / float(w.getframerate()) + + +def vram_mb() -> float: + return torch.cuda.max_memory_allocated() / (1024 * 1024) if torch.cuda.is_available() else 0.0 + + +def run_tdt(tdt, wav: str, timestamps: bool) -> dict: + torch.cuda.reset_peak_memory_stats() + t0 = time.monotonic() + try: + with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16): + with torch.no_grad(): + hyps = tdt.transcribe([wav], timestamps=timestamps) + elapsed = time.monotonic() - t0 + h = hyps[0] + text = (getattr(h, "text", None) or "").strip() + ts = getattr(h, "timestamp", None) + ts_summary = None + if isinstance(ts, dict): + ts_summary = {k: (len(v) if hasattr(v, "__len__") else str(type(v).__name__)) + for k, v in ts.items()} + return {"ok": True, "text": text, "chars": len(text), + "duration_infer": round(elapsed, 2), + "vram_peak_mb": round(vram_mb(), 0), + "timestamp_summary": ts_summary} + except Exception as e: + return {"ok": False, "error": f"{type(e).__name__}: {e}", + "duration_infer": round(time.monotonic() - t0, 2)} + + +def run_canary(model, wav: str) -> dict: + torch.cuda.reset_peak_memory_stats() + t0 = time.monotonic() + try: + with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16): + with torch.no_grad(): + hyps = model.transcribe([wav], source_lang="de", target_lang="de", + pnc="yes", batch_size=4) + elapsed = time.monotonic() - t0 + text = (getattr(hyps[0], "text", None) or str(hyps[0])).strip() + return {"ok": True, "text": text, "chars": len(text), + "duration_infer": round(elapsed, 2), + "vram_peak_mb": round(vram_mb(), 0)} + except Exception as e: + return {"ok": False, "error": f"{type(e).__name__}: {e}", + "duration_infer": round(time.monotonic() - t0, 2)} + + +def find_source() -> str | None: + for c in ("trim_full.m4a", "full.m4a", "source.m4a"): + p = os.path.join(AUDIO_DIR, c) + if os.path.exists(p): + return p + if not os.path.isdir(AUDIO_DIR): + return None + m4as = [f for f in os.listdir(AUDIO_DIR) if f.endswith(".m4a")] + if not m4as: + return None + m4as.sort(key=lambda f: os.path.getsize(os.path.join(AUDIO_DIR, f)), reverse=True) + return os.path.join(AUDIO_DIR, m4as[0]) + + +def main(): + open(RESULTS_PATH, "w").close() + open(LOG_PATH, "w").close() + + log("loading EncDecMultiTaskModel (Canary 1B v2)") + model = EncDecMultiTaskModel.from_pretrained("nvidia/canary-1b-v2") + model = model.to("cuda").to(torch.bfloat16) + model.eval() + log(f" vram after load: {vram_mb():.0f} MB") + + tdt = getattr(model, "timestamps_asr_model", None) + if tdt is None: + log("FATAL: model has no timestamps_asr_model attribute") + return + log(f" tdt class: {type(tdt).__name__}") + log(f" tdt device: {next(tdt.parameters()).device}, dtype: {next(tdt.parameters()).dtype}") + + src = find_source() + if src is None: + log(f"FATAL: no source audio found in {AUDIO_DIR}") + return + log(f"source audio: {src}") + + wav_full = transcode(src) + dur_full = wav_duration(wav_full) + log(f" full wav: {dur_full:.1f} s") + + wav_20 = transcode(src, duration_secs=20.0) + log(f" 20 s wav: {wav_duration(wav_20):.1f} s") + + wav_80 = transcode(src, duration_secs=80.0) if dur_full >= 80 else wav_full + log(f" 80 s wav: {wav_duration(wav_80):.1f} s") + + wav_160 = concat_wavs([wav_80], repeats=2) + log(f" 160 s concat wav: {wav_duration(wav_160):.1f} s") + + cases = [("20s", wav_20), ("80s", wav_80), ("160s_concat", wav_160)] + + def record(d: dict) -> None: + with open(RESULTS_PATH, "a") as f: + f.write(json.dumps(d, ensure_ascii=False) + "\n") + + for label, wav in cases: + dur = wav_duration(wav) + log(f"\n=== case {label} ({dur:.1f} s) ===") + + log(" TDT timestamps=False") + r = run_tdt(tdt, wav, timestamps=False) + record({"case": label, "duration_audio": dur, "model": "tdt", + "timestamps": False, **r}) + if r["ok"]: + log(f" -> {r['chars']} chars in {r['duration_infer']} s, " + f"{r['vram_peak_mb']} MB") + else: + log(f" FAIL: {r['error']}") + + log(" TDT timestamps=True") + r = run_tdt(tdt, wav, timestamps=True) + record({"case": label, "duration_audio": dur, "model": "tdt", + "timestamps": True, **r}) + if r["ok"]: + log(f" -> {r['chars']} chars, ts_summary={r['timestamp_summary']}") + else: + log(f" FAIL: {r['error']}") + + log(" Canary path1 (reference)") + r = run_canary(model, wav) + record({"case": label, "duration_audio": dur, "model": "canary_path1", + **r}) + if r["ok"]: + log(f" -> {r['chars']} chars in {r['duration_infer']} s") + else: + log(f" FAIL: {r['error']}") + + log("done") + + +if __name__ == "__main__": + main() diff --git a/canary/sweep.py b/canary/sweep.py new file mode 100644 index 0000000..68a0dd2 --- /dev/null +++ b/canary/sweep.py @@ -0,0 +1,425 @@ +"""Canary parameter sweep — direct NeMo calls, no HTTP layer. + +Runs many parameter combinations against a fixed set of audios and writes +JSONL + a human-readable summary. Designed to run inside the doctate-canary +Docker image (or its base) with the model volume mounted at /models. + +Outputs: + /sweep/results.jsonl — one JSON object per run + /sweep/log.txt — chronological log +""" +import copy +import json +import os +import subprocess +import sys +import tempfile +import time +import wave +from contextlib import contextmanager + +import torch +from omegaconf import OmegaConf + +from nemo.collections.asr.models import ASRModel +from nemo.collections.asr.parts.utils.streaming_utils import FrameBatchMultiTaskAED +from nemo.collections.asr.parts.utils.transcribe_utils import get_buffered_pred_feat_multitaskAED + + +AUDIO_DIR = "/sweep/audios" +OUT_DIR = "/sweep" +RESULTS_PATH = f"{OUT_DIR}/results.jsonl" +LOG_PATH = f"{OUT_DIR}/log.txt" + +# Subset for most axes — one short, one medium, one long (>40 s each). +SUBSET = [ + "2026-04-27T15-00-48Z.m4a", # 48.4 s + "2026-04-27T14-55-41Z.m4a", # 65.9 s + "2026-04-27T14-58-17Z.m4a", # 80.9 s +] + +# Single-audio subset for speed/VRAM-only axes. +SINGLE = ["2026-04-27T14-58-17Z.m4a"] # 80.9 s + +DTYPE_MAP = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32} + + +def log(msg: str) -> None: + line = f"{time.strftime('%H:%M:%S')} {msg}" + print(line, flush=True) + with open(LOG_PATH, "a") as f: + f.write(line + "\n") + + +def transcode_to_wav16k(src: str) -> str: + dst = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) + dst.close() + subprocess.run( + ["ffmpeg", "-y", "-loglevel", "error", "-i", src, + "-ac", "1", "-ar", "16000", "-f", "wav", dst.name], + check=True, + ) + return dst.name + + +def wav_duration(path: str) -> float: + with wave.open(path, "rb") as w: + return w.getnframes() / float(w.getframerate()) + + +def vram_mb() -> float: + if torch.cuda.is_available(): + return torch.cuda.max_memory_allocated() / (1024 * 1024) + return 0.0 + + +@contextmanager +def vram_track(): + torch.cuda.reset_peak_memory_stats() + try: + yield + finally: + pass + + +def load_model(precision: str): + log(f"loading model precision={precision}") + t0 = time.monotonic() + model = ASRModel.from_pretrained("nvidia/canary-1b-v2") + model = model.to("cuda") + if precision != "fp32": + model = model.to(DTYPE_MAP[precision]) + model.eval() + log(f"model loaded in {time.monotonic() - t0:.1f}s, vram={vram_mb():.0f}MB") + return model + + +def prep_cfg(model): + cfg = copy.deepcopy(model._cfg) + OmegaConf.set_struct(cfg.preprocessor, False) + cfg.preprocessor.dither = 0.0 + cfg.preprocessor.pad_to = 0 + OmegaConf.set_struct(cfg.preprocessor, True) + return cfg + + +def write_manifest(wav_path: str, duration: float, pnc: str = "yes", + src_lang: str = "de", tgt_lang: str = "de", + task: str = "asr") -> str: + f = tempfile.NamedTemporaryFile(suffix=".json", mode="w", delete=False) + json.dump({ + "audio_filepath": wav_path, + "duration": duration, + "taskname": task, + "source_lang": src_lang, + "target_lang": tgt_lang, + "pnc": pnc, + }, f) + f.write("\n") + f.close() + return f.name + + +def run_buffered(model, model_cfg, wav_path: str, duration: float, + chunk_len: float, total_buffer: float, batch_size: int, + precision: str, pnc: str = "yes", timestamps: bool = False): + manifest = write_manifest(wav_path, duration, pnc=pnc) + feature_stride = model_cfg.preprocessor["window_stride"] + model_stride_secs = feature_stride * 8 + + frame_asr = FrameBatchMultiTaskAED( + asr_model=model, + frame_len=chunk_len, + total_buffer=total_buffer, + batch_size=batch_size, + ) + amp_dtype = DTYPE_MAP[precision] + use_amp = precision != "fp32" + t0 = time.monotonic() + with vram_track(): + with torch.amp.autocast("cuda", enabled=use_amp, dtype=amp_dtype): + with torch.no_grad(): + hyps = get_buffered_pred_feat_multitaskAED( + frame_asr, model_cfg.preprocessor, model_stride_secs, + model.device, manifest=manifest, filepaths=None, + timestamps=timestamps, + ) + elapsed = time.monotonic() - t0 + peak_vram = vram_mb() + os.unlink(manifest) + text = (getattr(hyps[0], "text", None) or str(hyps[0])).strip() + return text, elapsed, peak_vram + + +def run_transcribe_path1(model, wav_path: str, batch_size: int, pnc: str, + timestamps: bool, precision: str): + """Path 1 — direct .transcribe() with batch_size=1 enables auto-chunking + in NeMo >= 2.5 (per docstring of speech_to_text_aed_chunked_infer.py).""" + amp_dtype = DTYPE_MAP[precision] + use_amp = precision != "fp32" + t0 = time.monotonic() + with vram_track(): + with torch.amp.autocast("cuda", enabled=use_amp, dtype=amp_dtype): + with torch.no_grad(): + hyps = model.transcribe( + [wav_path], + source_lang="de", target_lang="de", + pnc=pnc, timestamps=timestamps, + batch_size=batch_size, + ) + elapsed = time.monotonic() - t0 + peak_vram = vram_mb() + text = (getattr(hyps[0], "text", None) or str(hyps[0])).strip() + return text, elapsed, peak_vram + + +def write_result(record: dict) -> None: + with open(RESULTS_PATH, "a") as f: + f.write(json.dumps(record, ensure_ascii=False) + "\n") + + +def sweep(): + open(LOG_PATH, "w").close() + open(RESULTS_PATH, "w").close() + + log("=== preparing audios ===") + wavs = {} + for name in set(SUBSET + SINGLE): + src = os.path.join(AUDIO_DIR, name) + wav = transcode_to_wav16k(src) + dur = wav_duration(wav) + wavs[name] = (wav, dur) + log(f" {name} → {wav} ({dur:.1f}s)") + + # ===== precision axis (B.2) — separate model loads ===== + for precision in ["bf16", "fp16", "fp32"]: + try: + torch.cuda.empty_cache() + model = load_model(precision) + cfg = prep_cfg(model) + for name in SINGLE: + wav, dur = wavs[name] + axis = f"B2_precision={precision}" + log(f"running {axis} on {name}") + try: + text, t, vram = run_buffered( + model, cfg, wav, dur, + chunk_len=40.0, total_buffer=40.0, batch_size=8, + precision=precision, + ) + write_result({ + "axis": "precision", "precision": precision, + "chunk_len": 40.0, "overlap": 0, "batch_size": 8, + "decoding": "greedy", "pnc": "yes", "timestamps": False, + "audio": name, "duration_audio": dur, + "duration_infer": round(t, 2), "vram_peak_mb": round(vram, 0), + "chars": len(text), "text": text, + }) + except Exception as e: + log(f" FAIL {axis} on {name}: {e}") + write_result({"axis": "precision", "precision": precision, + "audio": name, "error": str(e)}) + del model + torch.cuda.empty_cache() + except Exception as e: + log(f"FAIL precision={precision}: {e}") + + # All remaining axes use bf16 default model. + torch.cuda.empty_cache() + model = load_model("bf16") + cfg = prep_cfg(model) + + # ===== chunk_len axis (A.1) ===== + for chunk_len in [10.0, 20.0, 40.0, 60.0]: + for name in SUBSET: + wav, dur = wavs[name] + axis = f"A1_chunk_len={chunk_len}" + log(f"running {axis} on {name}") + try: + text, t, vram = run_buffered( + model, cfg, wav, dur, + chunk_len=chunk_len, total_buffer=chunk_len, batch_size=8, + precision="bf16", + ) + write_result({ + "axis": "chunk_len", "chunk_len": chunk_len, + "overlap": 0, "batch_size": 8, "precision": "bf16", + "decoding": "greedy", "pnc": "yes", "timestamps": False, + "audio": name, "duration_audio": dur, + "duration_infer": round(t, 2), "vram_peak_mb": round(vram, 0), + "chars": len(text), "text": text, + }) + except Exception as e: + log(f" FAIL: {e}") + write_result({"axis": "chunk_len", "chunk_len": chunk_len, + "audio": name, "error": str(e)}) + + # ===== overlap axis (A.2) — chunk_len=40, total_buffer=chunk_len+overlap ===== + for overlap in [1.0, 2.0, 5.0, 10.0]: + chunk_len = 40.0 + total_buffer = chunk_len + overlap + for name in SUBSET: + wav, dur = wavs[name] + axis = f"A2_overlap={overlap}" + log(f"running {axis} on {name}") + try: + text, t, vram = run_buffered( + model, cfg, wav, dur, + chunk_len=chunk_len, total_buffer=total_buffer, + batch_size=8, precision="bf16", + ) + write_result({ + "axis": "overlap", "chunk_len": chunk_len, + "overlap": overlap, "batch_size": 8, "precision": "bf16", + "decoding": "greedy", "pnc": "yes", "timestamps": False, + "audio": name, "duration_audio": dur, + "duration_infer": round(t, 2), "vram_peak_mb": round(vram, 0), + "chars": len(text), "text": text, + }) + except Exception as e: + log(f" FAIL: {e}") + write_result({"axis": "overlap", "overlap": overlap, + "audio": name, "error": str(e)}) + + # ===== batch_size axis (B.1) ===== + for batch_size in [1, 2, 4, 8, 16]: + for name in SINGLE: + wav, dur = wavs[name] + axis = f"B1_batch={batch_size}" + log(f"running {axis} on {name}") + try: + text, t, vram = run_buffered( + model, cfg, wav, dur, + chunk_len=40.0, total_buffer=40.0, batch_size=batch_size, + precision="bf16", + ) + write_result({ + "axis": "batch_size", "batch_size": batch_size, + "chunk_len": 40.0, "overlap": 0, "precision": "bf16", + "decoding": "greedy", "pnc": "yes", "timestamps": False, + "audio": name, "duration_audio": dur, + "duration_infer": round(t, 2), "vram_peak_mb": round(vram, 0), + "chars": len(text), "text": text, + }) + except Exception as e: + log(f" FAIL: {e}") + write_result({"axis": "batch_size", "batch_size": batch_size, + "audio": name, "error": str(e)}) + + # ===== pnc axis (C.2) ===== + for pnc in ["no", "yes"]: + for name in SUBSET: + wav, dur = wavs[name] + axis = f"C2_pnc={pnc}" + log(f"running {axis} on {name}") + try: + text, t, vram = run_buffered( + model, cfg, wav, dur, + chunk_len=40.0, total_buffer=40.0, batch_size=8, + precision="bf16", pnc=pnc, + ) + write_result({ + "axis": "pnc", "pnc": pnc, + "chunk_len": 40.0, "overlap": 0, "batch_size": 8, + "precision": "bf16", "decoding": "greedy", + "timestamps": False, + "audio": name, "duration_audio": dur, + "duration_infer": round(t, 2), "vram_peak_mb": round(vram, 0), + "chars": len(text), "text": text, + }) + except Exception as e: + log(f" FAIL: {e}") + write_result({"axis": "pnc", "pnc": pnc, + "audio": name, "error": str(e)}) + + # ===== timestamps axis (C.3) — when on, NeMo recommends chunk_len=10 ===== + for ts_flag, chunk_len in [(False, 40.0), (True, 10.0)]: + for name in SUBSET: + wav, dur = wavs[name] + axis = f"C3_timestamps={ts_flag}" + log(f"running {axis} (chunk_len={chunk_len}) on {name}") + try: + text, t, vram = run_buffered( + model, cfg, wav, dur, + chunk_len=chunk_len, total_buffer=chunk_len, batch_size=8, + precision="bf16", timestamps=ts_flag, + ) + write_result({ + "axis": "timestamps", "timestamps": ts_flag, + "chunk_len": chunk_len, "overlap": 0, "batch_size": 8, + "precision": "bf16", "decoding": "greedy", "pnc": "yes", + "audio": name, "duration_audio": dur, + "duration_infer": round(t, 2), "vram_peak_mb": round(vram, 0), + "chars": len(text), "text": text, + }) + except Exception as e: + log(f" FAIL: {e}") + write_result({"axis": "timestamps", "timestamps": ts_flag, + "audio": name, "error": str(e)}) + + # ===== beam search (C.1) ===== + try: + from nemo.collections.asr.parts.submodules.multitask_decoding import MultiTaskDecodingConfig + for strategy, beam_size in [("beam", 4), ("greedy", 1)]: + decoding_cfg = MultiTaskDecodingConfig() + decoding_cfg.strategy = strategy + if hasattr(decoding_cfg, "beam"): + decoding_cfg.beam.beam_size = beam_size + try: + model.change_decoding_strategy(decoding_cfg) + except Exception as e: + log(f"change_decoding_strategy failed for {strategy}: {e}") + continue + for name in SUBSET: + wav, dur = wavs[name] + axis = f"C1_decoding={strategy}_beam={beam_size}" + log(f"running {axis} on {name}") + try: + text, t, vram = run_buffered( + model, cfg, wav, dur, + chunk_len=40.0, total_buffer=40.0, batch_size=8, + precision="bf16", + ) + write_result({ + "axis": "decoding", "decoding": strategy, + "beam_size": beam_size, + "chunk_len": 40.0, "overlap": 0, "batch_size": 8, + "precision": "bf16", "pnc": "yes", "timestamps": False, + "audio": name, "duration_audio": dur, + "duration_infer": round(t, 2), "vram_peak_mb": round(vram, 0), + "chars": len(text), "text": text, + }) + except Exception as e: + log(f" FAIL: {e}") + write_result({"axis": "decoding", "decoding": strategy, + "audio": name, "error": str(e)}) + except Exception as e: + log(f"beam search axis skipped: {e}") + + # ===== Path-1 (.transcribe() auto-chunking) — D.1 ===== + log("=== Path 1: model.transcribe() with auto-chunking ===") + for name in SUBSET: + wav, dur = wavs[name] + axis = "D1_path1_transcribe" + log(f"running {axis} on {name}") + try: + text, t, vram = run_transcribe_path1( + model, wav, batch_size=1, pnc="yes", + timestamps=False, precision="bf16", + ) + write_result({ + "axis": "path1", "path": "transcribe_auto_chunking", + "batch_size": 1, "precision": "bf16", + "audio": name, "duration_audio": dur, + "duration_infer": round(t, 2), "vram_peak_mb": round(vram, 0), + "chars": len(text), "text": text, + }) + except Exception as e: + log(f" FAIL: {e}") + write_result({"axis": "path1", "audio": name, "error": str(e)}) + + log("=== sweep complete ===") + + +if __name__ == "__main__": + sweep() diff --git a/canary/sweep_chunklen.py b/canary/sweep_chunklen.py new file mode 100644 index 0000000..87f4683 --- /dev/null +++ b/canary/sweep_chunklen.py @@ -0,0 +1,125 @@ +"""Mini-sweep: does chunk_len == audio_len fix Path 2 for short audios? + +If yes → Path 2 is OK as long as audio fits in exactly one chunk. + The bug is in the multi-chunk merging or pad/split logic. +If no → Path 2 has a structural problem with single-chunk inputs + independent of length matching. Confirms "different inference + pipeline" theory. +""" +import copy +import json +import os +import subprocess +import tempfile +import time +import wave + +import torch +from omegaconf import OmegaConf + +from nemo.collections.asr.models import ASRModel +from nemo.collections.asr.parts.utils.streaming_utils import FrameBatchMultiTaskAED +from nemo.collections.asr.parts.utils.transcribe_utils import get_buffered_pred_feat_multitaskAED + + +AUDIO_DIR = "/sweep/threshold_audios" +RESULTS_PATH = "/sweep/chunklen_results.jsonl" + + +def transcode(src): + dst = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) + dst.close() + subprocess.run(["ffmpeg", "-y", "-loglevel", "error", "-i", src, + "-ac", "1", "-ar", "16000", "-f", "wav", dst.name], check=True) + return dst.name + + +def wav_dur(p): + with wave.open(p, "rb") as w: + return w.getnframes() / float(w.getframerate()) + + +def write_manifest(wav, dur): + f = tempfile.NamedTemporaryFile(suffix=".json", mode="w", delete=False) + json.dump({"audio_filepath": wav, "duration": dur, "taskname": "asr", + "source_lang": "de", "target_lang": "de", "pnc": "yes"}, f) + f.write("\n") + f.close() + return f.name + + +def run_path2(model, cfg, wav, dur, chunk_len): + manifest = write_manifest(wav, dur) + feature_stride = cfg.preprocessor["window_stride"] + model_stride = feature_stride * 8 + fa = FrameBatchMultiTaskAED(asr_model=model, frame_len=chunk_len, + total_buffer=chunk_len, batch_size=8) + t0 = time.monotonic() + with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16): + with torch.no_grad(): + hyps = get_buffered_pred_feat_multitaskAED( + fa, cfg.preprocessor, model_stride, model.device, + manifest=manifest, filepaths=None, timestamps=False, + ) + elapsed = time.monotonic() - t0 + os.unlink(manifest) + return (getattr(hyps[0], "text", None) or str(hyps[0])).strip(), elapsed + + +def run_path1(model, wav): + t0 = time.monotonic() + with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16): + with torch.no_grad(): + hyps = model.transcribe([wav], source_lang="de", target_lang="de", + pnc="yes", batch_size=4) + elapsed = time.monotonic() - t0 + return (getattr(hyps[0], "text", None) or str(hyps[0])).strip(), elapsed + + +def main(): + open(RESULTS_PATH, "w").close() + print("loading model bf16", flush=True) + model = ASRModel.from_pretrained("nvidia/canary-1b-v2").to("cuda").to(torch.bfloat16) + model.eval() + cfg = copy.deepcopy(model._cfg) + OmegaConf.set_struct(cfg.preprocessor, False) + cfg.preprocessor.dither = 0.0 + cfg.preprocessor.pad_to = 0 + OmegaConf.set_struct(cfg.preprocessor, True) + print("model loaded", flush=True) + + audios = ["trim_20s.m4a", "trim_30s.m4a"] + chunk_lens = { + "trim_20s.m4a": [21.0, 25.0, 40.0], # exact, slightly larger, default + "trim_30s.m4a": [31.0, 35.0, 40.0], + } + + def record(d): + with open(RESULTS_PATH, "a") as f: + f.write(json.dumps(d, ensure_ascii=False) + "\n") + + for name in audios: + src = os.path.join(AUDIO_DIR, name) + wav = transcode(src) + dur = wav_dur(wav) + print(f"\n=== {name} ({dur:.1f}s) ===", flush=True) + + # baseline path1 + text, t = run_path1(model, wav) + print(f"path1: {len(text)} chars in {t:.1f}s", flush=True) + record({"audio": name, "duration": dur, "path": "path1", + "chunk_len": None, "chars": len(text), + "duration_infer": round(t, 2), "text": text}) + + for cl in chunk_lens[name]: + text, t = run_path2(model, cfg, wav, dur, cl) + print(f"path2 chunk_len={cl}: {len(text)} chars in {t:.1f}s", flush=True) + record({"audio": name, "duration": dur, "path": "path2", + "chunk_len": cl, "chars": len(text), + "duration_infer": round(t, 2), "text": text}) + + print("done", flush=True) + + +if __name__ == "__main__": + main() diff --git a/canary/sweep_threshold.py b/canary/sweep_threshold.py new file mode 100644 index 0000000..a8aaf67 --- /dev/null +++ b/canary/sweep_threshold.py @@ -0,0 +1,178 @@ +"""Threshold sweep — varying audio length around the 40 s Encoder-Window. + +Compares three transcription paths on artificially trimmed audios: + - path1_default: model.transcribe([wav]) — what main.py uses for ≤ 25 s + - path1_bs1: model.transcribe([wav], batch_size=1) — NeMo ≥ 2.5 auto-chunking + - path2: FrameBatchMultiTaskAED with chunk_len=40 — what main.py + uses for > 25 s + +Audios are 20, 30, 38 and 45 s slices of the 80.9 s source. The 25 s +threshold in main.py sits in the middle, so this sweep tells us: + - Does Path 1 truncate already at 30/38 s, or only > 40 s? + - Does the threshold need to move? +""" +import copy +import json +import os +import subprocess +import tempfile +import time +import wave + +import torch +from omegaconf import OmegaConf + +from nemo.collections.asr.models import ASRModel +from nemo.collections.asr.parts.utils.streaming_utils import FrameBatchMultiTaskAED +from nemo.collections.asr.parts.utils.transcribe_utils import get_buffered_pred_feat_multitaskAED + + +AUDIO_DIR = "/sweep/threshold_audios" +OUT_DIR = "/sweep" +RESULTS_PATH = f"{OUT_DIR}/threshold_results.jsonl" +LOG_PATH = f"{OUT_DIR}/threshold_log.txt" + + +def log(msg: str) -> None: + line = f"{time.strftime('%H:%M:%S')} {msg}" + print(line, flush=True) + with open(LOG_PATH, "a") as f: + f.write(line + "\n") + + +def transcode(src: str) -> str: + dst = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) + dst.close() + subprocess.run( + ["ffmpeg", "-y", "-loglevel", "error", "-i", src, + "-ac", "1", "-ar", "16000", "-f", "wav", dst.name], + check=True, + ) + return dst.name + + +def wav_duration(p: str) -> float: + with wave.open(p, "rb") as w: + return w.getnframes() / float(w.getframerate()) + + +def vram_mb() -> float: + return torch.cuda.max_memory_allocated() / (1024 * 1024) if torch.cuda.is_available() else 0 + + +def write_manifest(wav_path, duration, pnc="yes"): + f = tempfile.NamedTemporaryFile(suffix=".json", mode="w", delete=False) + json.dump({"audio_filepath": wav_path, "duration": duration, + "taskname": "asr", "source_lang": "de", "target_lang": "de", + "pnc": pnc}, f) + f.write("\n") + f.close() + return f.name + + +def run_path1(model, wav, batch_size): + torch.cuda.reset_peak_memory_stats() + t0 = time.monotonic() + with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16): + with torch.no_grad(): + hyps = model.transcribe( + [wav], source_lang="de", target_lang="de", + pnc="yes", batch_size=batch_size, + ) + return (getattr(hyps[0], "text", None) or str(hyps[0])).strip(), \ + time.monotonic() - t0, vram_mb() + + +def run_path2(model, model_cfg, wav, dur, chunk_len=40.0): + manifest = write_manifest(wav, dur) + feature_stride = model_cfg.preprocessor["window_stride"] + model_stride_secs = feature_stride * 8 + frame_asr = FrameBatchMultiTaskAED( + asr_model=model, frame_len=chunk_len, + total_buffer=chunk_len, batch_size=8, + ) + torch.cuda.reset_peak_memory_stats() + t0 = time.monotonic() + with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16): + with torch.no_grad(): + hyps = get_buffered_pred_feat_multitaskAED( + frame_asr, model_cfg.preprocessor, model_stride_secs, + model.device, manifest=manifest, filepaths=None, timestamps=False, + ) + elapsed = time.monotonic() - t0 + os.unlink(manifest) + return (getattr(hyps[0], "text", None) or str(hyps[0])).strip(), \ + elapsed, vram_mb() + + +def main(): + open(LOG_PATH, "w").close() + open(RESULTS_PATH, "w").close() + + log("loading model bf16") + model = ASRModel.from_pretrained("nvidia/canary-1b-v2") + model = model.to("cuda").to(torch.bfloat16) + model.eval() + cfg = copy.deepcopy(model._cfg) + OmegaConf.set_struct(cfg.preprocessor, False) + cfg.preprocessor.dither = 0.0 + cfg.preprocessor.pad_to = 0 + OmegaConf.set_struct(cfg.preprocessor, True) + log(f"model loaded, vram={vram_mb():.0f}MB") + + audios = sorted(os.listdir(AUDIO_DIR)) + wavs = {} + for name in audios: + if not name.endswith(".m4a"): + continue + src = os.path.join(AUDIO_DIR, name) + wav = transcode(src) + dur = wav_duration(wav) + wavs[name] = (wav, dur) + log(f" {name} → {dur:.1f}s") + + def record(d): + with open(RESULTS_PATH, "a") as f: + f.write(json.dumps(d, ensure_ascii=False) + "\n") + + for name, (wav, dur) in wavs.items(): + # path 1 default (no batch_size override → triggers no auto-chunking + # because input is a list, not a single string) + log(f"path1_default on {name}") + try: + text, t, vram = run_path1(model, wav, batch_size=4) + record({"audio": name, "duration_audio": dur, "path": "path1_default", + "batch_size": 4, "duration_infer": round(t, 2), + "vram_peak_mb": round(vram, 0), "chars": len(text), "text": text}) + except Exception as e: + log(f" FAIL: {e}") + record({"audio": name, "path": "path1_default", "error": str(e)}) + + # path 1 with batch_size=1 (auto-chunking trigger per NeMo docs) + log(f"path1_bs1 on {name}") + try: + text, t, vram = run_path1(model, wav, batch_size=1) + record({"audio": name, "duration_audio": dur, "path": "path1_bs1", + "batch_size": 1, "duration_infer": round(t, 2), + "vram_peak_mb": round(vram, 0), "chars": len(text), "text": text}) + except Exception as e: + log(f" FAIL: {e}") + record({"audio": name, "path": "path1_bs1", "error": str(e)}) + + # path 2 (always chunked, chunk_len=40) + log(f"path2 on {name}") + try: + text, t, vram = run_path2(model, cfg, wav, dur, chunk_len=40.0) + record({"audio": name, "duration_audio": dur, "path": "path2", + "chunk_len": 40.0, "batch_size": 8, + "duration_infer": round(t, 2), + "vram_peak_mb": round(vram, 0), "chars": len(text), "text": text}) + except Exception as e: + log(f" FAIL: {e}") + record({"audio": name, "path": "path2", "error": str(e)}) + + log("done") + + +if __name__ == "__main__": + main() diff --git a/docs/canary.md b/docs/canary.md new file mode 100644 index 0000000..399821c --- /dev/null +++ b/docs/canary.md @@ -0,0 +1,734 @@ +# Canary 1B v2 — Setup, Sweep-Reproduktion und Decoding-Schalter + +Selbsterhaltende Anleitung für **NVIDIA Canary 1B v2** in doctate. +Alles, was zur Installation, zur Reproduktion der bisherigen +Sweep-Ergebnisse und zur Konfiguration der Decoding-Schalter nötig +ist, steht in diesem Dokument. Detail- und Hintergrund-Material ist +als Appendix angehängt. + +## 1. Was und warum + +- **Modell:** `nvidia/canary-1b-v2`. FastConformer-Encoder + AED + (Attention-Encoder-Decoder), ca. 1 Mrd. Parameter. Vier Sprachen + (en/de/fr/es), beste Deutsch-Performance unter den freien Modellen + ≤ 12 GB VRAM. +- **Status in doctate:** zweiter ASR-Service neben Whisper, **nicht** + Drop-in-Replacement. Whisper bleibt am `/asr`-Endpoint, Canary läuft + separat als nativer FastAPI-Service auf Port 9002 für Vergleichs- + und Forschungsläufe. +- **Lizenz:** CC-BY-4.0 (kommerzielle Nutzung erlaubt, Attribution + nötig). + +## 2. Installation — Containerservice auf Minerva + +Dies ist der Produktionspfad. Der Service läuft 24/7 auf +`minerva.lan:9002`. Modellgewichte (~3 GB) werden im Volume +`/opt/stacks/doctate-canary/models` gecached, sodass Restarts schnell +sind. + +### 2.1 Voraussetzungen auf der Zielmaschine + +- Linux mit Docker und `docker compose`-Plugin +- NVIDIA-Treiber + `nvidia-container-toolkit` (für GPU-Zugriff aus + Containern) +- Mindestens 12 GB GPU-VRAM für bf16-Inferenz +- Mindestens 5 GB freier Plattenplatz (3 GB Modell, ca. 4.5 GB + Container-Image) +- SSH-Zugriff für `deploy.sh` + +### 2.2 Container-Stack + +Der Stack besteht aus fünf Files unter `canary/` im Repo (Inhalte +siehe Appendix A): + +| Datei | Zweck | +|---|---| +| `Dockerfile` | Image-Definition (cuda-devel + Python + ffmpeg + Pip-Deps) | +| `requirements.txt` | NeMo, FastAPI, HF-Hub mit Versions-Pins | +| `docker-compose.yml` | Service-Definition (Port 9002, GPU-Reservierung, Modell-Volume) | +| `main.py` | FastAPI-App mit `/health`, `/info`, `POST /inference` | +| `deploy.sh` | rsync + remote build + Health-Polling (10-min-Timeout) | + +### 2.3 Deployment + +Aus dem Dev-Repo: + +```bash +cd /home/brummel/dev/doctate/canary +./deploy.sh minerva.lan +``` + +`deploy.sh` führt aus: + +1. `rsync` der Container-Files nach `minerva:/opt/stacks/doctate-canary/` +2. `docker compose down` auf der Remote +3. `docker pull nvidia/cuda:12.6.1-devel-ubuntu22.04` (umgeht einen + TLS-Cert-Bug im Legacy-Builder, der sonst beim Pull innerhalb von + `docker build` auftritt) +4. `docker build -t doctate-canary .` +5. `docker compose up -d` +6. Pollt `/health` 120 × 5 s = 10 min lang. Beim Erststart muss das + Modell aus HuggingFace geladen werden — das dauert mehrere Minuten. + +Bei Erfolg gibt `deploy.sh` zusätzlich `/info` zurück, sonst die +letzten 60 Container-Log-Zeilen. + +### 2.4 API + +Der Service hat eine **native** Canary-API (kein Whisper-Mimik): + +| Methode | Pfad | Zweck | +|---|---|---| +| `GET` | `/health` | `{"status":"ok","model":...,"device":...,"precision":...}` | +| `GET` | `/info` | Erweiterte Metadaten | +| `POST` | `/inference` | Multipart-Upload, Inferenz, Antwort als Text oder JSON | + +Form-Felder von `POST /inference`: + +| Feld | Default | Werte | +|---|---|---| +| `file` | — | Upload (mp3/m4a/opus/wav/...). Intern via ffmpeg → 16 kHz mono WAV | +| `language` | `de` | `en`/`de`/`fr`/`es` | +| `pnc` | `yes` | `yes`/`no` (Punctuation/Capitalization) | +| `timestamps` | `no` | `yes`/`no` (segment-level, nur in `json`-Antwort) | +| `response_format` | `text` | `text` (PlainText) / `json` | + +Smoke-Test: + +```bash +curl http://minerva.lan:9002/health +curl -F file=@dictation.m4a -F language=de \ + http://minerva.lan:9002/inference +``` + +### 2.5 Env Vars + +| Variable | Default | Zweck | +|---|---|---| +| `CANARY_MODEL` | `nvidia/canary-1b-v2` | beliebiges NeMo-ASR-Modell | +| `CANARY_DEVICE` | `cuda` | `cuda` oder `cpu` | +| `CANARY_PRECISION` | `bf16` | `bf16` / `fp16` / `fp32` | +| `CANARY_MODELS_DIR` | `/models` | HF-Cache (volume-persistent) | +| `CANARY_CHUNK_LEN_SECS` | `40.0` | Chunk-Fensterlänge für Long-Form | +| `CANARY_CHUNKED_THRESHOLD_SECS` | `25.0` | ab welcher Audiolänge gechunkt wird | +| `HF_HOME` | wird auf `CANARY_MODELS_DIR` gesetzt | HuggingFace-Cache | + +### 2.6 VRAM-Budget + +Canary 1B v2 in bf16 belegt **ca. 9.7 GB** auf RTX 3060 mit +`batch_size=8` für die buffered Pipeline. Das passt **nicht** +gleichzeitig mit Whisper + Ollama auf eine 12-GB-Karte. Für +Sweep-Läufe (Abschnitt 3) muss der Service kurzzeitig pausiert +werden: + +```bash +ssh minerva.lan "cd /opt/stacks/doctate-canary && docker compose stop" +# ... Sweep-Run hier ... +ssh minerva.lan "cd /opt/stacks/doctate-canary && docker compose start" +``` + +## 3. Reproduktion der Sweep-Ergebnisse vom 2026-04-30 + +Der Validation-Sweep hat **101 Einzeldiktate**, **47 +Per-Case-Concats** und **1 globales 38.7-min-Concat** durch Canary +gejagt, um die `dur=0`-Halluzinations-Heuristik gegen ein echtes +Korpus zu validieren. + +**Ergebnis** (Disposition siehe Abschnitt 5): + +| Quantity | Value | +|---|---:| +| dur=0 Vorkommen | 81 | +| davon echte Halluzinationen | 2 | +| False Positives (echtes Transkript) | 79 | +| Precision | **≈ 2.5 %** | + +Diese Schritte regenerieren die Daten von Grund auf. Die Sweep- +Skripte liegen in `experiments/canary_sweep/`. + +### 3.1 Cases einsammeln + +```bash +mkdir -p experiments/canary_sweep/cases experiments/canary_sweep/concat +cd /home/brummel/dev/doctate/tmpdata/dr_mueller +find . -name "*.m4a" | while read m4a_rel; do + case_uuid=$(echo "$m4a_rel" | cut -d/ -f2) + stem=$(basename "$m4a_rel" .m4a) + dest="/home/brummel/dev/doctate/experiments/canary_sweep/cases/$case_uuid" + mkdir -p "$dest" + cp "./$case_uuid/$stem.m4a" "$dest/$stem.m4a" + [ -f "./$case_uuid/$stem.json" ] && cp "./$case_uuid/$stem.json" "$dest/$stem.whisper.json" +done +``` + +Resultat: `cases//.m4a` (101 Files) plus 53 zugehörige +`.whisper.json` aus dem Server-State (Rest sind ältere +Aufnahmen, deren Whisper-Output nur in `document.md` zusammengeführt +wurde). + +### 3.2 Per-Case- und globalen Concat bauen + +`build_per_case_concat.py` und `build_concat.py` ffmpeg-stitchen +alle Audios zu einem WAV pro Case bzw. zu einem 38.7-min-Konglomerat +über alle Cases. Beide Skripte schreiben jeweils ein +`concat.manifest.json` mit der Reihenfolge und den Time-Offsets der +Einzelschnipsel: + +```bash +cd experiments/canary_sweep +python build_per_case_concat.py # cases//concat.wav +python build_concat.py # concat/all.wav (38.7 min) +``` + +Beide sind idempotent. + +### 3.3 GPU-Konflikt klären, Sweep ausführen + +Der laufende `doctate-canary`-Service belegt fast die ganze GPU. Vor +dem Sweep einmal stoppen: + +```bash +ssh minerva.lan "cd /opt/stacks/doctate-canary && docker compose stop" +``` + +Daten + Skripte hochladen: + +```bash +rsync -av experiments/canary_sweep/ minerva.lan:/tmp/canary_sweep/ +``` + +**Sweep über die 101 Einzelaudios** — Mode A (Single-Shot, mit +Word-Confidence): + +```bash +ssh minerva.lan "docker run --rm --name canary-sweep --gpus all \ + -v /tmp/canary_sweep:/work \ + -v /opt/stacks/doctate-canary/models:/models \ + -e HF_HOME=/models \ + doctate-canary:latest \ + python3 /work/run_canary.py /work/cases /work/cases" +``` + +**Per-Case-Concats** — adaptiv (≤ 25 s → Single-Shot, > 25 s → +Buffered): + +```bash +ssh minerva.lan "docker run --rm --name canary-percase --gpus all \ + -v /tmp/canary_sweep:/work \ + -v /opt/stacks/doctate-canary/models:/models \ + -e HF_HOME=/models \ + doctate-canary:latest \ + python3 /work/run_canary_per_case.py /work/cases" +``` + +**Globaler 38.7-min-Concat** — Buffered ist Pflicht (Single-Shot +geht in OOM): + +```bash +ssh minerva.lan "docker run --rm --name canary-concat-buffered --gpus all \ + -v /tmp/canary_sweep:/work \ + -v /opt/stacks/doctate-canary/models:/models \ + -e HF_HOME=/models \ + doctate-canary:latest \ + python3 /work/run_canary_buffered.py \ + /work/concat/all.wav /work/concat/all.canary.json" +``` + +Service wieder hochfahren: + +```bash +ssh minerva.lan "cd /opt/stacks/doctate-canary && docker compose start" +``` + +### 3.4 Outputs zurücksyncen + +```bash +rsync -av --include='*/' \ + --include='*.canary.json' --include='concat.canary.json' \ + --exclude='*' \ + minerva.lan:/tmp/canary_sweep/cases/ \ + experiments/canary_sweep/cases/ + +rsync -av minerva.lan:/tmp/canary_sweep/concat/all.canary.json \ + experiments/canary_sweep/concat/ +``` + +### 3.5 dur=0-Inventar erzeugen + +`dur_zero_inventory.py` walks alle 149 `*.canary.json` Files, +extrahiert jedes Wort mit `start_offset == end_offset`, und gibt +ein strukturiertes JSON sowie einen Markdown-Report aus: + +```bash +cd experiments/canary_sweep +python dur_zero_inventory.py +``` + +Schreibt `dur_zero_inventory.json` (alle 81 Records mit Kontext) und +`dur_zero_inventory.md` (gruppiert nach File mit 3-Wörter-Kontext um +jeden Treffer). + +## 4. Decoding-Schalter — was, wann, wie + +Canary 1B v2 hat zwei Inferenz-Pipelines mit **unterschiedlichen +Output-Schemata**. Die Wahl bestimmt, welche Felder im Hypothesis- +Objekt gefüllt werden. + +### 4.1 Pipeline-Wahl + +| Pipeline | Audiolänge | Aufruf | Hypothesis-Felder | +|---|---|---|---| +| **Single-Shot (Mode A)** | ≤ 25 s | `model.transcribe([wav], …, return_hypotheses=True)` | text, score, timestamp (word+segment), tokens, **word_confidence**, **token_confidence** | +| **Buffered** | > 25 s | `FrameBatchMultiTaskAED` + `get_buffered_pred_feat_multitaskAED` | text, score, timestamp (word+segment) — **kein** word_confidence, keine tokens | + +Die Schwelle ist `CHUNKED_THRESHOLD_SECS=25.0` im Service. Hauptlimit +der buffered Pipeline: NeMo's `_join_hypotheses` in +`nemo/collections/asr/parts/utils/streaming_utils.py` mergt nur +`text`, `y_sequence` und `timestamp` über die Chunks — Confidence- +Felder werden wegen fehlender `_join_word_confidence`-Methode +verworfen. Das ist eine NeMo-Implementations-Lücke, kein +Konfigurationsfehler. Die Pro-Chunk-Confidence ginge intern, die +Streaming-Aggregation wirft sie nur weg. + +### 4.2 Timestamps an/aus + +**Schalter:** kwarg `timestamps=True` auf `model.transcribe()` bzw. +auf `get_buffered_pred_feat_multitaskAED()`. + +**Was sie liefern:** + +```json +{ + "timestamp": { + "word": [{"word": "Therapie.", "start_offset": 46, "end_offset": 51, + "start": 3.68, "end": 4.08}, …], + "segment": [{"segment": "Therapie.", …}, …] + } +} +``` + +`*_offset` sind Encoder-Frames (FastConformer-Stride 80 ms), +`start`/`end` sind Sekunden. Die Word-Timestamps stammen aus dem +**CTC-Submodell-Forced-Alignment**, nicht aus dem AED selbst. + +**Kosten:** Vernachlässigbar. Das CTC-Submodell läuft eh, der +Schalter erhält nur dessen Output, anstatt ihn zu verwerfen. Im +buffered Pfad gilt allerdings die NeMo-Empfehlung +`chunk_len_in_secs=10.0` für genauere Word-Timestamps; mit unseren +40-s-Chunks sind Word-Offsets weniger präzise (innerhalb eines +Chunks ok, an Chunk-Übergängen verlieren sie Genauigkeit). + +**Default im Service:** `timestamps=no` (segment-level, nur im JSON- +Response wenn `response_format=json`). Wer Word-Timestamps für +Forschungszwecke braucht (z.B. dur=0-Inventar), muss am +`transcribe()`-Call selbst flippen — siehe Sweep-Skripte in +Abschnitt 3. + +### 4.3 Confidence an/aus + +**Schalter:** vier Stellen, alle vier sind nötig — eine fehlt → Wert +ist `None`: + +```python +# 1. Decoding-Strategie auf greedy umstellen. +# Beam liefert keine Confidence — confidence_cfg wird im +# Beam-Pfad ignoriert. +cfg = OmegaConf.to_container(model.cfg.decoding, resolve=True) +cfg["strategy"] = "greedy" + +# 2. confidence_cfg-Flags setzen +cfg.setdefault("confidence_cfg", {}) +cfg["confidence_cfg"]["preserve_word_confidence"] = True +cfg["confidence_cfg"]["preserve_token_confidence"] = True +cfg["confidence_cfg"]["preserve_frame_confidence"] = True +cfg["confidence_cfg"]["aggregation"] = "min" + +# 3. Greedy-spezifische Flags +cfg.setdefault("greedy", {}) +cfg["greedy"]["preserve_token_confidence"] = True +cfg["greedy"]["preserve_alignments"] = True +model.change_decoding_strategy(OmegaConf.create(cfg)) + +# 4. transcribe() mit return_hypotheses=True (DER stille +# Stolperdraht!) +hyps = model.transcribe([wav], …, return_hypotheses=True) +``` + +**Der stille Stolperdraht:** `return_hypotheses` ist standardmäßig +`False`. Dann liefert NeMo Strings statt `Hypothesis`-Objekten — +ohne dass irgendein Flag oder Log darauf hinweist. Egal welche +Confidence-Flags man setzt: ohne `return_hypotheses=True` ist das +Feld `None`. Das herauszufinden hat in der Diagnostik-Phase mehrere +konfundierte Ablations-Runden gekostet. + +**Wert:** Tsallis-Entropie mit `aggregation=min`, theoretisch im +Bereich [0, 1] mit höher = sicherer. In der Praxis Magnituden um +3·10⁻⁵ als Median, einzelne Wörter bei float32-Min (1.4·10⁻⁴⁵). +Numerisch nutzbar nur über die relative Reihenfolge, nicht über +absolute Schwellen. + +**Wichtig — Confidence ist als Halluzinations-Signal nicht +brauchbar.** Echtes medizinisches Vokabular ("Bisoprolol", +"Ramipril") und Dosierungsdigits ("100", "101") haben dieselben +extrem niedrigen Confidence-Werte wie tatsächliche Halluzinationen. +Confidence kann "unsicheren echten Inhalt" und "halluzinierten +Inhalt" nicht trennen. Detail siehe Appendix C. + +### 4.4 Was zusammen geht — Ablations-Tabelle + +Die kondensierte Wahrheitstabelle für die wichtigen Knöpfe: + +| Strategie | Chunking | timestamps | return_hypotheses | text | confidence | xatt | +|---|---|---|---|---|---|---| +| beam | ON | True | False (default) | full | None | None | +| greedy | ON | True | False | full | None | None | +| greedy | ON | True | **True** | full | **gefüllt** | None | +| greedy | OFF | False | True | **degradiert** | gefüllt | gefüllt | + +**Lehre:** Confidence + Timestamps + voller Text gehen zusammen, +sobald `strategy=greedy` und `return_hypotheses=True`. +Cross-Attention (`xatt_scores`) ist die einzige Information, die +`enable_chunking=False` erfordert — und damit verliert man bei +Audios > 40 s die Textqualität, was Cross-Attention für Production +disqualifiziert. + +### 4.5 Cross-Attention (informativ, nicht in Production) + +Falls jemand jemals `xatt_scores` braucht: zusätzlich zu den +Confidence-Flags `MultiTaskTranscriptionConfig(enable_chunking=False)` +via `override_config` auf `transcribe()`. Liefert pro Output-Token +eine Verteilung über Encoder-Frames; Shannon-Entropy pro Token +unterscheidet (auf einer einzelnen 76-s-Aufnahme) Halluzination +(Entropie ~4.7) von echtem Inhalt (~3.0). Validation auf breiterem +Korpus fehlt; nicht weiterverfolgt, weil dur=0 + Confidence schon +als Signale gefallen sind. + +## 5. Bekannte Sackgassen + +| Idee | Verworfen weil | +|---|---| +| **`dur=0` als Halluzinations-Filter oder Advisory-Marker** | 79/81 FP auf dr_mueller — siehe Appendix C | +| **Word-Confidence** als Filter oder Marker | medizinisches Vokabular und Digits haben dieselben niedrigen Werte wie Halluzinationen — siehe Appendix C | +| Whisper `suppress_tokens=\d+` gegen Digit-Halluzinationen | erzeugt Halluzinations-Loops; `\d` allein lässt Digit-Sequenzen ganz verschwinden | +| Whisper-Hotwords aus einzelnen Test-Cases | halluziniert in anderen Cases (Curve-Fitting auf Einzelbeispielen) | +| Buffered Pipeline mit Confidence-Output | NeMo `_join_hypotheses` mergt Confidence-Felder nicht (siehe Abschnitt 4.1) | + +Falls Halluzinations-Detektion irgendwann nötig wird: das Signal +muss aus *außerhalb* der Per-Word-Telemetrie eines einzelnen ASR- +Modells kommen. Die naheliegenden Kandidaten: + +- LLM-Konsolidierung mit medizinischem Domänen-Wissen (geschieht + implizit bereits, könnte explizit gemacht werden) +- Cross-Modell-Vergleich: AED-Text vs CTC-Standalone, oder Canary + vs Whisper, mit Divergenz als Signal +- Domänen-Regeln: nicht-existierende Medikamentennamen, + unplausible Dosierungswerte etc. — aktuell vom Gazetteer + pre-LLM erledigt + +## 6. Domänen-Stolperfallen + +### 6.1 "1-0-1"-Notation + +Deutsche medizinische Diktate sprechen Dosierungsschemata als +Digit-Triple ("eins null eins" = morgens 1, mittags 0, abends 1). +Varianten: "1-0-0", "½-0-½", "1-0-0-1" (Vier-Slot). Canarys BPE- +Tokenizer kollabiert das oft zu einem einzigen Token "101", "100" +etc. — derselbe Mechanismus wie der allgemeine Digit-Collapse- +Effekt. + +**Konsequenz:** Bare 3-Digit-Tokens im Medikationskontext sind +**echter Inhalt**, nicht Halluzination. Eine Pipeline-Stufe, die +"100" oder "101" als Halluzination markiert (z.B. weil Confidence +niedrig ist), zerstört reale Dosierungsinformation — patient- +sicherheitsrelevant. + +**Validierungsbeispiel** (case_c414cf52, Ramipril 5 mg 1-0-1): +Der AED produzierte zunächst "100." für gesprochenes "1-0-1" +(Digit-Collapse mit falscher Korrektur), dann eine Selbst-Korrektur +mit "5 milligramm 101" — beide erscheinen im Transkript. Das "101" +ist die korrekte Wiedergabe, das "100." der Fehler. Ein Filter, +der nur niedrig-confidente Digits droppt, würde beide vernichten. + +### 6.2 Stille zwischen Schnipsel-Concats + +Beim globalen 38.7-min-Concat (Buffered Pipeline) meldet NeMo für +mehrere Chunks `text of utterance with ID: audio_X is empty`. Das +sind die Übergangs-Stellen zwischen aneinandergehängten Audios mit +Stille — kein Bug, sondern erwartet. Die Output-Hypothese hat +trotzdem 1555 Wörter, also keine Information verloren. + +--- + +## Appendix A — Container-Files (vollständige Inhalte) + +### Dockerfile + +```dockerfile +FROM nvidia/cuda:12.6.1-devel-ubuntu22.04 + +ENV DEBIAN_FRONTEND=noninteractive \ + PIP_NO_CACHE_DIR=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 python3-pip python3-dev \ + ffmpeg \ + libsndfile1 \ + git \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --upgrade pip && \ + pip install -r requirements.txt + +COPY main.py . + +ENV CANARY_MODEL=nvidia/canary-1b-v2 \ + CANARY_DEVICE=cuda \ + CANARY_PRECISION=bf16 \ + CANARY_MODELS_DIR=/models \ + HF_HOME=/models + +VOLUME ["/models"] +EXPOSE 9002 + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "9002", "--workers", "1"] +``` + +### requirements.txt + +``` +nemo_toolkit[asr]>=2.7.3,<3.0 +huggingface_hub>=0.26.0,<1.0 +fastapi==0.115.0 +uvicorn[standard]==0.30.6 +python-multipart==0.0.9 +``` + +**Begründung der Pins:** `nemo_toolkit ≥ 2.7.3` ist Pflicht, weil in +NeMo 2.4.x ein Bug verhinderte, dass `FrameBatchMultiTaskAED` mit +Canary 1B v2 funktioniert (`CanaryBPETokenizer.vocabulary` fehlte). +In 2.5+ ist der Fall `tokenizer.vocab_size`-Fallback eingebaut. Wir +pinnen nach unten auf 2.7.3 (in Production verifiziert) und nach +oben auf < 3.0 (kein erwartet-stabiler Bruch). + +### docker-compose.yml + +```yaml +services: + doctate-canary: + image: doctate-canary:latest + container_name: doctate-canary + environment: + - CANARY_MODEL=nvidia/canary-1b-v2 + - CANARY_PRECISION=bf16 + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=compute,utility + volumes: + - /opt/stacks/doctate-canary/models:/models + ports: + - "9002:9002" + shm_size: 2g + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + restart: unless-stopped +``` + +### deploy.sh + +```bash +#!/usr/bin/env bash +set -euo pipefail +HOST="${1:-minerva.lan}" +REMOTE_DIR="/opt/stacks/doctate-canary" +LOCAL_DIR="$(cd "$(dirname "$0")" && pwd)" + +rsync -av --exclude 'deploy.sh' --exclude 'models/' \ + "$LOCAL_DIR/" "$HOST:$REMOTE_DIR/" + +ssh "$HOST" "cd $REMOTE_DIR && \ + docker compose down && \ + docker pull nvidia/cuda:12.6.1-devel-ubuntu22.04 && \ + docker build -t doctate-canary . && \ + docker compose up -d" + +for i in $(seq 1 120); do + if ssh "$HOST" "curl -sf http://localhost:9002/health" 2>/dev/null; then + echo " OK"; exit 0 + fi + sleep 5 +done +ssh "$HOST" "docker logs doctate-canary --tail 60" +exit 1 +``` + +## Appendix B — Service-Schlüssel-Code (`main.py`-Auszug) + +Diese beiden Funktionen aus `canary/main.py` sind die +Implementierungs-Quelle für die Pipeline-Wahl in Abschnitt 4.1: + +```python +def _transcribe_buffered(wav_path, duration, language, pnc, timestamps): + """Long-audio path: explicit chunked inference via FrameBatchMultiTaskAED. + + Writes a temporary single-line JSONL manifest because language + and pnc must be passed via manifest fields when using the + buffered API. + """ + manifest_tmp = tempfile.NamedTemporaryFile(suffix=".json", mode="w", delete=False) + json.dump({"audio_filepath": wav_path, "duration": duration, + "taskname": "asr", "source_lang": language, + "target_lang": language, "pnc": pnc}, manifest_tmp) + manifest_tmp.write("\n"); manifest_tmp.close() + + frame_asr = FrameBatchMultiTaskAED( + asr_model=model, frame_len=CHUNK_LEN_SECS, + total_buffer=CHUNK_LEN_SECS, batch_size=8, + ) + with torch.amp.autocast(model.device.type, + enabled=use_amp, dtype=DTYPE_MAP[PRECISION]): + with torch.no_grad(): + hyps = get_buffered_pred_feat_multitaskAED( + frame_asr, _model_cfg.preprocessor, MODEL_STRIDE_SECS, + model.device, manifest=manifest_tmp.name, + filepaths=None, timestamps=timestamps) + return hyps + +# Im /inference-Handler: +chunked = duration > CHUNKED_THRESHOLD_SECS +if chunked: + hyps = _transcribe_buffered(wav_path, duration, language, pnc, ts_flag) +else: + hyps = model.transcribe([wav_path], source_lang=language, + target_lang=language, pnc=pnc, + timestamps=ts_flag) +``` + +Pre-Conditions auf das Modell-Config-Objekt, einmal beim Service- +Start zu setzen (sonst inkonsistente Feature-Extraktion zwischen +Single-Shot- und Buffered-Pfad): + +```python +_model_cfg = copy.deepcopy(model._cfg) +OmegaConf.set_struct(_model_cfg.preprocessor, False) +_model_cfg.preprocessor.dither = 0.0 +_model_cfg.preprocessor.pad_to = 0 +OmegaConf.set_struct(_model_cfg.preprocessor, True) +assert _model_cfg.preprocessor.normalize == "per_feature" +MODEL_STRIDE_SECS = float(_model_cfg.preprocessor["window_stride"]) * 8 +``` + +## Appendix C — `dur=0` Validation auf dr_mueller + +### Methodik + +1. Cohorte: 101 m4a-Aufnahmen aus 48 Case-Verzeichnissen. +2. Drei Inferenz-Pässe: + - **Single Audios** (101 Files): Mode A — Single-Shot mit + `confidence_cfg`, `return_hypotheses=True`, `timestamps=True`. + - **Per-Case-Concats** (47 Files): adaptiv (≤ 25 s → + Single-Shot, > 25 s → Buffered). + - **Globaler Concat** (1 File, 38.7 min): Buffered. +3. `dur_zero_inventory.py` walks alle 149 `*.canary.json` und + extrahiert jedes Wort mit `start_offset == end_offset`. +4. **Ground-Truth-Annotation durch User**: Jedes flagged Vorkommen + im 3-Wörter-Kontext gegen die zugehörige Audio-Datei gehört. + +### Resultat + +| Quantity | Value | +|---|---:| +| dur=0 Vorkommen | 81 | +| True Positives (echte Halluzinationen) | 2 | +| False Positives (echtes Transkript) | 79 | +| Precision | **≈ 2.5 %** | + +Verteilung nach Quelle × Pipeline: + +| Quelle | Pipeline | n_zero_dur | +|---|---|---:| +| single_audio | single_shot | 29 | +| per_case_concat | single_shot | 1 | +| per_case_concat | buffered | 21 | +| global_concat | buffered | 30 | + +Die zwei True Positives sind beide in case_c414cf52: die Wörter +"5" und "milligramm" im Cluster `Bisoprolol 5 mg 100 und Ramipril +5 mg 100. [5 milligramm 101] wird vorerst pausiert ...`. Beachte: +"101" in diesem Cluster ist **echter Inhalt** (1-0-1-Schema für +Ramipril, siehe Abschnitt 6.1). + +Die 79 False Positives sind dominant unbetonte deutsche +Funktionswörter. Top-Formen (count ≥ 2): + +``` +6 und 4 einer 4 sich 3 wie 3 für +3 mit 3 einen 2 wird 2 von 2 es +2 um 2 pro 2 einmal 2 im +``` + +Diese Wörter werden im CTC-Viterbi-Forced-Alignment auf 0-Frame- +Seams zwischen Nachbarwörter gepresst, weil ihre akustische +Realisierung zu kurz oder zu reduziert ist, um eigene Encoder- +Frames zu beanspruchen. Strukturell sieht das identisch zu einer +Halluzination aus — die Alignment-Mathematik allein kann die +beiden nicht trennen. + +### Disposition + +`dur=0` ist als Halluzinations-Signal nicht verwendbar: + +- **Auto-Filter:** Bei 79/81 FP würde das Droppen geflagged Wörter + in jedem Diktat echten Transkript zerstören, mit dem Upside, in + einer einzigen Aufnahme zwei Halluzinationen zu erwischen. + Net-negativ. +- **Advisory `[?word]` Marker** für die LLM-Konsolidierung: + Gleiches Problem mit niedrigeren Stakes. 79/81 korrekte Wörter + als unsicher zu markieren verschmutzt den Prompt und trainiert + die LLM, den Marker zu ignorieren — was seinen Nutzen für die + echten Halluzinationen kompromittiert. + +Word-Confidence verhält sich auf demselben Korpus analog: das +Top-6-niedrigste-Confidence-Wort sind "SC", "Bisoprolol", +"Thorazemit", "5", "ausärztliche", "einmal" — eine Mischung aus +echtem medizinischen Vokabular, echten Dosierungsdigits und +einzelnen tatsächlichen Halluzinationen. Confidence kann "unsicher +echt" und "halluziniert" nicht trennen. + +### Implikation für die Pipeline-Architektur + +Halluzinations-Detektion über Per-Word-Telemetrie eines einzelnen +ASRs ist ein Sackgasse. Kandidaten für künftige Versuche, falls +nötig: + +- Cross-Modell-Agreement (Canary AED vs Canary CTC-Standalone vs + Whisper) als Divergenz-Signal +- LLM-seitige semantische Plausibilitätsprüfung (passiert implizit, + könnte explizit gemacht werden) +- Domänenregeln (Medikamenten-Existenz, Dosierungs-Plausibilität) + — aktuell vom Gazetteer pre-LLM erledigt + +## Appendix D — Container-Image bauen ohne `deploy.sh` + +Manuell auf einer beliebigen GPU-Maschine: + +```bash +cd canary/ +docker build -t doctate-canary . +docker run --rm --gpus all \ + -v $(pwd)/models:/models \ + -p 9002:9002 \ + -e HF_HOME=/models \ + doctate-canary +``` + +Beim Erststart lädt der Container ~3 GB aus HuggingFace ins Volume +`./models`. Nachfolgende Restarts nutzen den Cache. diff --git a/experiments/.gitignore b/experiments/.gitignore index e7c5dc5..b428562 100644 --- a/experiments/.gitignore +++ b/experiments/.gitignore @@ -1,5 +1,6 @@ /target/ /_data/ +/canary_sweep/ # Fall-spezifische Inputs (Audios, Diktat-Text, Befund-Snapshots, # vocab-Test-Erweiterungen) bleiben strikt lokal — Patientendaten und