diff --git a/whisper/Dockerfile b/whisper/Dockerfile new file mode 100644 index 0000000..db4d919 --- /dev/null +++ b/whisper/Dockerfile @@ -0,0 +1,21 @@ +FROM nvidia/cuda:12.1.0-cudnn8-runtime-ubuntu22.04 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 python3-pip ffmpeg && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY main.py . + +ENV WHISPER_MODEL=large-v3-turbo \ + WHISPER_COMPUTE_TYPE=float16 \ + WHISPER_MODELS_DIR=/models + +VOLUME ["/models"] +EXPOSE 9001 + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "9001", "--workers", "1"] diff --git a/whisper/README.md b/whisper/README.md new file mode 100644 index 0000000..869fa8d --- /dev/null +++ b/whisper/README.md @@ -0,0 +1,101 @@ +# doctate-whisper + +Thin FastAPI wrapper around [faster-whisper](https://github.com/SYSTRAN/faster-whisper). + +## Why a custom service? + +Existing Whisper HTTP wrappers (`ahmetoner/whisper-asr-webservice`, +`speaches`, `linuxserver/faster-whisper`, `hwdsl2/docker-whisper`) hardcode +the faster-whisper defaults. Those defaults include: + +- `condition_on_previous_text=True` — classic cascading-hallucination source +- temperature fallback cascade `[0.0, 0.2, …, 1.0]` — when confidence drops, + Whisper gets "creative" + +On real cardiology dictations we saw **2/13 runs with catastrophic +hallucinations** (Russian/Chinese fragments, repetition loops) using +ahmetoner's defaults. None of the available wrappers expose these +parameters as request fields or env vars. + +This service fixes them to safe values that never change. + +## Anti-hallucination params (fixed) + +| Param | Value | Reason | +|---|---|---| +| `temperature` | `0.0` | No fallback cascade; deterministic output | +| `condition_on_previous_text` | `False` | Prevents cascading hallucinations | +| `vad_filter` | `True` | Silence → no transcription (not garbage) | +| `vad_parameters.min_silence_duration_ms` | `500` | VAD sensitivity | +| `no_speech_threshold` | `0.6` | Drop silent segments | +| `log_prob_threshold` | `-1.0` | Drop low-confidence segments | +| `compression_ratio_threshold` | `2.4` | Anti-repetition | +| `beam_size` / `best_of` | `5` / `5` | Default quality beam search | + +## Endpoints + +- `POST /asr` — multipart `audio_file`, query `output=txt|json`, `language=de`, form `initial_prompt` +- `GET /health` — liveness: `{"status":"ok","model":...,"device":...}` +- `GET /info` — debug: model, device, compute_type, offline flag + +Compatible with ahmetoner's `/asr` interface so the Axum server needs no change. + +## Env vars + +| Var | Default | Purpose | +|---|---|---| +| `WHISPER_MODEL` | `large-v3-turbo` | Any faster-whisper model name | +| `WHISPER_COMPUTE_TYPE` | `float16` | `float16`, `int8_float16`, `int8` (CPU) | +| `WHISPER_DEVICE` | `cuda` | `cuda` or `cpu` | +| `WHISPER_MODELS_DIR` | `/models` | Cache location | +| `WHISPER_OFFLINE` | `0` | `1` → no HuggingFace download | + +## Model choice + +**Default is `large-v3-turbo`**: same 32-layer encoder as `large-v3`, +only decoder distilled to 4 layers. OpenAI benchmarks: equal quality on +European languages. ~1.6 GB VRAM vs. ~3 GB — leaves room for Ollama +(gemma4, 9 GB) in a 12 GB card. + +Switch to `large-v3` at any time via `WHISPER_MODEL=large-v3` + container restart. + +## Build + +```bash +docker build -t doctate-whisper ./whisper +``` + +Takes 2–5 minutes. First run of the container pulls ~1.6 GB from +HuggingFace into `/models` (volume-persisted). + +## Deploy to minerva via Dockge + +Transport the image without a registry: + +```bash +docker save doctate-whisper | ssh minerva 'docker load' +``` + +Then add the compose file in Dockge and start. + +## Integration with Axum server + +In `server/.env`: + +``` +WHISPER_URL=http://minerva.lan:9001 +``` + +No other change. The Axum server speaks ahmetoner's interface; we mirror it. + +## Offline mode + +```bash +# Pre-pull on a host with internet: +docker run --rm -v /opt/stacks/doctate-whisper/models:/models doctate-whisper \ + python3 -c "from faster_whisper import WhisperModel; WhisperModel('large-v3-turbo', download_root='/models')" + +# Then set in docker-compose.yml: +environment: + - WHISPER_OFFLINE=1 +``` diff --git a/whisper/docker-compose.yml b/whisper/docker-compose.yml new file mode 100644 index 0000000..3c6887e --- /dev/null +++ b/whisper/docker-compose.yml @@ -0,0 +1,21 @@ +services: + doctate-whisper: + image: doctate-whisper:latest + container_name: doctate-whisper + environment: + - WHISPER_MODEL=large-v3-turbo + - WHISPER_COMPUTE_TYPE=float16 + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=compute,utility + volumes: + - /opt/stacks/doctate-whisper/models:/models + ports: + - "9001:9001" + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + restart: unless-stopped diff --git a/whisper/main.py b/whisper/main.py new file mode 100644 index 0000000..121f34c --- /dev/null +++ b/whisper/main.py @@ -0,0 +1,129 @@ +"""Thin FastAPI wrapper around faster-whisper. + +Exposes the same /asr interface as ahmetoner/whisper-asr-webservice so the +Axum server stays unchanged, but hardcodes anti-hallucination parameters +that no off-the-shelf wrapper exposes. +""" +import json +import logging +import os +import tempfile +import time +from typing import Optional + +from fastapi import FastAPI, File, Form, Query, UploadFile +from fastapi.responses import JSONResponse, PlainTextResponse +from faster_whisper import WhisperModel + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", +) +log = logging.getLogger("whisper") + +MODEL_NAME = os.getenv("WHISPER_MODEL", "large-v3-turbo") +COMPUTE_TYPE = os.getenv("WHISPER_COMPUTE_TYPE", "float16") +DEVICE = os.getenv("WHISPER_DEVICE", "cuda") +MODELS_DIR = os.getenv("WHISPER_MODELS_DIR", "/models") +OFFLINE = os.getenv("WHISPER_OFFLINE", "0") == "1" + +log.info("Loading model=%s device=%s compute_type=%s", MODEL_NAME, DEVICE, COMPUTE_TYPE) +model = WhisperModel( + MODEL_NAME, + device=DEVICE, + compute_type=COMPUTE_TYPE, + download_root=MODELS_DIR, + local_files_only=OFFLINE, +) +log.info("Model loaded") + +app = FastAPI(title="doctate-whisper", version="0.1.0") + + +@app.get("/health") +def health(): + return {"status": "ok", "model": MODEL_NAME, "device": DEVICE} + + +@app.get("/info") +def info(): + return { + "model": MODEL_NAME, + "device": DEVICE, + "compute_type": COMPUTE_TYPE, + "offline": OFFLINE, + } + + +@app.post("/asr") +async def asr( + audio_file: UploadFile = File(...), + output: str = Query("txt"), + language: Optional[str] = Query(None), + initial_prompt: Optional[str] = Form(None), +): + suffix = os.path.splitext(audio_file.filename or "")[1] or ".bin" + tmp = tempfile.NamedTemporaryFile(suffix=suffix, delete=False) + try: + tmp.write(await audio_file.read()) + tmp.flush() + tmp.close() + + t0 = time.monotonic() + segments_gen, info_obj = model.transcribe( + tmp.name, + language=language, + task="transcribe", + beam_size=5, + best_of=5, + # Fixed anti-hallucination params — the whole point of this service. + temperature=0.0, + condition_on_previous_text=False, + vad_filter=True, + vad_parameters={"min_silence_duration_ms": 500}, + no_speech_threshold=0.6, + log_prob_threshold=-1.0, + compression_ratio_threshold=2.4, + initial_prompt=initial_prompt, + ) + segments = list(segments_gen) + infer_secs = time.monotonic() - t0 + + text = "".join(s.text for s in segments).strip() + worst_logprob = min((s.avg_logprob for s in segments), default=0.0) + + log.info( + json.dumps({ + "event": "transcribe", + "duration_audio": info_obj.duration, + "duration_infer": round(infer_secs, 2), + "segments": len(segments), + "worst_logprob": round(worst_logprob, 3), + "language": info_obj.language, + }) + ) + + if output == "json": + return JSONResponse({ + "text": text, + "language": info_obj.language, + "duration": info_obj.duration, + "segments": [ + { + "start": s.start, + "end": s.end, + "text": s.text, + "avg_logprob": s.avg_logprob, + "compression_ratio": s.compression_ratio, + "no_speech_prob": s.no_speech_prob, + "temperature": s.temperature, + } + for s in segments + ], + }) + return PlainTextResponse(text) + finally: + try: + os.unlink(tmp.name) + except OSError: + pass diff --git a/whisper/requirements.txt b/whisper/requirements.txt new file mode 100644 index 0000000..ad11b3c --- /dev/null +++ b/whisper/requirements.txt @@ -0,0 +1,5 @@ +faster-whisper==1.0.3 +requests==2.32.3 +fastapi==0.115.0 +uvicorn[standard]==0.30.6 +python-multipart==0.0.9