feat: Add hunspell dictionary support to gazetteer
Integrates the `spellbook` crate to allow for a Hunspell dictionary to be used as a veto mechanism within the gazetteer. This prevents common German words from being incorrectly rewritten to vocabulary entries if they fall within the edit distance threshold. For example, "Kaktus" will no longer be rewritten to "Lantus" if a Hunspell dictionary is provided and contains "Kaktus". The `hunspell_dict_path` configuration option is added, defaulting to `/usr/share/hunspell/de_DE`. This path should point to the stem of the Hunspell dictionary files (e.g., `/usr/share/hunspell/de_DE.aff` and `/usr/share/hunspell/de_DE.dic`). The gazetteer now supports an optional `DictChecker` trait, allowing for pluggable dictionary implementations. `SpellbookDict` is the initial implementation using the `spellbook` crate. Includes new tests to verify the dict veto functionality, including handling of case sensitivity and exact matches. An ignored test is added for live verification against system-installed Hunspell dictionaries.
This commit is contained in:
+24
-2
@@ -10,7 +10,7 @@ use tracing_subscriber::EnvFilter;
|
||||
|
||||
use doctate_server::analyze;
|
||||
use doctate_server::config::Config;
|
||||
use doctate_server::gazetteer::Gazetteer;
|
||||
use doctate_server::gazetteer::{Gazetteer, SpellbookDict};
|
||||
use doctate_server::transcribe;
|
||||
use doctate_server::AppState;
|
||||
|
||||
@@ -69,7 +69,7 @@ async fn main() {
|
||||
// every AI output (Whisper, oneliner, analyze). Missing / unreadable
|
||||
// vocab directory is non-fatal — the pipeline runs without
|
||||
// normalization.
|
||||
let vocab = Arc::new(match Gazetteer::load_dir(&config.vocab_dir).await {
|
||||
let base_gazetteer = match Gazetteer::load_dir(&config.vocab_dir).await {
|
||||
Ok(g) => {
|
||||
info!(
|
||||
dir = %config.vocab_dir.display(),
|
||||
@@ -86,6 +86,28 @@ async fn main() {
|
||||
);
|
||||
Gazetteer::empty()
|
||||
}
|
||||
};
|
||||
|
||||
// Dict-veto: preserve tokens that are real German words, even if
|
||||
// they sit at DL ≤ 2 from a vocab entry (e.g. `Kaktus` stays
|
||||
// `Kaktus`, does not become `Lantus`). Missing / unreadable
|
||||
// hunspell files are non-fatal — pipeline runs without the veto.
|
||||
let vocab = Arc::new(match SpellbookDict::load(&config.hunspell_dict_path) {
|
||||
Ok(dict) => {
|
||||
info!(
|
||||
stem = %config.hunspell_dict_path.display(),
|
||||
"Hunspell dict loaded for gazetteer veto"
|
||||
);
|
||||
base_gazetteer.with_dict(Box::new(dict))
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
stem = %config.hunspell_dict_path.display(),
|
||||
error = %e,
|
||||
"Hunspell dict unavailable; gazetteer running without dict veto"
|
||||
);
|
||||
base_gazetteer
|
||||
}
|
||||
});
|
||||
|
||||
// Transcription pipeline: channel + worker + recovery scan.
|
||||
|
||||
Reference in New Issue
Block a user