Add scripts to build anatomy vocabulary
This commit is contained in:
Executable
+42
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Filter a word list to entries NOT recognised by hunspell-de. Uses
|
||||
# `hunspell -l` (list-unknown-words mode) — a single batch invocation,
|
||||
# orders of magnitude faster than per-word lookups.
|
||||
#
|
||||
# Multi-word input lines are split into words by hunspell; only the
|
||||
# individual unknown words are kept. That matches our gazetteer's
|
||||
# token-level matching model (see gazetteer/mod.rs).
|
||||
#
|
||||
# Dependencies: hunspell, hunspell-de-de (Debian: apt install hunspell
|
||||
# hunspell-de-de).
|
||||
#
|
||||
# Usage: ./filter-hunspell-de.sh <input> <output>
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $# -ne 2 ]]; then
|
||||
echo "Usage: $0 <input> <output>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
INPUT="$1"
|
||||
OUTPUT="$2"
|
||||
|
||||
if ! command -v hunspell >/dev/null 2>&1; then
|
||||
echo "Error: hunspell is not installed." >&2
|
||||
echo " Debian/Ubuntu: sudo apt install hunspell hunspell-de-de" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Sanity-check: is the de_DE dictionary available?
|
||||
if ! echo "Haus" | hunspell -d de_DE -a >/dev/null 2>&1; then
|
||||
echo "Error: hunspell de_DE dictionary not available." >&2
|
||||
echo " Debian/Ubuntu: sudo apt install hunspell-de-de" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# `hunspell -l` prints every unknown word from stdin, one per line.
|
||||
# Pipe through sort -u to drop duplicates (multi-word titles can share
|
||||
# tokens) and produce a stable output order.
|
||||
hunspell -d de_DE -l < "$INPUT" | sort -u > "$OUTPUT"
|
||||
Reference in New Issue
Block a user