43 lines
1.3 KiB
Bash
Executable File
43 lines
1.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Filter a word list to entries NOT recognised by hunspell-de. Uses
|
|
# `hunspell -l` (list-unknown-words mode) — a single batch invocation,
|
|
# orders of magnitude faster than per-word lookups.
|
|
#
|
|
# Multi-word input lines are split into words by hunspell; only the
|
|
# individual unknown words are kept. That matches our gazetteer's
|
|
# token-level matching model (see gazetteer/mod.rs).
|
|
#
|
|
# Dependencies: hunspell, hunspell-de-de (Debian: apt install hunspell
|
|
# hunspell-de-de).
|
|
#
|
|
# Usage: ./filter-hunspell-de.sh <input> <output>
|
|
|
|
set -euo pipefail
|
|
|
|
if [[ $# -ne 2 ]]; then
|
|
echo "Usage: $0 <input> <output>" >&2
|
|
exit 1
|
|
fi
|
|
|
|
INPUT="$1"
|
|
OUTPUT="$2"
|
|
|
|
if ! command -v hunspell >/dev/null 2>&1; then
|
|
echo "Error: hunspell is not installed." >&2
|
|
echo " Debian/Ubuntu: sudo apt install hunspell hunspell-de-de" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Sanity-check: is the de_DE dictionary available?
|
|
if ! echo "Haus" | hunspell -d de_DE -a >/dev/null 2>&1; then
|
|
echo "Error: hunspell de_DE dictionary not available." >&2
|
|
echo " Debian/Ubuntu: sudo apt install hunspell-de-de" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# `hunspell -l` prints every unknown word from stdin, one per line.
|
|
# Pipe through sort -u to drop duplicates (multi-word titles can share
|
|
# tokens) and produce a stable output order.
|
|
hunspell -d de_DE -l < "$INPUT" | sort -u > "$OUTPUT"
|