#!/usr/bin/env bash
#
# Filter a word list to entries NOT recognised by hunspell-de. Uses
# `hunspell -l` (list-unknown-words mode) — a single batch invocation,
# orders of magnitude faster than per-word lookups.
#
# Multi-word input lines are split into words by hunspell; only the
# individual unknown words are kept. That matches our gazetteer's
# token-level matching model (see gazetteer/mod.rs).
#
# Dependencies: hunspell + German dictionary. Package names vary:
#   Arch/CachyOS:    pacman -S hunspell hunspell-de
#   Debian/Ubuntu:   apt install hunspell hunspell-de-de
#   Fedora:          dnf install hunspell hunspell-de
#
# Usage: ./filter-hunspell-de.sh <input> <output>

set -euo pipefail

if [[ $# -ne 2 ]]; then
    echo "Usage: $0 <input> <output>" >&2
    exit 1
fi

INPUT="$1"
OUTPUT="$2"

# Minimum entry length. Must match MIN_TOKEN_LEN in src/gazetteer/mod.rs.
# 5 is the current value — appropriate for curated medication lists where
# the 4-char eponym-vs-German-word collision class is not present.
MIN_LEN=5

if ! command -v hunspell >/dev/null 2>&1; then
    echo "Error: hunspell is not installed. Install it via your package" >&2
    echo "  manager (e.g. pacman -S hunspell, apt install hunspell)." >&2
    exit 1
fi

# Sanity-check: is the de_DE dictionary available?
if ! echo "Haus" | hunspell -d de_DE -a >/dev/null 2>&1; then
    echo "Error: hunspell de_DE dictionary not available. Install:" >&2
    echo "  Arch/CachyOS:  pacman -S hunspell-de" >&2
    echo "  Debian/Ubuntu: apt install hunspell-de-de" >&2
    echo "  Fedora:        dnf install hunspell-de" >&2
    exit 1
fi

# `hunspell -l` prints every unknown word from stdin, one per line.
# Drop words shorter than MIN_LEN, then sort -u for dedup + stable order.
hunspell -d de_DE -l < "$INPUT" \
    | awk -v min="$MIN_LEN" 'length($0) >= min' \
    | sort -u > "$OUTPUT"