#!/usr/bin/env bash
#
# Filter a word list to entries NOT recognised by hunspell-de. Uses
# `hunspell -l` (list-unknown-words mode) — a single batch invocation,
# orders of magnitude faster than per-word lookups.
#
# Multi-word input lines are split into words by hunspell; only the
# individual unknown words are kept. That matches our gazetteer's
# token-level matching model (see gazetteer/mod.rs).
#
# Dependencies: hunspell, hunspell-de-de (Debian: apt install hunspell
# hunspell-de-de).
#
# Usage: ./filter-hunspell-de.sh <input> <output>

set -euo pipefail

if [[ $# -ne 2 ]]; then
    echo "Usage: $0 <input> <output>" >&2
    exit 1
fi

INPUT="$1"
OUTPUT="$2"

if ! command -v hunspell >/dev/null 2>&1; then
    echo "Error: hunspell is not installed." >&2
    echo "  Debian/Ubuntu: sudo apt install hunspell hunspell-de-de" >&2
    exit 1
fi

# Sanity-check: is the de_DE dictionary available?
if ! echo "Haus" | hunspell -d de_DE -a >/dev/null 2>&1; then
    echo "Error: hunspell de_DE dictionary not available." >&2
    echo "  Debian/Ubuntu: sudo apt install hunspell-de-de" >&2
    exit 1
fi

# `hunspell -l` prints every unknown word from stdin, one per line.
# Pipe through sort -u to drop duplicates (multi-word titles can share
# tokens) and produce a stable output order.
hunspell -d de_DE -l < "$INPUT" | sort -u > "$OUTPUT"