Add scripts to build anatomy vocabulary

server/scripts/build-anatomy.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+#
+# Build vocab/anatomy.txt from scratch:
+#   1. Crawl de.wikipedia.org Kategorie:Anatomie → raw/anatomy-wiki.txt
+#   2. Filter against hunspell-de          → vocab/anatomy.txt
+#
+# Idempotent: safe to re-run. Commit vocab/anatomy.txt after a successful
+# build so the server uses a deterministic artefact, not a live crawl.
+#
+# Usage: (from anywhere)  bash path/to/build-anatomy.sh
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+VOCAB_DIR="$(cd "$SCRIPT_DIR/../vocab" && pwd)"
+RAW_DIR="$VOCAB_DIR/raw"
+
+mkdir -p "$RAW_DIR"
+
+RAW_FILE="$RAW_DIR/anatomy-wiki.txt"
+OUT_FILE="$VOCAB_DIR/anatomy.txt"
+
+echo "[1/2] Fetching Wikipedia-de Kategorie:Anatomie..."
+"$SCRIPT_DIR/fetch-wiki-anatomy.sh" > "$RAW_FILE"
+echo "      raw titles: $(wc -l < "$RAW_FILE")"
+
+echo "[2/2] Filtering against hunspell-de..."
+"$SCRIPT_DIR/filter-hunspell-de.sh" "$RAW_FILE" "$OUT_FILE"
+echo "      kept unknown: $(wc -l < "$OUT_FILE")"
+
+echo "Done. Updated: $OUT_FILE"

server/scripts/fetch-wiki-anatomy.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+#
+# Crawl the German Wikipedia "Kategorie:Anatomie" (recursively into
+# subcategories) and print each page title on stdout, one per line,
+# sorted and deduplicated. Progress info goes to stderr.
+#
+# Dependencies: curl, jq.
+# Rate limit: ~10 requests/sec (sleep 0.1 between requests).
+# Depth limit: MAX_DEPTH — prevents infinite recursion via cross-links.
+#
+# Usage: ./fetch-wiki-anatomy.sh > raw/anatomy-wiki.txt
+
+set -euo pipefail
+
+USER_AGENT="doctate-vocab-builder (mould73@gmail.com)"
+API="https://de.wikipedia.org/w/api.php"
+ROOT_CATEGORY="Kategorie:Anatomie"
+MAX_DEPTH=4
+
+for tool in curl jq; do
+    if ! command -v "$tool" >/dev/null 2>&1; then
+        echo "Error: $tool is required but not installed." >&2
+        exit 1
+    fi
+done
+
+PAGES=$(mktemp)
+SEEN=$(mktemp)
+trap 'rm -f "$PAGES" "$SEEN"' EXIT
+
+# Fetch all pages under one category, recursing into subcategories.
+# Args: category_title depth
+fetch_category() {
+    local cat="$1"
+    local depth="$2"
+
+    if [[ $depth -gt $MAX_DEPTH ]]; then
+        return
+    fi
+    if grep -qFx "$cat" "$SEEN"; then
+        return
+    fi
+    echo "$cat" >> "$SEEN"
+
+    echo "[depth=$depth] $cat" >&2
+
+    local cmcontinue=""
+    while true; do
+        local response
+        if [[ -n "$cmcontinue" ]]; then
+            response=$(curl -sS -A "$USER_AGENT" -G "$API" \
+                --data-urlencode "action=query" \
+                --data-urlencode "list=categorymembers" \
+                --data-urlencode "cmtitle=$cat" \
+                --data-urlencode "cmlimit=500" \
+                --data-urlencode "cmtype=page|subcat" \
+                --data-urlencode "format=json" \
+                --data-urlencode "cmcontinue=$cmcontinue")
+        else
+            response=$(curl -sS -A "$USER_AGENT" -G "$API" \
+                --data-urlencode "action=query" \
+                --data-urlencode "list=categorymembers" \
+                --data-urlencode "cmtitle=$cat" \
+                --data-urlencode "cmlimit=500" \
+                --data-urlencode "cmtype=page|subcat" \
+                --data-urlencode "format=json")
+        fi
+
+        # Collect page titles (ns=0 is "Main" namespace, i.e. articles).
+        echo "$response" | jq -r '.query.categorymembers[] | select(.ns == 0) | .title' >> "$PAGES"
+
+        # Recurse into subcategories (ns=14).
+        local subcats
+        subcats=$(echo "$response" | jq -r '.query.categorymembers[] | select(.ns == 14) | .title')
+        while IFS= read -r subcat; do
+            [[ -z "$subcat" ]] && continue
+            fetch_category "$subcat" $((depth + 1))
+        done <<< "$subcats"
+
+        cmcontinue=$(echo "$response" | jq -r '.continue.cmcontinue // empty')
+        if [[ -z "$cmcontinue" ]]; then
+            break
+        fi
+
+        sleep 0.1
+    done
+}
+
+fetch_category "$ROOT_CATEGORY" 0
+
+echo "Collected $(wc -l < "$PAGES") raw titles, emitting sorted-unique..." >&2
+sort -u "$PAGES"

server/scripts/filter-hunspell-de.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+#
+# Filter a word list to entries NOT recognised by hunspell-de. Uses
+# `hunspell -l` (list-unknown-words mode) — a single batch invocation,
+# orders of magnitude faster than per-word lookups.
+#
+# Multi-word input lines are split into words by hunspell; only the
+# individual unknown words are kept. That matches our gazetteer's
+# token-level matching model (see gazetteer/mod.rs).
+#
+# Dependencies: hunspell, hunspell-de-de (Debian: apt install hunspell
+# hunspell-de-de).
+#
+# Usage: ./filter-hunspell-de.sh <input> <output>
+
+set -euo pipefail
+
+if [[ $# -ne 2 ]]; then
+    echo "Usage: $0 <input> <output>" >&2
+    exit 1
+fi
+
+INPUT="$1"
+OUTPUT="$2"
+
+if ! command -v hunspell >/dev/null 2>&1; then
+    echo "Error: hunspell is not installed." >&2
+    echo "  Debian/Ubuntu: sudo apt install hunspell hunspell-de-de" >&2
+    exit 1
+fi
+
+# Sanity-check: is the de_DE dictionary available?
+if ! echo "Haus" | hunspell -d de_DE -a >/dev/null 2>&1; then
+    echo "Error: hunspell de_DE dictionary not available." >&2
+    echo "  Debian/Ubuntu: sudo apt install hunspell-de-de" >&2
+    exit 1
+fi
+
+# `hunspell -l` prints every unknown word from stdin, one per line.
+# Pipe through sort -u to drop duplicates (multi-word titles can share
+# tokens) and produce a stable output order.
+hunspell -d de_DE -l < "$INPUT" | sort -u > "$OUTPUT"