From 3da5a36dc8dad85e4f6466d728d7f208fecd8da0 Mon Sep 17 00:00:00 2001 From: Brummel Date: Thu, 16 Apr 2026 17:34:38 +0200 Subject: [PATCH] Add scripts to build anatomy vocabulary --- server/scripts/build-anatomy.sh | 31 ++++++++++ server/scripts/fetch-wiki-anatomy.sh | 92 ++++++++++++++++++++++++++++ server/scripts/filter-hunspell-de.sh | 42 +++++++++++++ 3 files changed, 165 insertions(+) create mode 100755 server/scripts/build-anatomy.sh create mode 100755 server/scripts/fetch-wiki-anatomy.sh create mode 100755 server/scripts/filter-hunspell-de.sh diff --git a/server/scripts/build-anatomy.sh b/server/scripts/build-anatomy.sh new file mode 100755 index 0000000..9a085dc --- /dev/null +++ b/server/scripts/build-anatomy.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# +# Build vocab/anatomy.txt from scratch: +# 1. Crawl de.wikipedia.org Kategorie:Anatomie → raw/anatomy-wiki.txt +# 2. Filter against hunspell-de → vocab/anatomy.txt +# +# Idempotent: safe to re-run. Commit vocab/anatomy.txt after a successful +# build so the server uses a deterministic artefact, not a live crawl. +# +# Usage: (from anywhere) bash path/to/build-anatomy.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VOCAB_DIR="$(cd "$SCRIPT_DIR/../vocab" && pwd)" +RAW_DIR="$VOCAB_DIR/raw" + +mkdir -p "$RAW_DIR" + +RAW_FILE="$RAW_DIR/anatomy-wiki.txt" +OUT_FILE="$VOCAB_DIR/anatomy.txt" + +echo "[1/2] Fetching Wikipedia-de Kategorie:Anatomie..." +"$SCRIPT_DIR/fetch-wiki-anatomy.sh" > "$RAW_FILE" +echo " raw titles: $(wc -l < "$RAW_FILE")" + +echo "[2/2] Filtering against hunspell-de..." +"$SCRIPT_DIR/filter-hunspell-de.sh" "$RAW_FILE" "$OUT_FILE" +echo " kept unknown: $(wc -l < "$OUT_FILE")" + +echo "Done. Updated: $OUT_FILE" diff --git a/server/scripts/fetch-wiki-anatomy.sh b/server/scripts/fetch-wiki-anatomy.sh new file mode 100755 index 0000000..e5ed4b8 --- /dev/null +++ b/server/scripts/fetch-wiki-anatomy.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# +# Crawl the German Wikipedia "Kategorie:Anatomie" (recursively into +# subcategories) and print each page title on stdout, one per line, +# sorted and deduplicated. Progress info goes to stderr. +# +# Dependencies: curl, jq. +# Rate limit: ~10 requests/sec (sleep 0.1 between requests). +# Depth limit: MAX_DEPTH — prevents infinite recursion via cross-links. +# +# Usage: ./fetch-wiki-anatomy.sh > raw/anatomy-wiki.txt + +set -euo pipefail + +USER_AGENT="doctate-vocab-builder (mould73@gmail.com)" +API="https://de.wikipedia.org/w/api.php" +ROOT_CATEGORY="Kategorie:Anatomie" +MAX_DEPTH=4 + +for tool in curl jq; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "Error: $tool is required but not installed." >&2 + exit 1 + fi +done + +PAGES=$(mktemp) +SEEN=$(mktemp) +trap 'rm -f "$PAGES" "$SEEN"' EXIT + +# Fetch all pages under one category, recursing into subcategories. +# Args: category_title depth +fetch_category() { + local cat="$1" + local depth="$2" + + if [[ $depth -gt $MAX_DEPTH ]]; then + return + fi + if grep -qFx "$cat" "$SEEN"; then + return + fi + echo "$cat" >> "$SEEN" + + echo "[depth=$depth] $cat" >&2 + + local cmcontinue="" + while true; do + local response + if [[ -n "$cmcontinue" ]]; then + response=$(curl -sS -A "$USER_AGENT" -G "$API" \ + --data-urlencode "action=query" \ + --data-urlencode "list=categorymembers" \ + --data-urlencode "cmtitle=$cat" \ + --data-urlencode "cmlimit=500" \ + --data-urlencode "cmtype=page|subcat" \ + --data-urlencode "format=json" \ + --data-urlencode "cmcontinue=$cmcontinue") + else + response=$(curl -sS -A "$USER_AGENT" -G "$API" \ + --data-urlencode "action=query" \ + --data-urlencode "list=categorymembers" \ + --data-urlencode "cmtitle=$cat" \ + --data-urlencode "cmlimit=500" \ + --data-urlencode "cmtype=page|subcat" \ + --data-urlencode "format=json") + fi + + # Collect page titles (ns=0 is "Main" namespace, i.e. articles). + echo "$response" | jq -r '.query.categorymembers[] | select(.ns == 0) | .title' >> "$PAGES" + + # Recurse into subcategories (ns=14). + local subcats + subcats=$(echo "$response" | jq -r '.query.categorymembers[] | select(.ns == 14) | .title') + while IFS= read -r subcat; do + [[ -z "$subcat" ]] && continue + fetch_category "$subcat" $((depth + 1)) + done <<< "$subcats" + + cmcontinue=$(echo "$response" | jq -r '.continue.cmcontinue // empty') + if [[ -z "$cmcontinue" ]]; then + break + fi + + sleep 0.1 + done +} + +fetch_category "$ROOT_CATEGORY" 0 + +echo "Collected $(wc -l < "$PAGES") raw titles, emitting sorted-unique..." >&2 +sort -u "$PAGES" diff --git a/server/scripts/filter-hunspell-de.sh b/server/scripts/filter-hunspell-de.sh new file mode 100755 index 0000000..35ab0e3 --- /dev/null +++ b/server/scripts/filter-hunspell-de.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# +# Filter a word list to entries NOT recognised by hunspell-de. Uses +# `hunspell -l` (list-unknown-words mode) — a single batch invocation, +# orders of magnitude faster than per-word lookups. +# +# Multi-word input lines are split into words by hunspell; only the +# individual unknown words are kept. That matches our gazetteer's +# token-level matching model (see gazetteer/mod.rs). +# +# Dependencies: hunspell, hunspell-de-de (Debian: apt install hunspell +# hunspell-de-de). +# +# Usage: ./filter-hunspell-de.sh + +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +INPUT="$1" +OUTPUT="$2" + +if ! command -v hunspell >/dev/null 2>&1; then + echo "Error: hunspell is not installed." >&2 + echo " Debian/Ubuntu: sudo apt install hunspell hunspell-de-de" >&2 + exit 1 +fi + +# Sanity-check: is the de_DE dictionary available? +if ! echo "Haus" | hunspell -d de_DE -a >/dev/null 2>&1; then + echo "Error: hunspell de_DE dictionary not available." >&2 + echo " Debian/Ubuntu: sudo apt install hunspell-de-de" >&2 + exit 1 +fi + +# `hunspell -l` prints every unknown word from stdin, one per line. +# Pipe through sort -u to drop duplicates (multi-word titles can share +# tokens) and produce a stable output order. +hunspell -d de_DE -l < "$INPUT" | sort -u > "$OUTPUT"