Add scripts to build anatomy vocabulary

This commit is contained in:
2026-04-16 17:34:38 +02:00
parent 8a394ec8c0
commit 3da5a36dc8
3 changed files with 165 additions and 0 deletions
+31
View File
@@ -0,0 +1,31 @@
#!/usr/bin/env bash
#
# Build vocab/anatomy.txt from scratch:
# 1. Crawl de.wikipedia.org Kategorie:Anatomie → raw/anatomy-wiki.txt
# 2. Filter against hunspell-de → vocab/anatomy.txt
#
# Idempotent: safe to re-run. Commit vocab/anatomy.txt after a successful
# build so the server uses a deterministic artefact, not a live crawl.
#
# Usage: (from anywhere) bash path/to/build-anatomy.sh
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
VOCAB_DIR="$(cd "$SCRIPT_DIR/../vocab" && pwd)"
RAW_DIR="$VOCAB_DIR/raw"
mkdir -p "$RAW_DIR"
RAW_FILE="$RAW_DIR/anatomy-wiki.txt"
OUT_FILE="$VOCAB_DIR/anatomy.txt"
echo "[1/2] Fetching Wikipedia-de Kategorie:Anatomie..."
"$SCRIPT_DIR/fetch-wiki-anatomy.sh" > "$RAW_FILE"
echo " raw titles: $(wc -l < "$RAW_FILE")"
echo "[2/2] Filtering against hunspell-de..."
"$SCRIPT_DIR/filter-hunspell-de.sh" "$RAW_FILE" "$OUT_FILE"
echo " kept unknown: $(wc -l < "$OUT_FILE")"
echo "Done. Updated: $OUT_FILE"