Add scripts to build anatomy vocabulary
This commit is contained in:
Executable
+31
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Build vocab/anatomy.txt from scratch:
|
||||
# 1. Crawl de.wikipedia.org Kategorie:Anatomie → raw/anatomy-wiki.txt
|
||||
# 2. Filter against hunspell-de → vocab/anatomy.txt
|
||||
#
|
||||
# Idempotent: safe to re-run. Commit vocab/anatomy.txt after a successful
|
||||
# build so the server uses a deterministic artefact, not a live crawl.
|
||||
#
|
||||
# Usage: (from anywhere) bash path/to/build-anatomy.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
VOCAB_DIR="$(cd "$SCRIPT_DIR/../vocab" && pwd)"
|
||||
RAW_DIR="$VOCAB_DIR/raw"
|
||||
|
||||
mkdir -p "$RAW_DIR"
|
||||
|
||||
RAW_FILE="$RAW_DIR/anatomy-wiki.txt"
|
||||
OUT_FILE="$VOCAB_DIR/anatomy.txt"
|
||||
|
||||
echo "[1/2] Fetching Wikipedia-de Kategorie:Anatomie..."
|
||||
"$SCRIPT_DIR/fetch-wiki-anatomy.sh" > "$RAW_FILE"
|
||||
echo " raw titles: $(wc -l < "$RAW_FILE")"
|
||||
|
||||
echo "[2/2] Filtering against hunspell-de..."
|
||||
"$SCRIPT_DIR/filter-hunspell-de.sh" "$RAW_FILE" "$OUT_FILE"
|
||||
echo " kept unknown: $(wc -l < "$OUT_FILE")"
|
||||
|
||||
echo "Done. Updated: $OUT_FILE"
|
||||
Reference in New Issue
Block a user