93 lines
2.9 KiB
Bash
Executable File
93 lines
2.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Crawl the German Wikipedia "Kategorie:Anatomie" (recursively into
|
|
# subcategories) and print each page title on stdout, one per line,
|
|
# sorted and deduplicated. Progress info goes to stderr.
|
|
#
|
|
# Dependencies: curl, jq.
|
|
# Rate limit: ~10 requests/sec (sleep 0.1 between requests).
|
|
# Depth limit: MAX_DEPTH — prevents infinite recursion via cross-links.
|
|
#
|
|
# Usage: ./fetch-wiki-anatomy.sh > raw/anatomy-wiki.txt
|
|
|
|
set -euo pipefail
|
|
|
|
USER_AGENT="doctate-vocab-builder (mould73@gmail.com)"
|
|
API="https://de.wikipedia.org/w/api.php"
|
|
ROOT_CATEGORY="Kategorie:Anatomie"
|
|
MAX_DEPTH=4
|
|
|
|
for tool in curl jq; do
|
|
if ! command -v "$tool" >/dev/null 2>&1; then
|
|
echo "Error: $tool is required but not installed." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
PAGES=$(mktemp)
|
|
SEEN=$(mktemp)
|
|
trap 'rm -f "$PAGES" "$SEEN"' EXIT
|
|
|
|
# Fetch all pages under one category, recursing into subcategories.
|
|
# Args: category_title depth
|
|
fetch_category() {
|
|
local cat="$1"
|
|
local depth="$2"
|
|
|
|
if [[ $depth -gt $MAX_DEPTH ]]; then
|
|
return
|
|
fi
|
|
if grep -qFx "$cat" "$SEEN"; then
|
|
return
|
|
fi
|
|
echo "$cat" >> "$SEEN"
|
|
|
|
echo "[depth=$depth] $cat" >&2
|
|
|
|
local cmcontinue=""
|
|
while true; do
|
|
local response
|
|
if [[ -n "$cmcontinue" ]]; then
|
|
response=$(curl -sS -A "$USER_AGENT" -G "$API" \
|
|
--data-urlencode "action=query" \
|
|
--data-urlencode "list=categorymembers" \
|
|
--data-urlencode "cmtitle=$cat" \
|
|
--data-urlencode "cmlimit=500" \
|
|
--data-urlencode "cmtype=page|subcat" \
|
|
--data-urlencode "format=json" \
|
|
--data-urlencode "cmcontinue=$cmcontinue")
|
|
else
|
|
response=$(curl -sS -A "$USER_AGENT" -G "$API" \
|
|
--data-urlencode "action=query" \
|
|
--data-urlencode "list=categorymembers" \
|
|
--data-urlencode "cmtitle=$cat" \
|
|
--data-urlencode "cmlimit=500" \
|
|
--data-urlencode "cmtype=page|subcat" \
|
|
--data-urlencode "format=json")
|
|
fi
|
|
|
|
# Collect page titles (ns=0 is "Main" namespace, i.e. articles).
|
|
echo "$response" | jq -r '.query.categorymembers[] | select(.ns == 0) | .title' >> "$PAGES"
|
|
|
|
# Recurse into subcategories (ns=14).
|
|
local subcats
|
|
subcats=$(echo "$response" | jq -r '.query.categorymembers[] | select(.ns == 14) | .title')
|
|
while IFS= read -r subcat; do
|
|
[[ -z "$subcat" ]] && continue
|
|
fetch_category "$subcat" $((depth + 1))
|
|
done <<< "$subcats"
|
|
|
|
cmcontinue=$(echo "$response" | jq -r '.continue.cmcontinue // empty')
|
|
if [[ -z "$cmcontinue" ]]; then
|
|
break
|
|
fi
|
|
|
|
sleep 0.1
|
|
done
|
|
}
|
|
|
|
fetch_category "$ROOT_CATEGORY" 0
|
|
|
|
echo "Collected $(wc -l < "$PAGES") raw titles, emitting sorted-unique..." >&2
|
|
sort -u "$PAGES"
|