From 64f436335c5d56fc2d9ac03e1ec809857d7c2168 Mon Sep 17 00:00:00 2001 From: Neal Date: Fri, 4 Sep 2026 08:45:16 -0400 Subject: [PATCH 1/8] add function for printing progress. Fixes #72 --- exe/findingaid-cache-regen/fa-full-regen | 37 ++++++++++++++++++++--- exe/findingaid-cache-regen/fetch-ead-arks | 14 ++++++--- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/exe/findingaid-cache-regen/fa-full-regen b/exe/findingaid-cache-regen/fa-full-regen index 5343ff8..eaf3f53 100755 --- a/exe/findingaid-cache-regen/fa-full-regen +++ b/exe/findingaid-cache-regen/fa-full-regen @@ -9,9 +9,38 @@ export ARKS_FILE trap 'rm -f "$ARKS_FILE"' EXIT -echo "Attempting generation of a list of finding aids to reprocess" +echo "Attempting generation of a list of finding aids to regenerate" fetch-ead-arks -echo "List generated" -echo "Reprocessing finding aids" -xargs -n 1 -P 4 fa-regen < "$ARKS_FILE" +FOUND=$(wc -l < "$ARKS_FILE") + +PROCESSED=0 + +if [ "$FOUND" -le 0 ]; then + echo "$FOUND findingaid records found. Exiting..." + exit 0 +fi + +function print_progress { + PROCESSED=$((PROCESSED + 1)) + local PROGRESS_PERCENTAGE=$(( 100 * PROCESSED / FOUND )) + local PROGRESS_STR="[" + local HASH_COUNT=$(( PROGRESS_PERCENTAGE / 5 )) + local i + for i in {1..20}; do + if [ "$i" -le $HASH_COUNT ]; then + PROGRESS_STR+="#" + else + PROGRESS_STR+=" " + fi + done + PROGRESS_STR+="] " + PROGRESS_STR+="($PROGRESS_PERCENTAGE)%" + echo -ne "$PROGRESS_STR\r" +} + +xargs -n 1 -P 4 fa-regen < "$ARKS_FILE" | while read -r _; do + print_progress +done +printf '\n' + echo "Finding aids reprocessed" diff --git a/exe/findingaid-cache-regen/fetch-ead-arks b/exe/findingaid-cache-regen/fetch-ead-arks index b43778b..641860e 100755 --- a/exe/findingaid-cache-regen/fetch-ead-arks +++ b/exe/findingaid-cache-regen/fetch-ead-arks @@ -5,12 +5,18 @@ set -euo pipefail # SOLR_URL: e.g. https://solrhost.example.com/solr/select # ARKS_FILE: where ARKs should be written -COUNT_URL="${SOLR_URL}?wt=json&fl=id&fq=format:collections" -numFound=$(curl -s "$COUNT_URL" | jq -r .response.numFound) -echo "Found $numFound collections" +COUNT_URL="${SOLR_URL}?wt=json&fl=id&fq=format:collections&rows=0" +NUM_FOUND=$(curl -s "$COUNT_URL" | jq -r '.response.numFound') + +if ! [[ "$NUM_FOUND" =~ ^[0-9]+$ ]]; then + echo "fa-get-count: unexpected response from Solr: $NUM_FOUND" >&2 + exit 1 +fi + +echo "$NUM_FOUND records found" echo "Fetching ids" -EADS_URL="${SOLR_URL}?wt=json&fl=id&fq=format:collections&rows=${numFound}" +EADS_URL="${SOLR_URL}?wt=json&fl=id&fq=format:collections&rows=${NUM_FOUND}" curl -s "$EADS_URL" | jq -r '.response.docs[] | .id' > "$ARKS_FILE" echo "IDs written to $ARKS_FILE" From ee353c90a0737b555d7918810d8500935f7af59d Mon Sep 17 00:00:00 2001 From: Neal Date: Fri, 4 Sep 2026 14:22:24 -0400 Subject: [PATCH 2/8] add error reporting --- exe/findingaid-cache-regen/fa-full-regen | 50 +++++++++++++++++++++-- exe/findingaid-cache-regen/fa-regen | 13 +++++- exe/findingaid-cache-regen/fetch-ead-arks | 2 - 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/exe/findingaid-cache-regen/fa-full-regen b/exe/findingaid-cache-regen/fa-full-regen index eaf3f53..140fd02 100755 --- a/exe/findingaid-cache-regen/fa-full-regen +++ b/exe/findingaid-cache-regen/fa-full-regen @@ -1,13 +1,43 @@ #!/bin/bash set -euo pipefail -# The environment should provide these variables -# SOLR_URL: e.g. https://solrhost.example.com/solr/select +usage() { + cat <<'EOF' +Usage: fa-full-regen [-v|--verbose] + +Regenerate the cached page for every finding aid Solr lists as a collection. + +Options: + -v, --verbose List every ARK that could not be regenerated. Without it only + the count is reported + -h, --help Show this help. + +Environment: + SOLR_URL Required, e.g. https://solrhost.example.com/solr/select +EOF +} + +VERBOSE=0 + +while [ $# -gt 0 ]; do + case "$1" in + -v|--verbose) VERBOSE=1 ;; + -h|--help) usage; exit 0 ;; + *) + echo "fa-full-regen: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac + shift +done ARKS_FILE="$(mktemp)" export ARKS_FILE -trap 'rm -f "$ARKS_FILE"' EXIT +ERRORS_FILE="$(mktemp)" + +trap 'rm -f "$ARKS_FILE" "$ERRORS_FILE"' EXIT echo "Attempting generation of a list of finding aids to regenerate" fetch-ead-arks @@ -38,9 +68,21 @@ function print_progress { echo -ne "$PROGRESS_STR\r" } -xargs -n 1 -P 4 fa-regen < "$ARKS_FILE" | while read -r _; do +xargs -n 1 -P 4 fa-regen < "$ARKS_FILE" 2> "$ERRORS_FILE" | while read -r _; do print_progress done printf '\n' +# Missing EADs are a data condition, not a failed run, so report and exit 0. +if [ -s "$ERRORS_FILE" ]; then + MISSING=$(wc -l < "$ERRORS_FILE") + if [ "$VERBOSE" -eq 1 ]; then + echo "$MISSING of $FOUND finding aids could not be regenerated:" >&2 + cat "$ERRORS_FILE" >&2 + else + echo "$MISSING of $FOUND finding aids could not be regenerated" \ + "(re-run with --verbose for the list)" >&2 + fi +fi + echo "Finding aids reprocessed" diff --git a/exe/findingaid-cache-regen/fa-regen b/exe/findingaid-cache-regen/fa-regen index 52c59d0..7afeb5e 100755 --- a/exe/findingaid-cache-regen/fa-regen +++ b/exe/findingaid-cache-regen/fa-regen @@ -6,4 +6,15 @@ TARGET_URL=http://web ID=$1 BASE="$TARGET_URL/fa/findingaid/?id=$ID&invalidate_cache=1" echo "$BASE" -curl "$BASE" >/dev/null + +# A finding aid whose EAD is missing from the pairtree redirects to / +STATUS=$(curl -sS -o /dev/null -w '%{http_code}' "$BASE") || STATUS=000 + +case "$STATUS" in + 2??) ;; + 302) echo "$ID: not found (redirected to /); no EAD in the pairtree" >&2 ;; + 000) echo "$ID: request to $TARGET_URL failed" >&2 ;; + *) echo "$ID: unexpected HTTP $STATUS" >&2 ;; +esac + +exit 0 diff --git a/exe/findingaid-cache-regen/fetch-ead-arks b/exe/findingaid-cache-regen/fetch-ead-arks index 641860e..1546be7 100755 --- a/exe/findingaid-cache-regen/fetch-ead-arks +++ b/exe/findingaid-cache-regen/fetch-ead-arks @@ -13,8 +13,6 @@ if ! [[ "$NUM_FOUND" =~ ^[0-9]+$ ]]; then exit 1 fi -echo "$NUM_FOUND records found" - echo "Fetching ids" EADS_URL="${SOLR_URL}?wt=json&fl=id&fq=format:collections&rows=${NUM_FOUND}" curl -s "$EADS_URL" | jq -r '.response.docs[] | .id' > "$ARKS_FILE" From 3089635d4e5b355e9525cb9c2a10d640f584f31f Mon Sep 17 00:00:00 2001 From: Neal Date: Fri, 11 Sep 2026 16:01:13 -0400 Subject: [PATCH 3/8] add explicit env variable checks --- exe/findingaid-cache-regen/fetch-ead-arks | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/exe/findingaid-cache-regen/fetch-ead-arks b/exe/findingaid-cache-regen/fetch-ead-arks index 1546be7..f54d7e5 100755 --- a/exe/findingaid-cache-regen/fetch-ead-arks +++ b/exe/findingaid-cache-regen/fetch-ead-arks @@ -1,9 +1,15 @@ #!/bin/bash set -euo pipefail -# The environment should provide these variables -# SOLR_URL: e.g. https://solrhost.example.com/solr/select -# ARKS_FILE: where ARKs should be written +if [[ -z "$SOLR_URL" ]]; then + echo "fetch-ead-arks: SOLR_URL must be set, e.g. https://solrhost.example.com/solr/select" >&2 + exit 1 +fi + +if [[ -z "$ARKS_FILE" ]]; then + echo "fetch-ead-arks: ARKS_FILE must be set to the file ARKs should be written to" >&2 + exit 1 +fi COUNT_URL="${SOLR_URL}?wt=json&fl=id&fq=format:collections&rows=0" NUM_FOUND=$(curl -s "$COUNT_URL" | jq -r '.response.numFound') From a6584f28bde750337952c6f3cb4ed23cc47d3225 Mon Sep 17 00:00:00 2001 From: Neal Date: Fri, 11 Sep 2026 16:01:59 -0400 Subject: [PATCH 4/8] add symlinks to development stage and required deps --- Dockerfile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 180eac4..bee332b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,9 @@ FROM php:8.3-fpm-alpine AS development # add other deps for dev here RUN apk add --no-cache \ libzip-dev \ - bash + bash \ + jq \ + curl COPY --from=composer:2.8 /usr/bin/composer /usr/bin/composer COPY --from=jsmin /usr/bin/jsmin /usr/bin/jsmin @@ -30,6 +32,13 @@ RUN composer install --no-interaction COPY /exe ./exe +RUN chmod +x /opt/findingaid/exe/findingaid-cache-regen/fa-regen \ + /opt/findingaid/exe/findingaid-cache-regen/fa-full-regen \ + /opt/findingaid/exe/findingaid-cache-regen/fetch-ead-arks && \ + ln -s /opt/findingaid/exe/findingaid-cache-regen/fa-regen /usr/local/bin/fa-regen && \ + ln -s /opt/findingaid/exe/findingaid-cache-regen/fa-full-regen /usr/local/bin/fa-full-regen && \ + ln -s /opt/findingaid/exe/findingaid-cache-regen/fetch-ead-arks /usr/local/bin/fetch-ead-arks + COPY entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh From a048ad688fd18fbc8f07dc79f5cc00fe5ff0cde0 Mon Sep 17 00:00:00 2001 From: Neal Date: Fri, 11 Sep 2026 16:04:38 -0400 Subject: [PATCH 5/8] change wording. set -u already reports unset variables --- exe/findingaid-cache-regen/fetch-ead-arks | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exe/findingaid-cache-regen/fetch-ead-arks b/exe/findingaid-cache-regen/fetch-ead-arks index f54d7e5..d0ec866 100755 --- a/exe/findingaid-cache-regen/fetch-ead-arks +++ b/exe/findingaid-cache-regen/fetch-ead-arks @@ -2,12 +2,12 @@ set -euo pipefail if [[ -z "$SOLR_URL" ]]; then - echo "fetch-ead-arks: SOLR_URL must be set, e.g. https://solrhost.example.com/solr/select" >&2 + echo "fetch-ead-arks: SOLR_URL must not be empty, e.g. https://solrhost.example.com/solr/select" >&2 exit 1 fi if [[ -z "$ARKS_FILE" ]]; then - echo "fetch-ead-arks: ARKS_FILE must be set to the file ARKs should be written to" >&2 + echo "fetch-ead-arks: ARKS_FILE must not be empty" >&2 exit 1 fi From 94a2f1836155e645dc54f39e5c2c45de2e4507e8 Mon Sep 17 00:00:00 2001 From: Neal Date: Fri, 11 Sep 2026 16:07:05 -0400 Subject: [PATCH 6/8] fix old naming --- exe/findingaid-cache-regen/fetch-ead-arks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exe/findingaid-cache-regen/fetch-ead-arks b/exe/findingaid-cache-regen/fetch-ead-arks index d0ec866..e029637 100755 --- a/exe/findingaid-cache-regen/fetch-ead-arks +++ b/exe/findingaid-cache-regen/fetch-ead-arks @@ -15,7 +15,7 @@ COUNT_URL="${SOLR_URL}?wt=json&fl=id&fq=format:collections&rows=0" NUM_FOUND=$(curl -s "$COUNT_URL" | jq -r '.response.numFound') if ! [[ "$NUM_FOUND" =~ ^[0-9]+$ ]]; then - echo "fa-get-count: unexpected response from Solr: $NUM_FOUND" >&2 + echo "fetch-ead-arks: unexpected response from Solr: $NUM_FOUND" >&2 exit 1 fi From 075ce4dfb9458b85934bfe5e3150f78200d689b3 Mon Sep 17 00:00:00 2001 From: Neal Date: Fri, 11 Sep 2026 16:29:33 -0400 Subject: [PATCH 7/8] made reporting more clear --- exe/findingaid-cache-regen/fa-full-regen | 4 +++- exe/findingaid-cache-regen/fetch-ead-arks | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/exe/findingaid-cache-regen/fa-full-regen b/exe/findingaid-cache-regen/fa-full-regen index 140fd02..3494997 100755 --- a/exe/findingaid-cache-regen/fa-full-regen +++ b/exe/findingaid-cache-regen/fa-full-regen @@ -50,6 +50,8 @@ if [ "$FOUND" -le 0 ]; then exit 0 fi +echo "$FOUND findingaid IDs found. Regenerating findingaids..." + function print_progress { PROCESSED=$((PROCESSED + 1)) local PROGRESS_PERCENTAGE=$(( 100 * PROCESSED / FOUND )) @@ -85,4 +87,4 @@ if [ -s "$ERRORS_FILE" ]; then fi fi -echo "Finding aids reprocessed" +echo "$((FOUND - MISSING)) finding aids reprocessed" diff --git a/exe/findingaid-cache-regen/fetch-ead-arks b/exe/findingaid-cache-regen/fetch-ead-arks index e029637..e26cb5e 100755 --- a/exe/findingaid-cache-regen/fetch-ead-arks +++ b/exe/findingaid-cache-regen/fetch-ead-arks @@ -23,4 +23,4 @@ echo "Fetching ids" EADS_URL="${SOLR_URL}?wt=json&fl=id&fq=format:collections&rows=${NUM_FOUND}" curl -s "$EADS_URL" | jq -r '.response.docs[] | .id' > "$ARKS_FILE" -echo "IDs written to $ARKS_FILE" +echo "List of IDs fetched successfully" From d744351aae693ff4ccce30c4977c647b2b53f04d Mon Sep 17 00:00:00 2001 From: Neal Date: Fri, 11 Sep 2026 16:30:25 -0400 Subject: [PATCH 8/8] move percent inside parens --- exe/findingaid-cache-regen/fa-full-regen | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exe/findingaid-cache-regen/fa-full-regen b/exe/findingaid-cache-regen/fa-full-regen index 3494997..e0006ac 100755 --- a/exe/findingaid-cache-regen/fa-full-regen +++ b/exe/findingaid-cache-regen/fa-full-regen @@ -66,7 +66,7 @@ function print_progress { fi done PROGRESS_STR+="] " - PROGRESS_STR+="($PROGRESS_PERCENTAGE)%" + PROGRESS_STR+="($PROGRESS_PERCENTAGE%)" echo -ne "$PROGRESS_STR\r" }