From 0fdc09a91759691d2f85097b2689276356e910df Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Fri, 21 Aug 2026 09:07:10 +1000 Subject: [PATCH 1/3] CI: add timeout and retry to apt-get install steps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bare `sudo apt-get install -y` with no `timeout-minutes` and no retry let a stalled Ubuntu mirror hang CI for hours — two 358-minute hangs on 2026-08-18 across two edition repos ten minutes apart, and four more hangs on 2026-08-19 each cleared only by cancel-and-rerun. Every apt-installing step now carries a step-level `timeout-minutes` and runs up to three attempts with a per-attempt `timeout`, so a stalled fetch fails fast and a transient mirror outage is ridden out rather than burning the job. Package lists are unchanged. Tracked in QuantEcon/project-translation#43. --- .github/workflows/cache.yml | 20 +++++++++----------- .github/workflows/ci.yml | 30 ++++++++++++++++++------------ .github/workflows/publish.yml | 20 +++++++++----------- 3 files changed, 36 insertions(+), 34 deletions(-) diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml index 036203a9..3e5cc913 100644 --- a/.github/workflows/cache.yml +++ b/.github/workflows/cache.yml @@ -25,18 +25,16 @@ jobs: run: | sudo apt-get -qq update && sudo apt-get install -y graphviz - name: Install latex dependencies + timeout-minutes: 30 run: | - sudo apt-get -qq update - sudo apt-get install -y \ - texlive-latex-recommended \ - texlive-latex-extra \ - texlive-fonts-recommended \ - texlive-fonts-extra \ - texlive-xetex \ - latexmk \ - xindy \ - dvipng \ - cm-super + # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # hangs this step for hours (two 358-minute hangs on 2026-08-18). + pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" + for attempt in 1 2 3; do + timeout 9m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 + done - name: Build HTML shell: bash -l {0} run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 466ee0cd..00097b11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,21 +18,27 @@ jobs: environment-file: environment.yml activate-environment: quantecon - name: Graphics Support #TODO: Review if graphviz is needed + timeout-minutes: 10 run: | - sudo apt-get -qq update && sudo apt-get install -y graphviz + # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # hangs this step for hours (two 358-minute hangs on 2026-08-18). + pkgs="graphviz" + for attempt in 1 2 3; do + timeout 3m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 + done - name: Install latex dependencies + timeout-minutes: 30 run: | - sudo apt-get -qq update - sudo apt-get install -y \ - texlive-latex-recommended \ - texlive-latex-extra \ - texlive-fonts-recommended \ - texlive-fonts-extra \ - texlive-xetex \ - latexmk \ - xindy \ - dvipng \ - cm-super + # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # hangs this step for hours (two 358-minute hangs on 2026-08-18). + pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" + for attempt in 1 2 3; do + timeout 9m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 + done - name: Display Conda Environment Versions shell: bash -l {0} run: conda list diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 70178d09..b4ec75f9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -37,18 +37,16 @@ jobs: environment-file: environment.yml activate-environment: quantecon - name: Install latex dependencies + timeout-minutes: 30 run: | - sudo apt-get -qq update - sudo apt-get install -y \ - texlive-latex-recommended \ - texlive-latex-extra \ - texlive-fonts-recommended \ - texlive-fonts-extra \ - texlive-xetex \ - latexmk \ - xindy \ - dvipng \ - cm-super + # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # hangs this step for hours (two 358-minute hangs on 2026-08-18). + pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" + for attempt in 1 2 3; do + timeout 9m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 + done - name: Display Conda Environment Versions shell: bash -l {0} run: conda list From 541e2bbf952f3caf6c057a972d9de569ccf513d9 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Fri, 21 Aug 2026 09:40:39 +1000 Subject: [PATCH 2/3] CI: harden the cache.yml graphviz step too; two attempts with wider per-attempt timeouts (4m/14m) Addresses Copilot review on QuantEcon/lecture-python.zh-cn#263: the cache.yml graphviz step was missed by a name match, and three 3m/9m attempts could never let a slow-but-progressing install finish. Now two attempts at 4m (graphviz) / 14m (texlive) inside the unchanged 10/30-minute step budgets. --- .github/workflows/cache.yml | 16 ++++++++++++---- .github/workflows/ci.yml | 12 ++++++------ .github/workflows/publish.yml | 6 +++--- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml index 3e5cc913..443041f2 100644 --- a/.github/workflows/cache.yml +++ b/.github/workflows/cache.yml @@ -22,17 +22,25 @@ jobs: environment-file: environment.yml activate-environment: quantecon - name: graphviz Support # TODO: required? + timeout-minutes: 10 run: | - sudo apt-get -qq update && sudo apt-get install -y graphviz + # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # hangs this step for hours (two 358-minute hangs on 2026-08-18). + pkgs="graphviz" + for attempt in 1 2; do + timeout 4m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi + echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 + done - name: Install latex dependencies timeout-minutes: 30 run: | # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" - for attempt in 1 2 3; do - timeout 9m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break - if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + for attempt in 1 2; do + timeout 14m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done - name: Build HTML diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00097b11..824dd71a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,9 +23,9 @@ jobs: # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="graphviz" - for attempt in 1 2 3; do - timeout 3m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break - if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + for attempt in 1 2; do + timeout 4m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done - name: Install latex dependencies @@ -34,9 +34,9 @@ jobs: # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" - for attempt in 1 2 3; do - timeout 9m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break - if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + for attempt in 1 2; do + timeout 14m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done - name: Display Conda Environment Versions diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b4ec75f9..bdbb071e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -42,9 +42,9 @@ jobs: # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" - for attempt in 1 2 3; do - timeout 9m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break - if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + for attempt in 1 2; do + timeout 14m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done - name: Display Conda Environment Versions From cc3f54a3b81033d635c1852c84c4d0da57dd89bc Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Fri, 21 Aug 2026 09:43:58 +1000 Subject: [PATCH 3/3] CI: run timeout inside sudo, per apt command, so the kill reaches apt-get Addresses Copilot review on QuantEcon/lecture-intro.zh-cn#301: with timeout wrapping sudo, a timed-out attempt terminated sudo and the bash -c shell but could orphan apt-get holding the dpkg lock, making the retry block instead of retry. timeout is now the direct parent of each apt-get (inside sudo, -k 30s escalation): update 1m/2m, install 3m/12m for graphviz/texlive. Step budgets unchanged. --- .github/workflows/cache.yml | 10 ++++++---- .github/workflows/ci.yml | 10 ++++++---- .github/workflows/publish.yml | 5 +++-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml index 443041f2..4674c06b 100644 --- a/.github/workflows/cache.yml +++ b/.github/workflows/cache.yml @@ -24,22 +24,24 @@ jobs: - name: graphviz Support # TODO: required? timeout-minutes: 10 run: | - # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # Retried with per-command timeouts (timeout inside sudo, so the kill reaches + # apt-get and the dpkg lock is released): a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="graphviz" for attempt in 1 2; do - timeout 4m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + sudo timeout -k 30s 1m apt-get -qq update && sudo timeout -k 30s 3m apt-get install -y $pkgs && break if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done - name: Install latex dependencies timeout-minutes: 30 run: | - # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # Retried with per-command timeouts (timeout inside sudo, so the kill reaches + # apt-get and the dpkg lock is released): a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" for attempt in 1 2; do - timeout 14m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + sudo timeout -k 30s 2m apt-get -qq update && sudo timeout -k 30s 12m apt-get install -y $pkgs && break if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 824dd71a..726cde0a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,22 +20,24 @@ jobs: - name: Graphics Support #TODO: Review if graphviz is needed timeout-minutes: 10 run: | - # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # Retried with per-command timeouts (timeout inside sudo, so the kill reaches + # apt-get and the dpkg lock is released): a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="graphviz" for attempt in 1 2; do - timeout 4m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + sudo timeout -k 30s 1m apt-get -qq update && sudo timeout -k 30s 3m apt-get install -y $pkgs && break if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done - name: Install latex dependencies timeout-minutes: 30 run: | - # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # Retried with per-command timeouts (timeout inside sudo, so the kill reaches + # apt-get and the dpkg lock is released): a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" for attempt in 1 2; do - timeout 14m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + sudo timeout -k 30s 2m apt-get -qq update && sudo timeout -k 30s 12m apt-get install -y $pkgs && break if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bdbb071e..6d3f6f2b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -39,11 +39,12 @@ jobs: - name: Install latex dependencies timeout-minutes: 30 run: | - # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # Retried with per-command timeouts (timeout inside sudo, so the kill reaches + # apt-get and the dpkg lock is released): a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" for attempt in 1 2; do - timeout 14m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + sudo timeout -k 30s 2m apt-get -qq update && sudo timeout -k 30s 12m apt-get install -y $pkgs && break if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done