From 619da6c2050fa227e280ec8902c282a88a5ea9be Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Fri, 21 Aug 2026 09:07:23 +1000 Subject: [PATCH 1/3] CI: add timeout and retry to apt-get install steps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bare `sudo apt-get install -y` with no `timeout-minutes` and no retry let a stalled Ubuntu mirror hang CI for hours — two 358-minute hangs on 2026-08-18 across two edition repos ten minutes apart, and four more hangs on 2026-08-19 each cleared only by cancel-and-rerun. Every apt-installing step now carries a step-level `timeout-minutes` and runs up to three attempts with a per-attempt `timeout`, so a stalled fetch fails fast and a transient mirror outage is ridden out rather than burning the job. Package lists are unchanged. Tracked in QuantEcon/project-translation#43. --- .github/workflows/ci.yml | 20 +++++++++----------- .github/workflows/publish.yml | 20 +++++++++----------- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 077258c8..e258f6b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,18 +36,16 @@ jobs: pip install numpyro python scripts/test-jax-install.py - name: Install latex dependencies + timeout-minutes: 30 run: | - sudo apt-get -qq update - sudo apt-get install -y \ - texlive-latex-recommended \ - texlive-latex-extra \ - texlive-fonts-recommended \ - texlive-fonts-extra \ - texlive-xetex \ - latexmk \ - xindy \ - dvipng \ - cm-super + # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # hangs this step for hours (two 358-minute hangs on 2026-08-18). + pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" + for attempt in 1 2 3; do + timeout 9m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 + done - name: Display Conda Environment Versions shell: bash -l {0} run: conda list diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 83baca77..615bfd51 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -52,18 +52,16 @@ jobs: run: | nvidia-smi - name: Install latex dependencies + timeout-minutes: 30 run: | - sudo apt-get -qq update - sudo apt-get install -y \ - texlive-latex-recommended \ - texlive-latex-extra \ - texlive-fonts-recommended \ - texlive-fonts-extra \ - texlive-xetex \ - latexmk \ - xindy \ - dvipng \ - cm-super + # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # hangs this step for hours (two 358-minute hangs on 2026-08-18). + pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" + for attempt in 1 2 3; do + timeout 9m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 + done - name: Display Conda Environment Versions shell: bash -l {0} run: conda list From 20a55688d802ea408221a37fc76aaa0038358023 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Fri, 21 Aug 2026 09:40:41 +1000 Subject: [PATCH 2/3] CI: harden the cache.yml graphviz step too; two attempts with wider per-attempt timeouts (4m/14m) Addresses Copilot review on QuantEcon/lecture-python.zh-cn#263: the cache.yml graphviz step was missed by a name match, and three 3m/9m attempts could never let a slow-but-progressing install finish. Now two attempts at 4m (graphviz) / 14m (texlive) inside the unchanged 10/30-minute step budgets. --- .github/workflows/ci.yml | 6 +++--- .github/workflows/publish.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e258f6b2..b3cd8a59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,9 +41,9 @@ jobs: # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" - for attempt in 1 2 3; do - timeout 9m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break - if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + for attempt in 1 2; do + timeout 14m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done - name: Display Conda Environment Versions diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 615bfd51..9fea39d5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -57,9 +57,9 @@ jobs: # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" - for attempt in 1 2 3; do - timeout 9m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break - if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + for attempt in 1 2; do + timeout 14m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done - name: Display Conda Environment Versions From c794cb86c704c32199651321a97156378875f5b8 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Fri, 21 Aug 2026 09:44:01 +1000 Subject: [PATCH 3/3] CI: run timeout inside sudo, per apt command, so the kill reaches apt-get Addresses Copilot review on QuantEcon/lecture-intro.zh-cn#301: with timeout wrapping sudo, a timed-out attempt terminated sudo and the bash -c shell but could orphan apt-get holding the dpkg lock, making the retry block instead of retry. timeout is now the direct parent of each apt-get (inside sudo, -k 30s escalation): update 1m/2m, install 3m/12m for graphviz/texlive. Step budgets unchanged. --- .github/workflows/ci.yml | 5 +++-- .github/workflows/publish.yml | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3cd8a59..bf66a3a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,11 +38,12 @@ jobs: - name: Install latex dependencies timeout-minutes: 30 run: | - # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # Retried with per-command timeouts (timeout inside sudo, so the kill reaches + # apt-get and the dpkg lock is released): a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" for attempt in 1 2; do - timeout 14m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + sudo timeout -k 30s 2m apt-get -qq update && sudo timeout -k 30s 12m apt-get install -y $pkgs && break if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9fea39d5..de7e5c66 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -54,11 +54,12 @@ jobs: - name: Install latex dependencies timeout-minutes: 30 run: | - # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # Retried with per-command timeouts (timeout inside sudo, so the kill reaches + # apt-get and the dpkg lock is released): a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" for attempt in 1 2; do - timeout 14m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + sudo timeout -k 30s 2m apt-get -qq update && sudo timeout -k 30s 12m apt-get install -y $pkgs && break if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done