From c1252ca9d11b795cccfcf5f9ce0a7284c0ecdf39 Mon Sep 17 00:00:00 2001 From: cvanelteren Date: Thu, 10 Sep 2026 15:30:28 +1000 Subject: [PATCH 1/5] Create one master baseline and compare against that --- .github/workflows/build-ultraplot.yml | 121 +------------------------- .github/workflows/main.yml | 68 +++++++++++++++ 2 files changed, 72 insertions(+), 117 deletions(-) diff --git a/.github/workflows/build-ultraplot.yml b/.github/workflows/build-ultraplot.yml index 7778a0a5b..efec7498c 100644 --- a/.github/workflows/build-ultraplot.yml +++ b/.github/workflows/build-ultraplot.yml @@ -133,124 +133,11 @@ jobs: cache-environment: true cache-downloads: false - - name: Resolve baseline reference - id: baseline-ref - run: | - if [ "${IS_PR}" = "true" ]; then - BASE_REF="${{ github.event.pull_request.base.ref }}" - git fetch origin "${BASE_REF}" - BASE_SHA="$(git rev-parse "origin/${BASE_REF}")" - else - BASE_REF="${GITHUB_REF_NAME:-main}" - BASE_SHA="${GITHUB_SHA}" - fi - echo "base_ref=${BASE_REF}" >> "${GITHUB_OUTPUT}" - echo "base_sha=${BASE_SHA}" >> "${GITHUB_OUTPUT}" - echo "Resolved baseline ref=${BASE_REF} sha=${BASE_SHA}" - - - name: Check baseline version support - id: baseline-support - run: | - git show '${{ steps.baseline-ref.outputs.base_sha }}:pyproject.toml' > '${{ runner.temp }}/baseline-pyproject.toml' - python tools/ci/version_support.py \ - --baseline-pyproject '${{ runner.temp }}/baseline-pyproject.toml' \ - --python-version '${{ inputs.python-version }}' \ - --matplotlib-version '${{ inputs.matplotlib-version }}' >> "$GITHUB_OUTPUT" - - - name: Explain unavailable baseline - if: steps.baseline-support.outputs.baseline-supported != 'true' - run: | - echo '::notice::Image comparison skipped: the base commit does not support this Python/Matplotlib pair. PR tests still run in the coverage job.' - echo 'Image comparison skipped for Python ${{ inputs.python-version }} / Matplotlib ${{ inputs.matplotlib-version }}: base commit ${{ steps.baseline-ref.outputs.base_sha }} does not support this pair. The coverage job still tests the PR on this pair.' >> "$GITHUB_STEP_SUMMARY" - - # Cache Baseline Figures (Restore step) - - name: Cache Baseline Figures - id: cache-baseline - uses: actions/cache@v6 - if: env.IS_PR == 'true' && steps.baseline-support.outputs.baseline-supported == 'true' + - name: Download canonical baseline + uses: actions/download-artifact@v7 with: - path: ./ultraplot/tests/baseline # The directory to cache - # Key is based on OS, Python/Matplotlib versions, and the base commit SHA - key: ${{ runner.os }}-baseline-base-v5-hs${{ env.PYTHONHASHSEED }}-${{ steps.baseline-ref.outputs.base_sha }}-${{ inputs.python-version }}-${{ inputs.matplotlib-version }} - restore-keys: | - ${{ runner.os }}-baseline-base-v5-hs${{ env.PYTHONHASHSEED }}-${{ steps.baseline-ref.outputs.base_sha }}-${{ inputs.python-version }}-${{ inputs.matplotlib-version }}- - - # Conditional Baseline Generation (Only runs on cache miss) - - name: Generate baseline from main - # Skip this step if the cache was found (cache-hit is true) - if: steps.baseline-support.outputs.baseline-supported == 'true' && (steps.cache-baseline.outputs.cache-hit != 'true' || env.IS_PR != 'true') - run: | - mkdir -p ultraplot/tests/baseline - echo "TEST_MODE=${TEST_MODE}" - echo "IS_PR=${IS_PR}" - echo "PR_BASE_REF=${{ steps.baseline-ref.outputs.base_ref }}" - echo "PR_BASE_SHA=${{ steps.baseline-ref.outputs.base_sha }}" - echo "TEST_NODEIDS=${TEST_NODEIDS}" - # Save PR-selected nodeids for reuse after checkout (if provided) - if [ "${TEST_MODE}" = "selected" ] && [ -n "${TEST_NODEIDS}" ]; then - python -c 'import json,os; raw=os.environ.get("TEST_NODEIDS","").strip(); parsed=json.loads(raw) if raw and raw!="[]" else []; parsed=[parsed] if isinstance(parsed,str) else parsed; nodeids=[item for item in parsed if isinstance(item,str) and item]; open("/tmp/pr_selected_nodeids.txt","w",encoding="utf-8").write("".join(f"{nodeid}\n" for nodeid in nodeids)); print(f"Selected nodeids parsed: {len(nodeids)}")' - else - : > /tmp/pr_selected_nodeids.txt - fi - # Checkout the resolved base-branch tip for PR baseline generation. - if [ "${IS_PR}" = "true" ]; then - git checkout "${{ steps.baseline-ref.outputs.base_sha }}" - fi - - # Install the Ultraplot version from the base branch's code - pip install --no-build-isolation --no-deps . - - # Generate the baseline images and hash library - python -c "import ultraplot as plt; plt.config.Configurator()._save_yaml('ultraplot.yml')" - if [ "${TEST_MODE}" = "selected" ] && [ -s /tmp/pr_selected_nodeids.txt ]; then - status=0 - FILTERED_NODEIDS=() - while IFS= read -r nodeid; do - if [ -z "$nodeid" ]; then - continue - fi - path="${nodeid%%::*}" - if [ -f "$path" ]; then - FILTERED_NODEIDS+=("$nodeid") - fi - done < /tmp/pr_selected_nodeids.txt - echo "FILTERED_NODEIDS_BASE_COUNT=${#FILTERED_NODEIDS[@]}" - if [ "${#FILTERED_NODEIDS[@]}" -eq 0 ]; then - echo "No valid nodeids found on base; skipping baseline generation." - else - echo "=== Memory before baseline generation ===" && free -h - pytest -n ${PYTEST_WORKERS} --dist loadfile --tb=short --disable-warnings -W ignore \ - --mpl-generate-path=./ultraplot/tests/baseline/ \ - --mpl-default-style="./ultraplot.yml" \ - "${FILTERED_NODEIDS[@]}" || status=$? - echo "=== Memory after baseline generation ===" && free -h - if [ "$status" -eq 4 ] || [ "$status" -eq 5 ]; then - echo "No tests collected from selected nodeids on base; skipping baseline generation." - status=0 - fi - fi - # Return to the PR branch before continuing - if [ "${IS_PR}" = "true" ]; then - echo "Checking out PR branch: ${{ github.sha }}" - git checkout ${{ github.sha }} || echo "Warning: git checkout failed, but continuing" - fi - if [ "$status" -ne 0 ]; then - echo "Baseline generation failed with status $status" - exit "$status" - fi - else - echo "=== Memory before baseline generation ===" && free -h - pytest -n ${PYTEST_WORKERS} --dist loadfile --tb=short --disable-warnings -W ignore \ - --mpl-generate-path=./ultraplot/tests/baseline/ \ - --mpl-default-style="./ultraplot.yml" \ - ultraplot/tests - echo "=== Memory after baseline generation ===" && free -h - # Return to the PR branch for the rest of the job - if [ "${IS_PR}" = "true" ]; then - echo "Checking out PR branch: ${{ github.sha }}" - git checkout ${{ github.sha }} || echo "Warning: git checkout failed, but continuing" - fi - fi + name: visual-baseline + path: ultraplot/tests/baseline # Image Comparison (Uses cached or newly generated baseline) - name: Image Comparison Ultraplot diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 173ed9fd8..bcce63cee 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -209,11 +209,79 @@ jobs: files: ./coverage.xml name: codecov-pr-python${{ matrix.python-version }}-mpl${{ matrix.matplotlib-version }} + visual-baseline: + needs: + - run-if-changes + - select-tests + if: always() && needs.run-if-changes.outputs.run == 'true' + runs-on: ubuntu-latest + + env: + PYTHONHASHSEED: "0" + MPLBACKEND: Agg + + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Resolve baseline commit + id: baseline + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + BASE_SHA="${{ github.event.pull_request.base.sha }}" + else + BASE_SHA="${{ github.event.before }}" + fi + + echo "sha=${BASE_SHA}" >> "$GITHUB_OUTPUT" + git checkout "${BASE_SHA}" + + - uses: mamba-org/setup-micromamba@v3.2.1 + with: + environment-file: ./environment.yml + init-shell: bash + condarc-file: ./.github/micromamba-condarc.yml + create-args: >- + --verbose + python=3.12 + matplotlib=3.10.9 + cache-environment: true + cache-downloads: false + + - name: Generate canonical visual baseline + shell: bash -el {0} + run: | + pip install --no-build-isolation --no-deps . + + python -c \ + "import ultraplot as plt; plt.config.Configurator()._save_yaml('ultraplot.yml')" + + mkdir -p ultraplot/tests/baseline + + pytest \ + -n 4 \ + --dist loadfile \ + --tb=short \ + --disable-warnings \ + -W ignore \ + -m mpl_image_compare \ + --mpl-generate-path=./ultraplot/tests/baseline \ + --mpl-default-style="./ultraplot.yml" \ + ultraplot/tests + + - name: Upload canonical baseline + uses: actions/upload-artifact@v7 + with: + name: visual-baseline + path: ultraplot/tests/baseline + if-no-files-found: error build: needs: - get-versions - run-if-changes - select-tests + - visual-baseline if: always() && needs.run-if-changes.outputs.run == 'true' && needs.get-versions.result == 'success' && needs.select-tests.result == 'success' strategy: matrix: From 41af4845a5a5726790661863b2cf4118403fab91 Mon Sep 17 00:00:00 2001 From: cvanelteren Date: Thu, 10 Sep 2026 15:45:07 +1000 Subject: [PATCH 2/5] attempt fix --- .github/workflows/main.yml | 353 ++++++++++++++++++++++++++++--------- 1 file changed, 265 insertions(+), 88 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bcce63cee..0159f590d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -68,23 +68,35 @@ jobs: echo "Test map cache miss; generating map from tests." pip install --no-build-isolation --no-deps . mkdir -p .ci + pytest -q --tb=short --disable-warnings -n auto -p pytest_cov \ - --cov=ultraplot --cov-branch --cov-context=test --cov-report= \ + --cov=ultraplot \ + --cov-branch \ + --cov-context=test \ + --cov-report= \ ultraplot/tests - python tools/ci/build_test_map.py --coverage-file .coverage --output .ci/test-map.json --root . + + python tools/ci/build_test_map.py \ + --coverage-file .coverage \ + --output .ci/test-map.json \ + --root . - name: Select impacted tests id: select run: | if [ "${{ github.event_name }}" != "pull_request" ]; then - echo "mode=full" >> $GITHUB_OUTPUT - echo "tests=[]" >> $GITHUB_OUTPUT + echo "mode=full" >> "$GITHUB_OUTPUT" + echo "tests=[]" >> "$GITHUB_OUTPUT" exit 0 fi - git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} > .ci/changed.txt + git diff --name-only \ + ${{ github.event.pull_request.base.sha }} \ + ${{ github.sha }} > .ci/changed.txt + echo "Changed files:" cat .ci/changed.txt || true + echo "Test map exists:" if [ -f .ci/test-map.json ]; then echo "yes (size=$(wc -c < .ci/test-map.json))" @@ -106,11 +118,21 @@ jobs: python - <<'PY' > .ci/selection.out import json - data = json.load(open(".ci/selection.json", "r", encoding="utf-8")) + + with open(".ci/selection.json", encoding="utf-8") as fh: + data = json.load(fh) + print(f"mode={data['mode']}") - print("tests=" + json.dumps(data.get("tests", []), separators=(",", ":"))) + print( + "tests=" + + json.dumps( + data.get("tests", []), + separators=(",", ":"), + ) + ) PY - cat .ci/selection.out >> $GITHUB_OUTPUT + + cat .ci/selection.out >> "$GITHUB_OUTPUT" get-versions: runs-on: ubuntu-latest @@ -136,22 +158,44 @@ jobs: echo "Detected Python versions: $(echo "$OUTPUT" | jq -c '.python_versions')" echo "Detected Matplotlib versions: $(echo "$OUTPUT" | jq -c '.matplotlib_versions')" echo "Detected test matrix: $(echo "$OUTPUT" | jq -c '.test_matrix')" - python tools/ci/version_support.py --format github-output >> $GITHUB_OUTPUT - coverage: - name: Coverage Python ${{ matrix.python-version }} / MPL ${{ matrix.matplotlib-version }} + python tools/ci/version_support.py \ + --format github-output >> "$GITHUB_OUTPUT" + + # + # Generate ONE canonical set of images. + # + # test-matrix[0] is deliberately the oldest supported + # Python + Matplotlib combination. + # + visual-baseline: + name: Generate canonical visual baseline runs-on: ubuntu-latest + needs: - run-if-changes - get-versions - if: always() && needs.run-if-changes.outputs.run == 'true' && needs.get-versions.result == 'success' && github.event_name == 'pull_request' - strategy: - fail-fast: false - matrix: - include: ${{ fromJson(needs.get-versions.outputs.test-matrix) }} + + if: >- + always() && + needs.run-if-changes.outputs.run == 'true' && + needs.get-versions.result == 'success' + defaults: run: shell: bash -el {0} + + env: + PYTEST_WORKERS: 4 + PYTHONHASHSEED: "0" + MPLBACKEND: Agg + + BASELINE_PYTHON: >- + ${{ fromJson(needs.get-versions.outputs.test-matrix)[0]['python-version'] }} + + BASELINE_MATPLOTLIB: >- + ${{ fromJson(needs.get-versions.outputs.test-matrix)[0]['matplotlib-version'] }} + steps: - name: Set up swap space uses: pierotofy/set-swap-space@master @@ -162,136 +206,260 @@ jobs: with: fetch-depth: 0 - - name: Prepare compatible coverage environment - run: >- - python3 tools/ci/version_support.py - --matplotlib-version '${{ matrix.matplotlib-version }}' - --environment-output '${{ runner.temp }}/ultraplot-environment.yml' + # + # Build the environment description while still on the PR/current + # branch, i.e. from the version contract that produced our matrix. + # + - name: Prepare baseline environment + run: | + echo "Baseline Python: ${BASELINE_PYTHON}" + echo "Baseline Matplotlib: ${BASELINE_MATPLOTLIB}" - - uses: mamba-org/setup-micromamba@v3.2.1 + python3 tools/ci/version_support.py \ + --matplotlib-version "${BASELINE_MATPLOTLIB}" \ + --environment-output "${RUNNER_TEMP}/ultraplot-baseline-environment.yml" + + - name: Create baseline environment + uses: mamba-org/setup-micromamba@v3.2.1 with: - environment-file: ${{ runner.temp }}/ultraplot-environment.yml + environment-file: ${{ runner.temp }}/ultraplot-baseline-environment.yml init-shell: bash condarc-file: ./.github/micromamba-condarc.yml post-cleanup: none create-args: >- --verbose - python=${{ matrix.python-version }} - matplotlib=${{ matrix.matplotlib-version }} + python=${{ env.BASELINE_PYTHON }} + matplotlib=${{ env.BASELINE_MATPLOTLIB }} cache-environment: true cache-downloads: false - - name: Build Ultraplot + # + # For a PR, the expected images must come from the base commit, + # not from the PR itself. + # + # For a push to main/devel we use the pushed commit. This still + # lets the newer matrix entries be compared against rendering + # produced by the minimum supported environment. + # + - name: Checkout baseline commit + id: baseline run: | - pip install --no-build-isolation --no-deps . + if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then + BASE_SHA="${{ github.event.pull_request.base.sha }}" + else + BASE_SHA="${GITHUB_SHA}" + fi - - name: Run full coverage suite + echo "sha=${BASE_SHA}" >> "$GITHUB_OUTPUT" + echo "Generating baseline from ${BASE_SHA}" + + git checkout --detach "${BASE_SHA}" + + - name: Install baseline UltraPlot run: | + pip install --no-build-isolation --no-deps . + python - <<'PY' import sys - from importlib.metadata import version - import matplotlib, mpltern, pycirclize - assert sys.version_info[:2] == tuple(map(int, '${{ matrix.python-version }}'.split('.'))) - assert matplotlib.__version__.startswith('${{ matrix.matplotlib-version }}.') - print('Python executable:', sys.executable) - for package in ('matplotlib', 'mpltern', 'pycirclize', 'cartopy'): - print(f'{package}: {version(package)}') + import matplotlib + + print("Python:", sys.version) + print("Matplotlib:", matplotlib.__version__) PY - pytest -q --tb=short --disable-warnings -n auto -p pytest_cov \ - --cov=ultraplot --cov-branch --cov-context=test \ - --cov-report=xml:coverage.xml --cov-report= \ + + - name: Generate visual baseline + run: | + rm -rf ultraplot/tests/baseline + mkdir -p ultraplot/tests/baseline + + python -c \ + "import ultraplot as plt; plt.config.Configurator()._save_yaml('ultraplot.yml')" + + echo "Generating baseline from:" + echo " commit: ${{ steps.baseline.outputs.sha }}" + echo " Python: ${BASELINE_PYTHON}" + echo " Matplotlib: ${BASELINE_MATPLOTLIB}" + + pytest \ + -n "${PYTEST_WORKERS}" \ + --dist loadfile \ + --tb=short \ + --disable-warnings \ + -W ignore \ + -m mpl_image_compare \ + --mpl-generate-path=./ultraplot/tests/baseline \ + --mpl-default-style=./ultraplot.yml \ ultraplot/tests - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v7 + - name: Show generated baseline + run: | + echo "Generated baseline images:" + find ultraplot/tests/baseline \ + -type f \ + -name '*.png' \ + | sort + + COUNT=$(find ultraplot/tests/baseline -type f -name '*.png' | wc -l) + + echo "" + echo "Total images: ${COUNT}" + + if [ "${COUNT}" -eq 0 ]; then + echo "::error::No baseline images were generated" + exit 1 + fi + + - name: Upload visual baseline + uses: actions/upload-artifact@v7 with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml - name: codecov-pr-python${{ matrix.python-version }}-mpl${{ matrix.matplotlib-version }} + name: visual-baseline + path: ultraplot/tests/baseline/ + if-no-files-found: error + compression-level: 0 + overwrite: true + + coverage: + name: Coverage Python ${{ matrix.python-version }} / MPL ${{ matrix.matplotlib-version }} + + runs-on: ubuntu-latest - visual-baseline: needs: - run-if-changes - - select-tests - if: always() && needs.run-if-changes.outputs.run == 'true' - runs-on: ubuntu-latest + - get-versions - env: - PYTHONHASHSEED: "0" - MPLBACKEND: Agg + if: >- + always() && + needs.run-if-changes.outputs.run == 'true' && + needs.get-versions.result == 'success' && + github.event_name == 'pull_request' + + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.get-versions.outputs.test-matrix) }} + + defaults: + run: + shell: bash -el {0} steps: + - name: Set up swap space + uses: pierotofy/set-swap-space@master + with: + swap-size-gb: 10 + - uses: actions/checkout@v7 with: fetch-depth: 0 - - name: Resolve baseline commit - id: baseline - run: | - if [ "${{ github.event_name }}" = "pull_request" ]; then - BASE_SHA="${{ github.event.pull_request.base.sha }}" - else - BASE_SHA="${{ github.event.before }}" - fi - - echo "sha=${BASE_SHA}" >> "$GITHUB_OUTPUT" - git checkout "${BASE_SHA}" + - name: Prepare compatible coverage environment + run: >- + python3 tools/ci/version_support.py + --matplotlib-version '${{ matrix.matplotlib-version }}' + --environment-output '${{ runner.temp }}/ultraplot-environment.yml' - uses: mamba-org/setup-micromamba@v3.2.1 with: - environment-file: ./environment.yml + environment-file: ${{ runner.temp }}/ultraplot-environment.yml init-shell: bash condarc-file: ./.github/micromamba-condarc.yml + post-cleanup: none create-args: >- --verbose - python=3.12 - matplotlib=3.10.9 + python=${{ matrix.python-version }} + matplotlib=${{ matrix.matplotlib-version }} cache-environment: true cache-downloads: false - - name: Generate canonical visual baseline - shell: bash -el {0} + - name: Build Ultraplot run: | pip install --no-build-isolation --no-deps . - python -c \ - "import ultraplot as plt; plt.config.Configurator()._save_yaml('ultraplot.yml')" + - name: Run full coverage suite + run: | + python - <<'PY' + import sys + from importlib.metadata import version - mkdir -p ultraplot/tests/baseline + import matplotlib + import mpltern + import pycirclize + + expected_python = tuple( + map(int, '${{ matrix.python-version }}'.split('.')) + ) + + assert sys.version_info[:2] == expected_python + assert matplotlib.__version__.startswith( + '${{ matrix.matplotlib-version }}.' + ) + + print('Python executable:', sys.executable) + + for package in ( + 'matplotlib', + 'mpltern', + 'pycirclize', + 'cartopy', + ): + print(f'{package}: {version(package)}') + PY pytest \ - -n 4 \ - --dist loadfile \ + -q \ --tb=short \ --disable-warnings \ - -W ignore \ - -m mpl_image_compare \ - --mpl-generate-path=./ultraplot/tests/baseline \ - --mpl-default-style="./ultraplot.yml" \ + -n auto \ + -p pytest_cov \ + --cov=ultraplot \ + --cov-branch \ + --cov-context=test \ + --cov-report=xml:coverage.xml \ + --cov-report= \ ultraplot/tests - - name: Upload canonical baseline - uses: actions/upload-artifact@v7 + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v7 with: - name: visual-baseline - path: ultraplot/tests/baseline - if-no-files-found: error + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + name: >- + codecov-pr-python${{ matrix.python-version }}-mpl${{ matrix.matplotlib-version }} + + # + # IMPORTANT: + # + # Every matrix invocation waits for visual-baseline. + # Therefore visual-baseline is guaranteed to exist before the + # reusable workflow attempts to download it. + # build: needs: - get-versions - run-if-changes - select-tests - visual-baseline - if: always() && needs.run-if-changes.outputs.run == 'true' && needs.get-versions.result == 'success' && needs.select-tests.result == 'success' + + if: >- + always() && + needs.run-if-changes.outputs.run == 'true' && + needs.get-versions.result == 'success' && + needs.select-tests.result == 'success' && + needs.visual-baseline.result == 'success' + strategy: matrix: include: ${{ fromJson(needs.get-versions.outputs.test-matrix) }} fail-fast: false max-parallel: 4 + uses: ./.github/workflows/build-ultraplot.yml + concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.python-version }}-${{ matrix.matplotlib-version }} + group: >- + ${{ github.workflow }}-${{ github.ref }}-${{ matrix.python-version }}-${{ matrix.matplotlib-version }} cancel-in-progress: false + with: python-version: ${{ matrix.python-version }} matplotlib-version: ${{ matrix.matplotlib-version }} @@ -302,6 +470,7 @@ jobs: needs: - build - coverage + - visual-baseline - run-if-changes if: always() runs-on: ubuntu-latest @@ -309,11 +478,19 @@ jobs: - run: | if [[ '${{ needs.run-if-changes.outputs.run }}' == 'false' ]]; then echo "No changes detected, tests skipped." + exit 0 + fi + + if [[ '${{ needs.visual-baseline.result }}' != 'success' ]]; then + echo "Visual baseline generation failed." + exit 1 + fi + + if [[ '${{ needs.build.result }}' == 'success' ]] && \ + [[ '${{ needs.coverage.result }}' == 'success' || \ + '${{ needs.coverage.result }}' == 'skipped' ]]; then + echo "All tests passed successfully!" else - if [[ '${{ needs.build.result }}' == 'success' && ( '${{ needs.coverage.result }}' == 'success' || '${{ needs.coverage.result }}' == 'skipped' ) ]]; then - echo "All tests passed successfully!" - else - echo "Tests failed!" - exit 1 - fi + echo "Tests failed!" + exit 1 fi From 283d6d08850e2e25f427c3b68ea242d14ff0cda4 Mon Sep 17 00:00:00 2001 From: cvanelteren Date: Thu, 10 Sep 2026 15:51:56 +1000 Subject: [PATCH 3/5] pack artefact --- .github/workflows/main.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0159f590d..58f5e0a0a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -418,13 +418,19 @@ jobs: --cov-report= \ ultraplot/tests - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v7 + - name: Pack visual baseline + run: | + tar -C ultraplot/tests \ + -cf "${RUNNER_TEMP}/visual-baseline.tar" \ + baseline + + - name: Upload visual baseline + uses: actions/upload-artifact@v7 with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml - name: >- - codecov-pr-python${{ matrix.python-version }}-mpl${{ matrix.matplotlib-version }} + name: visual-baseline + path: ${{ runner.temp }}/visual-baseline.tar + if-no-files-found: error + compression-level: 0 # # IMPORTANT: From fdfb0ba87cc3b050587b3ce7ca63d4e2d21bf232 Mon Sep 17 00:00:00 2001 From: cvanelteren Date: Thu, 10 Sep 2026 15:58:11 +1000 Subject: [PATCH 4/5] attempt 3 --- .github/workflows/build-ultraplot.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-ultraplot.yml b/.github/workflows/build-ultraplot.yml index efec7498c..39bcc1070 100644 --- a/.github/workflows/build-ultraplot.yml +++ b/.github/workflows/build-ultraplot.yml @@ -139,7 +139,25 @@ jobs: name: visual-baseline path: ultraplot/tests/baseline - # Image Comparison (Uses cached or newly generated baseline) + - name: Download canonical visual baseline + uses: actions/download-artifact@v7 + with: + name: visual-baseline + path: ${{ runner.temp }}/visual-baseline + + - name: Extract canonical visual baseline + run: | + rm -rf ultraplot/tests/baseline + mkdir -p ultraplot/tests + + tar -C ultraplot/tests \ + -xf "${RUNNER_TEMP}/visual-baseline/visual-baseline.tar" + + echo "Downloaded baseline:" + find ultraplot/tests/baseline -type f | head + echo "Total images: $(find ultraplot/tests/baseline -type f -name '*.png' | wc -l)" + + # Image Comparison (Uses cached or newly generated baseline) - name: Image Comparison Ultraplot if: steps.baseline-support.outputs.baseline-supported == 'true' run: | From ec5c7db30ad89bf6ccac519fa7090f344bf736d4 Mon Sep 17 00:00:00 2001 From: cvanelteren Date: Thu, 10 Sep 2026 16:09:49 +1000 Subject: [PATCH 5/5] attempt 4 --- .github/workflows/build-ultraplot.yml | 33 +++++++++++++++++---------- .github/workflows/main.yml | 27 +++++++++------------- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build-ultraplot.yml b/.github/workflows/build-ultraplot.yml index 39bcc1070..ec5410c05 100644 --- a/.github/workflows/build-ultraplot.yml +++ b/.github/workflows/build-ultraplot.yml @@ -87,7 +87,6 @@ jobs: compare-baseline: name: Compare baseline Python ${{ inputs.python-version }} with MPL ${{ inputs.matplotlib-version }} runs-on: ubuntu-latest - continue-on-error: true env: IS_PR: ${{ github.event_name == 'pull_request' }} TEST_MODE: ${{ inputs.test-mode }} @@ -133,12 +132,6 @@ jobs: cache-environment: true cache-downloads: false - - name: Download canonical baseline - uses: actions/download-artifact@v7 - with: - name: visual-baseline - path: ultraplot/tests/baseline - - name: Download canonical visual baseline uses: actions/download-artifact@v7 with: @@ -153,13 +146,29 @@ jobs: tar -C ultraplot/tests \ -xf "${RUNNER_TEMP}/visual-baseline/visual-baseline.tar" - echo "Downloaded baseline:" - find ultraplot/tests/baseline -type f | head - echo "Total images: $(find ultraplot/tests/baseline -type f -name '*.png' | wc -l)" + COUNT=$(find ultraplot/tests/baseline -type f -name '*.png' | wc -l) + echo "Extracted ${COUNT} baseline images" + if [ "${COUNT}" -eq 0 ]; then + echo "::error::Downloaded baseline contains no PNG files" + exit 1 + fi - # Image Comparison (Uses cached or newly generated baseline) + - name: Prepare selected test nodeids + run: | + : > /tmp/pr_selected_nodeids.txt + if [ "${TEST_MODE}" = "selected" ] && [ -n "${TEST_NODEIDS}" ]; then + python - <<'PY' + import json + import os + + nodeids = json.loads(os.environ.get("TEST_NODEIDS") or "[]") + with open("/tmp/pr_selected_nodeids.txt", "w", encoding="utf-8") as fh: + for nodeid in nodeids: + fh.write(f"{nodeid}\n") + print(f"Prepared {len(nodeids)} selected nodeids") + PY + fi - name: Image Comparison Ultraplot - if: steps.baseline-support.outputs.baseline-supported == 'true' run: | set -uo pipefail # This workflow runs in a login shell (bash -el), which executes diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 58f5e0a0a..5d5f7d263 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -309,14 +309,23 @@ jobs: exit 1 fi + # pytest-mpl parameter IDs may contain characters (e.g. ) + # that upload-artifact refuses as filenames. Archive the directory first. + - name: Pack visual baseline + run: | + rm -f "${RUNNER_TEMP}/visual-baseline.tar" + tar -C ultraplot/tests \ + -cf "${RUNNER_TEMP}/visual-baseline.tar" \ + baseline + ls -lh "${RUNNER_TEMP}/visual-baseline.tar" + - name: Upload visual baseline uses: actions/upload-artifact@v7 with: name: visual-baseline - path: ultraplot/tests/baseline/ + path: ${{ runner.temp }}/visual-baseline.tar if-no-files-found: error compression-level: 0 - overwrite: true coverage: name: Coverage Python ${{ matrix.python-version }} / MPL ${{ matrix.matplotlib-version }} @@ -418,20 +427,6 @@ jobs: --cov-report= \ ultraplot/tests - - name: Pack visual baseline - run: | - tar -C ultraplot/tests \ - -cf "${RUNNER_TEMP}/visual-baseline.tar" \ - baseline - - - name: Upload visual baseline - uses: actions/upload-artifact@v7 - with: - name: visual-baseline - path: ${{ runner.temp }}/visual-baseline.tar - if-no-files-found: error - compression-level: 0 - # # IMPORTANT: #