diff --git a/.github/workflows/build-ultraplot.yml b/.github/workflows/build-ultraplot.yml index 7778a0a5b..ec5410c05 100644 --- a/.github/workflows/build-ultraplot.yml +++ b/.github/workflows/build-ultraplot.yml @@ -87,7 +87,6 @@ jobs: compare-baseline: name: Compare baseline Python ${{ inputs.python-version }} with MPL ${{ inputs.matplotlib-version }} runs-on: ubuntu-latest - continue-on-error: true env: IS_PR: ${{ github.event_name == 'pull_request' }} TEST_MODE: ${{ inputs.test-mode }} @@ -133,128 +132,43 @@ jobs: cache-environment: true cache-downloads: false - - name: Resolve baseline reference - id: baseline-ref - run: | - if [ "${IS_PR}" = "true" ]; then - BASE_REF="${{ github.event.pull_request.base.ref }}" - git fetch origin "${BASE_REF}" - BASE_SHA="$(git rev-parse "origin/${BASE_REF}")" - else - BASE_REF="${GITHUB_REF_NAME:-main}" - BASE_SHA="${GITHUB_SHA}" - fi - echo "base_ref=${BASE_REF}" >> "${GITHUB_OUTPUT}" - echo "base_sha=${BASE_SHA}" >> "${GITHUB_OUTPUT}" - echo "Resolved baseline ref=${BASE_REF} sha=${BASE_SHA}" + - name: Download canonical visual baseline + uses: actions/download-artifact@v7 + with: + name: visual-baseline + path: ${{ runner.temp }}/visual-baseline - - name: Check baseline version support - id: baseline-support + - name: Extract canonical visual baseline run: | - git show '${{ steps.baseline-ref.outputs.base_sha }}:pyproject.toml' > '${{ runner.temp }}/baseline-pyproject.toml' - python tools/ci/version_support.py \ - --baseline-pyproject '${{ runner.temp }}/baseline-pyproject.toml' \ - --python-version '${{ inputs.python-version }}' \ - --matplotlib-version '${{ inputs.matplotlib-version }}' >> "$GITHUB_OUTPUT" + rm -rf ultraplot/tests/baseline + mkdir -p ultraplot/tests - - name: Explain unavailable baseline - if: steps.baseline-support.outputs.baseline-supported != 'true' - run: | - echo '::notice::Image comparison skipped: the base commit does not support this Python/Matplotlib pair. PR tests still run in the coverage job.' - echo 'Image comparison skipped for Python ${{ inputs.python-version }} / Matplotlib ${{ inputs.matplotlib-version }}: base commit ${{ steps.baseline-ref.outputs.base_sha }} does not support this pair. The coverage job still tests the PR on this pair.' >> "$GITHUB_STEP_SUMMARY" + tar -C ultraplot/tests \ + -xf "${RUNNER_TEMP}/visual-baseline/visual-baseline.tar" - # Cache Baseline Figures (Restore step) - - name: Cache Baseline Figures - id: cache-baseline - uses: actions/cache@v6 - if: env.IS_PR == 'true' && steps.baseline-support.outputs.baseline-supported == 'true' - with: - path: ./ultraplot/tests/baseline # The directory to cache - # Key is based on OS, Python/Matplotlib versions, and the base commit SHA - key: ${{ runner.os }}-baseline-base-v5-hs${{ env.PYTHONHASHSEED }}-${{ steps.baseline-ref.outputs.base_sha }}-${{ inputs.python-version }}-${{ inputs.matplotlib-version }} - restore-keys: | - ${{ runner.os }}-baseline-base-v5-hs${{ env.PYTHONHASHSEED }}-${{ steps.baseline-ref.outputs.base_sha }}-${{ inputs.python-version }}-${{ inputs.matplotlib-version }}- + COUNT=$(find ultraplot/tests/baseline -type f -name '*.png' | wc -l) + echo "Extracted ${COUNT} baseline images" + if [ "${COUNT}" -eq 0 ]; then + echo "::error::Downloaded baseline contains no PNG files" + exit 1 + fi - # Conditional Baseline Generation (Only runs on cache miss) - - name: Generate baseline from main - # Skip this step if the cache was found (cache-hit is true) - if: steps.baseline-support.outputs.baseline-supported == 'true' && (steps.cache-baseline.outputs.cache-hit != 'true' || env.IS_PR != 'true') + - name: Prepare selected test nodeids run: | - mkdir -p ultraplot/tests/baseline - echo "TEST_MODE=${TEST_MODE}" - echo "IS_PR=${IS_PR}" - echo "PR_BASE_REF=${{ steps.baseline-ref.outputs.base_ref }}" - echo "PR_BASE_SHA=${{ steps.baseline-ref.outputs.base_sha }}" - echo "TEST_NODEIDS=${TEST_NODEIDS}" - # Save PR-selected nodeids for reuse after checkout (if provided) + : > /tmp/pr_selected_nodeids.txt if [ "${TEST_MODE}" = "selected" ] && [ -n "${TEST_NODEIDS}" ]; then - python -c 'import json,os; raw=os.environ.get("TEST_NODEIDS","").strip(); parsed=json.loads(raw) if raw and raw!="[]" else []; parsed=[parsed] if isinstance(parsed,str) else parsed; nodeids=[item for item in parsed if isinstance(item,str) and item]; open("/tmp/pr_selected_nodeids.txt","w",encoding="utf-8").write("".join(f"{nodeid}\n" for nodeid in nodeids)); print(f"Selected nodeids parsed: {len(nodeids)}")' - else - : > /tmp/pr_selected_nodeids.txt - fi - # Checkout the resolved base-branch tip for PR baseline generation. - if [ "${IS_PR}" = "true" ]; then - git checkout "${{ steps.baseline-ref.outputs.base_sha }}" - fi - - # Install the Ultraplot version from the base branch's code - pip install --no-build-isolation --no-deps . + python - <<'PY' + import json + import os - # Generate the baseline images and hash library - python -c "import ultraplot as plt; plt.config.Configurator()._save_yaml('ultraplot.yml')" - if [ "${TEST_MODE}" = "selected" ] && [ -s /tmp/pr_selected_nodeids.txt ]; then - status=0 - FILTERED_NODEIDS=() - while IFS= read -r nodeid; do - if [ -z "$nodeid" ]; then - continue - fi - path="${nodeid%%::*}" - if [ -f "$path" ]; then - FILTERED_NODEIDS+=("$nodeid") - fi - done < /tmp/pr_selected_nodeids.txt - echo "FILTERED_NODEIDS_BASE_COUNT=${#FILTERED_NODEIDS[@]}" - if [ "${#FILTERED_NODEIDS[@]}" -eq 0 ]; then - echo "No valid nodeids found on base; skipping baseline generation." - else - echo "=== Memory before baseline generation ===" && free -h - pytest -n ${PYTEST_WORKERS} --dist loadfile --tb=short --disable-warnings -W ignore \ - --mpl-generate-path=./ultraplot/tests/baseline/ \ - --mpl-default-style="./ultraplot.yml" \ - "${FILTERED_NODEIDS[@]}" || status=$? - echo "=== Memory after baseline generation ===" && free -h - if [ "$status" -eq 4 ] || [ "$status" -eq 5 ]; then - echo "No tests collected from selected nodeids on base; skipping baseline generation." - status=0 - fi - fi - # Return to the PR branch before continuing - if [ "${IS_PR}" = "true" ]; then - echo "Checking out PR branch: ${{ github.sha }}" - git checkout ${{ github.sha }} || echo "Warning: git checkout failed, but continuing" - fi - if [ "$status" -ne 0 ]; then - echo "Baseline generation failed with status $status" - exit "$status" - fi - else - echo "=== Memory before baseline generation ===" && free -h - pytest -n ${PYTEST_WORKERS} --dist loadfile --tb=short --disable-warnings -W ignore \ - --mpl-generate-path=./ultraplot/tests/baseline/ \ - --mpl-default-style="./ultraplot.yml" \ - ultraplot/tests - echo "=== Memory after baseline generation ===" && free -h - # Return to the PR branch for the rest of the job - if [ "${IS_PR}" = "true" ]; then - echo "Checking out PR branch: ${{ github.sha }}" - git checkout ${{ github.sha }} || echo "Warning: git checkout failed, but continuing" - fi + nodeids = json.loads(os.environ.get("TEST_NODEIDS") or "[]") + with open("/tmp/pr_selected_nodeids.txt", "w", encoding="utf-8") as fh: + for nodeid in nodeids: + fh.write(f"{nodeid}\n") + print(f"Prepared {len(nodeids)} selected nodeids") + PY fi - - # Image Comparison (Uses cached or newly generated baseline) - name: Image Comparison Ultraplot - if: steps.baseline-support.outputs.baseline-supported == 'true' run: | set -uo pipefail # This workflow runs in a login shell (bash -el), which executes diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 173ed9fd8..5d5f7d263 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -68,23 +68,35 @@ jobs: echo "Test map cache miss; generating map from tests." pip install --no-build-isolation --no-deps . mkdir -p .ci + pytest -q --tb=short --disable-warnings -n auto -p pytest_cov \ - --cov=ultraplot --cov-branch --cov-context=test --cov-report= \ + --cov=ultraplot \ + --cov-branch \ + --cov-context=test \ + --cov-report= \ ultraplot/tests - python tools/ci/build_test_map.py --coverage-file .coverage --output .ci/test-map.json --root . + + python tools/ci/build_test_map.py \ + --coverage-file .coverage \ + --output .ci/test-map.json \ + --root . - name: Select impacted tests id: select run: | if [ "${{ github.event_name }}" != "pull_request" ]; then - echo "mode=full" >> $GITHUB_OUTPUT - echo "tests=[]" >> $GITHUB_OUTPUT + echo "mode=full" >> "$GITHUB_OUTPUT" + echo "tests=[]" >> "$GITHUB_OUTPUT" exit 0 fi - git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} > .ci/changed.txt + git diff --name-only \ + ${{ github.event.pull_request.base.sha }} \ + ${{ github.sha }} > .ci/changed.txt + echo "Changed files:" cat .ci/changed.txt || true + echo "Test map exists:" if [ -f .ci/test-map.json ]; then echo "yes (size=$(wc -c < .ci/test-map.json))" @@ -106,11 +118,21 @@ jobs: python - <<'PY' > .ci/selection.out import json - data = json.load(open(".ci/selection.json", "r", encoding="utf-8")) + + with open(".ci/selection.json", encoding="utf-8") as fh: + data = json.load(fh) + print(f"mode={data['mode']}") - print("tests=" + json.dumps(data.get("tests", []), separators=(",", ":"))) + print( + "tests=" + + json.dumps( + data.get("tests", []), + separators=(",", ":"), + ) + ) PY - cat .ci/selection.out >> $GITHUB_OUTPUT + + cat .ci/selection.out >> "$GITHUB_OUTPUT" get-versions: runs-on: ubuntu-latest @@ -136,22 +158,199 @@ jobs: echo "Detected Python versions: $(echo "$OUTPUT" | jq -c '.python_versions')" echo "Detected Matplotlib versions: $(echo "$OUTPUT" | jq -c '.matplotlib_versions')" echo "Detected test matrix: $(echo "$OUTPUT" | jq -c '.test_matrix')" - python tools/ci/version_support.py --format github-output >> $GITHUB_OUTPUT + + python tools/ci/version_support.py \ + --format github-output >> "$GITHUB_OUTPUT" + + # + # Generate ONE canonical set of images. + # + # test-matrix[0] is deliberately the oldest supported + # Python + Matplotlib combination. + # + visual-baseline: + name: Generate canonical visual baseline + runs-on: ubuntu-latest + + needs: + - run-if-changes + - get-versions + + if: >- + always() && + needs.run-if-changes.outputs.run == 'true' && + needs.get-versions.result == 'success' + + defaults: + run: + shell: bash -el {0} + + env: + PYTEST_WORKERS: 4 + PYTHONHASHSEED: "0" + MPLBACKEND: Agg + + BASELINE_PYTHON: >- + ${{ fromJson(needs.get-versions.outputs.test-matrix)[0]['python-version'] }} + + BASELINE_MATPLOTLIB: >- + ${{ fromJson(needs.get-versions.outputs.test-matrix)[0]['matplotlib-version'] }} + + steps: + - name: Set up swap space + uses: pierotofy/set-swap-space@master + with: + swap-size-gb: 10 + + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + # + # Build the environment description while still on the PR/current + # branch, i.e. from the version contract that produced our matrix. + # + - name: Prepare baseline environment + run: | + echo "Baseline Python: ${BASELINE_PYTHON}" + echo "Baseline Matplotlib: ${BASELINE_MATPLOTLIB}" + + python3 tools/ci/version_support.py \ + --matplotlib-version "${BASELINE_MATPLOTLIB}" \ + --environment-output "${RUNNER_TEMP}/ultraplot-baseline-environment.yml" + + - name: Create baseline environment + uses: mamba-org/setup-micromamba@v3.2.1 + with: + environment-file: ${{ runner.temp }}/ultraplot-baseline-environment.yml + init-shell: bash + condarc-file: ./.github/micromamba-condarc.yml + post-cleanup: none + create-args: >- + --verbose + python=${{ env.BASELINE_PYTHON }} + matplotlib=${{ env.BASELINE_MATPLOTLIB }} + cache-environment: true + cache-downloads: false + + # + # For a PR, the expected images must come from the base commit, + # not from the PR itself. + # + # For a push to main/devel we use the pushed commit. This still + # lets the newer matrix entries be compared against rendering + # produced by the minimum supported environment. + # + - name: Checkout baseline commit + id: baseline + run: | + if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then + BASE_SHA="${{ github.event.pull_request.base.sha }}" + else + BASE_SHA="${GITHUB_SHA}" + fi + + echo "sha=${BASE_SHA}" >> "$GITHUB_OUTPUT" + echo "Generating baseline from ${BASE_SHA}" + + git checkout --detach "${BASE_SHA}" + + - name: Install baseline UltraPlot + run: | + pip install --no-build-isolation --no-deps . + + python - <<'PY' + import sys + import matplotlib + + print("Python:", sys.version) + print("Matplotlib:", matplotlib.__version__) + PY + + - name: Generate visual baseline + run: | + rm -rf ultraplot/tests/baseline + mkdir -p ultraplot/tests/baseline + + python -c \ + "import ultraplot as plt; plt.config.Configurator()._save_yaml('ultraplot.yml')" + + echo "Generating baseline from:" + echo " commit: ${{ steps.baseline.outputs.sha }}" + echo " Python: ${BASELINE_PYTHON}" + echo " Matplotlib: ${BASELINE_MATPLOTLIB}" + + pytest \ + -n "${PYTEST_WORKERS}" \ + --dist loadfile \ + --tb=short \ + --disable-warnings \ + -W ignore \ + -m mpl_image_compare \ + --mpl-generate-path=./ultraplot/tests/baseline \ + --mpl-default-style=./ultraplot.yml \ + ultraplot/tests + + - name: Show generated baseline + run: | + echo "Generated baseline images:" + find ultraplot/tests/baseline \ + -type f \ + -name '*.png' \ + | sort + + COUNT=$(find ultraplot/tests/baseline -type f -name '*.png' | wc -l) + + echo "" + echo "Total images: ${COUNT}" + + if [ "${COUNT}" -eq 0 ]; then + echo "::error::No baseline images were generated" + exit 1 + fi + + # pytest-mpl parameter IDs may contain characters (e.g. ) + # that upload-artifact refuses as filenames. Archive the directory first. + - name: Pack visual baseline + run: | + rm -f "${RUNNER_TEMP}/visual-baseline.tar" + tar -C ultraplot/tests \ + -cf "${RUNNER_TEMP}/visual-baseline.tar" \ + baseline + ls -lh "${RUNNER_TEMP}/visual-baseline.tar" + + - name: Upload visual baseline + uses: actions/upload-artifact@v7 + with: + name: visual-baseline + path: ${{ runner.temp }}/visual-baseline.tar + if-no-files-found: error + compression-level: 0 coverage: name: Coverage Python ${{ matrix.python-version }} / MPL ${{ matrix.matplotlib-version }} + runs-on: ubuntu-latest + needs: - run-if-changes - get-versions - if: always() && needs.run-if-changes.outputs.run == 'true' && needs.get-versions.result == 'success' && github.event_name == 'pull_request' + + if: >- + always() && + needs.run-if-changes.outputs.run == 'true' && + needs.get-versions.result == 'success' && + github.event_name == 'pull_request' + strategy: fail-fast: false matrix: include: ${{ fromJson(needs.get-versions.outputs.test-matrix) }} + defaults: run: shell: bash -el {0} + steps: - name: Set up swap space uses: pierotofy/set-swap-space@master @@ -190,40 +389,78 @@ jobs: python - <<'PY' import sys from importlib.metadata import version - import matplotlib, mpltern, pycirclize - assert sys.version_info[:2] == tuple(map(int, '${{ matrix.python-version }}'.split('.'))) - assert matplotlib.__version__.startswith('${{ matrix.matplotlib-version }}.') + + import matplotlib + import mpltern + import pycirclize + + expected_python = tuple( + map(int, '${{ matrix.python-version }}'.split('.')) + ) + + assert sys.version_info[:2] == expected_python + assert matplotlib.__version__.startswith( + '${{ matrix.matplotlib-version }}.' + ) + print('Python executable:', sys.executable) - for package in ('matplotlib', 'mpltern', 'pycirclize', 'cartopy'): + + for package in ( + 'matplotlib', + 'mpltern', + 'pycirclize', + 'cartopy', + ): print(f'{package}: {version(package)}') PY - pytest -q --tb=short --disable-warnings -n auto -p pytest_cov \ - --cov=ultraplot --cov-branch --cov-context=test \ - --cov-report=xml:coverage.xml --cov-report= \ - ultraplot/tests - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v7 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml - name: codecov-pr-python${{ matrix.python-version }}-mpl${{ matrix.matplotlib-version }} + pytest \ + -q \ + --tb=short \ + --disable-warnings \ + -n auto \ + -p pytest_cov \ + --cov=ultraplot \ + --cov-branch \ + --cov-context=test \ + --cov-report=xml:coverage.xml \ + --cov-report= \ + ultraplot/tests + # + # IMPORTANT: + # + # Every matrix invocation waits for visual-baseline. + # Therefore visual-baseline is guaranteed to exist before the + # reusable workflow attempts to download it. + # build: needs: - get-versions - run-if-changes - select-tests - if: always() && needs.run-if-changes.outputs.run == 'true' && needs.get-versions.result == 'success' && needs.select-tests.result == 'success' + - visual-baseline + + if: >- + always() && + needs.run-if-changes.outputs.run == 'true' && + needs.get-versions.result == 'success' && + needs.select-tests.result == 'success' && + needs.visual-baseline.result == 'success' + strategy: matrix: include: ${{ fromJson(needs.get-versions.outputs.test-matrix) }} fail-fast: false max-parallel: 4 + uses: ./.github/workflows/build-ultraplot.yml + concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.python-version }}-${{ matrix.matplotlib-version }} + group: >- + ${{ github.workflow }}-${{ github.ref }}-${{ matrix.python-version }}-${{ matrix.matplotlib-version }} cancel-in-progress: false + with: python-version: ${{ matrix.python-version }} matplotlib-version: ${{ matrix.matplotlib-version }} @@ -234,6 +471,7 @@ jobs: needs: - build - coverage + - visual-baseline - run-if-changes if: always() runs-on: ubuntu-latest @@ -241,11 +479,19 @@ jobs: - run: | if [[ '${{ needs.run-if-changes.outputs.run }}' == 'false' ]]; then echo "No changes detected, tests skipped." + exit 0 + fi + + if [[ '${{ needs.visual-baseline.result }}' != 'success' ]]; then + echo "Visual baseline generation failed." + exit 1 + fi + + if [[ '${{ needs.build.result }}' == 'success' ]] && \ + [[ '${{ needs.coverage.result }}' == 'success' || \ + '${{ needs.coverage.result }}' == 'skipped' ]]; then + echo "All tests passed successfully!" else - if [[ '${{ needs.build.result }}' == 'success' && ( '${{ needs.coverage.result }}' == 'success' || '${{ needs.coverage.result }}' == 'skipped' ) ]]; then - echo "All tests passed successfully!" - else - echo "Tests failed!" - exit 1 - fi + echo "Tests failed!" + exit 1 fi