diff --git a/.github/actions/build-wheel/action.yml b/.github/actions/build-wheel/action.yml index 53e76036..a945a68f 100644 --- a/.github/actions/build-wheel/action.yml +++ b/.github/actions/build-wheel/action.yml @@ -22,7 +22,11 @@ inputs: required: false default: '' set-version: - description: 'Force version, passed to the build as OVERRIDE_GIT_DESCRIBE' + description: 'Force the package version, passed to the build as OVERRIDE_GIT_DESCRIBE' + required: false + default: '' + set-duckdb-version: + description: 'Force the DuckDB version, passed to the build as OVERRIDE_DUCKDB_GIT_DESCRIBE' required: false default: '' ccache-save: @@ -46,7 +50,11 @@ runs: - name: Set build environment shell: bash + env: + SET_VERSION: ${{ inputs.set-version }} + SET_DUCKDB_VERSION: ${{ inputs.set-duckdb-version }} run: | + set -eo pipefail echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV # The Visual Studio generator (the CMake default on windows) ignores # CMAKE__COMPILER_LAUNCHER, so ccache never sees the build. @@ -55,19 +63,29 @@ runs: if [[ "${{ inputs.cibw-system }}" == "win" ]]; then echo "CMAKE_GENERATOR=Ninja" >> $GITHUB_ENV fi + # Version overrides the build has to see, collected as NAME=value words. + # OVERRIDE_GIT_DESCRIBE forces the package version, OVERRIDE_DUCKDB_GIT_DESCRIBE + # forces the version DuckDB is built with. They are independent. + overrides="" + if [[ -n "${SET_VERSION}" ]]; then + overrides="${overrides} OVERRIDE_GIT_DESCRIBE=${SET_VERSION}" + fi + if [[ -n "${SET_DUCKDB_VERSION}" ]]; then + overrides="${overrides} OVERRIDE_DUCKDB_GIT_DESCRIBE=${SET_DUCKDB_VERSION}" + fi # CIBW_ENVIRONMENT is set ONLY for manylinux, where env must cross into # the container. The env var REPLACES any [tool.cibuildwheel.] # environment table from pyproject; setting it on macos silently dropped # MACOSX_DEPLOYMENT_TARGET and shipped wheels tagged macosx_10_9. On # macos and windows the build runs on the host, so plain job env works. - if [[ -n "${{ inputs.set-version }}" ]]; then - if [[ "${{ inputs.cibw-system }}" == "manylinux" ]]; then - echo "CIBW_ENVIRONMENT=CCACHE_DIR=/host${{ github.workspace }}/.ccache OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV - else - echo "OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV - fi - elif [[ "${{ inputs.cibw-system }}" == "manylinux" ]]; then - echo "CIBW_ENVIRONMENT=CCACHE_DIR=/host${{ github.workspace }}/.ccache" >> $GITHUB_ENV + # Every override has to ride along on that single CIBW_ENVIRONMENT line, + # or it never reaches the build inside the container. + if [[ "${{ inputs.cibw-system }}" == "manylinux" ]]; then + echo "CIBW_ENVIRONMENT=CCACHE_DIR=/host${{ github.workspace }}/.ccache${overrides}" >> $GITHUB_ENV + else + for override in ${overrides}; do + echo "${override}" >> $GITHUB_ENV + done fi # Ninja needs cl.exe on PATH. The Visual Studio generator activated the diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index f2990ceb..0ae1eeda 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -26,7 +26,11 @@ on: required: false set-version: type: string - description: Force version (vX.Y.Z, optionally -postN or a PEP440 pre-release suffix like -a1/-alpha1/-b2/-rc3) + description: Force the package version (vX.Y.Z, optionally -postN or a PEP440 pre-release suffix like -a1/-alpha1/-b2/-rc3) + required: false + set-duckdb-version: + type: string + description: Force the version DuckDB is built with. Handed to DuckDB's build verbatim, which validates it. required: false workflow_call: inputs: @@ -48,7 +52,11 @@ on: description: Override the DuckDB submodule commit or ref to build against required: false set-version: - description: Force version (vX.Y.Z, optionally -postN or a PEP440 pre-release suffix like -a1/-alpha1/-b2/-rc3) + description: Force the package version (vX.Y.Z, optionally -postN or a PEP440 pre-release suffix like -a1/-alpha1/-b2/-rc3) + required: false + type: string + set-duckdb-version: + description: Force the version DuckDB is built with. Handed to DuckDB's build verbatim, which validates it. required: false type: string @@ -69,6 +77,7 @@ jobs: duckdb-python-sha: ${{ inputs.duckdb-python-sha != '' && inputs.duckdb-python-sha || github.sha }} duckdb-sha: ${{ inputs.duckdb-sha }} set-version: ${{ inputs.set-version }} + set-duckdb-version: ${{ inputs.set-duckdb-version }} build_wheels: name: Build and test releases @@ -79,3 +88,4 @@ jobs: duckdb-python-sha: ${{ inputs.duckdb-python-sha != '' && inputs.duckdb-python-sha || github.sha }} duckdb-sha: ${{ inputs.duckdb-sha }} set-version: ${{ inputs.set-version }} + set-duckdb-version: ${{ inputs.set-duckdb-version }} diff --git a/.github/workflows/packaging_sdist.yml b/.github/workflows/packaging_sdist.yml index e75514a2..a574370c 100644 --- a/.github/workflows/packaging_sdist.yml +++ b/.github/workflows/packaging_sdist.yml @@ -16,7 +16,11 @@ on: description: Override the DuckDB submodule commit or ref to build against required: false set-version: - description: Force version (vX.Y.Z, optionally -postN or a PEP440 pre-release suffix like -a1/-alpha1/-b2/-rc3) + description: Force the package version (vX.Y.Z, optionally -postN or a PEP440 pre-release suffix like -a1/-alpha1/-b2/-rc3) + required: false + type: string + set-duckdb-version: + description: Force the version DuckDB is built with. Handed to DuckDB's build verbatim, which validates it. required: false type: string outputs: @@ -51,9 +55,19 @@ jobs: git fetch origin git checkout ${{ inputs.duckdb-sha }} - - name: Set OVERRIDE_GIT_DESCRIBE - if: ${{ inputs.set-version != '' }} - run: echo "OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV + # Inputs go through env, never interpolated into the script body. + - name: Set the version overrides + env: + SET_VERSION: ${{ inputs.set-version }} + SET_DUCKDB_VERSION: ${{ inputs.set-duckdb-version }} + run: | + set -euo pipefail + if [[ -n "${SET_VERSION}" ]]; then + echo "OVERRIDE_GIT_DESCRIBE=${SET_VERSION}" >> $GITHUB_ENV + fi + if [[ -n "${SET_DUCKDB_VERSION}" ]]; then + echo "OVERRIDE_DUCKDB_GIT_DESCRIBE=${SET_DUCKDB_VERSION}" >> $GITHUB_ENV + fi - name: Install Astral UV uses: astral-sh/setup-uv@v7 @@ -82,11 +96,55 @@ jobs: tests_dir="${tests_root}${{ inputs.testsuite == 'fast' && '/fast' || '/' }}" uv run --verbose pytest -c ${{ github.workspace }}/pyproject.toml $tests_dir + # Reads both versions back out of the installed sdist and fails if they are + # not what was asked for. This is the only place that proves a forced version + # actually reached the build. It has silently not reached it before: the env + # var never crossed into the manylinux container, and this step itself read a + # duckdb.duckdb_version attribute that does not exist and published an empty + # string for months. - id: versioning + env: + SET_VERSION: ${{ inputs.set-version }} + SET_DUCKDB_VERSION: ${{ inputs.set-duckdb-version }} run: | + set -euo pipefail cd ${{ runner.temp }} - echo "pkg_version=$( .venv/bin/python -c 'import duckdb; print(duckdb.__version__)' )" >> $GITHUB_OUTPUT - echo "duckdb_version=$( .venv/bin/python -c 'import duckdb; print(duckdb.duckdb_version)' )" >> $GITHUB_OUTPUT + pkg_version=$( .venv/bin/python -c 'import duckdb; print(duckdb.__version__)' ) + duckdb_version=$( .venv/bin/python -c 'import duckdb; print(duckdb.__duckdb_version__)' ) + echo "package version: '${pkg_version}'" + echo "duckdb version: '${duckdb_version}'" + + if [[ -z "${pkg_version}" || -z "${duckdb_version}" ]]; then + echo "::error::Version introspection produced an empty string" + exit 1 + fi + + if [[ -n "${SET_DUCKDB_VERSION}" ]]; then + if [[ "${SET_DUCKDB_VERSION}" == *-g* ]]; then + # A full git describe. DuckDB rewrites those to vX.Y.Z-devN, so there + # is nothing to compare against. + echo "::notice::set-duckdb-version is a git describe string, skipping the exact match" + elif [[ "v${duckdb_version}" != "${SET_DUCKDB_VERSION}" ]]; then + echo "::error::DuckDB was asked to build as ${SET_DUCKDB_VERSION} but reports v${duckdb_version}. The override did not reach the build." + exit 1 + fi + fi + + if [[ -n "${SET_VERSION}" ]]; then + if [[ "${SET_VERSION}" == *-g* ]]; then + echo "::notice::set-version is a git describe string, skipping the exact match" + else + expected=$(PYTHONPATH=${{ github.workspace }} python3 -c \ + 'import os; from duckdb_packaging._versioning import git_tag_to_pep440; print(git_tag_to_pep440(os.environ["SET_VERSION"]))') + if [[ "${pkg_version}" != "${expected}" ]]; then + echo "::error::The package was asked to build as ${SET_VERSION} (${expected}) but reports ${pkg_version}. The override did not reach the build." + exit 1 + fi + fi + fi + + echo "pkg_version=${pkg_version}" >> $GITHUB_OUTPUT + echo "duckdb_version=${duckdb_version}" >> $GITHUB_OUTPUT - uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index ceac5382..b86ec800 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -39,7 +39,11 @@ on: description: Override the DuckDB submodule commit or ref to build against required: false set-version: - description: Force version (vX.Y.Z, optionally -postN or a PEP440 pre-release suffix like -a1/-alpha1/-b2/-rc3) + description: Force the package version (vX.Y.Z, optionally -postN or a PEP440 pre-release suffix like -a1/-alpha1/-b2/-rc3) + required: false + type: string + set-duckdb-version: + description: Force the version DuckDB is built with. Handed to DuckDB's build verbatim, which validates it. required: false type: string @@ -82,6 +86,7 @@ jobs: testsuite: ${{ inputs.testsuite }} duckdb-sha: ${{ inputs.duckdb-sha }} set-version: ${{ inputs.set-version }} + set-duckdb-version: ${{ inputs.set-duckdb-version }} ccache-save: 'true' sanity_wheels: @@ -112,6 +117,7 @@ jobs: testsuite: ${{ inputs.testsuite }} duckdb-sha: ${{ inputs.duckdb-sha }} set-version: ${{ inputs.set-version }} + set-duckdb-version: ${{ inputs.set-duckdb-version }} ccache-save: 'false' build_wheels: @@ -148,4 +154,5 @@ jobs: testsuite: ${{ inputs.testsuite }} duckdb-sha: ${{ inputs.duckdb-sha }} set-version: ${{ inputs.set-version }} + set-duckdb-version: ${{ inputs.set-duckdb-version }} ccache-save: 'false' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 35ea8b89..e8ecee06 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,7 @@ -# Release is called by duckdb's InvokeCI -> NotifyExternalRepositories job +# Called by DuckDB core when its release artifacts are ready. On main that is +# Main.yml -> duckdb-workflow-trigger (core_ready), which sends duckdb-version. +# On release branches it is still InvokeCI -> NotifyExternalRepositories, which +# sends only duckdb-sha. We also dispatch it by hand for stable releases. name: Release on: workflow_dispatch: @@ -13,11 +16,18 @@ on: required: true duckdb-version: type: string - description: The DuckDB version associated with duckdb-sha + description: >- + Build DuckDB with this exact version identifier (vX.Y.Z, or a prerelease like + vX.Y.Z-alphaN or vX.Y.Z-rcN). Handed to DuckDB's build verbatim, which is the + only thing that validates it. Empty means derive it from the submodule. required: false - stable-version: + duckdb-python-version: type: string - description: Release a tagged version (vX.Y.Z, optionally -postN or a PEP440 pre-release suffix like -a1/-alpha1/-b2/-rc3) + description: >- + Release this version of the Python package (vX.Y.Z, optionally -postN or a + PEP440 pre-release suffix like -a1/-alpha1/-b2/-rc3). Setting it makes this a + stable release: the submodule pin must equal duckdb-sha and the package goes + to the prod index. Requires duckdb-version. Empty means a nightly. required: false pypi-index: type: choice @@ -41,28 +51,76 @@ defaults: shell: bash jobs: + # Two independent version identifiers reach this workflow. duckdb-version sets + # the version DuckDB is built with, and nothing else. duckdb-python-version sets + # the version of the package we publish, and setting it is what makes this run a + # stable release instead of a nightly. + # # A stable release must be built from a commit whose submodule pin equals the # duckdb-sha input. Otherwise the build is dirty and the released artifacts # correspond to no commit in this repository. # - # The gate is applied per step, not on the job, on purpose: a skipped job + # The pin gate is applied per step, not on the job, on purpose: a skipped job # propagates the skip down the needs chain, and the downstream jobs # (workflow_state, submodule_pr) have no !failure() && !cancelled() guard, so # a job-level skip here silently skips the whole nightly (no wheels, no # publish, no bump PR) while the run still reports success. With the gate on # the steps, the job always completes success and the steps no-op on - # nightlies (empty stable-version). - verify_stable_pin: - name: Verify submodule pin for stable release + # nightlies (empty duckdb-python-version). + verify_release_inputs: + name: Verify release inputs runs-on: ubuntu-latest steps: + - name: Validate the version inputs + env: + DUCKDB_VERSION: ${{ inputs.duckdb-version }} + DUCKDB_PYTHON_VERSION: ${{ inputs.duckdb-python-version }} + run: | + set -euo pipefail + version_re='^v([0-9]+\.[0-9]+\.[0-9]+)' + + duckdb_base="" + if [[ -n "${DUCKDB_VERSION}" ]]; then + if [[ ! "${DUCKDB_VERSION}" =~ ${version_re} ]]; then + echo "::error::duckdb-version (${DUCKDB_VERSION}) must start with a vX.Y.Z version" + exit 1 + fi + duckdb_base="${BASH_REMATCH[1]}" + fi + + if [[ -z "${DUCKDB_PYTHON_VERSION}" ]]; then + echo "::notice::Nightly release${DUCKDB_VERSION:+, building DuckDB as ${DUCKDB_VERSION}}" + exit 0 + fi + + if [[ -z "${DUCKDB_VERSION}" ]]; then + echo "::error::duckdb-python-version is set but duckdb-version is not. A stable release states the DuckDB version it ships, we never infer that." + exit 1 + fi + + if [[ ! "${DUCKDB_PYTHON_VERSION}" =~ ${version_re} ]]; then + echo "::error::duckdb-python-version (${DUCKDB_PYTHON_VERSION}) must start with a vX.Y.Z version" + exit 1 + fi + python_base="${BASH_REMATCH[1]}" + + # duckdb-python follows DuckDB's major, minor and patch. Only the suffix + # may differ, for example DuckDB v1.5.4 republished as v1.5.4-post1. A + # mismatch on the rest is a typo in one of the two inputs. + if [[ "${duckdb_base}" != "${python_base}" ]]; then + echo "::error::duckdb-version (${DUCKDB_VERSION}) and duckdb-python-version (${DUCKDB_PYTHON_VERSION}) disagree on major.minor.patch: ${duckdb_base} vs ${python_base}" + exit 1 + fi + + echo "::notice::Stable release ${DUCKDB_PYTHON_VERSION}, shipping DuckDB ${DUCKDB_VERSION}" + - uses: actions/checkout@v4 - if: ${{ inputs.stable-version != '' }} + if: ${{ inputs.duckdb-python-version != '' }} with: ref: ${{ inputs.duckdb-python-sha != '' && inputs.duckdb-python-sha || github.sha }} - name: Compare the pin to duckdb-sha - if: ${{ inputs.stable-version != '' }} + if: ${{ inputs.duckdb-python-version != '' }} env: GH_TOKEN: ${{ github.token }} run: | @@ -78,14 +136,15 @@ jobs: build_sdist: name: Build an sdist and determine versions - needs: [verify_stable_pin] + needs: [verify_release_inputs] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/packaging_sdist.yml with: testsuite: all duckdb-python-sha: ${{ inputs.duckdb-python-sha != '' && inputs.duckdb-python-sha || github.sha }} duckdb-sha: ${{ inputs.duckdb-sha }} - set-version: ${{ inputs.stable-version }} + set-version: ${{ inputs.duckdb-python-version }} + set-duckdb-version: ${{ inputs.duckdb-version }} submodule_pr: name: Create or update PR to bump submodule to given SHA @@ -135,7 +194,7 @@ jobs: else pypi_state=VERSION_FOUND fi - if [[ -z "${{ inputs.stable-version }}" ]]; then + if [[ -z "${{ inputs.duckdb-python-version }}" ]]; then age=${result#age = } if [ "${age}" -ge "${{ inputs.nightly-stale-after-days }}" ]; then echo "::warning title=Stale nightly for ${{ github.ref_name }}::Nightly is ${age} days old (max=${{ inputs.nightly-stale-after-days }})" @@ -150,7 +209,7 @@ jobs: if [[ test == "${{ inputs.pypi-index }}" ]]; then ci_env=pypi-test elif [[ prod == "${{ inputs.pypi-index }}" ]]; then - ci_env=pypi-prod${{ inputs.stable-version == '' && '-nightly' || '' }} + ci_env=pypi-prod${{ inputs.duckdb-python-version == '' && '-nightly' || '' }} else echo "::error::Invalid value for inputs.pypi-index: ${{ inputs.pypi-index }}" exit 1 @@ -189,7 +248,8 @@ jobs: testsuite: all duckdb-python-sha: ${{ inputs.duckdb-python-sha != '' && inputs.duckdb-python-sha || github.sha }} duckdb-sha: ${{ inputs.duckdb-sha }} - set-version: ${{ inputs.stable-version }} + set-version: ${{ inputs.duckdb-python-version }} + set-duckdb-version: ${{ inputs.duckdb-version }} # Renders the built artifact set as a matrix in the run summary, so the # maintainer can check completeness at a glance and link one summary from the @@ -338,7 +398,7 @@ jobs: fi echo "* Package index: ${pypi_host}" >> $GITHUB_STEP_SUMMARY echo "* Vendored DuckDB Version: ${{ needs.build_sdist.outputs.duckdb-version }} (${dsha:0:10})" >> $GITHUB_STEP_SUMMARY - echo "* Dispatch DuckDB Version: ${{ inputs.duckdb-version }} (${{ inputs.duckdb-sha }})" >> $GITHUB_STEP_SUMMARY + echo "* Requested DuckDB Version: ${{ inputs.duckdb-version != '' && inputs.duckdb-version || '(derived from the submodule)' }} (${{ inputs.duckdb-sha }})" >> $GITHUB_STEP_SUMMARY echo "* S3 upload status: ${{ needs.upload_s3.result == 'success' && needs.workflow_state.outputs.s3_url || needs.upload_s3.result }}" >> $GITHUB_STEP_SUMMARY echo "* CI Environment: ${{ needs.workflow_state.outputs.ci_env }}" >> $GITHUB_STEP_SUMMARY diff --git a/duckdb_packaging/_versioning.py b/duckdb_packaging/_versioning.py index acf705ab..e195d5e7 100644 --- a/duckdb_packaging/_versioning.py +++ b/duckdb_packaging/_versioning.py @@ -21,6 +21,10 @@ PRE_RELEASE_KINDS = ("a", "b", "rc") +# The post component of a version in git tag form. The only part we strip +# before handing a forced version to DuckDB, see duckdb_describe_from_override. +POST_COMPONENT_RE = re.compile(r"-post[0-9]+", re.IGNORECASE) + # PEP440 alternative pre-release spellings and their canonical form _PRE_KIND_ALIASES = { "a": "a", @@ -171,21 +175,25 @@ def create_git_tag(version: str, message: str | None = None, repo_path: pathlib. subprocess.run(cmd, check=True, cwd=cwd) -def duckdb_tag_from_pep440(version: str) -> str: - """Map a forced package version to a DuckDB version tag. +def duckdb_describe_from_override(override: str) -> str: + """Map a forced package version to the version string DuckDB gets built with. + + The value passes through untouched apart from a post component. Post + releases repackage the same engine, so DuckDB never sees the post suffix. - Post releases repackage the stable engine, so the post suffix is dropped. - Pre-release suffixes pass through: DuckDB's build validates what it - supports and fails on versions it does not (yet) accept. + Everything else reaches DuckDB byte for byte. Its CMake is the only + authority on which version strings are valid, and it fails loud on the ones + it does not support. Normalizing here would corrupt them: PEP440 spells an + alpha "a1", DuckDB only accepts "alpha1" and rejects "a1". Args: - version: PEP440 version string of the Python package + override: Forced version in git tag form (e.g. "v1.3.1", "v1.3.1-post1", + "v1.3.1-alpha1", "v1.3.1-rc2-5-g1234567") Returns: - DuckDB git tag (e.g. "v1.3.1", "v1.3.1-rc2" or "v1.3.1-a1") + The version string for DuckDB's build (e.g. "v1.3.1", "v1.3.1-alpha1") """ - major, minor, patch, _post, pre = parse_version(version) - return pep440_to_git_tag(format_version(major, minor, patch, pre=pre)) + return POST_COMPONENT_RE.sub("", override, count=1) def get_git_describe( diff --git a/duckdb_packaging/build_backend.py b/duckdb_packaging/build_backend.py index 9bbb004c..a1026743 100644 --- a/duckdb_packaging/build_backend.py +++ b/duckdb_packaging/build_backend.py @@ -32,14 +32,22 @@ build_wheel as skbuild_build_wheel, ) -from duckdb_packaging._versioning import duckdb_tag_from_pep440, get_git_describe -from duckdb_packaging.setuptools_scm_version import MAIN_BRANCH_VERSIONING, forced_version_from_env +from duckdb_packaging._versioning import get_git_describe +from duckdb_packaging.setuptools_scm_version import ( + MAIN_BRANCH_VERSIONING, + forced_duckdb_version_from_env, + forced_version_from_env, +) _DUCKDB_VERSION_FILENAME = "duckdb_version.txt" _LOGGING_FORMAT = "[duckdb_pytooling.build_backend] {}" _SKBUILD_CMAKE_OVERRIDE_GIT_DESCRIBE = "cmake.define.OVERRIDE_GIT_DESCRIBE" -# The below will check whether we should set a specific version in our build, and if so, set the version -_FORCED_PEP440_VERSION = forced_version_from_env() + +# Load bearing at import time: this turns OVERRIDE_GIT_DESCRIBE into the +# SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DUCKDB that setuptools_scm reads, and drops the +# pretend variables we do not support. It has to run before scikit-build-core asks +# setuptools_scm for a version. +forced_version_from_env() def _log(msg: str) -> None: @@ -198,9 +206,8 @@ def build_sdist(sdist_directory: str, config_settings: dict[str, list[str] | str msg = "Not in a git repository, can't create an sdist" raise RuntimeError(msg) submodule_path = _duckdb_submodule_path() - if _FORCED_PEP440_VERSION is not None: - duckdb_version = duckdb_tag_from_pep440(_FORCED_PEP440_VERSION) - else: + duckdb_version = forced_duckdb_version_from_env() + if duckdb_version is None: duckdb_version = get_git_describe(repo_path=submodule_path, since_minor=MAIN_BRANCH_VERSIONING) _write_duckdb_long_version(duckdb_version) return skbuild_build_sdist(sdist_directory, config_settings=config_settings) @@ -229,16 +236,15 @@ def build_wheel( RuntimeError: If not in a git repository or sdist environment. """ # First figure out the duckdb version we should use - duckdb_version = None + config_settings = config_settings or {} + duckdb_version = forced_duckdb_version_from_env() if not _in_git_repository(): if not _in_sdist(): msg = "Not in a git repository nor in an sdist, can't build a wheel" raise RuntimeError(msg) - _log("Building duckdb wheel from sdist. Reading duckdb version from file.") - config_settings = config_settings or {} - duckdb_version = _read_duckdb_long_version() - elif _FORCED_PEP440_VERSION is not None: - duckdb_version = duckdb_tag_from_pep440(_FORCED_PEP440_VERSION) + if duckdb_version is None: + _log("Building duckdb wheel from sdist. Reading duckdb version from file.") + duckdb_version = _read_duckdb_long_version() # We add the found version to the OVERRIDE_GIT_DESCRIBE cmake var if duckdb_version is not None: diff --git a/duckdb_packaging/setuptools_scm_version.py b/duckdb_packaging/setuptools_scm_version.py index a5f86e96..9206b237 100644 --- a/duckdb_packaging/setuptools_scm_version.py +++ b/duckdb_packaging/setuptools_scm_version.py @@ -9,14 +9,21 @@ from typing import Protocol # Import from our own versioning module to avoid duplication -from ._versioning import format_version, git_tag_to_pep440, parse_version +from ._versioning import duckdb_describe_from_override, format_version, git_tag_to_pep440, parse_version # MAIN_BRANCH_VERSIONING should be 'True' on main branch only MAIN_BRANCH_VERSIONING = True SCM_PRETEND_ENV_VAR = "SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DUCKDB" SCM_GLOBAL_PRETEND_ENV_VAR = "SETUPTOOLS_SCM_PRETEND_VERSION" + +# Two independent version overrides. OVERRIDE_GIT_DESCRIBE forces the version of +# this package, and DuckDB inherits it, because a stable release uses one version +# identifier for both. OVERRIDE_DUCKDB_GIT_DESCRIBE forces the version DuckDB is +# built with and nothing else, which is what a nightly that vendors a specific +# DuckDB version needs: it keeps deriving its own version from git. OVERRIDE_GIT_DESCRIBE_ENV_VAR = "OVERRIDE_GIT_DESCRIBE" +OVERRIDE_DUCKDB_GIT_DESCRIBE_ENV_VAR = "OVERRIDE_DUCKDB_GIT_DESCRIBE" class _VersionObject(Protocol): @@ -108,6 +115,30 @@ def forced_version_from_env() -> str | None: return pep440_version +def forced_duckdb_version_from_env() -> str | None: + """The version DuckDB was explicitly asked to build as, if any. + + OVERRIDE_DUCKDB_GIT_DESCRIBE wins and is passed on verbatim. Otherwise a + forced package version carries over, minus its post suffix. Returns None when + neither is set, and the caller derives the version from git instead. + + Returns: + The version string to build DuckDB with, or None to derive it. + """ + forced_duckdb = os.getenv(OVERRIDE_DUCKDB_GIT_DESCRIBE_ENV_VAR) + if forced_duckdb: + print(f"[versioning] Found {OVERRIDE_DUCKDB_GIT_DESCRIBE_ENV_VAR}={forced_duckdb}") + return forced_duckdb + + forced_package = os.getenv(OVERRIDE_GIT_DESCRIBE_ENV_VAR) + if forced_package: + duckdb_version = duckdb_describe_from_override(forced_package) + print(f"[versioning] Derived DuckDB version {duckdb_version} from {OVERRIDE_GIT_DESCRIBE_ENV_VAR}") + return duckdb_version + + return None + + def _git_describe_override_to_pep_440(override_value: str) -> str: """Process the OVERRIDE_GIT_DESCRIBE value.""" describe_pattern = re.compile( diff --git a/tests/fast/test_versioning.py b/tests/fast/test_versioning.py index c67c9059..bc520121 100644 --- a/tests/fast/test_versioning.py +++ b/tests/fast/test_versioning.py @@ -10,7 +10,7 @@ duckdb_packaging = pytest.importorskip("duckdb_packaging") from duckdb_packaging._versioning import ( # noqa: E402 - duckdb_tag_from_pep440, + duckdb_describe_from_override, format_version, get_current_version, get_git_describe, @@ -21,6 +21,7 @@ from duckdb_packaging.setuptools_scm_version import ( # noqa: E402 _bump_dev_version, _tag_to_version, + forced_duckdb_version_from_env, forced_version_from_env, version_scheme, ) @@ -167,24 +168,34 @@ def test_roundtrip_conversion(self): assert converted_back == version -class TestDuckDBTagFromPep440(unittest.TestCase): - """Test the mapping of forced package versions to DuckDB version tags.""" +class TestDuckDBDescribeFromOverride(unittest.TestCase): + """Test the mapping of a forced package version to what DuckDB gets built with.""" def test_stable_version(self): - assert duckdb_tag_from_pep440("1.2.3") == "v1.2.3" + assert duckdb_describe_from_override("v1.2.3") == "v1.2.3" def test_post_version_strips_post(self): - """Post releases repackage the stable engine.""" - assert duckdb_tag_from_pep440("1.2.3.post1") == "v1.2.3" - - def test_pre_release_passes_through(self): - """Pre-releases pass through normalized, DuckDB's build validates them.""" - assert duckdb_tag_from_pep440("1.2.3rc2") == "v1.2.3-rc2" - assert duckdb_tag_from_pep440("1.2.3a1") == "v1.2.3-a1" - assert duckdb_tag_from_pep440("1.2.3b2") == "v1.2.3-b2" - # alternative spellings normalize - assert duckdb_tag_from_pep440("1.2.3alpha1") == "v1.2.3-a1" - assert duckdb_tag_from_pep440("1.2.3pre1") == "v1.2.3-rc1" + """Post releases repackage the same engine, so DuckDB never sees the post.""" + assert duckdb_describe_from_override("v1.2.3-post1") == "v1.2.3" + assert duckdb_describe_from_override("v1.2.3-post10") == "v1.2.3" + + def test_post_version_with_distance_keeps_the_distance(self): + assert duckdb_describe_from_override("v1.2.3-post1-3-g1234567") == "v1.2.3-3-g1234567" + + def test_pre_release_spelling_is_preserved(self): + """The whole point: DuckDB accepts -alphaN and rejects the PEP440 -aN.""" + assert duckdb_describe_from_override("v2.0.0-alpha38426") == "v2.0.0-alpha38426" + assert duckdb_describe_from_override("v1.2.3-rc2") == "v1.2.3-rc2" + + def test_unsupported_spellings_pass_through_untouched(self): + """We never normalize. DuckDB's CMake is the only authority, and it fails loud.""" + assert duckdb_describe_from_override("v1.2.3-a1") == "v1.2.3-a1" + assert duckdb_describe_from_override("v1.2.3-b2") == "v1.2.3-b2" + assert duckdb_describe_from_override("v1.2.3-beta2") == "v1.2.3-beta2" + + def test_full_describe_passes_through(self): + assert duckdb_describe_from_override("v1.2.3-5-g1234567") == "v1.2.3-5-g1234567" + assert duckdb_describe_from_override("v2.0.0-alpha1-42-gabc123") == "v2.0.0-alpha1-42-gabc123" class TestSetupToolsScmIntegration(unittest.TestCase): @@ -336,3 +347,54 @@ def test_override_git_describe_invalid(self): """Test OVERRIDE_GIT_DESCRIBE with invalid format.""" with pytest.raises(ValueError, match="Invalid git describe override"): forced_version_from_env() + + +class TestForcedDuckDBVersionFromEnv(unittest.TestCase): + """Test which version DuckDB gets built with, per the two env overrides. + + The two overrides are independent. A nightly that vendors a specific DuckDB + version sets only OVERRIDE_DUCKDB_GIT_DESCRIBE and keeps deriving its own + package version from git. A stable release sets OVERRIDE_GIT_DESCRIBE and + DuckDB inherits it. + """ + + @patch.dict("os.environ", {"OVERRIDE_GIT_DESCRIBE": "", "OVERRIDE_DUCKDB_GIT_DESCRIBE": ""}) + def test_neither_set_derives_from_git(self): + assert forced_duckdb_version_from_env() is None + + @patch.dict("os.environ", {"OVERRIDE_GIT_DESCRIBE": "", "OVERRIDE_DUCKDB_GIT_DESCRIBE": "v2.0.0-alpha38426"}) + def test_duckdb_override_passes_through_verbatim(self): + """The nightly case. DuckDB accepts -alphaN, so it must survive untouched.""" + assert forced_duckdb_version_from_env() == "v2.0.0-alpha38426" + + @patch.dict("os.environ", {"OVERRIDE_GIT_DESCRIBE": "v1.5.4", "OVERRIDE_DUCKDB_GIT_DESCRIBE": ""}) + def test_package_override_carries_over(self): + """The stable release case, one version identifier for both.""" + assert forced_duckdb_version_from_env() == "v1.5.4" + + @patch.dict("os.environ", {"OVERRIDE_GIT_DESCRIBE": "v1.5.4-post1", "OVERRIDE_DUCKDB_GIT_DESCRIBE": ""}) + def test_package_override_drops_post(self): + """A post release repackages the same engine.""" + assert forced_duckdb_version_from_env() == "v1.5.4" + + @patch.dict("os.environ", {"OVERRIDE_GIT_DESCRIBE": "v2.0.0-alpha1", "OVERRIDE_DUCKDB_GIT_DESCRIBE": ""}) + def test_package_override_keeps_the_alpha_spelling(self): + """Regression: normalizing this to v2.0.0-a1 makes DuckDB's CMake fail.""" + assert forced_duckdb_version_from_env() == "v2.0.0-alpha1" + + @patch.dict( + "os.environ", + {"OVERRIDE_GIT_DESCRIBE": "v1.5.4-post1", "OVERRIDE_DUCKDB_GIT_DESCRIBE": "v2.0.0-alpha38426"}, + ) + def test_duckdb_override_wins(self): + assert forced_duckdb_version_from_env() == "v2.0.0-alpha38426" + + @patch.dict( + "os.environ", + {"OVERRIDE_GIT_DESCRIBE": "v1.5.4", "OVERRIDE_DUCKDB_GIT_DESCRIBE": "", "MAIN_BRANCH_VERSIONING": "0"}, + ) + def test_package_version_is_untouched_by_the_duckdb_override(self): + """The two channels do not interfere: forcing DuckDB leaves the package alone.""" + with patch.dict("os.environ", {"OVERRIDE_DUCKDB_GIT_DESCRIBE": "v2.0.0-alpha38426"}): + assert forced_version_from_env() == "1.5.4" + assert forced_duckdb_version_from_env() == "v2.0.0-alpha38426"