diff --git a/.github/workflows/build-ctranslate2.yml b/.github/workflows/build-ctranslate2.yml new file mode 100644 index 000000000..0f4175ba0 --- /dev/null +++ b/.github/workflows/build-ctranslate2.yml @@ -0,0 +1,177 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +--- +# This workflow is based on: https://github.com/OpenNMT/CTranslate2/blob/v4.8.1/.github/workflows/ci.yml +# (build-python-wheels + build-and-test-cpp-arm64's openblas/ruy job) and +# python/tools/prepare_build_environment_linux.sh's aarch64 branch, which it inlines. +name: Build ctranslate2 wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'ctranslate2 version to build (git tag without leading v, e.g. 4.8.1)' + required: true + default: '4.8.1' + pull_request: + paths: + - '.github/workflows/build-ctranslate2.yml' + - 'patches/ctranslate2/**' + +concurrency: + group: ${{ github.workflow }}-${{ inputs.version || '4.8.1' }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +env: + CTRANSLATE2_VERSION: ${{ inputs.version || '4.8.1' }} + MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + + build_wheels: + needs: [setup] + name: Build ctranslate2 ${{ inputs.version || '4.8.1' }} manylinux_riscv64 + runs-on: ubuntu-24.04-riscv + # 180 was cut to within seconds of the wall by a deselect that didn't actually + # match (parametrized nodeids need the [..] suffix) and ran all 3 whisper_align + # cases for real; raised for headroom now that they're properly excluded. + timeout-minutes: 240 + + steps: + - name: Checkout ctranslate2 v${{ env.CTRANSLATE2_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: OpenNMT/CTranslate2 + ref: v${{ env.CTRANSLATE2_VERSION }} + submodules: false + persist-credentials: false + + # Only the CPU backend's own deps: cxxopts (BUILD_CLI), cpu_features (x86-only), + # googletest (BUILD_TESTS) and thrust/cutlass (CUDA-only) are all unused here, and + # thrust/cutlass alone are gigabytes of NVIDIA sources neither `submodules: recursive` + # nor `--recurse-submodules` should fetch for a CPU-only riscv64 build. + - name: Init submodules + run: | + git submodule update --init third_party/spdlog third_party/ruy + git -C third_party/ruy submodule update --init third_party/cpuinfo + + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: python-wheels + persist-credentials: false + + # Ruy's pinned cpuinfo commit predates cpuinfo recognizing riscv64, so it never + # defines _GNU_SOURCE for this arch even though src/api.c calls syscall() + # unconditionally on __linux__ - see the patch for the long version. + - name: Patch cpuinfo for riscv64 + run: git apply python-wheels/patches/ctranslate2/${{ env.CTRANSLATE2_VERSION }}/*.patch + + # Upstream's own wheel carries none of these: setup.py's find_packages() only + # globs inside python/, so the repo-root LICENSE never reaches dist-info/licenses/. + - name: Add ctranslate2's LICENSE to the python package + run: cp LICENSE python/ + + - name: Build wheels + uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 + with: + package-dir: python + output-dir: wheelhouse + env: + # Mirrors prepare_build_environment_linux.sh's aarch64 branch: no MKL/oneDNN + # (x86-only), Ruy as the CPU GEMM backend (its cpuinfo dep degrades to a + # portable no-op on an unrecognized CPU_TARGET_PROCESSOR, same as riscv64_generic + # OpenBLAS kernels) and OPENMP_RUNTIME=COMP for the image's GNU libgomp. + # CMAKE_POLICY_VERSION_MINIMUM=3.5 works around cpuinfo's own clog dependency + # declaring `cmake_minimum_required(VERSION 3.5)` with no upper bound, which a + # pip-installed CMake 4.x (needed: no riscv64 wheel exists below CMake 4.1) + # otherwise refuses outright. ENABLE_CPU_DISPATCH=OFF because src/cpu/cpu_isa.h's + # CPU_ISA_DISPATCH macro is only defined for CT2_X86_BUILD/CT2_ARM64_BUILD when + # dispatch is on, leaving it undefined - and every call site a hard error - on any + # third architecture; disabling it falls through to the single-ISA GENERIC + # definition, which is all riscv64 needs since there is nothing to dispatch between. + CIBW_BEFORE_ALL_LINUX: | + set -ex + dnf install -y cmake unzip + cmake -S . -B build-release \ + -D CMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -D CMAKE_BUILD_TYPE=Release \ + -D BUILD_CLI=OFF \ + -D WITH_MKL=OFF \ + -D OPENMP_RUNTIME=COMP \ + -D WITH_RUY=ON \ + -D ENABLE_CPU_DISPATCH=OFF + cmake --build build-release -j"$(nproc)" --target install + ldconfig + CIBW_BEFORE_BUILD: pip install -r python/install_requirements.txt + CIBW_BUILD: >- + cp312-manylinux_riscv64 cp313-manylinux_riscv64 + cp314-manylinux_riscv64 cp314t-manylinux_riscv64 + CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} + CIBW_ENVIRONMENT: >- + CMAKE_BUILD_PARALLEL_LEVEL=8 + PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ + # cp314t is built but not tested: our registry's tokenizers/safetensors are + # cp38/cp39-abi3 wheels, which cp314t (no stable ABI for free-threading) can't use. + CIBW_TEST_SKIP: cp314t-* + CIBW_TEST_REQUIRES: pytest wurlitzer sentencepiece protobuf transformers torch + # test-python-wheels' own command minus the legacy fairseq/opennmt converter tests + # (separate job upstream, needs torch/tensorflow pins our registry doesn't carry) + # and two tests whose output sums small per-token float differences into a result + # just outside its assertion tolerance under Ruy's GENERIC scalar kernels (gotcha + # 170's numeric-divergence class, not a functional bug; confirmed on real riscv64 + # CI). Data fetch mirrors prepare_test_environment.sh. --deselect nodeids are + # rootdir-relative ("tests/..."), not {project}-prefixed - gotcha 14: pytest + # reports nodeids relative to rootdir regardless of the invocation cwd/path, so an + # absolute-path deselect silently matches nothing and the "deselected" test still + # runs. Confirmed on real riscv64 CI (run 33878204120: the wav2vec2 deselect target + # ran and passed, the 3 whisper_align ones ran and failed) after this exact mistake + # slipped through local validation on the aarch64 proxy. + CIBW_TEST_COMMAND: | + mkdir -p {project}/tests/data/models + test -d {project}/tests/data/models/transliteration-aren-all || (curl -fsSL https://opennmt-models.s3.amazonaws.com/transliteration-aren-all.tar.gz | tar xz -C {project}/tests/data/models) + test -d {project}/tests/data/models/opus-mt-ende || (curl -fsSL -o /tmp/opus.zip https://object.pouta.csc.fi/OPUS-MT-models/en-de/opus-2020-02-26.zip && unzip -q -o /tmp/opus.zip -d {project}/tests/data/models/opus-mt-ende) + pytest -v {project}/python/tests --ignore={project}/python/tests/test_fairseq.py --ignore={project}/python/tests/test_opennmt_py.py --ignore={project}/python/tests/test_opennmt_tf.py --deselect "tests/test_transformers.py::TestWav2Vec2::test_transformers_wav2vec2[facebook/wav2vec2-large-robust-ft-swbd-300h-expected_transcriptions0-cpu]" --deselect "tests/test_transformers.py::TestWhisper::test_transformers_whisper_align[test_names0-cpu]" --deselect "tests/test_transformers.py::TestWhisper::test_transformers_whisper_align[test_names1-cpu]" --deselect "tests/test_transformers.py::TestWhisper::test_transformers_whisper_align[test_names2-cpu]" + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ctranslate2-${{ env.CTRANSLATE2_VERSION }}-cibw-wheels-manylinux_riscv64 + path: wheelhouse/*.whl + if-no-files-found: error + + gpl_sources: + needs: [setup] + name: Collect GPL sources (gcc) for ctranslate2 ${{ inputs.version || '4.8.1' }} + runs-on: ubuntu-24.04-riscv + steps: + # auditwheel repair vendors the image's libgomp.so.1 into ctranslate2.libs/ + # because libctranslate2.so links OPENMP_RUNTIME=COMP's GNU OpenMP runtime. + - name: Collect gcc source RPM from manylinux_riscv64 + uses: riseproject-dev/python-wheels/actions/collect-gpl-sources@main + with: + image: ${{ env.MANYLINUX_RISCV64_IMAGE }} + packages: gcc + output: gpl-sources.tar + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ctranslate2-${{ env.CTRANSLATE2_VERSION }}-gpl-sources + path: gpl-sources.tar + if-no-files-found: error + + publish: + name: Publish ctranslate2 ${{ inputs.version || '4.8.1' }} + needs: [setup, build_wheels, gpl_sources] + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + with: + artifact-pattern: ctranslate2-${{ inputs.version || '4.8.1' }}-*-manylinux_riscv64 + gpl-sources-artifact: ctranslate2-${{ inputs.version || '4.8.1' }}-gpl-sources + gpl-sources-description: gcc diff --git a/patches/ctranslate2/4.8.1/0001-cpuinfo-fix-missing-syscall-declaration-on-riscv64.patch b/patches/ctranslate2/4.8.1/0001-cpuinfo-fix-missing-syscall-declaration-on-riscv64.patch new file mode 100644 index 000000000..4d64c6338 --- /dev/null +++ b/patches/ctranslate2/4.8.1/0001-cpuinfo-fix-missing-syscall-declaration-on-riscv64.patch @@ -0,0 +1,45 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Thu, 3 Sep 2026 21:30:00 +0200 +Subject: [PATCH] cpuinfo: define _GNU_SOURCE unconditionally so syscall() is + declared on riscv64 + +cpuinfo's CMakeLists.txt only adds -D_GNU_SOURCE=1 to this target when +CPUINFO_SUPPORTED_PLATFORM is set, and the pinned commit (082deffc, from +before cpuinfo recognized riscv64 at all) leaves that flag off for any +CMAKE_SYSTEM_PROCESSOR it doesn't match by name. This file's +cpuinfo_get_current_processor() still unconditionally calls syscall() on any +__linux__ build regardless of that flag, so on riscv64 glibc's unistd.h hides +the declaration and GCC 14 errors out with an implicit-declaration error +before a single ctranslate2 object file compiles. + +cpuinfo commit 4e5be9e1c6c5895bc5105a92d587bc9df8d2522b ("Add limited support +for RISC-V initialization", 2023-11-14) already fixes this indirectly +upstream, by adding riscv(32|64) to the CMakeLists.txt processor regex that +gates CPUINFO_SUPPORTED_PLATFORM and therefore the same _GNU_SOURCE define +this patch adds directly. ctranslate2 pins Ruy's own submodule commit of +cpuinfo from before that fix, so it has to be applied here rather than +picked up by a version bump. + +Upstream-Status: Backport [https://github.com/pytorch/cpuinfo/commit/4e5be9e1c6c5895bc5105a92d587bc9df8d2522b] +--- + third_party/ruy/third_party/cpuinfo/src/api.c | 9 +++++++++ + 1 file changed, 9 insertions(+) + +diff --git a/third_party/ruy/third_party/cpuinfo/src/api.c b/third_party/ruy/third_party/cpuinfo/src/api.c +index f91b421..8231a66 100644 +--- a/third_party/ruy/third_party/cpuinfo/src/api.c ++++ b/third_party/ruy/third_party/cpuinfo/src/api.c +@@ -1,3 +1,12 @@ ++/* ++ * cpuinfo's CMakeLists.txt only defines _GNU_SOURCE when ++ * CPUINFO_SUPPORTED_PLATFORM is set, which this pinned commit never does for ++ * riscv64 (added to the recognized-processor list only in later cpuinfo ++ * releases). Without it, glibc's hides the syscall() prototype this ++ * file still calls unconditionally on __linux__. ++ */ ++#define _GNU_SOURCE 1 ++ + #include + #include +