From 631fb7cdaa82f313be391a2345f4c9e3ff7588a1 Mon Sep 17 00:00:00 2001 From: gongchensu Date: Thu, 27 Aug 2026 15:55:03 +0800 Subject: [PATCH] feat(hygon): add reproducible development image Add a DTK-based Hygon development image with the build and test dependencies required by InfiniRT, InfiniOps, InfiniCore, and InfiniLM. Pin Xmake and its package recipes, and pre-populate matching Boost and pybind11 packages so fresh containers do not depend on mutable recipes or repeated package builds. Document the isolated mainline build, Qwen3 inference, operator tracing, runtime dependency checks, and per-run cleanup workflow. --- README.md | 1 + images/hygon/Dockerfile | 82 +++++++++ images/hygon/README.md | 342 +++++++++++++++++++++++++++++++++++ images/hygon/xmake-cache.lua | 4 + 4 files changed, 429 insertions(+) create mode 100644 images/hygon/Dockerfile create mode 100644 images/hygon/README.md create mode 100644 images/hygon/xmake-cache.lua diff --git a/README.md b/README.md index 42b00dd..7947755 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ helpers, GitHub Actions matrix converter, reusable workflow, and tests. │ ├── metax/ │ ├── moore/ │ ├── cambricon/ +│ ├── hygon/ │ └── ascend/ └── tests/ ``` diff --git a/images/hygon/Dockerfile b/images/hygon/Dockerfile new file mode 100644 index 0000000..1bcd65c --- /dev/null +++ b/images/hygon/Dockerfile @@ -0,0 +1,82 @@ +ARG BASE_IMAGE=harbor.sourcefind.cn:5443/dcu/admin/base/custom:sglang-deepseek-v4-dev-zkjh +FROM ${BASE_IMAGE} + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ENV DEBIAN_FRONTEND=noninteractive + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY +ARG http_proxy +ARG https_proxy +ARG no_proxy + +ARG APT_MIRROR +RUN if [ -n "$APT_MIRROR" ]; then \ + sed -i "s|http://[^[:space:]]*/ubuntu|${APT_MIRROR}|g" \ + /etc/apt/sources.list \ + /etc/apt/sources.list.d/*.sources 2>/dev/null || true; \ + fi && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + clang-format \ + cmake \ + curl \ + git \ + libboost-stacktrace-dev \ + libclang-dev \ + ninja-build \ + patchelf \ + pkg-config \ + python3-dev \ + && rm -rf /var/lib/apt/lists/* + +# Keep the DTK-enabled PyTorch supplied by the base image when project +# dependencies are installed later in a development container. +RUN torch_version="$(python3 -m pip show torch | awk '/^Version:/ {print $2}')" && \ + test -n "$torch_version" && \ + printf 'torch==%s\n' "$torch_version" > /etc/pip-constraints.txt +ENV PIP_CONSTRAINT=/etc/pip-constraints.txt + +RUN python3 -m pip install --no-cache-dir \ + libclang \ + pybind11 \ + pytest \ + pytest-cov \ + pytest-mock \ + pytest-xdist \ + pyyaml \ + ruff==0.15.7 \ + scikit-build-core + +ARG XMAKE_VERSION=v3.0.5 +ARG XMAKE_SHA256=4075ef1b4cba7f5eb5c55a4451761ef081086bb3142e7d4b130cf9c7484e68f0 +ARG XMAKE_DOWNLOAD_URL=https://github.com/xmake-io/xmake/releases/download/${XMAKE_VERSION}/xmake-bundle-${XMAKE_VERSION}.linux.x86_64 +ENV XMAKE_ROOT=y +RUN curl --retry 5 --retry-all-errors -fsSL \ + "${XMAKE_DOWNLOAD_URL}" -o /usr/local/bin/xmake && \ + printf '%s %s\n' "$XMAKE_SHA256" /usr/local/bin/xmake \ + | sha256sum --check - && \ + chmod +x /usr/local/bin/xmake + +# Pin the package recipes used by the validated environment and pre-populate +# the Xmake package cache. Otherwise a fresh container resolves a newer Boost +# archive from GitHub and every developer has to download and build it again. +ARG XMAKE_REPO_URL=https://gitee.com/crapromer/xmake-repo.git +ARG XMAKE_REPO_COMMIT=674a71c5905870e6e7e162bf689903822bbbf16e +COPY images/hygon/xmake-cache.lua /opt/xmake-cache-project/xmake.lua +RUN git clone "${XMAKE_REPO_URL}" /opt/xmake-repo && \ + git -C /opt/xmake-repo checkout "${XMAKE_REPO_COMMIT}" && \ + xmake repo --clear --global && \ + xmake repo --add --global xmake-repo /opt/xmake-repo && \ + xmake require -y -P /opt/xmake-cache-project + +ENV DTK_ROOT=/opt/dtk +ENV HYGON_ARCH=gfx936 +ENV CUDA_HOME=/opt/dtk/cuda/cuda +ENV PATH=/opt/dtk/cuda/cuda/bin:/root/.local/bin:${PATH} + +WORKDIR /workspace/codes diff --git a/images/hygon/README.md b/images/hygon/README.md new file mode 100644 index 0000000..2947825 --- /dev/null +++ b/images/hygon/README.md @@ -0,0 +1,342 @@ +# Hygon development image + +This image contains the Hygon DTK toolchain and the build/test dependencies for +InfiniRT, InfiniOps, InfiniCore, and InfiniLM. Source code is mounted from the +host and is not included in the image. + +The default base image matches the environment used to validate Hygon Qwen3 +inference: + +```text +harbor.sourcefind.cn:5443/dcu/admin/base/custom:sglang-deepseek-v4-dev-zkjh +DTK 26.04 RC4, Python 3.10, DTK PyTorch 2.9.0, Xmake 3.0.5 +``` + +The image pins the Xmake recipe repository to commit +`674a71c5905870e6e7e162bf689903822bbbf16e` and pre-builds Boost 1.89.0 and +pybind11 3.0.1 with the same options required by InfiniCore. This prevents a +fresh development container from resolving different package recipes or +downloading and rebuilding these packages during `xmake f`. + +Access to the internal image registry, Hygon devices, `/opt/hyhal`, and the +Qwen3-0.6B model is required. + +## Build the image + +Run from the root of the `InfiniTensor/ci` checkout: + +```bash +docker build \ + -f images/hygon/Dockerfile \ + -t infinitensor/infini-dev:hygon-dtk2604 . +``` + +After the image has been published to the team registry, developers should pull +that immutable tag (or digest) instead of rebuilding it locally: + +```bash +docker pull /infinitensor/infini-dev:hygon-dtk2604 +``` + +Replace `infinitensor/infini-dev:hygon-dtk2604` in the commands below with the +published image name. + +To use another compatible DTK image, pass +`--build-arg BASE_IMAGE=` explicitly and validate the full workflow +below before publishing it. + +## Prepare source checkouts + +The Hygon integration is available on each upstream default branch. Prepare the +source on the host before starting the container. No Git network access is +required inside the container after the recursive clones complete. + +Set the source root whether using existing checkouts or cloning them now: + +```bash +export SOURCE_ROOT=/home/zhuyue/codes +mkdir -p "$SOURCE_ROOT" +cd "$SOURCE_ROOT" +``` + +For a first-time setup, clone the four upstream repositories: + +```bash +git clone --branch master --single-branch --recursive \ + https://github.com/InfiniTensor/InfiniRT.git +git clone --branch master --single-branch --recursive \ + https://github.com/InfiniTensor/InfiniOps.git +git clone --branch main --single-branch --recursive \ + https://github.com/InfiniTensor/InfiniCore.git +git clone --branch main --single-branch --recursive \ + https://github.com/InfiniTensor/InfiniLM.git +``` + +Existing checkouts may be used instead. Update and select `master` for InfiniRT +and InfiniOps, and `main` for InfiniCore and InfiniLM, using the remote names +configured in each checkout. All native build trees and library install prefixes +below are kept outside the existing checkouts. + +Create a new run directory for every validation. It holds all build trees, +install prefixes, and logs, so stale CMake, Xmake, or installed-library state +from an earlier run cannot participate. + +```bash +export RUNS_ROOT=/home/zhuyue/infini-hygon-runs +mkdir -p "$RUNS_ROOT" +export RUN_ROOT="$(mktemp -d "$RUNS_ROOT/$(date +%Y%m%d-%H%M%S)-XXXXXX")" + +for repo in InfiniRT InfiniOps InfiniCore InfiniLM; do + test -d "$SOURCE_ROOT/$repo/.git" || { + echo "Missing checkout: $SOURCE_ROOT/$repo" >&2 + exit 1 + } +done + +printf 'SOURCE_ROOT=%s\nRUN_ROOT=%s\n' "$SOURCE_ROOT" "$RUN_ROOT" +``` + +## Start the development container + +Run this command on the Hygon host. Mount both directories at the same absolute +paths inside the container because CMake records absolute paths in its build +tree. `--rm` discards the container's writable layer when it exits. + +```bash +docker run --rm -it \ + --name infini-hygon-dev \ + --network host \ + --ipc host \ + --device /dev/kfd \ + --device /dev/mkfd \ + --device /dev/dri \ + --group-add video \ + --group-add render \ + --ulimit memlock=-1:-1 \ + --ulimit stack=67108864:67108864 \ + -v /opt/hyhal:/opt/hyhal:ro \ + -v "${SOURCE_ROOT:?SOURCE_ROOT is not set}:${SOURCE_ROOT}" \ + -v "${RUN_ROOT:?RUN_ROOT is not set}:${RUN_ROOT}" \ + -v /data-aisoft:/data-aisoft:ro \ + -e SOURCE_ROOT="${SOURCE_ROOT:?SOURCE_ROOT is not set}" \ + -e RUN_ROOT="${RUN_ROOT:?RUN_ROOT is not set}" \ + -w "${SOURCE_ROOT:?SOURCE_ROOT is not set}" \ + infinitensor/infini-dev:hygon-dtk2604 \ + bash +``` + +## Configure the workspace + +Run the remaining commands inside the container: + +```bash +if [ -z "${SOURCE_ROOT:-}" ] || [ -z "${RUN_ROOT:-}" ]; then + echo "SOURCE_ROOT or RUN_ROOT was not passed to the container" >&2 + exit 1 +fi + +test -d "$SOURCE_ROOT" || { + echo "Mounted source root does not exist: $SOURCE_ROOT" >&2 + exit 1 +} +test -d "$RUN_ROOT" || { + echo "Mounted run root does not exist: $RUN_ROOT" >&2 + exit 1 +} + +export BUILD_ROOT="$RUN_ROOT/build" +export PREFIX_ROOT="$RUN_ROOT/prefix" +export LOG_ROOT="$RUN_ROOT/logs" + +export INFINI_RT_ROOT="$PREFIX_ROOT/InfiniRT" +export INFINI_OPS_ROOT="$PREFIX_ROOT/InfiniOps" +export INFINI_ROOT="$PREFIX_ROOT/InfiniCore" +export DTK_ROOT=/opt/dtk +export HYGON_ARCH=gfx936 +export CUDA_HOME="$DTK_ROOT/cuda/cuda" + +export PATH="$CUDA_HOME/bin:$PATH" +export TORCH_PACKAGE_ROOT="$(python3 -c 'import os, torch; print(os.path.dirname(torch.__file__))')" +export TORCH_BUNDLED_LIBS="$(dirname "$TORCH_PACKAGE_ROOT")/torch.libs" +export LD_LIBRARY_PATH="$CUDA_HOME/lib64:$TORCH_PACKAGE_ROOT/lib:$TORCH_BUNDLED_LIBS:$INFINI_ROOT/lib:$INFINI_OPS_ROOT/lib:$INFINI_RT_ROOT/lib:${LD_LIBRARY_PATH:-}" + +test ! -e "$BUILD_ROOT" || { + echo "Build directory already exists; create a new RUN_ROOT: $BUILD_ROOT" >&2 + exit 1 +} +mkdir -p "$BUILD_ROOT" "$PREFIX_ROOT" "$LOG_ROOT" + +# The container runs as root while the mounted repositories belong to the host +# user. Trust only the four explicit mounted checkouts. +for repo in InfiniRT InfiniOps InfiniCore InfiniLM; do + git config --global --add safe.directory "$SOURCE_ROOT/$repo" +done + +for repo in InfiniRT InfiniOps InfiniCore InfiniLM; do + git -C "$SOURCE_ROOT/$repo" status --short --branch + git -C "$SOURCE_ROOT/$repo" rev-parse HEAD +done +``` + +## Build and test standalone InfiniRT + +```bash +cmake -S "$SOURCE_ROOT/InfiniRT" -B "$BUILD_ROOT/InfiniRT" \ + -DWITH_CPU=ON \ + -DWITH_HYGON=ON \ + -DDTK_ROOT="$DTK_ROOT" \ + -DHYGON_ARCH="$HYGON_ARCH" \ + -DINFINI_RT_BUILD_TESTING=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$INFINI_RT_ROOT" + +cmake --build "$BUILD_ROOT/InfiniRT" -j"$(nproc)" && \ +LD_LIBRARY_PATH="$BUILD_ROOT/InfiniRT/src:$LD_LIBRARY_PATH" \ + ctest --test-dir "$BUILD_ROOT/InfiniRT" --output-on-failure && \ +cmake --install "$BUILD_ROOT/InfiniRT" +``` + +The build-tree library is deliberately placed first only for `ctest`. This +prevents an older `libinfinirt.so` under `INFINI_ROOT` from shadowing the +standalone library being tested. + +## Build and install InfiniOps + +InfiniCore consumes the installed InfiniOps CMake package on the current main +branch. Build InfiniOps separately against the standalone InfiniRT prefix: + +```bash +unset INFINI_OPS_OPS +unset INFINI_OPS_TRACE_CALLS + +cmake -S "$SOURCE_ROOT/InfiniOps" -B "$BUILD_ROOT/InfiniOps" \ + -DWITH_CPU=ON \ + -DWITH_HYGON=ON \ + -DAUTO_DETECT_DEVICES=OFF \ + -DDTK_ROOT="$DTK_ROOT" \ + -DHYGON_ARCH="$HYGON_ARCH" \ + -DWITH_TORCH=ON \ + -DINFINI_OPS_TORCH_OPS=argmax,index_select \ + -DGENERATE_OPERATOR_CALL_INSTANTIATIONS=ON \ + -DGENERATE_PYTHON_BINDINGS=OFF \ + -DINFINI_RT_ROOT="$INFINI_RT_ROOT" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$INFINI_OPS_ROOT" + +cmake --build "$BUILD_ROOT/InfiniOps" --target infiniops -j"$(nproc)" && \ +cmake --install "$BUILD_ROOT/InfiniOps" + +test -f "$INFINI_OPS_ROOT/lib/libinfiniops.so" +test -f "$INFINI_OPS_ROOT/lib/cmake/InfiniOps/InfiniOpsConfig.cmake" +``` + +## Build InfiniCore with installed InfiniOps and InfiniRT + +```bash +cd "$SOURCE_ROOT/InfiniCore" + +unset INFINI_OPS_OPS +unset INFINI_OPS_TRACE_CALLS + +xmake f -y -c \ + -o "$BUILD_ROOT/InfiniCore" \ + --hygon-dcu=y \ + --hygon-arch="$HYGON_ARCH" \ + --cuda="$CUDA_HOME" \ + --cpu=y \ + --ccl=n \ + --graph=n \ + --aten=n \ + --infiniops=y \ + --infiniops-root="$INFINI_OPS_ROOT" \ + --infinirt-root="$INFINI_RT_ROOT" \ + --flash-attn= \ + -m release \ + -v + +xmake build -j"$(nproc)" && \ +xmake install && \ +xmake build -j"$(nproc)" _infinicore && \ +xmake install _infinicore && \ +python3 -m pip install -e . +``` + +## Install InfiniLM and run Qwen3-0.6B + +```bash +cd "$SOURCE_ROOT/InfiniLM" + +python3 -m pip install -e . +python3 -m pip install \ + transformers==5.15.0 \ + janus \ + datamodel-code-generator \ + xxhash + +export INFINI_OPS_TRACE_CALLS=1 +LOG_FILE="$LOG_ROOT/hygon-qwen3-infiniops.log" + +python3 examples/test_infer.py \ + --device hygon \ + --model=/data-aisoft/mechdancer/models/Qwen3-0.6B \ + >"$LOG_FILE" 2>&1 + +tail -n 40 "$LOG_FILE" +grep '^\[INFINI_OPS_TRACE_CALLS\]' "$LOG_FILE" \ + | sed 's/^\[INFINI_OPS_TRACE_CALLS\] //' \ + | sort | uniq -c +``` + +`examples/test_infer.py` uses static caching and `enable_graph=False` unless +explicitly requested otherwise. The trace lines show calls that entered the +canonical InfiniOps dispatcher, including the operator, device, and selected +implementation. + +For Qwen3-0.6B greedy decoding, the validated run includes Hygon native +(`implementation=0`) calls for `Add`, `Argmax`, `CausalSoftmax`, `Copy`, +`Embedding`, `FusedAddRmsNorm`, `Gemm`, `RmsNorm`, `RotaryEmbedding`, and +`SiluAndMul`. `IndexSelect` uses the InfiniOps ATen provider +(`implementation=8`). + +## Verify the standalone InfiniRT dependency + +```bash +readelf -d "$INFINI_ROOT/lib/libinfiniops.so" \ + | grep -E 'NEEDED|RPATH|RUNPATH' + +test -f "$INFINI_ROOT/lib/libinfiniops_infinirt.so" + +if ldd "$INFINI_ROOT/lib/libinfiniops.so" | grep -q 'not found'; then + ldd "$INFINI_ROOT/lib/libinfiniops.so" | grep 'not found' + exit 1 +fi +``` + +The dynamic section should contain `libinfiniops_infinirt.so` and `$ORIGIN`. +The private name allows InfiniOps to use standalone InfiniRT while InfiniCore +continues to load its legacy runtime where still required by the current ABI. + +This workflow intentionally leaves `INFINI_OPS_OPS` unset and disables the +InfiniCore ATen backend, FlashAttention, CCL, and graph mode. InfiniOps itself +builds the explicitly selected `argmax` and `index_select` ATen candidates; +Hygon native wins dispatch for `Argmax`, while `IndexSelect` selects ATen. The +workflow does not move or enable an NVIDIA `topk_softmax` linked provider. + +## Repeat or remove a run + +Start the next clean validation with a new timestamped `RUN_ROOT`; do not reuse +an earlier run directory. The source checkouts may be reused because all native +build trees and prefixes are outside them. The `_infinicore` extension installed +for editable Python use is overwritten by each fresh build. The image's pinned +Xmake package cache is intentional toolchain content, not project build state. + +After the container exits, the following host-side command removes exactly one +run, including its build trees, installed libraries, and logs. Inspect +`RUN_ROOT` before running it because this data is not recoverable: + +```bash +test -n "${RUNS_ROOT:-}" && \ +test -n "${RUN_ROOT:-}" && \ +test "$(dirname "$RUN_ROOT")" = "$RUNS_ROOT" && \ +rm -rf -- "$RUN_ROOT" +``` diff --git a/images/hygon/xmake-cache.lua b/images/hygon/xmake-cache.lua new file mode 100644 index 0000000..4cb7070 --- /dev/null +++ b/images/hygon/xmake-cache.lua @@ -0,0 +1,4 @@ +set_project("hygon-development-image-cache") + +add_requires("boost 1.89.0", {configs = {stacktrace = true}}) +add_requires("pybind11 v3.0.1")