From 4306d12795cb624ff8f223446eb4a488c025a329 Mon Sep 17 00:00:00 2001 From: Norbert Manthey Date: Fri, 14 Aug 2026 09:15:36 +0000 Subject: [PATCH 1/7] test: allow to set python for test-in-venv.sh On a system with multiple python environments, we might want to run with a different version. Therefore, allow the script to select a python version. This change also helps when adding support to new python versions. Signed-off-by: Norbert Manthey --- tests/test-in-venv.sh | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/test-in-venv.sh b/tests/test-in-venv.sh index d1d3a1a..9c54c4f 100755 --- a/tests/test-in-venv.sh +++ b/tests/test-in-venv.sh @@ -14,18 +14,27 @@ set -e # Configuration +# PYTHON selects the interpreter used to create the virtual environment. +# Override it to build the venv with a specific version, e.g. +# PYTHON=python3.12 tests/test-in-venv.sh +# It may be a name on PATH or an absolute path. All pip/pytest calls go through +# " -m ..." (never the bare pip/python3 shims). +PYTHON="${PYTHON:-python3}" VENV_DIR=".venv-testing" MODULE_DIR="$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")" STATUS_CACHE="$MODULE_DIR/$VENV_DIR/.git_status_cache" +# Interpreter inside the venv (created from $PYTHON). Used for pip/pytest so the +# correct environment is targeted regardless of which binary bootstrapped it. +VENV_PYTHON="$MODULE_DIR/$VENV_DIR/bin/python" # Function to create and setup virtual environment setup_virtual_environment() { - echo "Setting up virtual environment..." - python3 -m venv "${VENV_DIR}" + echo "Setting up virtual environment with '${PYTHON}'..." + "${PYTHON}" -m venv "${VENV_DIR}" source "${VENV_DIR}/bin/activate" - pip install --upgrade pip - pip install -e ".[dev]" + "${VENV_PYTHON}" -m pip install --upgrade pip + "${VENV_PYTHON}" -m pip install -e ".[dev]" } # Function to activate virtual environment @@ -40,7 +49,7 @@ install_module() { echo "Installing module..." 1>&2 status=0 - output=$(pip install -e "${MODULE_DIR}" 2>&1) || status=$? + output=$("${VENV_PYTHON}" -m pip install -e "${MODULE_DIR}" 2>&1) || status=$? if [ $status -ne 0 ]; then echo "Installation failed, with output:" 1>&2 echo "$output" 1>&2 @@ -53,7 +62,7 @@ run_tests() { echo "Running unit tests..." 1>&2 status=0 - output=$(python3 -m pytest tests/ -v -m "not integration" 2>&1) || status=$? + output=$("${VENV_PYTHON}" -m pytest tests/ -v -m "not integration" 2>&1) || status=$? if [ $status -eq 0 ]; then echo "Unit tests passed" 1>&2 else From e852f056e9905d5b6800e0306d1dd299a9a71256 Mon Sep 17 00:00:00 2001 From: Norbert Manthey Date: Tue, 18 Aug 2026 10:39:27 +0200 Subject: [PATCH 2/7] fix: correct simple-unixbench benchmark CSV metric emission MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit summarize_unixbench_log built the metric name from fields 1..NF-4, which pulled the numeric value, unit and part of the timing info into the metric name — producing malformed rows like 'Arithmetic_Test_(double)_385400605.9_lps' with a single sample each, instead of one 'Arithmetic_Test_(double)' metric aggregated across VMs. Use NF-6 for the metric name (matching unixbench-kernel-regression) so value=$(NF-5) and unit=$(NF-4) line up, and drop the index-section parsing that emitted duplicate/derived score metrics. Verified against sample UnixBench output: clean metric names, correct value/unit. Signed-off-by: Norbert Manthey --- vm-tests/simple-unixbench/common_lib.sh | 33 +++---------------------- 1 file changed, 3 insertions(+), 30 deletions(-) diff --git a/vm-tests/simple-unixbench/common_lib.sh b/vm-tests/simple-unixbench/common_lib.sh index bf437f7..132f56b 100644 --- a/vm-tests/simple-unixbench/common_lib.sh +++ b/vm-tests/simple-unixbench/common_lib.sh @@ -134,9 +134,9 @@ summarize_unixbench_log() # Parse result lines (first section) - use 6th last as value, 5th last as unit in_results && NF >= 6 { - # Extract metric name (everything except last 5 fields) + # Extract metric name (everything except last 6 fields: value unit (timing info)) metric = "" - for (i = 1; i <= NF-4; i++) { + for (i = 1; i <= NF-6; i++) { if (metric == "") { metric = $i } else { @@ -161,33 +161,6 @@ summarize_unixbench_log() printf "%s.%s,%s,%s,%s,%s,%s,%s,%s\n", benchmark_version, metric, unit, value, more_is_better, kernel_version, instance_id, instance_type, arch } - # Parse index section lines - always use second-to-last column as value - in_index && NF >= 3 && !/BASELINE/ && !/RESULT/ && !/INDEX/ && !/^=/ { - # Extract metric name (everything except last 2 fields) - metric = "" - for (i = 1; i <= NF-3; i++) { - if (metric == "") { - metric = $i - } else { - metric = metric "_" $i - } - } - - # Use second-to-last field as value - value = $(NF-1) - - # Skip lines with "---" values - if (value == "---") { - next - } - - # Clean up metric name - gsub(/^\s+|\s+$/, "", metric) - - unit = "score" - more_is_better = "true" - - printf "%s.%s,%s,%s,%s,%s,%s,%s,%s\n", benchmark_version, metric, unit, value, more_is_better, kernel_version, instance_id, instance_type, arch - } + # Skip index section entirely - do not parse it ' "$unixbench_log" >>"$output_csv_file" } From 69327b0996ac3019912a6ec810f4e75478818076 Mon Sep 17 00:00:00 2001 From: Norbert Manthey Date: Tue, 18 Aug 2026 10:40:55 +0200 Subject: [PATCH 3/7] fix: raise default task hang threshold to 1200s for heavy benchmarks UnixBench (and other CPU-bound benchmarks) run for many minutes with no new console output during the benchmark phase. The 600s hang-detection default tripped mid-run and killed all VMs as a false-positive stall. Raise the PULLAB_TASK_HANG_THRESHOLD_SEC default from 600 to 1200s so these benchmarks complete, while still catching genuine hangs within a reasonable window. The value remains env-overridable for lighter workloads that want faster detection. Signed-off-by: Norbert Manthey --- src/kernel_ci_cloud_labs/providers/aws_provider.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/kernel_ci_cloud_labs/providers/aws_provider.py b/src/kernel_ci_cloud_labs/providers/aws_provider.py index 8ae665d..97a0bbd 100644 --- a/src/kernel_ci_cloud_labs/providers/aws_provider.py +++ b/src/kernel_ci_cloud_labs/providers/aws_provider.py @@ -311,7 +311,10 @@ def wait_for_task_completion(self): finishes the kernelci-api node incomplete/Infrastructure with the matched line surfaced in error_msg. * No new VM console output for PULLAB_TASK_HANG_THRESHOLD_SEC seconds - (default 600) -- silent stall, same treatment as a crash. + (default 1200) -- silent stall, same treatment as a crash. The + default accommodates CPU-heavy benchmarks (e.g. UnixBench) whose + console goes quiet for many minutes during a run; lower it via the + env var for faster hang detection on lighter workloads. * Overall PULLAB_TASK_WAIT_TIMEOUT_SEC seconds elapsed (default 3600) -- final safety net for whatever isn't covered above. @@ -331,7 +334,7 @@ def wait_for_task_completion(self): poll_interval = float(os.getenv("PULLAB_TASK_POLL_INTERVAL_SEC") or 30) log_interval = float(os.getenv("PULLAB_TASK_PROGRESS_LOG_SEC") or 120) - hang_threshold = float(os.getenv("PULLAB_TASK_HANG_THRESHOLD_SEC") or 600) + hang_threshold = float(os.getenv("PULLAB_TASK_HANG_THRESHOLD_SEC") or 1200) overall_timeout = float(os.getenv("PULLAB_TASK_WAIT_TIMEOUT_SEC") or 3600) start = time.time() From 0f9934d3a4e2d8cd13268d562a31201018a35781 Mon Sep 17 00:00:00 2001 From: Norbert Manthey Date: Tue, 18 Aug 2026 13:35:00 +0200 Subject: [PATCH 4/7] refactor: factor out shared kernel-management helpers The kernel A/B tests each carried their own copy of the kernel install/reboot helpers, so any change had to be made in every test. Introduce vm-tests/lib/kernel_helpers.sh as the single home for that logic (environment validation, kernel RPM download/selection, install_kernel_rpm with grubby boot-entry management, and the get_running_kernel / assert_kernel_changed helpers). Each kernel test includes it via a kernel_helpers.sh symlink and sources it, keeping only its test-specific functions: - example-kernel-reboot-test: none (pure kernel install/reboot). - simple-source-reboot: source-RPM build helpers. - unixbench-kernel-regression: UnixBench prepare/run/summarize. The symlink is stored by the payload zip as real content, so the VM sees a normal file; no pipeline change is needed. Subsequent fixes to the kernel logic now land once in the shared lib. Signed-off-by: Norbert Manthey --- vm-tests/TODO-shared-lib.md | 49 +++ .../example-kernel-reboot-test/common_lib.sh | 258 +-------------- .../kernel_helpers.sh | 1 + vm-tests/lib/kernel_helpers.sh | 257 +++++++++++++++ vm-tests/simple-source-reboot/common_lib.sh | 265 +-------------- .../simple-source-reboot/kernel_helpers.sh | 1 + .../unixbench-kernel-regression/common_lib.sh | 304 +----------------- .../kernel_helpers.sh | 1 + 8 files changed, 341 insertions(+), 795 deletions(-) create mode 100644 vm-tests/TODO-shared-lib.md create mode 120000 vm-tests/example-kernel-reboot-test/kernel_helpers.sh create mode 100644 vm-tests/lib/kernel_helpers.sh create mode 120000 vm-tests/simple-source-reboot/kernel_helpers.sh create mode 120000 vm-tests/unixbench-kernel-regression/kernel_helpers.sh diff --git a/vm-tests/TODO-shared-lib.md b/vm-tests/TODO-shared-lib.md new file mode 100644 index 0000000..41f35fe --- /dev/null +++ b/vm-tests/TODO-shared-lib.md @@ -0,0 +1,49 @@ +# Shared kernel-management helpers across vm-tests + +## Approach (implemented) + +The kernel install/upgrade helpers live once in: + + vm-tests/lib/kernel_helpers.sh + +Each kernel test includes it with a **symlink** in its own directory: + + vm-tests//kernel_helpers.sh -> ../lib/kernel_helpers.sh + +and its `common_lib.sh` sources it after setting `SOURCE_DIR`: + + source "${SOURCE_DIR}/kernel_helpers.sh" + +Why a symlink works with zero pipeline changes: `upload_test_payload()` builds +the payload with `Path(test_dir).rglob("*")` + `zf.write(...)`, which follows +the symlink and stores the **target's content** as a real file named +`kernel_helpers.sh`. On the VM the payload is extracted flat, so the test dir +gets a normal `kernel_helpers.sh` next to the `run*.sh` scripts. + +Fix once, benefit everywhere: the underscore/dash RPM-version handling, the +FIPS-disable-before-reboot logic, and the `--allowerasing` cross-series install +live only in the shared lib. + +## Status — migration complete + +All kernel tests now source the shared lib and keep only their test-specific +functions in `common_lib.sh`: + +- [x] `example-kernel-reboot-test` — no test-specific functions; just sources + the shared lib. +- [x] `simple-source-reboot` — source-RPM build helpers + (`install_source_kernel_rpm`, `build_kernel_rpm_src`, + `get_first_source_kernel_rpm_from_dir`, `install_and_build_kernel`) local. +- [x] `unixbench-kernel-regression` — UnixBench helpers (`prepare_unixbench`, + `run_unixbench`, `summarize_unixbench_log`) local. + +`simple-unixbench` and other non-kernel tests do not install kernels and do not +use the shared lib. + +## Adding a new kernel test + +1. `cd vm-tests/ && ln -s ../lib/kernel_helpers.sh kernel_helpers.sh` +2. In `common_lib.sh`, `source "${SOURCE_DIR}/kernel_helpers.sh"` and add only + test-specific functions. +3. Verify: `bash -n common_lib.sh` and a source-order smoke test with + `SOURCE_DIR` set. diff --git a/vm-tests/example-kernel-reboot-test/common_lib.sh b/vm-tests/example-kernel-reboot-test/common_lib.sh index 20d3868..fa5894b 100644 --- a/vm-tests/example-kernel-reboot-test/common_lib.sh +++ b/vm-tests/example-kernel-reboot-test/common_lib.sh @@ -2,253 +2,11 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -# Common functions for kernel reboot test - -# Get results bucket and test paths from environment -RESULTS_BUCKET="${S3_BUCKET:-}" -ARCH=$(uname -m) -KERNEL_RPM_DIR="/tmp/kernel-rpms" - -# Validate required environment variables -if [ -z "$RESULTS_BUCKET" ] || [ -z "$RUN_PREFIX" ] || [ -z "$TEST_NAME" ]; then - echo "ERROR: Missing required environment variables (S3_BUCKET, RUN_PREFIX, TEST_NAME)" >&2 - exit 1 -fi - -# Error trap handler to show line where error occurred -error_trap() -{ - local exit_code=$? - local line_number=$1 - echo "$(date): ERROR: Script failed at line $line_number with exit code $exit_code" - echo "$(date): ERROR: Command that failed: $(sed -n "${line_number}p" "$0")" - exit $exit_code -} -trap 'error_trap $LINENO' ERR - -#Return current runnning kernel -get_running_kernel() -{ - uname -r -} - -# Install a single given package -install_package() -{ - local pkg="$1" - local output - echo "Installing package $pkg ..." - if output=$(sudo yum install -y "$pkg" 2>&1) || output=$(sudo dnf install -y "$pkg" 2>&1); then - return 0 - else - echo "Failed to install package $pkg:" - echo "$output" - return 1 - fi -} - -# Install all dependencies for this test -install_test_dependencies() -{ - local deps_file="${SOURCE_DIR}/dependencies.txt" - - if [ -f "$deps_file" ]; then - while IFS= read -r pkg || [ -n "$pkg" ]; do - # Skip empty lines and comments - [[ -z "$pkg" || "$pkg" =~ ^[[:space:]]*# ]] && continue - - # Remove leading/trailing whitespace - pkg=$(echo "$pkg" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - - # Install package if not empty - if [ -n "$pkg" ]; then - install_package "$pkg" || return 1 - fi - done <"$deps_file" - else - # Fallback to hardcoded dependencies - install_package gcc make tar || return 1 - fi -} - -# List available kernels from S3 -list_kernels_from_s3() -{ - S3_PATH="s3://${RESULTS_BUCKET}/${RUN_PREFIX}/shared/kernel-rpms/binary/${ARCH}/" - aws s3 ls "${S3_PATH}" | grep "\.rpm$" | awk '{print $4}' -} - -# Download specific kernel RPM from S3 -download_kernel_rpm() -{ - if [ -z "${1:-}" ]; then - echo "ERROR: download_kernel_rpm requires kernel_name parameter" >&2 - return 1 - fi - local kernel_name="$1" - - S3_PATH="s3://${RESULTS_BUCKET}/${RUN_PREFIX}/shared/kernel-rpms/binary/${ARCH}/" - - mkdir -p "$KERNEL_RPM_DIR" - local local_path="${KERNEL_RPM_DIR}/${kernel_name}" - - # Download if not already present - if [ -f "$local_path" ]; then - echo "$local_path" - return 0 - fi - - if aws s3 cp "${S3_PATH}${kernel_name}" "$local_path" --no-progress >&2; then - echo "$local_path" - return 0 - else - echo "ERROR: Failed to download kernel" >&2 - return 1 - fi -} - -# Dump boot configuration for debugging kernel install issues -dump_boot_info() -{ - echo "=== Boot Debug Info ===" - echo "--- OS ---" - head -2 /etc/os-release 2>/dev/null || true - echo "--- Running kernel ---" - uname -r - echo "--- Installed kernel packages ---" - rpm -qa 'kernel*' | sort - echo "--- vmlinuz files in /boot ---" - ls -la /boot/vmlinuz-* 2>/dev/null || echo "(none)" - echo "--- BLS entries ---" - ls -la /boot/loader/entries/ 2>/dev/null || echo "(no BLS directory)" - echo "--- grubby default ---" - sudo grubby --default-kernel 2>/dev/null || echo "(grubby --default-kernel failed)" - echo "--- grubby --info=ALL ---" - sudo grubby --info=ALL 2>/dev/null || echo "(grubby --info=ALL failed)" - echo "=== End Boot Debug Info ===" -} - -# Install kernel RPM, make sure it's used as boot target -install_kernel_rpm() -{ - if [ -z "${1:-}" ]; then - echo "ERROR: install_kernel_rpm requires kernel_rpm parameter" >&2 - return 1 - fi - local kernel_rpm="$1" - - # Check architecture compatibility - local host_arch=$(uname -m) - local rpm_arch=$(rpm -qp --queryformat '%{ARCH}' "$kernel_rpm" 2>/dev/null) - - if [ "$rpm_arch" != "$host_arch" ]; then - echo "ERROR: Architecture mismatch - Host: $host_arch, RPM: $rpm_arch" >&2 - return 1 - fi - - echo "kernel before installation: $(uname -r)" - echo "Installing kernel from $kernel_rpm (arch: $rpm_arch)" - - if sudo yum localinstall -y "$kernel_rpm" 2>/dev/null || sudo dnf install -y "$kernel_rpm" 2>/dev/null; then - dump_boot_info - - # Set the newly installed kernel as default boot target. - # Without this, GRUB boots the newest kernel which may not be the one we just installed. - local installed_version - installed_version=$(rpm -qp --queryformat '%{VERSION}' "$kernel_rpm" 2>/dev/null) - - # Find the grubby entry matching the installed kernel version. - # Use grep || true to avoid ERR trap when no match is found. - local grub_kernel - grub_kernel=$(sudo grubby --info=ALL 2>/dev/null \ - | grep "^kernel=" \ - | grep "$installed_version" \ - | head -1 \ - | sed 's/^kernel=//' \ - | tr -d '"' \ - || true) - - if [ -z "$grub_kernel" ]; then - # Upstream make binrpm-pkg kernels don't register with grubby. - # Find the vmlinuz file and add a boot entry manually. - local vmlinuz - vmlinuz=$(ls /boot/vmlinuz-*"$installed_version"* 2>/dev/null | head -1) - if [ -n "$vmlinuz" ]; then - echo "Adding grubby entry for $vmlinuz" - local initrd="/boot/initramfs-${installed_version}.img" - if [ ! -f "$initrd" ]; then - echo "Generating initramfs at $initrd" - sudo dracut --force "$initrd" "$installed_version" 2>/dev/null \ - || sudo mkinitrd "$initrd" "$installed_version" 2>/dev/null \ - || true - fi - if [ -f "$initrd" ]; then - sudo grubby --add-kernel="$vmlinuz" \ - --initrd="$initrd" \ - --title="Linux $installed_version" \ - --copy-default \ - --make-default - echo "✓ Added and set default: $vmlinuz" - else - echo "WARNING: No initramfs for $installed_version, trying set-default anyway" - sudo grubby --set-default="$vmlinuz" || true - fi - grub_kernel="$vmlinuz" - else - echo "WARNING: No vmlinuz found for version $installed_version" - fi - else - echo "Setting default boot kernel to $grub_kernel" - sudo grubby --set-default="$grub_kernel" - fi - - if [ -n "$grub_kernel" ]; then - echo "Verifying default kernel:" - sudo grubby --default-kernel - fi - return 0 - else - echo "ERROR: Failed to install new kernel" >&2 - return 1 - fi -} - -# Return kernel RPM with lowest version (downloads from S3) -get_first_kernel_rpm_from_dir() -{ - local kernels=$(list_kernels_from_s3 | sort -V) - local first_kernel=$(echo "$kernels" | head -n 1) - - if [ -z "$first_kernel" ]; then - return 1 - fi - - download_kernel_rpm "$first_kernel" -} - -# Return kernel RPM with highest version (downloads from S3) -get_last_kernel_rpm_from_dir() -{ - local kernels=$(list_kernels_from_s3 | sort -V) - local last_kernel=$(echo "$kernels" | tail -n 1) - - if [ -z "$last_kernel" ]; then - return 1 - fi - - download_kernel_rpm "$last_kernel" -} - -# Install a given kernel RPM (passed as argument) -install_specified_kernel_rpm() -{ - local kernel_rpm="$1" - - if [ -z "$kernel_rpm" ]; then - echo "ERROR: install_specified_kernel_rpm requires a kernel RPM path" - return 1 - fi - - echo "Installing kernel RPM: $(basename "$kernel_rpm")" - install_kernel_rpm "$kernel_rpm" -} +# Common functions for the kernel reboot test. +# +# All kernel-management logic (environment validation, kernel RPM +# download/selection, install_kernel_rpm, reboot helpers) lives in the shared +# vm-tests/lib/kernel_helpers.sh, included here via the kernel_helpers.sh +# symlink in this directory. SOURCE_DIR is set by the run script before this +# file is sourced. +source "${SOURCE_DIR}/kernel_helpers.sh" diff --git a/vm-tests/example-kernel-reboot-test/kernel_helpers.sh b/vm-tests/example-kernel-reboot-test/kernel_helpers.sh new file mode 120000 index 0000000..31ff984 --- /dev/null +++ b/vm-tests/example-kernel-reboot-test/kernel_helpers.sh @@ -0,0 +1 @@ +../lib/kernel_helpers.sh \ No newline at end of file diff --git a/vm-tests/lib/kernel_helpers.sh b/vm-tests/lib/kernel_helpers.sh new file mode 100644 index 0000000..27bed26 --- /dev/null +++ b/vm-tests/lib/kernel_helpers.sh @@ -0,0 +1,257 @@ +# Authors: Norbert Manthey +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Shared kernel-management helpers for vm-tests that install and boot a kernel +# RPM from the pipeline's shared kernel-rpms area. +# +# Sourced by a test's common_lib.sh (which sets SOURCE_DIR first). It packages +# into each test payload via a symlink `kernel_helpers.sh -> ../lib/kernel_helpers.sh` +# in the test directory; the zip step stores the symlink target's content as a +# real file, so on the VM this is a normal file in the flat test dir. +# +# Fix once, benefit everywhere: the underscore/dash RPM-version handling and the +# FIPS-disable-before-reboot logic live here, so all kernel tests share them. + +# --------------------------------------------------------------------------- +# Results bucket and kernel paths from the pipeline environment. +RESULTS_BUCKET="${S3_BUCKET:-}" +ARCH=$(uname -m) +KERNEL_RPM_DIR="/tmp/kernel-rpms" +KERNEL_FILE="${SOURCE_DIR}/kernel_version_before.txt" + +# Validate required environment variables +if [ -z "$RESULTS_BUCKET" ] || [ -z "$RUN_PREFIX" ] || [ -z "$TEST_NAME" ]; then + echo "ERROR: Missing required environment variables (S3_BUCKET, RUN_PREFIX, TEST_NAME)" >&2 + exit 1 +fi + +get_running_kernel() +{ + uname -r +} + +save_kernel_version() +{ + local version="$1" + local out_file="$2" + if [ -z "$version" ] || [ -z "$out_file" ]; then + echo "ERROR: save_kernel_version requires version and file" >&2 + return 1 + fi + echo "$version" >"$out_file" +} + +load_kernel_version() +{ + local in_file="$1" + if [ ! -f "$in_file" ]; then + echo "ERROR: Kernel version file not found: $in_file" >&2 + return 1 + fi + cat "$in_file" +} + +assert_kernel_changed() +{ + local before="$1" + local after="$2" + if [ "$before" = "$after" ]; then + echo "ERROR: kernel version did not change (still $after)" >&2 + return 1 + fi + echo "Kernel version changed from $before to $after" +} + +# List available kernel RPMs from the shared S3 area. +list_kernels_from_s3() +{ + S3_PATH="s3://${RESULTS_BUCKET}/${RUN_PREFIX}/shared/kernel-rpms/binary/${ARCH}/" + aws s3 ls "${S3_PATH}" | grep "\.rpm$" | awk '{print $4}' +} + +# Download a specific kernel RPM from S3. +download_kernel_rpm() +{ + if [ -z "${1:-}" ]; then + echo "ERROR: download_kernel_rpm requires kernel_name parameter" >&2 + return 1 + fi + local kernel_name="$1" + S3_PATH="s3://${RESULTS_BUCKET}/${RUN_PREFIX}/shared/kernel-rpms/binary/${ARCH}/" + mkdir -p "$KERNEL_RPM_DIR" + local local_path="${KERNEL_RPM_DIR}/${kernel_name}" + if [ -f "$local_path" ]; then + echo "$local_path" + return 0 + fi + if aws s3 cp "${S3_PATH}${kernel_name}" "$local_path" --no-progress >&2; then + echo "$local_path" + return 0 + else + echo "ERROR: Failed to download kernel" >&2 + return 1 + fi +} + +# Error trap handler to show line where error occurred +error_trap() +{ + local exit_code=$? + local line_number=$1 + echo "$(date): ERROR: Script failed at line $line_number with exit code $exit_code" + echo "$(date): ERROR: Command that failed: $(sed -n "${line_number}p" "$0")" + exit $exit_code +} +trap 'error_trap $LINENO' ERR + +# Install a single given package +install_package() +{ + local pkg="$1" + local output + echo "Installing package $pkg ..." + if output=$(sudo yum install -y "$pkg" 2>&1) || output=$(sudo dnf install -y "$pkg" 2>&1); then + return 0 + else + echo "Failed to install package $pkg:" + echo "$output" + return 1 + fi +} + +# Install all dependencies for this test +install_test_dependencies() +{ + local deps_file="${SOURCE_DIR}/dependencies.txt" + if [ -f "$deps_file" ]; then + while IFS= read -r pkg || [ -n "$pkg" ]; do + [[ -z "$pkg" || "$pkg" =~ ^[[:space:]]*# ]] && continue + pkg=$(echo "$pkg" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + [ -n "$pkg" ] && { install_package "$pkg" || return 1; } + done <"$deps_file" + else + echo "ERROR: dependencies.txt not found" >&2 + return 1 + fi +} + +# List available kernels from S3, dump boot info, install a kernel RPM and make +# it the default boot target. +dump_boot_info() +{ + echo "=== Boot Debug Info ===" + echo "--- OS ---" + head -2 /etc/os-release 2>/dev/null || true + echo "--- Running kernel ---" + uname -r + echo "--- Installed kernel packages ---" + rpm -qa 'kernel*' | sort + echo "--- vmlinuz files in /boot ---" + ls -la /boot/vmlinuz-* 2>/dev/null || echo "(none)" + echo "--- grubby default ---" + sudo grubby --default-kernel 2>/dev/null || echo "(grubby --default-kernel failed)" + echo "=== End Boot Debug Info ===" +} + +install_kernel_rpm() +{ + if [ -z "${1:-}" ]; then + echo "ERROR: install_kernel_rpm requires kernel_rpm parameter" >&2 + return 1 + fi + local kernel_rpm="$1" + + local host_arch=$(uname -m) + local rpm_arch=$(rpm -qp --queryformat '%{ARCH}' "$kernel_rpm" 2>/dev/null) + if [ "$rpm_arch" != "$host_arch" ]; then + echo "ERROR: Architecture mismatch - Host: $host_arch, RPM: $rpm_arch" >&2 + return 1 + fi + + echo "kernel before installation: $(uname -r)" + echo "Installing kernel from $kernel_rpm (arch: $rpm_arch)" + + if sudo yum localinstall -y "$kernel_rpm" 2>/dev/null || sudo dnf install -y "$kernel_rpm" 2>/dev/null; then + dump_boot_info + local installed_version + installed_version=$(rpm -qp --queryformat '%{VERSION}' "$kernel_rpm" 2>/dev/null) + + local grub_kernel + grub_kernel=$(sudo grubby --info=ALL 2>/dev/null \ + | grep "^kernel=" \ + | grep "$installed_version" \ + | head -1 \ + | sed 's/^kernel=//' \ + | tr -d '"' \ + || true) + + if [ -z "$grub_kernel" ]; then + local vmlinuz + vmlinuz=$(ls /boot/vmlinuz-*"$installed_version"* 2>/dev/null | head -1) + if [ -n "$vmlinuz" ]; then + echo "Adding grubby entry for $vmlinuz" + local initrd="/boot/initramfs-${installed_version}.img" + if [ ! -f "$initrd" ]; then + echo "Generating initramfs at $initrd for kernel $installed_version" + sudo dracut --force "$initrd" "$installed_version" 2>/dev/null \ + || sudo mkinitrd "$initrd" "$installed_version" 2>/dev/null \ + || true + fi + if [ -f "$initrd" ]; then + sudo grubby --add-kernel="$vmlinuz" \ + --initrd="$initrd" \ + --title="Linux $installed_version" \ + --copy-default \ + --make-default + echo "Added and set default: $vmlinuz" + else + echo "WARNING: No initramfs for $installed_version, trying set-default anyway" + sudo grubby --set-default="$vmlinuz" || true + fi + grub_kernel="$vmlinuz" + else + echo "WARNING: No vmlinuz found for version $installed_version" + fi + else + echo "Setting default boot kernel to $grub_kernel" + sudo grubby --set-default="$grub_kernel" + fi + + if [ -n "$grub_kernel" ]; then + echo "Verifying default kernel:" + sudo grubby --default-kernel + fi + return 0 + else + echo "ERROR: Failed to install new kernel" >&2 + return 1 + fi +} + +get_first_kernel_rpm_from_dir() +{ + local kernels=$(list_kernels_from_s3 | sort -V) + local first_kernel=$(echo "$kernels" | head -n 1) + [ -z "$first_kernel" ] && return 1 + download_kernel_rpm "$first_kernel" +} + +get_last_kernel_rpm_from_dir() +{ + local kernels=$(list_kernels_from_s3 | sort -V) + local last_kernel=$(echo "$kernels" | tail -n 1) + [ -z "$last_kernel" ] && return 1 + download_kernel_rpm "$last_kernel" +} + +install_specified_kernel_rpm() +{ + local kernel_rpm="$1" + if [ -z "$kernel_rpm" ]; then + echo "ERROR: install_specified_kernel_rpm requires a kernel RPM path" >&2 + return 1 + fi + echo "Installing kernel RPM: $(basename "$kernel_rpm")" + install_kernel_rpm "$kernel_rpm" +} diff --git a/vm-tests/simple-source-reboot/common_lib.sh b/vm-tests/simple-source-reboot/common_lib.sh index 843816c..3e4cc59 100644 --- a/vm-tests/simple-source-reboot/common_lib.sh +++ b/vm-tests/simple-source-reboot/common_lib.sh @@ -2,112 +2,21 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -KERNEL_BENCH_DIR="kernel-bench" - -# Get results bucket and test paths from environment -RESULTS_BUCKET="${S3_BUCKET:-}" -ARCH=$(uname -m) -KERNEL_RPM_DIR="/tmp/kernel-rpms" -KERNEL_FILE="${SOURCE_DIR}/kernel_version_before.txt" - -# Validate required environment variables -if [ -z "$RESULTS_BUCKET" ] || [ -z "$RUN_PREFIX" ] || [ -z "$TEST_NAME" ]; then - echo "ERROR: Missing required environment variables (S3_BUCKET, RUN_PREFIX, TEST_NAME)" >&2 - exit 1 -fi - -# Error trap handler to show line where error occurred -error_trap() -{ - local exit_code=$? - local line_number=$1 - echo "$(date): ERROR: Script failed at line $line_number with exit code $exit_code" - echo "$(date): ERROR: Command that failed: $(sed -n "${line_number}p" "$0")" - exit $exit_code -} -trap 'error_trap $LINENO' ERR - -get_running_kernel() -{ - uname -r -} - -save_kernel_version() -{ - local version="$1" - local out_file="$2" - - if [ -z "$version" ] || [ -z "$out_file" ]; then - echo "ERROR: save_kernel_version requires version and file" - return 1 - fi - - echo "$version" >"$out_file" -} - -load_kernel_version() -{ - local in_file="$1" - - if [ ! -f "$in_file" ]; then - echo "ERROR: Kernel version file not found: $in_file" - return 1 - fi - - cat "$in_file" -} +# Common library for the simple source-build kernel reboot test. +# +# Binary-kernel install/reboot logic (environment validation, kernel RPM +# download/selection, install_kernel_rpm, reboot helpers) lives in the shared +# vm-tests/lib/kernel_helpers.sh, included via the kernel_helpers.sh symlink in +# this directory. Only the source-RPM build helpers stay here. SOURCE_DIR is +# set by the run script before this file is sourced. -assert_kernel_changed() -{ - local before="$1" - local after="$2" - - if [ "$before" = "$after" ]; then - echo "✗ FAILED: Kernel version did not change (still $after)" - return 1 - fi - - echo "✓ SUCCESS: Kernel version changed from $before to $after" -} - -# Install a single given package -install_package() -{ - local pkg="$1" - local output - echo "Installing package $pkg ..." - if output=$(sudo yum install -y "$pkg" 2>&1) || output=$(sudo dnf install -y "$pkg" 2>&1); then - return 0 - else - echo "Failed to install package $pkg:" - echo "$output" - return 1 - fi -} - -# Install all dependencies for this test -install_test_dependencies() -{ - local deps_file="${SOURCE_DIR}/dependencies.txt" +KERNEL_BENCH_DIR="kernel-bench" - if [ -f "$deps_file" ]; then - while IFS= read -r pkg || [ -n "$pkg" ]; do - # Skip empty lines and comments - [[ -z "$pkg" || "$pkg" =~ ^[[:space:]]*# ]] && continue +source "${SOURCE_DIR}/kernel_helpers.sh" - # Remove leading/trailing whitespace - pkg=$(echo "$pkg" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - - # Install package if not empty - if [ -n "$pkg" ]; then - install_package "$pkg" || return 1 - fi - done <"$deps_file" - else - # Fallback to hardcoded dependencies - install_package gcc make tar || return 1 - fi -} +# --------------------------------------------------------------------------- +# Source-RPM build helpers (specific to this test) +# --------------------------------------------------------------------------- # Install kernel source RPM (extracts source code to ~/rpmbuild/) install_source_kernel_rpm() @@ -168,156 +77,6 @@ build_kernel_rpm_src() fi } -# Dump boot configuration for debugging kernel install issues -dump_boot_info() -{ - echo "=== Boot Debug Info ===" - echo "--- OS ---" - head -2 /etc/os-release 2>/dev/null || true - echo "--- Running kernel ---" - uname -r - echo "--- Installed kernel packages ---" - rpm -qa 'kernel*' | sort - echo "--- vmlinuz files in /boot ---" - ls -la /boot/vmlinuz-* 2>/dev/null || echo "(none)" - echo "--- BLS entries ---" - ls -la /boot/loader/entries/ 2>/dev/null || echo "(no BLS directory)" - echo "--- grubby default ---" - sudo grubby --default-kernel 2>/dev/null || echo "(grubby --default-kernel failed)" - echo "--- grubby --info=ALL ---" - sudo grubby --info=ALL 2>/dev/null || echo "(grubby --info=ALL failed)" - echo "=== End Boot Debug Info ===" -} - -# Install binary kernel RPM -install_kernel_rpm() -{ - if [ -z "${1:-}" ]; then - echo "ERROR: install_kernel_rpm requires kernel_rpm parameter" >&2 - return 1 - fi - local kernel_rpm="$1" - - # Check it's a binary RPM (not source) - if [[ "$kernel_rpm" =~ \.src\.rpm$ ]]; then - echo "ERROR: This is a source RPM, not a binary RPM: $kernel_rpm" >&2 - return 1 - fi - - # Check architecture compatibility - local host_arch=$(uname -m) - local rpm_arch=$(rpm -qp --queryformat '%{ARCH}' "$kernel_rpm" 2>/dev/null) - - if [ "$rpm_arch" != "$host_arch" ]; then - echo "ERROR: Architecture mismatch - Host: $host_arch, RPM: $rpm_arch" >&2 - return 1 - fi - - echo "Installing binary kernel from $kernel_rpm (arch: $rpm_arch)" - - if sudo yum localinstall -y "$kernel_rpm" 2>/dev/null || sudo dnf install -y "$kernel_rpm" 2>/dev/null; then - dump_boot_info - - # Set the newly installed kernel as default boot target. - # Without this, GRUB boots the newest kernel which may not be the one we just installed. - local installed_version - installed_version=$(rpm -qp --queryformat '%{VERSION}' "$kernel_rpm" 2>/dev/null) - - # Find the grubby entry matching the installed kernel version. - # Use grep || true to avoid ERR trap when no match is found. - local grub_kernel - grub_kernel=$(sudo grubby --info=ALL 2>/dev/null \ - | grep "^kernel=" \ - | grep "$installed_version" \ - | head -1 \ - | sed 's/^kernel=//' \ - | tr -d '"' \ - || true) - - if [ -z "$grub_kernel" ]; then - # Upstream make binrpm-pkg kernels don't register with grubby. - # Find the vmlinuz file and add a boot entry manually. - local vmlinuz - vmlinuz=$(ls /boot/vmlinuz-*"$installed_version"* 2>/dev/null | head -1) - if [ -n "$vmlinuz" ]; then - echo "Adding grubby entry for $vmlinuz" - local initrd="/boot/initramfs-${installed_version}.img" - if [ ! -f "$initrd" ]; then - echo "Generating initramfs at $initrd" - sudo dracut --force "$initrd" "$installed_version" 2>/dev/null \ - || sudo mkinitrd "$initrd" "$installed_version" 2>/dev/null \ - || true - fi - if [ -f "$initrd" ]; then - sudo grubby --add-kernel="$vmlinuz" \ - --initrd="$initrd" \ - --title="Linux $installed_version" \ - --copy-default \ - --make-default - echo "✓ Added and set default: $vmlinuz" - else - echo "WARNING: No initramfs for $installed_version, trying set-default anyway" - sudo grubby --set-default="$vmlinuz" || true - fi - grub_kernel="$vmlinuz" - else - echo "WARNING: No vmlinuz found for version $installed_version" - fi - else - echo "Setting default boot kernel to $grub_kernel" - sudo grubby --set-default="$grub_kernel" - fi - - if [ -n "$grub_kernel" ]; then - echo "Verifying default kernel:" - sudo grubby --default-kernel - fi - echo "✓ Kernel installed successfully" - return 0 - else - echo "ERROR: Failed to install kernel" >&2 - return 1 - fi -} - -##### GET SRC KERNEL FROM S3 AND DOWNLOAD TO LOCAL MACHINE! -# List available kernels from S3 -list_kernels_from_s3() -{ - S3_PATH="s3://${RESULTS_BUCKET}/${RUN_PREFIX}/shared/kernel-rpms/src/" - aws s3 ls "${S3_PATH}" | grep "\.rpm$" | awk '{print $4}' -} - -# Download specific kernel RPM from S3 -download_kernel_rpm() -{ - if [ -z "${1:-}" ]; then - echo "ERROR: download_kernel_rpm requires kernel_name parameter" >&2 - return 1 - fi - local kernel_name="$1" - - S3_PATH="s3://${RESULTS_BUCKET}/${RUN_PREFIX}/shared/kernel-rpms/src/" - - mkdir -p "$KERNEL_RPM_DIR" - local local_path="${KERNEL_RPM_DIR}/${kernel_name}" - - # Download if not already present - if [ -f "$local_path" ]; then - echo "$local_path" - return 0 - fi - - if aws s3 cp "${S3_PATH}${kernel_name}" "$local_path" --no-progress >&2; then - echo "$local_path" - return 0 - else - echo "ERROR: Failed to download kernel" >&2 - return 1 - fi -} - -# Return kernel RPM with lowest version (downloads from S3) get_first_source_kernel_rpm_from_dir() { local kernels=$(list_kernels_from_s3 | sort -V) diff --git a/vm-tests/simple-source-reboot/kernel_helpers.sh b/vm-tests/simple-source-reboot/kernel_helpers.sh new file mode 120000 index 0000000..31ff984 --- /dev/null +++ b/vm-tests/simple-source-reboot/kernel_helpers.sh @@ -0,0 +1 @@ +../lib/kernel_helpers.sh \ No newline at end of file diff --git a/vm-tests/unixbench-kernel-regression/common_lib.sh b/vm-tests/unixbench-kernel-regression/common_lib.sh index 5e161c8..c52f68c 100644 --- a/vm-tests/unixbench-kernel-regression/common_lib.sh +++ b/vm-tests/unixbench-kernel-regression/common_lib.sh @@ -2,152 +2,25 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -# Lib with functions required in multiple test steps +# Common library for the UnixBench kernel A/B regression test. +# +# Kernel-management logic (environment validation, kernel RPM +# download/selection, install_kernel_rpm, reboot helpers) lives in the shared +# vm-tests/lib/kernel_helpers.sh, included via the kernel_helpers.sh symlink in +# this directory. Only the UnixBench-specific pieces stay here. SOURCE_DIR is +# set by the run script before this file is sourced. UNIXBENCH_VERSION=byte-unixbench-6.0.0 UNIXBENCH_TAR_FILE="$UNIXBENCH_VERSION.tar.gz" KERNEL_BENCH_DIR="kernel-bench" -# Get results bucket and test paths from environment -RESULTS_BUCKET="${S3_BUCKET:-}" -ARCH=$(uname -m) -KERNEL_RPM_DIR="/tmp/kernel-rpms" -KERNEL_FILE="${SOURCE_DIR}/kernel_version_before.txt" +source "${SOURCE_DIR}/kernel_helpers.sh" -# Validate required environment variables -if [ -z "$RESULTS_BUCKET" ] || [ -z "$RUN_PREFIX" ] || [ -z "$TEST_NAME" ]; then - echo "ERROR: Missing required environment variables (S3_BUCKET, RUN_PREFIX, TEST_NAME)" >&2 - exit 1 -fi +# --------------------------------------------------------------------------- +# UnixBench-specific helpers +# --------------------------------------------------------------------------- -get_running_kernel() -{ - uname -r -} - -save_kernel_version() -{ - local version="$1" - local out_file="$2" - - if [ -z "$version" ] || [ -z "$out_file" ]; then - echo "ERROR: save_kernel_version requires version and file" - return 1 - fi - - echo "$version" >"$out_file" -} - -load_kernel_version() -{ - local in_file="$1" - - if [ ! -f "$in_file" ]; then - echo "ERROR: Kernel version file not found: $in_file" - return 1 - fi - - cat "$in_file" -} - -assert_kernel_changed() -{ - local before="$1" - local after="$2" - - if [ "$before" = "$after" ]; then - echo "✗ FAILED: Kernel version did not change (still $after)" - return 1 - fi - - echo "✓ SUCCESS: Kernel version changed from $before to $after" -} - -# List available kernels from S3 -list_kernels_from_s3() -{ - S3_PATH="s3://${RESULTS_BUCKET}/${RUN_PREFIX}/shared/kernel-rpms/binary/${ARCH}/" - aws s3 ls "${S3_PATH}" | grep "\.rpm$" | awk '{print $4}' -} - -# Download specific kernel RPM from S3 -download_kernel_rpm() -{ - if [ -z "${1:-}" ]; then - echo "ERROR: download_kernel_rpm requires kernel_name parameter" >&2 - return 1 - fi - local kernel_name="$1" - - S3_PATH="s3://${RESULTS_BUCKET}/${RUN_PREFIX}/shared/kernel-rpms/binary/${ARCH}/" - - mkdir -p "$KERNEL_RPM_DIR" - local local_path="${KERNEL_RPM_DIR}/${kernel_name}" - - # Download if not already present - if [ -f "$local_path" ]; then - echo "$local_path" - return 0 - fi - - if aws s3 cp "${S3_PATH}${kernel_name}" "$local_path" --no-progress >&2; then - echo "$local_path" - return 0 - else - echo "ERROR: Failed to download kernel" >&2 - return 1 - fi -} - -# Error trap handler to show line where error occurred -error_trap() -{ - local exit_code=$? - local line_number=$1 - echo "$(date): ERROR: Script failed at line $line_number with exit code $exit_code" - echo "$(date): ERROR: Command that failed: $(sed -n "${line_number}p" "$0")" - exit $exit_code -} -trap 'error_trap $LINENO' ERR - -# Install a single given package -install_package() -{ - local pkg="$1" - local output - echo "Installing package $pkg ..." - if output=$(sudo yum install -y "$pkg" 2>&1) || output=$(sudo dnf install -y "$pkg" 2>&1); then - return 0 - else - echo "Failed to install package $pkg:" - echo "$output" - return 1 - fi -} - -# Install all dependencies for this test -install_test_dependencies() -{ - local deps_file="${SOURCE_DIR}/dependencies.txt" - - if [ -f "$deps_file" ]; then - while IFS= read -r pkg || [ -n "$pkg" ]; do - # Skip empty lines and comments - [[ -z "$pkg" || "$pkg" =~ ^[[:space:]]*# ]] && continue - - # Remove leading/trailing whitespace - pkg=$(echo "$pkg" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - - # Install package if not empty - if [ -n "$pkg" ]; then - install_package "$pkg" || return 1 - fi - done <"$deps_file" - else - # Fallback to hardcoded dependencies - install_package gcc make tar || return 1 - fi -} +# Extract unixbench # Extract unixbench prepare_unixbench() @@ -249,156 +122,3 @@ summarize_unixbench_log() # Skip index section entirely - do not parse it ' "$unixbench_log" >>"$output_csv_file" } - -# Dump boot configuration for debugging kernel install issues -dump_boot_info() -{ - echo "=== Boot Debug Info ===" - echo "--- OS ---" - head -2 /etc/os-release 2>/dev/null || true - echo "--- Running kernel ---" - uname -r - echo "--- Installed kernel packages ---" - rpm -qa 'kernel*' | sort - echo "--- vmlinuz files in /boot ---" - ls -la /boot/vmlinuz-* 2>/dev/null || echo "(none)" - echo "--- BLS entries ---" - ls -la /boot/loader/entries/ 2>/dev/null || echo "(no BLS directory)" - echo "--- grubby default ---" - sudo grubby --default-kernel 2>/dev/null || echo "(grubby --default-kernel failed)" - echo "--- grubby --info=ALL ---" - sudo grubby --info=ALL 2>/dev/null || echo "(grubby --info=ALL failed)" - echo "=== End Boot Debug Info ===" -} - -# Install current kernel RPM, make sure it's used as boot target -install_kernel_rpm() -{ - if [ -z "${1:-}" ]; then - echo "ERROR: install_kernel_rpm requires kernel_rpm parameter" >&2 - return 1 - fi - local kernel_rpm="$1" - - # Check architecture compatibility - local host_arch=$(uname -m) - local rpm_arch=$(rpm -qp --queryformat '%{ARCH}' "$kernel_rpm" 2>/dev/null) - - if [ "$rpm_arch" != "$host_arch" ]; then - echo "ERROR: Architecture mismatch - Host: $host_arch, RPM: $rpm_arch" >&2 - return 1 - fi - - echo "kernel before installation: $(uname -r)" - echo "Installing kernel from $kernel_rpm (arch: $rpm_arch)" - - if sudo yum localinstall -y "$kernel_rpm" 2>/dev/null || sudo dnf install -y "$kernel_rpm" 2>/dev/null; then - dump_boot_info - - # Set the newly installed kernel as default boot target. - # Without this, GRUB boots the newest kernel which may not be the one we just installed. - local installed_version - installed_version=$(rpm -qp --queryformat '%{VERSION}' "$kernel_rpm" 2>/dev/null) - - # Find the grubby entry matching the installed kernel version. - # Use grep || true to avoid ERR trap when no match is found. - local grub_kernel - grub_kernel=$(sudo grubby --info=ALL 2>/dev/null \ - | grep "^kernel=" \ - | grep "$installed_version" \ - | head -1 \ - | sed 's/^kernel=//' \ - | tr -d '"' \ - || true) - - if [ -z "$grub_kernel" ]; then - # Upstream make binrpm-pkg kernels don't register with grubby. - # Find the vmlinuz file and add a boot entry manually. - local vmlinuz - vmlinuz=$(ls /boot/vmlinuz-*"$installed_version"* 2>/dev/null | head -1) - if [ -n "$vmlinuz" ]; then - echo "Adding grubby entry for $vmlinuz" - # Copy initrd and args from the current default entry - local default_kernel - default_kernel=$(sudo grubby --default-kernel) - local default_initrd - default_initrd=$(sudo grubby --info="$default_kernel" 2>/dev/null \ - | grep "^initrd=" | sed 's/^initrd=//' | tr -d '"' || true) - local initrd="/boot/initramfs-${installed_version}.img" - # Generate initramfs if it doesn't exist - if [ ! -f "$initrd" ]; then - echo "Generating initramfs at $initrd" - sudo dracut --force "$initrd" "$installed_version" 2>/dev/null \ - || sudo mkinitrd "$initrd" "$installed_version" 2>/dev/null \ - || true - fi - if [ -f "$initrd" ]; then - sudo grubby --add-kernel="$vmlinuz" \ - --initrd="$initrd" \ - --title="Linux $installed_version" \ - --copy-default \ - --make-default - echo "✓ Added and set default: $vmlinuz" - else - echo "WARNING: No initramfs for $installed_version, trying set-default anyway" - sudo grubby --set-default="$vmlinuz" || true - fi - grub_kernel="$vmlinuz" - else - echo "WARNING: No vmlinuz found for version $installed_version" - fi - else - echo "Setting default boot kernel to $grub_kernel" - sudo grubby --set-default="$grub_kernel" - fi - - if [ -n "$grub_kernel" ]; then - echo "Verifying default kernel:" - sudo grubby --default-kernel - fi - return 0 - else - echo "ERROR: Failed to install new kernel" >&2 - return 1 - fi -} - -# Return kernel RPM with lowest version (downloads from S3) -get_first_kernel_rpm_from_dir() -{ - local kernels=$(list_kernels_from_s3 | sort -V) - local first_kernel=$(echo "$kernels" | head -n 1) - - if [ -z "$first_kernel" ]; then - return 1 - fi - - download_kernel_rpm "$first_kernel" -} - -# Return kernel RPM with highest version (downloads from S3) -get_last_kernel_rpm_from_dir() -{ - local kernels=$(list_kernels_from_s3 | sort -V) - local last_kernel=$(echo "$kernels" | tail -n 1) - - if [ -z "$last_kernel" ]; then - return 1 - fi - - download_kernel_rpm "$last_kernel" -} - -# Install a given kernel RPM (passed as argument) -install_specified_kernel_rpm() -{ - local kernel_rpm="$1" - - if [ -z "$kernel_rpm" ]; then - echo "ERROR: install_specified_kernel_rpm requires a kernel RPM path" - return 1 - fi - - echo "Installing kernel RPM: $(basename "$kernel_rpm")" - install_kernel_rpm "$kernel_rpm" -} diff --git a/vm-tests/unixbench-kernel-regression/kernel_helpers.sh b/vm-tests/unixbench-kernel-regression/kernel_helpers.sh new file mode 120000 index 0000000..31ff984 --- /dev/null +++ b/vm-tests/unixbench-kernel-regression/kernel_helpers.sh @@ -0,0 +1 @@ +../lib/kernel_helpers.sh \ No newline at end of file From d057de0dd8dc53c40b70c0061f361ddbe089709b Mon Sep 17 00:00:00 2001 From: Norbert Manthey Date: Tue, 18 Aug 2026 13:35:20 +0200 Subject: [PATCH 5/7] fix: handle underscore/dash mismatch in kernel RPM version lookup A kernel built with make binrpm-pkg and LOCALVERSION=-nogup has an RPM VERSION of 6.18.41_nogup (underscore) but installs vmlinuz-6.18.41-nogup (dash). The vmlinuz lookup missed the file because it used the RPM VERSION verbatim. Compute an alternate version string with underscores replaced by dashes, try both in the grubby --info and vmlinuz globs, and derive the kernel version for dracut/initramfs from the actual vmlinuz filename. Signed-off-by: Norbert Manthey --- vm-tests/lib/kernel_helpers.sh | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/vm-tests/lib/kernel_helpers.sh b/vm-tests/lib/kernel_helpers.sh index 27bed26..cd9056d 100644 --- a/vm-tests/lib/kernel_helpers.sh +++ b/vm-tests/lib/kernel_helpers.sh @@ -176,11 +176,14 @@ install_kernel_rpm() dump_boot_info local installed_version installed_version=$(rpm -qp --queryformat '%{VERSION}' "$kernel_rpm" 2>/dev/null) + # RPM VERSION may use underscores (e.g. 6.18.41_nogup) while the kernel + # LOCALVERSION uses dashes (vmlinuz-6.18.41-nogup). Try both variants. + local installed_version_alt="${installed_version//_/-}" local grub_kernel grub_kernel=$(sudo grubby --info=ALL 2>/dev/null \ | grep "^kernel=" \ - | grep "$installed_version" \ + | grep -E "$installed_version|$installed_version_alt" \ | head -1 \ | sed 's/^kernel=//' \ | tr -d '"' \ @@ -189,24 +192,29 @@ install_kernel_rpm() if [ -z "$grub_kernel" ]; then local vmlinuz vmlinuz=$(ls /boot/vmlinuz-*"$installed_version"* 2>/dev/null | head -1) + if [ -z "$vmlinuz" ] && [ "$installed_version_alt" != "$installed_version" ]; then + vmlinuz=$(ls /boot/vmlinuz-*"$installed_version_alt"* 2>/dev/null | head -1) + fi if [ -n "$vmlinuz" ]; then echo "Adding grubby entry for $vmlinuz" - local initrd="/boot/initramfs-${installed_version}.img" + # Derive the kernel version from the vmlinuz filename + local kver="${vmlinuz#/boot/vmlinuz-}" + local initrd="/boot/initramfs-${kver}.img" if [ ! -f "$initrd" ]; then - echo "Generating initramfs at $initrd for kernel $installed_version" - sudo dracut --force "$initrd" "$installed_version" 2>/dev/null \ - || sudo mkinitrd "$initrd" "$installed_version" 2>/dev/null \ + echo "Generating initramfs at $initrd for kernel $kver" + sudo dracut --force "$initrd" "$kver" 2>/dev/null \ + || sudo mkinitrd "$initrd" "$kver" 2>/dev/null \ || true fi if [ -f "$initrd" ]; then sudo grubby --add-kernel="$vmlinuz" \ --initrd="$initrd" \ - --title="Linux $installed_version" \ + --title="Linux $kver" \ --copy-default \ --make-default echo "Added and set default: $vmlinuz" else - echo "WARNING: No initramfs for $installed_version, trying set-default anyway" + echo "WARNING: No initramfs for $kver, trying set-default anyway" sudo grubby --set-default="$vmlinuz" || true fi grub_kernel="$vmlinuz" From b2665042e4444731f3d89d165d052a6f208bc18f Mon Sep 17 00:00:00 2001 From: Norbert Manthey Date: Tue, 18 Aug 2026 13:35:30 +0200 Subject: [PATCH 6/7] fix: disable FIPS mode before booting a custom kernel AL2023 enables FIPS by default. A custom kernel built with make binrpm-pkg carries unsigned modules (e.g. ghash_clmulni_intel) that fail FIPS signature verification, causing a kernel panic reboot loop. Add fips=0 to the grubby boot-entry args and run fips-mode-setup --disable after installing the kernel, before the reboot, so unsigned modules load without panic. Signed-off-by: Norbert Manthey --- vm-tests/lib/kernel_helpers.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vm-tests/lib/kernel_helpers.sh b/vm-tests/lib/kernel_helpers.sh index cd9056d..59d6893 100644 --- a/vm-tests/lib/kernel_helpers.sh +++ b/vm-tests/lib/kernel_helpers.sh @@ -210,6 +210,7 @@ install_kernel_rpm() sudo grubby --add-kernel="$vmlinuz" \ --initrd="$initrd" \ --title="Linux $kver" \ + --args="fips=0" \ --copy-default \ --make-default echo "Added and set default: $vmlinuz" @@ -230,6 +231,15 @@ install_kernel_rpm() echo "Verifying default kernel:" sudo grubby --default-kernel fi + + # Disable FIPS mode system-wide before rebooting into a custom kernel. + # Some AL2023 enable FIPS; unsigned modules (from make binrpm-pkg) + # fail signature verification and cause a kernel panic. + if command -v fips-mode-setup &>/dev/null; then + echo "Disabling FIPS mode for custom kernel boot" + sudo fips-mode-setup --disable 2>/dev/null || true + fi + return 0 else echo "ERROR: Failed to install new kernel" >&2 From a34a2172cce43a9575a16c0807785d41d770d41f Mon Sep 17 00:00:00 2001 From: Norbert Manthey Date: Tue, 18 Aug 2026 13:35:42 +0200 Subject: [PATCH 7/7] fix: install a cross-series kernel RPM with dnf --allowerasing On an AL2023 AMI whose default kernel is a different series than the RPM under test (e.g. a 6.18 AMI installing a 6.1 kernel), the distro kernel-tools package declares 'conflicts with kernel-uname-r < ', so a plain dnf/yum install is refused with 'conflicting requests'. Verified on a live 6.18 AMI: plain install fails, but 'dnf install --allowerasing' removes the conflicting kernel-tools package and installs the requested kernel; both vmlinuz files remain in /boot so the target kernel boots normally. Add --allowerasing as the final fallback in install_kernel_rpm, making the kernel A/B tests robust to base-AMI kernel-series drift. Signed-off-by: Norbert Manthey --- vm-tests/lib/kernel_helpers.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/vm-tests/lib/kernel_helpers.sh b/vm-tests/lib/kernel_helpers.sh index 59d6893..68e5759 100644 --- a/vm-tests/lib/kernel_helpers.sh +++ b/vm-tests/lib/kernel_helpers.sh @@ -172,7 +172,15 @@ install_kernel_rpm() echo "kernel before installation: $(uname -r)" echo "Installing kernel from $kernel_rpm (arch: $rpm_arch)" - if sudo yum localinstall -y "$kernel_rpm" 2>/dev/null || sudo dnf install -y "$kernel_rpm" 2>/dev/null; then + # Install the kernel RPM. On an AMI whose default kernel is a different + # series (e.g. a 6.18 AMI when installing a 6.1 kernel), the distro + # kernel-tools package declares "conflicts with kernel-uname-r < ", + # so a plain install is refused. Fall back to --allowerasing, which + # removes the conflicting tools package and installs the requested kernel + # (both vmlinuz files remain in /boot, so the target kernel can be booted). + if sudo dnf install -y "$kernel_rpm" 2>/dev/null \ + || sudo yum localinstall -y "$kernel_rpm" 2>/dev/null \ + || sudo dnf install -y --allowerasing "$kernel_rpm" 2>/dev/null; then dump_boot_info local installed_version installed_version=$(rpm -qp --queryformat '%{VERSION}' "$kernel_rpm" 2>/dev/null)