test(monorepo): adds adhoc package selection for CI system tests

.kokoro/system.sh

@@ -161,6 +161,8 @@ reap_parallel_results() {
     fi
   done
 
+
+
   if [ "$failed_count" -gt 0 ]; then
     echo "=================================================="
     echo "@FAILED - DETAILED LOGS FOR FAILED PACKAGES"
@@ -183,6 +185,7 @@ reap_parallel_results() {
           cat "$LOG_DIR/$pkg.log"
         else
           echo "Warning: No log file found for failed package $pkg"
+
         fi
         echo ""
       fi
@@ -276,6 +279,14 @@ for path in `find 'packages' \
   fi
 done
 
+# --- Ad-hoc Testing Integration ---
+# If this is a Pull Request, check for the 'test:adhoc' label and override packages if active.
+# This fails open and is isolated to minimize impact on production.
+if [[ -n "${KOKORO_GITHUB_PULL_REQUEST_NUMBER}" ]] && [[ -f "ci/adhoc/setup_adhoc.sh" ]]; then
+    source ci/adhoc/setup_adhoc.sh
+fi
+# --- End Ad-hoc Testing Integration ---
+
 # Parallel Execution Logic
 MAX_JOBS=${MAX_JOBS:-4}
 
@@ -301,18 +312,20 @@ export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR
 # Stream package names to xargs for parallel execution
 # -P "$MAX_JOBS" controls concurrency
 # -I {} replaces {} with the package name
-printf '%s\n' "${PACKAGES_TO_TEST[@]}" \
-  | xargs -n 1 -P "$MAX_JOBS" \
+[ ${#PACKAGES_TO_TEST[@]} -eq 0 ] || printf '%s\0' "${PACKAGES_TO_TEST[@]}" \
+  | xargs -0 -n 1 -P "$MAX_JOBS" \
     bash -c '
       pkg="$0"
+
       # Determine log location: prefer Sponge artifacts directory if available
       if [ -n "$KOKORO_ARTIFACTS_DIR" ]; then
         pkg_log_dir="$KOKORO_ARTIFACTS_DIR/$pkg"
-        mkdir -p "$pkg_log_dir" || { touch "$LOG_DIR/$pkg.failed"; exit 1; }
+        mkdir -p "$pkg_log_dir" || { echo "Failed to mkdir $pkg_log_dir"; touch "$LOG_DIR/$pkg.failed"; exit 1; }
         log_file="$pkg_log_dir/sponge_log.log"
       else
         log_file="$LOG_DIR/$pkg.log"
       fi
+      echo "Log file for $pkg: $log_file"
 
       # Run test; if it fails, create a .failed file to signal failure to the reaper
       run_package_test "$pkg" > "$log_file" 2>&1 || touch "$LOG_DIR/$pkg.failed"

ci/adhoc/.package_groups.txt

@@ -0,0 +1,4 @@
+handwritten: google-cloud-translate
+handwritten: google-cloud-logging
+core: google-api-core
+core: google-cloud-core

ci/adhoc/.standalone_package_list.txt

@@ -0,0 +1,3 @@
+package: google-cloud-logging
+package: google-cloud-dns
+group: handwritten

ci/adhoc/README.md

@@ -0,0 +1,47 @@
+# Ad-Hoc Package Testing
+
+## Overview
+Ad-hoc package testing allows you to run CI tests for a specific subset of packages or predefined package groups without the need for intrusive and/or temporary mods to the package code to trigger a CI job run. Key use cases include:
+
+* **Downstream Dependency Smoke Tests:** If you update a core library (like `google-api-core`), the diff detector only sees the core library. Ad-hoc lets you explicitly include major downstream consumers (like `storage`) to verify compatibility.
+* **Debugging specific package failures:** If you want to look at just one OR two failing packages out of a larger group of failing packages, it can be helpful to run them in isolation in a separate PR (so that your prospective changes don't have to wait on all the other packages). This allows you to easily flag which packages you want to investigate by potentially starting with a baseline test with no changes (i.e. does this fail due to an externality OR due to a change in the code)?
+* **Testing CI infrastructure updates:** If you are changing `.kokoro/system.sh` or root scripts, the standard diff detector won't trigger tests because no package folders changed. Ad-hoc allows you to test your CI scripts using a single lightweight package without polluting package code with dummy comments.
+
+## How It Works
+The ad-hoc testing system reads configuration files in the `ci/adhoc/` directory to determine which packages to test. It is triggered via the `test:adhoc` GitHub label on Pull Requests.
+
+When triggered, the ad-hoc selected packages are **merged** with any packages automatically detected by the CI system (e.g., packages modified in the current PR). The final combined list is automatically deduplicated, ensuring each package is tested only once.
+
+## Configuration Files
+
+These files are located in the `ci/adhoc/` directory.
+
+### 1. `.standalone_package_list.txt`
+This file lists the specific packages or groups you want to test.
+
+*   **To test an individual package:** Add a line starting with `package: ` (be sure to include the colon and space) followed by the package directory name.
+    *   *Example:* `package: google-cloud-dns`
+*   **To test a group of packages:** Add a line starting with `group: ` (be sure to include the colon and space) followed by the group name. NOTE: groups are defined in the file: `.package_groups.txt`
+    *   *Example:* `group: handwritten`
+
+### 2. `.package_groups.txt`
+This file defines groups of commonly tested packages for convenience of the team. Groups such as all handwritten, all core, all hybrids, most widely used, etc. can be defined here.
+
+*   **Format:** Each package in a group should be on its own line, prefixed by the group name, colon, and a space.
+    *   *Example:*
+        ```text
+        handwritten: google-cloud-translate
+        handwritten: google-cloud-logging
+        core: google-api-core
+        ```
+
+#### 💡 Pro Tip
+You can mix packages and groups in `.standalone_package_list.txt`. The system will automatically expand groups and deduplicate the list!
+
+## Usage
+
+1.  **Edit Configuration:** Open `ci/adhoc/.standalone_package_list.txt` and add the packages or groups you want to test.
+2.  **Trigger Tests:**
+    *   **New PR:** Commit the changes and open a Pull Request form.
+    *   **Activate Label:** Add the `test:adhoc` label to your PR form in the GitHub UI. If you miss this step, simply applying the label won't magically launch the tests the way `kokoro-force-run` does. The label is only checked when a commit is detected.
+    *   **Existing PR:** Commit and push the changes to your branch. If the label is already present, pushing a new commit will trigger the tests.

ci/adhoc/adhoc_test_runner.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Script to determine ad-hoc packages to test.
+# This script is intended to be sourced from main test scripts.
+#
+# Precondition: This script assumes it is sourced from the project root (as set by system.sh).
+
+ADHOC_DIR="ci/adhoc"
+STANDALONE_LIST="${ADHOC_DIR}/.standalone_package_list.txt"
+GROUPS_FILE="${ADHOC_DIR}/.package_groups.txt"
+
+if [[ ! -f "$STANDALONE_LIST" ]]; then
+    echo "Warning: $STANDALONE_LIST not found."
+    return 0 2>/dev/null || exit 0
+fi
+
+if [[ ! -f "$GROUPS_FILE" ]]; then
+    echo "Warning: $GROUPS_FILE not found."
+    return 0 2>/dev/null || exit 0
+fi
+
+# Grab individual packages
+adhoc_packages=$(grep "^package:" "$STANDALONE_LIST" | cut -d':' -f2 | tr -d '\r' | xargs || true)
+
+# Grab requested groups
+requested_groups=$(grep "^group:" "$STANDALONE_LIST" | cut -d':' -f2 | tr -d '\r' | xargs || true)
+
+# Expand groups
+for group in $requested_groups; do
+    group_pkgs=$(grep "^$group:" "$GROUPS_FILE" | cut -d':' -f2 | tr -d '\r' | xargs || true)
+    adhoc_packages="$adhoc_packages $group_pkgs"
+done
+
+# Convert to unique list (deduplicate our adhoc packages)
+ADHOC_PACKAGES=$(echo "$adhoc_packages" | tr ' ' '\n' | sort -u | xargs)
+
+export ADHOC_PACKAGES

ci/adhoc/setup_adhoc.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# ci/adhoc/setup_adhoc.sh
+# This script is sourced by system.sh to check for ad-hoc status and update PACKAGES_TO_TEST.
+# It fails open and is isolated to minimize impact on production.
+
+TRIGGER_ADHOC="false"
+
+echo "Checking for adhoc test label on PR #${KOKORO_GITHUB_PULL_REQUEST_NUMBER}..."
+# Simple, unauthenticated call to check labels (capturing headers for rate limit debugging)
+CURL_OUTPUT=$(curl -s -i -H "User-Agent: Kokoro-AdHoc-Checker" "https://api.github.com/repos/googleapis/google-cloud-python/issues/${KOKORO_GITHUB_PULL_REQUEST_NUMBER}/labels" || true)
+
+# Separate headers and body
+# Using awk to find the blank line separating headers from body
+HEADERS=$(echo "$CURL_OUTPUT" | awk 'BEGIN{RS="\r\n\r\n"} NR==1' || true)
+LABELS_JSON=$(echo "$CURL_OUTPUT" | awk 'BEGIN{RS="\r\n\r\n"} NR==2' || true)
+
+# --- DEBUGGING: Extract and print Rate Limit Headers ---
+echo "---------------------------------------------------------------"
+echo "DEBUG: GitHub API Rate Limit Status:"
+echo "$HEADERS" | grep -i "^x-ratelimit-limit:" | tr -d '\r' || true
+echo "$HEADERS" | grep -i "^x-ratelimit-remaining:" | tr -d '\r' || true
+echo "$HEADERS" | grep -i "^x-ratelimit-used:" | tr -d '\r' || true
+
+# Convert reset time to human-readable if date is available, otherwise show raw
+RESET_EPOCH=$(echo "$HEADERS" | grep -i "^x-ratelimit-reset:" | awk '{print $2}' | tr -d '\r' || true)
+if [[ -n "$RESET_EPOCH" ]]; then
+    if date -d "@$RESET_EPOCH" >/dev/null 2>&1; then
+        RESET_HUMAN=$(date -d "@$RESET_EPOCH" "+%Y-%m-%d %H:%M:%S UTC")
+        echo "x-ratelimit-reset: $RESET_HUMAN ($RESET_EPOCH)" 
+    else
+        echo "x-ratelimit-reset: $RESET_EPOCH"
+    fi
+fi
+echo "---------------------------------------------------------------"
+
+if [[ -z "$LABELS_JSON" ]] || [[ "$HEADERS" == *"HTTP/1.1 4"* ]] || [[ "$HEADERS" == *"HTTP/1.1 5"* ]]; then
+    echo "==============================================================="
+    echo "WARNING: Failed to fetch labels or received error from GitHub API!"
+    echo "Ad-hoc tests will NOT be triggered."
+    echo "==============================================================="
+else
+    # Use jq to parse github labels (works as long as jq is available in python-multi image).
+    IS_ADHOC=$(echo "$LABELS_JSON" | jq -r 'if type == "array" then any(.name == "test:adhoc") else false end' 2>/dev/null || echo "false")
+
+    if [[ "$IS_ADHOC" == "true" ]]; then
+        TRIGGER_ADHOC="true"
+        echo "Adhoc test label 'test:adhoc' found!"
+    else
+        if [[ "$LABELS_JSON" != "["* ]]; then
+            API_ERR_MSG=$(echo "$LABELS_JSON" | jq -r '.message // "Unknown error"' 2>/dev/null || echo "Unknown error")
+            echo "==============================================================="
+            echo "WARNING: Failed to fetch PR labels from GitHub API!"
+            echo "Error Message: $API_ERR_MSG"
+            echo "This might be due to API Rate Limiting."
+            echo "Ad-hoc tests will NOT be triggered."
+            echo "==============================================================="
+        else
+            echo "Adhoc test label 'test:adhoc' not found."
+        fi
+    fi
+fi
+
+if [[ "$TRIGGER_ADHOC" == "true" ]]; then
+    echo "Running ad-hoc package selection..."
+    # Ensure the runner script exists before sourcing
+    if [[ -f "ci/adhoc/adhoc_test_runner.sh" ]]; then
+        source ci/adhoc/adhoc_test_runner.sh
+
+        echo "Deduplicating packages..."
+        # Deduplication using Associative Arrays (Requires Bash 4+)
+        declare -A unique_packages
+        for pkg in "${PACKAGES_TO_TEST[@]}"; do
+            [[ -n "$pkg" ]] && unique_packages["$pkg"]=1
+        done
+        for pkg in $ADHOC_PACKAGES; do
+            [[ -n "$pkg" ]] && unique_packages["$pkg"]=1
+        done
+
+        PACKAGES_TO_TEST=("${!unique_packages[@]}")
+
+        echo "Combined packages to test: ${PACKAGES_TO_TEST[*]}"
+    else
+        echo "==============================================================="
+        echo "WARNING: ci/adhoc/adhoc_test_runner.sh not found!"
+        echo "Ad-hoc packages will NOT be added."
+        echo "==============================================================="
+    fi
+fi