diff --git a/Makefile b/Makefile index 30f3af632..f817358a5 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ SHELL := /bin/bash #TODO: We need to rename all those python and config files as well +# Define comma for use in $(if) expressions (where commas are argument separators) +COMMA := , + ARTIFACT_DIR := $(if $(ARTIFACT_DIR),$(ARTIFACT_DIR),tests/test_results) PATH_TO_PLANTUML := ~/bin @@ -16,6 +19,7 @@ OGX_CONFIG ?= run.yaml OGX_CONTAINER_NAME ?= lightspeed-ogx OGX_IMAGE ?= lightspeed-ogx:local OGX_PORT ?= 8321 +LIGHTSPEED_PROVIDERS_DIR ?= $(shell [ -d ../lightspeed-providers ] && cd ../lightspeed-providers && pwd) CONTAINER_RUNTIME ?= $(shell command -v podman 2>/dev/null || command -v docker 2>/dev/null) .PHONY: run \ @@ -86,6 +90,9 @@ start-ogx-container: build-ogx-image ## Start OGX container -v $(PWD)/$(CONFIG):/opt/app-root/lightspeed-stack.yaml:ro,z \ -v $(PWD)/scripts/llama-stack-entrypoint.sh:/opt/app-root/enrich-entrypoint.sh:ro,z \ -v $(PWD)/src/llama_stack_configuration.py:/opt/app-root/llama_stack_configuration.py:ro,z \ + $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/lightspeed_stack_providers:/opt/app-root/providers/lightspeed_stack_providers:ro$(COMMA)z) \ + $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/resources/external_providers:/opt/app-root/src/.llama/providers.d:ro$(COMMA)z) \ + $(if $(LIGHTSPEED_PROVIDERS_DIR),-e EXTERNAL_PROVIDERS_DIR=/opt/app-root/src/.llama/providers.d) \ -e OPENAI_API_KEY \ -e BRAVE_SEARCH_API_KEY \ -e TAVILY_SEARCH_API_KEY \ @@ -164,7 +171,7 @@ test-e2e-local: ## Run end to end tests for the service (no script wrapper) # Tag-based subsets (@cfg_* on features/scenarios). Default runs all config groups; override for one shard, e.g. # E2E_BEHAVE_TAG_EXPR='not @skip and @cfg_authorized' make test-e2e-tagged-local -E2E_BEHAVE_TAG_EXPR ?= not @skip and (@cfg_default or @cfg_authorized or @cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth or @cfg_rbac or @cfg_rh_identity or @cfg_negative or @cfg_skills or @cfg_skills_directory or @cfg_byok_pdf or @cfg_tls or @cfg_degraded or @cfg_unified) +E2E_BEHAVE_TAG_EXPR ?= not @skip and (@cfg_default or @cfg_authorized or @cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth or @cfg_rbac or @cfg_rh_identity or @cfg_negative or @cfg_skills or @cfg_skills_directory or @cfg_byok_pdf or @cfg_tls or @cfg_degraded or @cfg_unified or @cfg_okp) test-e2e-tagged: ## Run e2e tests with E2E_BEHAVE_TAG_EXPR (default: all @cfg_*) script -q -e -c "uv run behave --color --format pretty --tags=\"$(E2E_BEHAVE_TAG_EXPR)\" -D dump_errors=true @tests/e2e/test_list.txt" diff --git a/tests/e2e-prow/rhoai/manifests/lightspeed/okp-solr.yaml b/tests/e2e-prow/rhoai/manifests/lightspeed/okp-solr.yaml new file mode 100644 index 000000000..bc730230c --- /dev/null +++ b/tests/e2e-prow/rhoai/manifests/lightspeed/okp-solr.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: Pod +metadata: + name: okp-solr-service + labels: + app: okp-solr +spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + imagePullSecrets: + - name: redhat-registry-pull-secret + containers: + - name: okp-solr + image: registry.redhat.io/offline-knowledge-portal/rhokp-rhel9:latest + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + ports: + - containerPort: 8080 + name: http + readinessProbe: + httpGet: + path: /solr + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /solr + port: 8080 + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "2Gi" + cpu: "1000m" +--- +apiVersion: v1 +kind: Service +metadata: + name: okp-solr-service-svc +spec: + selector: + app: okp-solr + ports: + - port: 8080 + targetPort: 8080 + name: http diff --git a/tests/e2e-prow/rhoai/pipeline-konflux.sh b/tests/e2e-prow/rhoai/pipeline-konflux.sh index 14020dbcf..f391dc898 100755 --- a/tests/e2e-prow/rhoai/pipeline-konflux.sh +++ b/tests/e2e-prow/rhoai/pipeline-konflux.sh @@ -110,6 +110,70 @@ oc create secret docker-registry quay-lightspeed-pull-secret \ # Link the secret to default service account for image pulls oc secrets link default quay-lightspeed-pull-secret --for=pull -n "$NAMESPACE" 2>/dev/null || echo "⚠️ Secret already linked to default SA" +# Create Red Hat registry pull secret for OKP images +# Credentials from Konflux secrets (mounted at /var/run/redhat-registry-*) +if [[ -d /var/run/redhat-registry-username ]] && [[ -d /var/run/redhat-registry-password ]]; then + log "Creating Red Hat registry pull secret..." + REDHAT_USERNAME="" + REDHAT_PASSWORD="" + + # Read username + shopt -s nullglob + for _f in /var/run/redhat-registry-username/*; do + [[ -f "$_f" ]] && REDHAT_USERNAME="$(cat "$_f")" && break + done + + # Read password + for _f in /var/run/redhat-registry-password/*; do + [[ -f "$_f" ]] && REDHAT_PASSWORD="$(cat "$_f")" && break + done + shopt -u nullglob + + if [[ -n "$REDHAT_USERNAME" ]] && [[ -n "$REDHAT_PASSWORD" ]]; then + # Get current pipeline Pod metadata for ownerReference (ensures cleanup after job completes) + PIPELINE_POD_NAME="${HOSTNAME}" + if PIPELINE_POD_UID=$(oc get pod "$PIPELINE_POD_NAME" -n "$NAMESPACE" -o jsonpath='{.metadata.uid}' 2>/dev/null); then + log "Setting ownerReference to pipeline Pod: $PIPELINE_POD_NAME" + + # Create secret with ownerReference using YAML (ensures automatic cleanup) + cat </dev/null && log "✅ Red Hat registry pull secret created" || log "⚠️ Secret exists or creation failed" + fi + + # Link to default service account + oc secrets link default redhat-registry-pull-secret --for=pull -n "$NAMESPACE" 2>/dev/null || echo "⚠️ Secret already linked to default SA" + else + log "⚠️ Red Hat registry credentials not found in /var/run - OKP image pull may fail" + fi +else + log "⚠️ Red Hat registry credential mounts not found - OKP image pull may fail" + log " (This is OK if not testing OKP features)" +fi + #======================================== # 4. DEPLOY MOCK SERVERS (JWKS & MCP) @@ -146,6 +210,22 @@ oc wait pod/mock-jwks pod/mock-mcp \ } log "✅ Mock servers deployed" +# Deploy OKP Solr server for RAG tests +log "Deploying OKP Solr server..." +oc apply -n "$NAMESPACE" -f "$PIPELINE_DIR/manifests/lightspeed/okp-solr.yaml" + +# Wait for OKP Solr to be ready +log "Waiting for OKP Solr to be ready..." +oc wait pod/okp-solr-service \ + -n "$NAMESPACE" --for=condition=Ready --timeout=180s || { + echo "⚠️ OKP Solr not ready, checking status..." + oc get pods -n "$NAMESPACE" | grep okp-solr || true + oc describe pod okp-solr-service -n "$NAMESPACE" 2>/dev/null | tail -30 || true + echo "❌ OKP Solr failed to become ready" + exit 1 +} +log "✅ OKP Solr deployed" + # e2e-tunnel-proxy and e2e-interception-proxy are deployed from proxy.feature steps # (see tests/e2e/features/steps/proxy.py + e2e-ops deploy-e2e-*-proxy). @@ -297,8 +377,10 @@ fi # Debug hook/port churn: export E2E_OPS_VERBOSE=1 before running pipeline.sh export E2E_LSC_PORT_FORWARD_PID_FILE="${E2E_LSC_PORT_FORWARD_PID_FILE:-/tmp/e2e-lightspeed-port-forward.pid}" export E2E_LLAMA_PORT_FORWARD_PID_FILE="${E2E_LLAMA_PORT_FORWARD_PID_FILE:-/tmp/e2e-llama-port-forward.pid}" +export E2E_OKP_PORT_FORWARD_PID_FILE="${E2E_OKP_PORT_FORWARD_PID_FILE:-/tmp/e2e-okp-port-forward.pid}" rm -f "$E2E_LSC_PORT_FORWARD_PID_FILE" rm -f "$E2E_LLAMA_PORT_FORWARD_PID_FILE" +rm -f "$E2E_OKP_PORT_FORWARD_PID_FILE" oc label pod lightspeed-stack-service pod=lightspeed-stack-service -n $NAMESPACE @@ -319,7 +401,7 @@ kill_listeners_on_ports() { fi done } -kill_listeners_on_ports 8080 8000 8321 +kill_listeners_on_ports 8080 8000 8321 8081 # Start port-forward for lightspeed-stack progress "Starting port-forward, then E2E tests" @@ -339,6 +421,12 @@ oc port-forward svc/llama-stack-service-svc 8321:8321 -n $NAMESPACE & PF_LLAMA_PID=$! echo "$PF_LLAMA_PID" >"$E2E_LLAMA_PORT_FORWARD_PID_FILE" +# Start port-forward for OKP Solr (RAG tests) +log "Starting port-forward for OKP Solr..." +oc port-forward svc/okp-solr-service-svc 8081:8080 -n $NAMESPACE & +PF_OKP_PID=$! +echo "$PF_OKP_PID" >"$E2E_OKP_PORT_FORWARD_PID_FILE" + # Wait for port-forward to be usable (app may not be listening immediately; port-forward can drop) log "Waiting for port-forward to lightspeed-stack to be ready..." for i in $(seq 1 36); do @@ -359,6 +447,7 @@ for i in $(seq 1 36); do kill $PF_LCS_PID 2>/dev/null || true kill $PF_JWKS_PID 2>/dev/null || true kill $PF_LLAMA_PID 2>/dev/null || true + kill $PF_OKP_PID 2>/dev/null || true exit 1 fi # If port-forward process died, restart it (e.g. "connection refused" / "lost connection to pod") @@ -385,6 +474,7 @@ for i in $(seq 1 36); do kill $PF_LCS_PID 2>/dev/null || true kill $PF_JWKS_PID 2>/dev/null || true kill $PF_LLAMA_PID 2>/dev/null || true + kill $PF_OKP_PID 2>/dev/null || true exit 1 fi if ! kill -0 $PF_LLAMA_PID 2>/dev/null; then @@ -396,10 +486,30 @@ for i in $(seq 1 36); do sleep 5 done +log "Waiting for OKP Solr port-forward (localhost:8081 /solr)..." +for i in $(seq 1 24); do + if curl -sf --max-time 5 http://localhost:8081/solr > /dev/null 2>&1; then + log "✅ OKP Solr port-forward ready after $(( i * 5 ))s" + break + fi + if [ $i -eq 24 ]; then + echo "⚠️ Port-forward to OKP Solr never became healthy (2 min) - OKP RAG tests may fail" | tee /dev/stderr + # Don't exit - OKP is optional, other tests can still run + fi + if ! kill -0 $PF_OKP_PID 2>/dev/null; then + log "OKP port-forward died, restarting (attempt $i)..." + oc port-forward svc/okp-solr-service-svc 8081:8080 -n $NAMESPACE & + PF_OKP_PID=$! + echo "$PF_OKP_PID" >"$E2E_OKP_PORT_FORWARD_PID_FILE" + fi + sleep 5 +done + export E2E_LSC_HOSTNAME="localhost" export E2E_JWKS_HOSTNAME="localhost" export E2E_LLAMA_HOSTNAME="localhost" export E2E_LLAMA_PORT="8321" +export E2E_OKP_URL="http://localhost:8081" # Same pattern as tests/e2e-prow/rhoai/pipeline.sh and .github/workflows/e2e_tests_*.yaml: # Behave {MODEL}/{PROVIDER} use these when set; avoids wrong fallbacks if /v1/models # discovery in before_all is empty (matches run-ci.yaml openai + E2E_OPENAI_MODEL). @@ -414,6 +524,7 @@ export E2E_DEFAULT_PROVIDER_OVERRIDE E2E_DEFAULT_MODEL_OVERRIDE log "LCS accessible at: http://$E2E_LSC_HOSTNAME:8080" log "Mock JWKS accessible at: http://$E2E_JWKS_HOSTNAME:8000" log "OGX (e2e client hooks) at: http://$E2E_LLAMA_HOSTNAME:$E2E_LLAMA_PORT" +log "OKP Solr (RAG tests) at: $E2E_OKP_URL" #======================================== # 7. RUN TESTS @@ -448,13 +559,22 @@ if [[ -n "${E2E_LLAMA_PORT_FORWARD_PID_FILE:-}" && -f "$E2E_LLAMA_PORT_FORWARD_P fi rm -f "$E2E_LLAMA_PORT_FORWARD_PID_FILE" fi +if [[ -n "${E2E_OKP_PORT_FORWARD_PID_FILE:-}" && -f "$E2E_OKP_PORT_FORWARD_PID_FILE" ]]; then + read -r _okp_pf <"$E2E_OKP_PORT_FORWARD_PID_FILE" 2>/dev/null || true + if [[ "${_okp_pf:-}" =~ ^[0-9]+$ ]]; then + kill -9 "$_okp_pf" 2>/dev/null || true + fi + rm -f "$E2E_OKP_PORT_FORWARD_PID_FILE" +fi kill $PF_LCS_PID 2>/dev/null || true kill $PF_JWKS_PID 2>/dev/null || true kill $PF_LLAMA_PID 2>/dev/null || true +kill $PF_OKP_PID 2>/dev/null || true wait $PF_LCS_PID 2>/dev/null || true wait $PF_JWKS_PID 2>/dev/null || true wait $PF_LLAMA_PID 2>/dev/null || true +wait $PF_OKP_PID 2>/dev/null || true set -e trap 'echo "❌ Pipeline failed at line $LINENO"; exit 1' ERR diff --git a/tests/e2e-prow/rhoai/pipeline.sh b/tests/e2e-prow/rhoai/pipeline.sh index c393e2c15..cb93e3695 100755 --- a/tests/e2e-prow/rhoai/pipeline.sh +++ b/tests/e2e-prow/rhoai/pipeline.sh @@ -92,6 +92,52 @@ oc create secret docker-registry quay-lightspeed-pull-secret \ # Link the secret to default service account for image pulls oc secrets link default quay-lightspeed-pull-secret --for=pull -n "$NAMESPACE" 2>/dev/null || echo "⚠️ Secret already linked to default SA" +# Create Red Hat registry pull secret for OKP images +# Credentials from Prow secrets (environment variables or mounted volumes) +if [[ -n "${REDHAT_REGISTRY_USERNAME:-}" ]] && [[ -n "${REDHAT_REGISTRY_PASSWORD:-}" ]]; then + echo "Creating Red Hat registry pull secret from environment..." + + # Get current pipeline Pod metadata for ownerReference (ensures cleanup after job completes) + PIPELINE_POD_NAME="${HOSTNAME}" + if PIPELINE_POD_UID=$(oc get pod "$PIPELINE_POD_NAME" -n "$NAMESPACE" -o jsonpath='{.metadata.uid}' 2>/dev/null); then + echo "Setting ownerReference to pipeline Pod: $PIPELINE_POD_NAME" + + # Create secret with ownerReference using YAML (ensures automatic cleanup) + cat </dev/null && echo "✅ Red Hat registry pull secret created" || echo "⚠️ Secret exists or creation failed" + fi + + # Link to default service account + oc secrets link default redhat-registry-pull-secret --for=pull -n "$NAMESPACE" 2>/dev/null || echo "⚠️ Secret already linked to default SA" +else + echo "⚠️ REDHAT_REGISTRY_USERNAME/PASSWORD not set - OKP image pull may fail" + echo " (This is OK if not testing OKP features)" +fi + #======================================== # 5. CONFIGMAPS @@ -215,6 +261,22 @@ oc wait pod/mock-jwks pod/mock-mcp \ } echo "✅ Mock servers deployed" +# Deploy OKP Solr server for RAG tests +echo "Deploying OKP Solr server..." +oc apply -n "$NAMESPACE" -f "$PIPELINE_DIR/manifests/lightspeed/okp-solr.yaml" + +# Wait for OKP Solr to be ready +echo "Waiting for OKP Solr to be ready..." +oc wait pod/okp-solr-service \ + -n "$NAMESPACE" --for=condition=Ready --timeout=180s || { + echo "⚠️ OKP Solr not ready, checking status..." + oc get pods -n "$NAMESPACE" | grep okp-solr || true + oc describe pod okp-solr-service -n "$NAMESPACE" 2>/dev/null | tail -30 || true + echo "❌ OKP Solr failed to become ready" + exit 1 +} +echo "✅ OKP Solr deployed" + #======================================== # 8. BUILD OGX IMAGE #======================================== @@ -358,9 +420,11 @@ oc describe pod llama-stack-service -n "$NAMESPACE" || true export E2E_LSC_PORT_FORWARD_PID_FILE="${E2E_LSC_PORT_FORWARD_PID_FILE:-/tmp/e2e-lightspeed-port-forward.pid}" export E2E_LLAMA_PORT_FORWARD_PID_FILE="${E2E_LLAMA_PORT_FORWARD_PID_FILE:-/tmp/e2e-llama-port-forward.pid}" export E2E_JWKS_PORT_FORWARD_PID_FILE="${E2E_JWKS_PORT_FORWARD_PID_FILE:-/tmp/e2e-jwks-port-forward.pid}" +export E2E_OKP_PORT_FORWARD_PID_FILE="${E2E_OKP_PORT_FORWARD_PID_FILE:-/tmp/e2e-okp-port-forward.pid}" rm -f "$E2E_LSC_PORT_FORWARD_PID_FILE" rm -f "$E2E_LLAMA_PORT_FORWARD_PID_FILE" rm -f "$E2E_JWKS_PORT_FORWARD_PID_FILE" +rm -f "$E2E_OKP_PORT_FORWARD_PID_FILE" oc label pod lightspeed-stack-service pod=lightspeed-stack-service -n $NAMESPACE @@ -370,16 +434,18 @@ oc expose pod lightspeed-stack-service \ --type=ClusterIP \ -n $NAMESPACE -# Kill any existing processes on ports 8080, 8000, and 8321 (lsof may be missing in minimal images) -echo "Checking for existing processes on ports 8080, 8000, and 8321..." +# Kill any existing processes on ports 8080, 8000, 8321, and 8081 (lsof may be missing in minimal images) +echo "Checking for existing processes on ports 8080, 8000, 8321, and 8081..." if command -v lsof >/dev/null 2>&1; then lsof -ti:8080 | xargs kill -9 2>/dev/null || true lsof -ti:8000 | xargs kill -9 2>/dev/null || true lsof -ti:8321 | xargs kill -9 2>/dev/null || true + lsof -ti:8081 | xargs kill -9 2>/dev/null || true elif command -v fuser >/dev/null 2>&1; then fuser -k 8080/tcp 2>/dev/null || true fuser -k 8000/tcp 2>/dev/null || true fuser -k 8321/tcp 2>/dev/null || true + fuser -k 8081/tcp 2>/dev/null || true fi # Start port-forward for lightspeed-stack @@ -401,6 +467,12 @@ oc port-forward svc/llama-stack-service-svc 8321:8321 -n $NAMESPACE & PF_LLAMA_PID=$! echo "$PF_LLAMA_PID" >"$E2E_LLAMA_PORT_FORWARD_PID_FILE" +# Start port-forward for OKP Solr (RAG tests) +echo "Starting port-forward for OKP Solr..." +oc port-forward svc/okp-solr-service-svc 8081:8080 -n $NAMESPACE & +PF_OKP_PID=$! +echo "$PF_OKP_PID" >"$E2E_OKP_PORT_FORWARD_PID_FILE" + # Wait for port-forward to be usable (app may not be listening immediately; port-forward can drop) echo "Waiting for port-forward to lightspeed-stack to be ready..." for i in $(seq 1 36); do @@ -422,6 +494,7 @@ for i in $(seq 1 36); do kill $PF_LCS_PID 2>/dev/null || true kill $PF_JWKS_PID 2>/dev/null || true kill $PF_LLAMA_PID 2>/dev/null || true + kill $PF_OKP_PID 2>/dev/null || true exit 1 fi # If port-forward process died, restart it (e.g. "connection refused" / "lost connection to pod") @@ -447,6 +520,7 @@ for i in $(seq 1 36); do kill $PF_LCS_PID 2>/dev/null || true kill $PF_JWKS_PID 2>/dev/null || true kill $PF_LLAMA_PID 2>/dev/null || true + kill $PF_OKP_PID 2>/dev/null || true exit 1 fi if ! kill -0 $PF_LLAMA_PID 2>/dev/null; then @@ -458,13 +532,35 @@ for i in $(seq 1 36); do sleep 5 done +# Wait for OKP Solr port-forward to be usable (non-fatal - OKP is optional) +echo "Waiting for OKP Solr port-forward (localhost:8081 /solr)..." +for i in $(seq 1 24); do + if curl -sf --max-time 5 http://localhost:8081/solr > /dev/null 2>&1; then + echo "✅ OKP Solr port-forward ready after $(( i * 5 ))s" + break + fi + if [ $i -eq 24 ]; then + echo "⚠️ Port-forward to OKP Solr never became healthy (2 min) - OKP RAG tests may fail" + # Don't exit - OKP is optional, other tests can still run + fi + if ! kill -0 $PF_OKP_PID 2>/dev/null; then + echo "OKP port-forward died, restarting (attempt $i)..." + oc port-forward svc/okp-solr-service-svc 8081:8080 -n $NAMESPACE & + PF_OKP_PID=$! + echo "$PF_OKP_PID" >"$E2E_OKP_PORT_FORWARD_PID_FILE" + fi + sleep 5 +done + export E2E_LSC_HOSTNAME="localhost" export E2E_JWKS_HOSTNAME="localhost" export E2E_DEFAULT_MODEL_OVERRIDE="$MODEL_NAME" export E2E_DEFAULT_PROVIDER_OVERRIDE="vllm" +export E2E_OKP_URL="http://localhost:8081" echo "LCS accessible at: http://$E2E_LSC_HOSTNAME:8080" echo "Mock JWKS accessible at: http://$E2E_JWKS_HOSTNAME:8000" echo "OGX accessible at: http://localhost:8321" +echo "OKP Solr (RAG tests) at: $E2E_OKP_URL" @@ -488,10 +584,12 @@ TEST_EXIT_CODE=$(cat "$E2E_EXIT_CODE_FILE" 2>/dev/null || echo 1) kill $PF_LCS_PID 2>/dev/null || true kill $PF_JWKS_PID 2>/dev/null || true kill $PF_LLAMA_PID 2>/dev/null || true +kill $PF_OKP_PID 2>/dev/null || true wait $PF_LCS_PID 2>/dev/null || true wait $PF_JWKS_PID 2>/dev/null || true wait $PF_LLAMA_PID 2>/dev/null || true -rm -f "$E2E_LSC_PORT_FORWARD_PID_FILE" "$E2E_LLAMA_PORT_FORWARD_PID_FILE" "$E2E_JWKS_PORT_FORWARD_PID_FILE" +wait $PF_OKP_PID 2>/dev/null || true +rm -f "$E2E_LSC_PORT_FORWARD_PID_FILE" "$E2E_LLAMA_PORT_FORWARD_PID_FILE" "$E2E_JWKS_PORT_FORWARD_PID_FILE" "$E2E_OKP_PORT_FORWARD_PID_FILE" set -e trap 'echo "❌ Pipeline failed at line $LINENO"; exit 1' ERR diff --git a/tests/e2e-prow/rhoai/scripts/e2e-ops.sh b/tests/e2e-prow/rhoai/scripts/e2e-ops.sh index ff2139bf5..9df97fb3a 100755 --- a/tests/e2e-prow/rhoai/scripts/e2e-ops.sh +++ b/tests/e2e-prow/rhoai/scripts/e2e-ops.sh @@ -23,6 +23,7 @@ # restart-both-services - Full OGX then lightspeed-stack restart (explicit only) # restart-port-forward - Re-establish port-forward for lightspeed # restart-llama-port-forward - Re-establish port-forward for OGX (8321) +# restart-okp-port-forward - Re-establish port-forward for OKP Solr (8081) # wait-for-pod [attempts] - Wait for a pod to be ready # update-configmap - Update ConfigMap from file # get-configmap-content - Get ConfigMap content (outputs to stdout) @@ -33,6 +34,10 @@ # delete-e2e-mock-tls-inference - Remove mock TLS pod + Service (manual cleanup) # restart-e2e-mock-tls-inference - Delete then deploy mock TLS (manual / recovery) # sync-mock-tls-certs-secret - Copy mock /certs into Secret for OGX mount +# deploy-okp-solr - Deploy OKP Solr service +# delete-okp-solr - Delete OKP Solr pod +# disrupt-okp-solr - Delete OKP Solr pod to disrupt connection +# restore-okp-solr - Restore OKP Solr pod set -e @@ -43,6 +48,7 @@ MANIFEST_DIR="$SCRIPT_DIR/../manifests/lightspeed" E2E_LSC_PORT_FORWARD_PID_FILE="${E2E_LSC_PORT_FORWARD_PID_FILE:-/tmp/e2e-lightspeed-port-forward.pid}" E2E_LLAMA_PORT_FORWARD_PID_FILE="${E2E_LLAMA_PORT_FORWARD_PID_FILE:-/tmp/e2e-llama-port-forward.pid}" E2E_JWKS_PORT_FORWARD_PID_FILE="${E2E_JWKS_PORT_FORWARD_PID_FILE:-/tmp/e2e-jwks-port-forward.pid}" +E2E_OKP_PORT_FORWARD_PID_FILE="${E2E_OKP_PORT_FORWARD_PID_FILE:-/tmp/e2e-okp-port-forward.pid}" # ============================================================================ # Helper functions @@ -208,6 +214,24 @@ kill_stale_jwks_forward() { free_local_tcp_port "$port" } +# Kill anything likely to hold the OKP Solr local forward (localhost:8081). +kill_stale_okp_forward() { + local port="${1:-8081}" + local saved_pf + if [[ -f "$E2E_OKP_PORT_FORWARD_PID_FILE" ]]; then + read -r saved_pf <"$E2E_OKP_PORT_FORWARD_PID_FILE" 2>/dev/null || true + if [[ "$saved_pf" =~ ^[0-9]+$ ]]; then + kill -9 "$saved_pf" 2>/dev/null || true + fi + fi + pkill -9 -f "port-forward.*okp-solr-service-svc.*8081:8080" 2>/dev/null || true + pkill -9 -f "oc port-forward svc/okp-solr-service-svc 8081:8080" 2>/dev/null || true + pkill -9 -f "port-forward pod/okp-solr-service.*8081:8080" 2>/dev/null || true + free_local_tcp_port "$port" + sleep 1 + free_local_tcp_port "$port" +} + # After oc port-forward dies in <2s, show recent oc stderr from the log file. e2e_ops_emit_port_forward_immediate_failure_diag() { echo "[e2e-ops] /tmp/port-forward.log (tail 25):" @@ -554,6 +578,25 @@ verify_llama_local_forward() { return 1 } +verify_okp_connectivity() { + local max_attempts="${1:-15}" + local http_code="" + local attempt + + for ((attempt=1; attempt<=max_attempts; attempt++)); do + http_code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 "http://localhost:8081/solr" 2>/dev/null) || http_code="000" + # OKP Solr returns various 200-399 codes for /solr endpoint + if [[ "$http_code" =~ ^[23][0-9][0-9]$ ]]; then + return 0 + fi + if [[ $attempt -lt $max_attempts ]]; then + sleep 2 + fi + done + echo "OKP Solr localhost:8081 connectivity check failed (HTTP: ${http_code:-unknown})" + return 1 +} + cmd_restart_llama_port_forward() { local local_port="${LOCAL_LLAMA_PORT:-8321}" local remote_port="${REMOTE_LLAMA_PORT:-8321}" @@ -676,6 +719,67 @@ cmd_restart_jwks_port_forward() { return 1 } +cmd_restart_okp_port_forward() { + local local_port="${LOCAL_OKP_PORT:-8081}" + local remote_port="${REMOTE_OKP_PORT:-8080}" + local max_attempts=6 + local pf_pid + local pf_resource + local okp_pf_log="/tmp/port-forward-okp.log" + + echo "Re-establishing OKP Solr port-forward on $local_port:$remote_port..." + + for ((attempt=1; attempt<=max_attempts; attempt++)); do + kill_stale_okp_forward "$local_port" + sleep 3 + + if [[ $attempt -le 2 ]]; then + pf_resource="svc/okp-solr-service-svc" + else + pf_resource="pod/okp-solr-service" + fi + echo "OKP port-forward attempt $attempt/$max_attempts -> $pf_resource" + + : >"$okp_pf_log" + nohup oc port-forward "$pf_resource" "$local_port:$remote_port" -n "$NAMESPACE" \ + "$okp_pf_log" 2>&1 & + pf_pid=$! + disown "$pf_pid" 2>/dev/null || true + sleep 3 + + if ! kill -0 "$pf_pid" 2>/dev/null; then + echo "OKP port-forward process exited immediately:" + if [[ -s "$okp_pf_log" ]]; then + tail -25 "$okp_pf_log" 2>/dev/null | sed 's/^/[e2e-ops] /' || true + fi + kill_stale_okp_forward "$local_port" + sleep 2 + continue + fi + sleep 4 + + if verify_okp_connectivity 12; then + echo "$pf_pid" >"$E2E_OKP_PORT_FORWARD_PID_FILE" + echo "✓ OKP Solr port-forward established (PID: $pf_pid)" + return 0 + fi + + if [[ $attempt -lt $max_attempts ]]; then + echo "OKP forward attempt $attempt failed (connectivity check failed), retrying..." + kill -9 "$pf_pid" 2>/dev/null || true + kill_stale_okp_forward "$local_port" + sleep 3 + fi + done + + echo "Failed to establish OKP Solr port-forward after $max_attempts attempts" + if [[ -s "$okp_pf_log" ]]; then + echo "Port-forward log (tail 30):" + tail -30 "$okp_pf_log" 2>/dev/null | sed 's/^/[e2e-ops] /' || true + fi + return 1 +} + cmd_wait_for_pod() { local pod_name="${1:?Pod name required}" local max_attempts="${2:-24}" @@ -989,6 +1093,49 @@ cmd_disrupt_llama_stack() { fi } +cmd_deploy_okp_solr() { + echo "Deploying OKP Solr service in namespace $NAMESPACE..." + oc apply -n "$NAMESPACE" -f "$MANIFEST_DIR/okp-solr.yaml" + wait_for_pod "okp-solr-service" 60 + echo "✓ OKP Solr service deployed and ready" +} + +cmd_delete_okp_solr() { + echo "Deleting OKP Solr pod from namespace $NAMESPACE..." + timeout 60 oc delete pod okp-solr-service -n "$NAMESPACE" --ignore-not-found=true --wait=true 2>/dev/null || { + oc delete pod okp-solr-service -n "$NAMESPACE" --ignore-not-found=true --force --grace-period=0 2>/dev/null || true + sleep 2 + } + echo "✓ OKP Solr pod deleted" +} + +cmd_disrupt_okp_solr() { + local pod_name="okp-solr-service" + + local phase + phase=$(oc get pod "$pod_name" -n "$NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null || echo "NotFound") + + if [[ "$phase" == "Running" ]]; then + oc delete pod "$pod_name" -n "$NAMESPACE" --wait=true + sleep 2 + echo "OKP Solr connection disrupted successfully (pod deleted)" + exit 0 + else + echo "OKP Solr pod was not running (phase: $phase)" + exit 2 + fi +} + +cmd_restore_okp_solr() { + echo "Restoring OKP Solr service in namespace $NAMESPACE..." + oc apply -n "$NAMESPACE" -f "$MANIFEST_DIR/okp-solr.yaml" + wait_for_pod "okp-solr-service" 60 + echo "✓ OKP Solr pod restored and ready" + + # Restart port-forward since pod was replaced + cmd_restart_okp_port_forward +} + # ============================================================================ # Main command dispatcher # ============================================================================ @@ -1012,6 +1159,9 @@ case "$COMMAND" in restart-jwks-port-forward) cmd_restart_jwks_port_forward ;; + restart-okp-port-forward) + cmd_restart_okp_port_forward + ;; restart-port-forward) cmd_restart_port_forward ;; @@ -1060,6 +1210,18 @@ case "$COMMAND" in dump-pod-logs) cmd_dump_pod_logs "$@" ;; + deploy-okp-solr) + cmd_deploy_okp_solr + ;; + delete-okp-solr) + cmd_delete_okp_solr + ;; + disrupt-okp-solr) + cmd_disrupt_okp_solr + ;; + restore-okp-solr) + cmd_restore_okp_solr + ;; *) echo "Usage: $0 [args...]" echo "" @@ -1068,6 +1230,7 @@ case "$COMMAND" in echo " restart-llama-stack - Restart/restore llama-stack pod" echo " restart-both-services - Full llama-stack + lightspeed-stack restart (explicit)" echo " restart-llama-port-forward - Re-establish port-forward for OGX (8321)" + echo " restart-okp-port-forward - Re-establish port-forward for OKP Solr (8081)" echo " restart-port-forward - Re-establish port-forward for lightspeed" echo " wait-for-pod [attempts] - Wait for a pod to be ready" echo " update-configmap - Update ConfigMap from file" @@ -1081,6 +1244,10 @@ case "$COMMAND" in echo " deploy-e2e-interception-proxy - Deploy in-cluster interception proxy pod" echo " deploy-e2e-mock-tls-inference - Deploy mock HTTPS inference (tls-*.feature)" echo " delete-e2e-mock-tls-inference - Remove mock TLS pod + Service" + echo " deploy-okp-solr - Deploy OKP Solr service" + echo " delete-okp-solr - Delete OKP Solr pod" + echo " disrupt-okp-solr - Delete OKP Solr pod to disrupt connection" + echo " restore-okp-solr - Restore OKP Solr pod" echo " restart-e2e-mock-tls-inference - Delete then deploy mock TLS (recovery)" echo " sync-mock-tls-certs-secret - Publish mock TLS /certs to Secret" echo " dump-pod-logs [tail-lines] - Print init + container logs" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-offline.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-offline.yaml new file mode 100644 index 000000000..1863a7daa --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-offline.yaml @@ -0,0 +1,40 @@ +# @cfg_okp +# OKP inline RAG — offline mode. +# Chunks use parent_id-based URLs (localhost:8081). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: true + config: + baseline: default + # profile: run.yaml # Commented out to allow auto-generation with OKP enrichment +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + inline: + sources: + - okp + max_chunks: 1 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: true + max_chunks: 5 diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-online.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-online.yaml new file mode 100644 index 000000000..2c7936500 --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-online.yaml @@ -0,0 +1,41 @@ +# @cfg_okp +# OKP inline RAG — online mode. +# Chunks use reference_url-based URLs (docs.redhat.com). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: true + config: + baseline: default + # profile: run.yaml # Commented out to allow auto-generation with OKP enrichment +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + inline: + sources: + - okp + max_chunks: 3 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: false + max_chunks: 3 + chunk_filter_query: "product:openshift_container_platform AND product_version:4.21" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-offline.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-offline.yaml new file mode 100644 index 000000000..4dd5d37d9 --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-offline.yaml @@ -0,0 +1,40 @@ +# @cfg_okp +# OKP tool RAG — offline mode. +# LLM calls file_search on demand; chunks use parent_id-based URLs (localhost:8081). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: true + config: + baseline: default + # profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + tool: + sources: + - okp + max_chunks: 2 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: true + max_chunks: 5 diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-online.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-online.yaml new file mode 100644 index 000000000..f39e5aac6 --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-online.yaml @@ -0,0 +1,40 @@ +# @cfg_okp +# OKP tool RAG — online mode. +# LLM calls file_search on demand; chunks use reference_url-based URLs (access.redhat.com). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: true + config: + baseline: default + # profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + tool: + sources: + - okp + max_chunks : 3 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: false + max_chunks: 5 diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml new file mode 100644 index 000000000..bc6015887 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml @@ -0,0 +1,39 @@ +# @cfg_okp +# OKP inline RAG — offline mode. +# Chunks use parent_id-based URLs (localhost:8081). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + inline: + sources: + - okp + max_chunks: 1 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: true + max_chunks: 5 \ No newline at end of file diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-okp-online.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-online.yaml new file mode 100644 index 000000000..94c9362b9 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-online.yaml @@ -0,0 +1,40 @@ +# @cfg_okp +# OKP inline RAG — online mode. +# Chunks use reference_url-based URLs (docs.redhat.com). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + inline: + sources: + - okp + max_chunks: 3 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: false + max_chunks: 3 + chunk_filter_query: "product:openshift_container_platform AND product_version:4.21" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-offline.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-offline.yaml new file mode 100644 index 000000000..82c981216 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-offline.yaml @@ -0,0 +1,39 @@ +# @cfg_okp +# OKP tool RAG — offline mode. +# LLM calls file_search on demand; chunks use parent_id-based URLs (localhost:8081). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + tool: + sources: + - okp + max_chunks: 2 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: true + max_chunks: 5 diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-online.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-online.yaml new file mode 100644 index 000000000..ec6c258e2 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-online.yaml @@ -0,0 +1,39 @@ +# @cfg_okp +# OKP tool RAG — online mode. +# LLM calls file_search on demand; chunks use reference_url-based URLs (access.redhat.com). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + tool: + sources: + - okp + max_chunks: 3 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: false + max_chunks: 5 diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index 273d80ec9..3c7b5bc45 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -304,6 +304,8 @@ def after_scenario(context: Context, scenario: Scenario) -> None: running before the scenario. - hostname_llama, port_llama (str/int, optional): host and port used for the OGX health check. + - okp_was_running (bool, optional): whether OKP server was running + before it was stopped by the scenario. scenario (Scenario): Behave scenario (unused; shield restore uses context flags). """ if is_prow_environment(): @@ -329,6 +331,80 @@ def after_scenario(context: Context, scenario: Scenario) -> None: except (TypeError, ValueError, RuntimeError, KeyboardInterrupt) as e: print(f"Warning: Could not re-register shield: {e}") + # Restart OKP server if it was stopped during the scenario + if getattr(context, "okp_was_running", False): + if is_prow_environment(): + # Prow/OpenShift: restore pod and raise on failure + from tests.e2e.utils.prow_utils import restore_okp_solr_pod + + restore_okp_solr_pod() + else: + # Docker mode: recreate or restart container + from tests.e2e.features.steps.okp_rag import ( + OKP_DEFAULT_URL, + OKP_IMAGE_NAME, + ) + + container_name = getattr(context, "okp_container_name", None) + if container_name: + # Check if container still exists (may be auto-removed if started with --rm) + check_result = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Running}}", container_name], + capture_output=True, + text=True, + check=False, + ) + + if check_result.returncode != 0: + # Container was removed (likely started with --rm flag) - recreate it + print( + f"OKP container '{container_name}' was removed, recreating from {OKP_IMAGE_NAME}..." + ) + subprocess.run( + [ + "docker", + "run", + "--rm", + "-d", + "-p", + "8081:8080", + OKP_IMAGE_NAME, + ], + capture_output=True, + text=True, + check=True, + ) + else: + # Container exists - just start it + subprocess.run( + ["docker", "start", container_name], + capture_output=True, + text=True, + check=True, + ) + + # Wait for the server to be ready + max_attempts = 30 + okp_ready = False + for attempt in range(max_attempts): + try: + resp = requests.get(OKP_DEFAULT_URL, timeout=5) + if resp.status_code < 500: + print("✓ OKP server restarted successfully") + okp_ready = True + break + # HTTP 5xx: delay and retry + if attempt < max_attempts - 1: + time.sleep(1) + except (requests.ConnectionError, requests.Timeout): + if attempt < max_attempts - 1: + time.sleep(1) + + if not okp_ready: + raise RuntimeError( + f"OKP server failed to become ready after {max_attempts} attempts" + ) + def _print_llama_stack_diagnostics() -> None: """Print container state, health, and recent logs to diagnose why OGX did not recover.""" diff --git a/tests/e2e/features/okp_rag.feature b/tests/e2e/features/okp_rag.feature index fb5c6654f..725417d20 100644 --- a/tests/e2e/features/okp_rag.feature +++ b/tests/e2e/features/okp_rag.feature @@ -1,4 +1,4 @@ -@cfg_okp @skip +@cfg_okp Feature: OKP(Solr) RAG retrieval tests # Offline Knowledge Portal (OKP) provides a Solr-backed RAG source to LSC. diff --git a/tests/e2e/features/steps/llm_query_response.py b/tests/e2e/features/steps/llm_query_response.py index 50ff3cbd4..3a4b811e8 100644 --- a/tests/e2e/features/steps/llm_query_response.py +++ b/tests/e2e/features/steps/llm_query_response.py @@ -95,7 +95,14 @@ def responses_output_should_include_one_of_types(context: Context) -> None: @step("I wait for the response to be completed") def wait_for_complete_response(context: Context) -> None: """Wait for the response to be complete.""" - context.response_data = _parse_streaming_response(context.response.text) + # Reuse cached parse result if it was created for the current response + if not ( + hasattr(context, "response_data") + and hasattr(context, "_response_data_source") + and context._response_data_source is context.response + ): + context.response_data = _parse_streaming_response(context.response.text) + context._response_data_source = context.response context.response.raise_for_status() assert context.response_data["finished"] is True context.use_streaming_response_data = True @@ -156,6 +163,33 @@ def ask_question_authorized(context: Context, endpoint: str) -> None: body = _read_streamed_response(resp) resp._content = body.encode(resp.encoding or "utf-8") context.response = resp + # Parse SSE events and make data available for assertions + # Preserve existing conversation_id if the new response doesn't have one + # (e.g., 403 errors won't have a 'start' event with conversation_id) + old_conversation_id = ( + context.response_data.get("conversation_id") + if hasattr(context, "response_data") + else None + ) + context.response_data = _parse_streaming_response(body) + # Extract conversation from terminal event if not found in start event (Responses API) + if not context.response_data.get("conversation_id"): + if old_conversation_id: + # Preserve from previous response (for 403 error scenarios) + context.response_data["conversation_id"] = old_conversation_id + else: + # Try to extract from terminal event (Responses API streaming format) + try: + terminal = parse_responses_sse_final_response_object(body) + context.response_data["conversation"] = terminal.get("conversation") + context.response_data["conversation_id"] = terminal.get( + "conversation" + ) + except AssertionError: + pass # No terminal event found (e.g., error responses) + # Mark that this parsed data is from the current response (for cache reuse) + context._response_data_source = resp + context.use_streaming_response_data = True else: context.response = request_with_transient_retry( method="POST", @@ -164,6 +198,8 @@ def ask_question_authorized(context: Context, endpoint: str) -> None: headers=context.auth_headers, timeout=DEFAULT_LLM_TIMEOUT, ) + # Reset flag so assertions read from response.json(), not stale response_data + context.use_streaming_response_data = False # Query length chosen to exceed typical model context windows (e.g. 128k tokens) @@ -186,15 +222,35 @@ def ask_question_too_long_authorized(context: Context, endpoint: str) -> None: @step("I store conversation details") def store_conversation_details(context: Context) -> None: - """Store details about the conversation.""" + """Store details about the conversation. + + Reuses cached parse result if it was created for the current response. + """ + # Reuse if already parsed for this response + if ( + hasattr(context, "response_data") + and hasattr(context, "_response_data_source") + and context._response_data_source is context.response + ): + return + try: context.response_data = json.loads(context.response.text) + context._response_data_source = context.response except json.JSONDecodeError: + # Streaming response - parse it context.response_data = _parse_streaming_response(context.response.text) + # Extract conversation from terminal event if not in start event (Responses API) if not context.response_data.get("conversation_id"): - terminal = parse_responses_sse_final_response_object(context.response.text) - context.response_data["conversation"] = terminal.get("conversation") - context.response_data["conversation_id"] = terminal.get("conversation") + try: + terminal = parse_responses_sse_final_response_object( + context.response.text + ) + context.response_data["conversation"] = terminal.get("conversation") + context.response_data["conversation_id"] = terminal.get("conversation") + except AssertionError: + pass # No terminal event found + context._response_data_source = context.response @step('I use "{endpoint}" to ask question with same conversation_id') @@ -220,7 +276,10 @@ def ask_question_in_same_conversation(context: Context, endpoint: str) -> None: def check_rag_chunks_present(context: Context) -> None: """Check that the response contains non-empty rag_chunks from inline RAG.""" assert context.response is not None - response_json = context.response.json() + if getattr(context, "use_streaming_response_data", False): + response_json = context.response_data + else: + response_json = context.response.json() assert "rag_chunks" in response_json, "rag_chunks field missing from response" assert ( len(response_json["rag_chunks"]) > 0 @@ -231,7 +290,10 @@ def check_rag_chunks_present(context: Context) -> None: def check_referenced_documents_present(context: Context) -> None: """Check that the response contains non-empty referenced_documents.""" assert context.response is not None - response_json = context.response.json() + if getattr(context, "use_streaming_response_data", False): + response_json = context.response_data + else: + response_json = context.response.json() assert ( "referenced_documents" in response_json ), "referenced_documents field missing from response" @@ -379,6 +441,7 @@ def _parse_streaming_response(response_text: str) -> dict: full_response_split: list[str] = [] tool_calls: list[dict[str, Any]] = [] tool_results: list[dict[str, Any]] = [] + referenced_documents: list[dict[str, Any]] = [] finished = False stream_error = ( None # {"status_code": int, "response": str, "cause": str} if event "error" @@ -402,6 +465,10 @@ def _parse_streaming_response(response_text: str) -> dict: full_response = data["data"]["token"] elif event == "end": finished = True + end_data = data.get("data") or {} + referenced_documents = end_data.get( + "referenced_documents", referenced_documents + ) elif event == "error": stream_error = data.get("data") or {} except json.JSONDecodeError: @@ -413,6 +480,7 @@ def _parse_streaming_response(response_text: str) -> dict: "response_complete": full_response, "tool_calls": tool_calls, "tool_results": tool_results, + "referenced_documents": referenced_documents, "finished": finished, "stream_error": stream_error, } diff --git a/tests/e2e/features/steps/okp_rag.py b/tests/e2e/features/steps/okp_rag.py new file mode 100644 index 000000000..6911dd2f9 --- /dev/null +++ b/tests/e2e/features/steps/okp_rag.py @@ -0,0 +1,585 @@ +"""Step definitions for OKP(Solr) RAG retrieval tests.""" + +import os +import subprocess +import time +from typing import Any + +import requests +from behave import given, then # pyright: ignore[reportAttributeAccessIssue] +from behave.runner import Context + +# ── Constants ── + +# OKP/Solr Docker container name +OKP_CONTAINER_NAME = os.getenv("E2E_OKP_CONTAINER", "okp-solr") + +# OKP Docker image name (for finding container by image) +OKP_IMAGE_NAME = os.getenv( + "E2E_OKP_IMAGE", "registry.redhat.io/offline-knowledge-portal/rhokp-rhel9" +) + +# Default OKP health check URL +OKP_DEFAULT_URL = os.getenv("E2E_OKP_URL", "http://localhost:8081") + +# Output item types that represent tool invocations +_TOOL_OUTPUT_TYPES = frozenset( + { + "file_search_call", + "function_call", + "mcp_call", + "web_search_call", + "mcp_list_tools", + } +) + + +# ── Response Body Extraction ── + + +def _get_response_body(context: Context) -> dict[str, Any]: + """Return the response body dict, handling both JSON and streaming formats.""" + if getattr(context, "use_streaming_response_data", False): + return context.response_data + return context.response.json() + + +def _get_response_text(context: Context) -> str: + """Extract response text from various response formats.""" + body = _get_response_body(context) + return ( + body.get("response") + or body.get("output_text") + or body.get("response_complete", "") + ) + + +# ── Data Extractors ── + + +def _get_rag_chunks(context: Context) -> list[dict[str, Any]]: + """Extract rag_chunks from query response or results from Responses API output.""" + body = _get_response_body(context) + if "rag_chunks" in body: + return body["rag_chunks"] + # Responses API: collect results from file_search_call output items + results: list[dict[str, Any]] = [] + for item in body.get("output", []): + if item.get("type") == "file_search_call": + results.extend(item.get("results") or []) + return results + + +def _get_referenced_documents(context: Context) -> list[dict[str, Any]]: + """Extract referenced_documents from response body.""" + body = _get_response_body(context) + return body.get("referenced_documents", []) + + +def _get_tool_calls(context: Context) -> list[dict[str, Any]]: + """Extract tool calls from query response or output items from Responses API.""" + body = _get_response_body(context) + if "tool_calls" in body: + return body["tool_calls"] + # Responses API: extract tool-type items from output + return [ + item + for item in body.get("output", []) + if item.get("type") in _TOOL_OUTPUT_TYPES + ] + + +def _get_file_search_results(context: Context) -> list[dict[str, Any]]: + """Extract file_search results from Responses API output items.""" + body = _get_response_body(context) + results: list[dict[str, Any]] = [] + for item in body.get("output", []): + if item.get("type") == "file_search_call": + results.extend(item.get("results") or []) + return results + + +# ── Generic Field Accessors ── + + +def _get_nested_field(item: dict[str, Any], field_path: str) -> Any: + """Get a field from item, supporting nested access via dot notation. + + Examples: + _get_nested_field(chunk, "score") -> chunk.get("score") + _get_nested_field(chunk, "attributes.reference_url") + -> chunk.get("attributes", {}).get("reference_url") + + Parameters: + item: Dictionary to extract field from. + field_path: Field path, using dots for nested access. + + Returns: + Field value or None if not found. + """ + keys = field_path.split(".") + value: Any = item + for key in keys: + if isinstance(value, dict): + value = value.get(key) + else: + return None + return value + + +# ── Generic Assertion Helpers ── + + +def _assert_count_matches(items: list, expected_count: int, item_type: str) -> None: + """Assert the number of items matches the expected count. + + Parameters: + items: List of items to check. + expected_count: Expected number of items. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If count doesn't match. + """ + actual_count = len(items) + assert ( + actual_count == expected_count + ), f"Expected {expected_count} {item_type}, but found {actual_count}" + + +def _assert_not_empty(items: list, item_type: str) -> None: + """Assert the collection is not empty. + + Parameters: + items: List of items to check. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If collection is empty. + """ + assert len(items) > 0, f"{item_type} is empty — no items were found" + + +def _assert_empty(items: list, item_type: str) -> None: + """Assert the collection is empty. + + Parameters: + items: List of items to check. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If collection is not empty. + """ + assert len(items) == 0, f"Expected no {item_type}, but found {len(items)}" + + +def _assert_field_not_empty( + items: list[dict[str, Any]], field_path: str, item_type: str +) -> None: + """Assert every item has a non-empty value for the specified field. + + Parameters: + items: List of items to check. + field_path: Field path to check (supports dot notation). + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If any item has empty or missing field. + """ + assert items, f"No {item_type} to check" + for i, item in enumerate(items): + value = _get_nested_field(item, field_path) + assert value is not None and value != "", ( + f"Expected non-empty {field_path} in {item_type}[{i}], " + f"but found {value!r}" + ) + + +def _assert_field_contains( + items: list[dict[str, Any]], field_path: str, substring: str, item_type: str +) -> None: + """Assert every item's field contains the expected substring (case-insensitive). + + Parameters: + items: List of items to check. + field_path: Field path to check (supports dot notation). + substring: Expected substring. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If any item's field doesn't contain substring. + """ + assert items, f"No {item_type} to check" + for i, item in enumerate(items): + value = _get_nested_field(item, field_path) + assert substring.lower() in str(value).lower(), ( + f"Expected {substring!r} in {item_type}[{i}].{field_path}, " + f"but found {value!r}" + ) + + +def _assert_field_matches( + items: list[dict[str, Any]], field_path: str, expected: Any, item_type: str +) -> None: + """Assert every item's field matches the expected value. + + For fields that might be nested (e.g., source in attributes), checks both + the direct field and the attributes.field path. + + Parameters: + items: List of items to check. + field_path: Field path to check (supports dot notation). + expected: Expected value. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If any item's field doesn't match expected value. + """ + assert items, f"No {item_type} to check" + for i, item in enumerate(items): + actual = _get_nested_field(item, field_path) + # Fallback: check if field exists in attributes + if actual is None and "." not in field_path: + actual = _get_nested_field(item, f"attributes.{field_path}") + assert actual == expected, ( + f"Expected {field_path}={expected!r} in {item_type}[{i}], " + f"but found {actual!r}" + ) + + +def _assert_has_fields( + items: list[dict[str, Any]], required_fields: set[str], item_type: str +) -> None: + """Assert every item has all required fields. + + Parameters: + items: List of items to check. + required_fields: Set of required field names. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If any item is missing required fields. + """ + assert items, f"No {item_type} to check" + for i, item in enumerate(items): + missing = required_fields - set(item.keys()) + assert not missing, ( + f"Expected fields {required_fields} in {item_type}[{i}], " + f"but missing {missing}. Available fields: {list(item.keys())}" + ) + + +# ── Docker/OKP Management ── + + +def _find_okp_container() -> str | None: + """Find OKP container by name or image. + + Returns: + Container name/ID if found, None otherwise. + """ + # Try by configured name first + result = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Running}}", OKP_CONTAINER_NAME], + capture_output=True, + text=True, + check=False, + ) + if result.returncode == 0: + return OKP_CONTAINER_NAME + + # Try by image name + result = subprocess.run( + ["docker", "ps", "-a", "-q", "--filter", f"ancestor={OKP_IMAGE_NAME}"], + capture_output=True, + text=True, + check=False, + ) + if result.returncode == 0 and result.stdout.strip(): + container_id = result.stdout.strip().split()[0] + return container_id + + return None + + +# ── Given steps ── + + +@given("OKP(Solr) server is running") +def okp_server_is_running(context: Context) -> None: + """Verify that the OKP(Solr) server is reachable.""" + url = OKP_DEFAULT_URL + try: + resp = requests.get(url, timeout=10) + assert ( + resp.status_code < 500 + ), f"OKP server at {url} returned status {resp.status_code}" + except requests.ConnectionError as exc: + assert False, f"OKP(Solr) server is not reachable at {url}: {exc}" + + +@given("The OKP(Solr) server is stopped") +def okp_server_is_stopped(context: Context) -> None: + """Stop the OKP(Solr) Docker container or pod to simulate unavailability.""" + from tests.e2e.utils.utils import is_prow_environment + + context.okp_was_running = False + context.okp_container_name = None + + if is_prow_environment(): + # Prow/OpenShift: use pod disruption + from tests.e2e.utils.prow_utils import disrupt_okp_solr_pod + + was_running = disrupt_okp_solr_pod() + if was_running: + context.okp_was_running = True + print("✓ OKP Solr pod disrupted in Prow environment") + else: + print("✓ OKP Solr pod was not running") + return + + # Docker mode: existing logic + container_name = _find_okp_container() + if not container_name: + print( + f"✓ OKP container not found (neither '{OKP_CONTAINER_NAME}' " + f"nor '{OKP_IMAGE_NAME}') - already unavailable" + ) + return + + # Check if container is running + result = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Running}}", container_name], + capture_output=True, + text=True, + check=False, + ) + + if result.returncode != 0: + print(f"✓ OKP container '{container_name}' inspection failed - unavailable") + return + + if "true" not in result.stdout.lower(): + print(f"✓ OKP container '{container_name}' already stopped") + return + + # Container is running - stop it + context.okp_was_running = True + context.okp_container_name = container_name + + stop_result = subprocess.run( + ["docker", "stop", container_name], + capture_output=True, + text=True, + check=False, + ) + if stop_result.returncode != 0: + assert ( + False + ), f"Failed to stop OKP container '{container_name}': {stop_result.stderr}" + + # Wait for the container to fully stop + time.sleep(5) + + # Verify the server is actually unreachable + max_attempts = 10 + for attempt in range(max_attempts): + try: + resp = requests.get(OKP_DEFAULT_URL, timeout=2) + if attempt < max_attempts - 1: + time.sleep(1) + else: + assert ( + False + ), f"OKP server still responding after stop: {resp.status_code}" + except requests.ConnectionError: + # Server is unreachable - success + print(f"✓ OKP server stopped and verified unreachable ({container_name})") + break + except requests.Timeout: + # Timeout is also acceptable - server not responding + print(f"✓ OKP server stopped (connection timeout) ({container_name})") + break + + +# ── Then Steps: rag_chunks Assertions ── + + +@then("The number of rag_chunk returned is {count:d}") +def check_rag_chunk_count(context: Context, count: int) -> None: + """Assert the number of rag_chunks matches the expected count.""" + chunks = _get_rag_chunks(context) + _assert_count_matches(chunks, count, "rag_chunks") + + +@then("Each rag_chunk has a non-empty score") +def check_rag_chunk_scores(context: Context) -> None: + """Assert every rag_chunk has a non-empty score.""" + chunks = _get_rag_chunks(context) + _assert_field_not_empty(chunks, "score", "rag_chunks") + + +@then('Each rag_chunk source is "{source}"') +def check_rag_chunk_source(context: Context, source: str) -> None: + """Assert every rag_chunk has the expected source.""" + chunks = _get_rag_chunks(context) + _assert_field_matches(chunks, "source", source, "rag_chunks") + + +@then('Each rag_chunk reference_url contains "{domain}"') +def check_rag_chunk_reference_url(context: Context, domain: str) -> None: + """Assert every rag_chunk's reference_url contains the expected domain.""" + chunks = _get_rag_chunks(context) + # Check both possible field paths for reference_url + assert chunks, "No rag_chunks to check" + for i, chunk in enumerate(chunks): + attrs = chunk.get("attributes") or {} + ref_url = attrs.get("reference_url") or attrs.get("doc_url") or "" + assert domain in str(ref_url), ( + f"Expected {domain!r} in rag_chunks[{i}].attributes.reference_url, " + f"but found {ref_url!r}" + ) + + +# ── Then Steps: referenced_documents Assertions ── + + +@then("Each referenced_document has fields doc_url, doc_title, source, and document_id") +def check_referenced_document_fields(context: Context) -> None: + """Assert every referenced_document has the required fields.""" + docs = _get_referenced_documents(context) + required_fields = {"doc_url", "doc_title", "source", "document_id"} + _assert_has_fields(docs, required_fields, "referenced_documents") + + +@then('Each referenced_document doc_url contains "{domain}"') +def check_referenced_document_doc_url(context: Context, domain: str) -> None: + """Assert every referenced_document doc_url contains the expected domain.""" + docs = _get_referenced_documents(context) + _assert_field_contains(docs, "doc_url", domain, "referenced_documents") + + +@then("Each referenced_document doc_title is not empty") +def check_referenced_document_doc_title(context: Context) -> None: + """Assert every referenced_document has a non-empty doc_title.""" + docs = _get_referenced_documents(context) + _assert_field_not_empty(docs, "doc_title", "referenced_documents") + + +@then('Each referenced_document doc_title contains "{substring}"') +def check_referenced_document_doc_title_contains( + context: Context, substring: str +) -> None: + """Assert every referenced_document doc_title contains the expected substring. + + Matching is case-insensitive. + """ + docs = _get_referenced_documents(context) + _assert_field_contains(docs, "doc_title", substring, "referenced_documents") + + +@then("The number of referenced_document returned is {count:d}") +def check_referenced_document_count(context: Context, count: int) -> None: + """Assert the number of referenced_documents matches the expected count.""" + docs = _get_referenced_documents(context) + _assert_count_matches(docs, count, "referenced_documents") + + +@then('Each referenced_document source is "{source}"') +def check_referenced_document_source(context: Context, source: str) -> None: + """Assert every referenced_document has the expected source.""" + docs = _get_referenced_documents(context) + _assert_field_matches(docs, "source", source, "referenced_documents") + + +@then("Each referenced_document has a non-empty document_id") +def check_referenced_document_id(context: Context) -> None: + """Assert every referenced_document has a non-empty document_id.""" + docs = _get_referenced_documents(context) + _assert_field_not_empty(docs, "document_id", "referenced_documents") + + +# ── Then Steps: tool_calls Assertions ── + + +@then("The response contains non-empty tool_calls") +def check_tool_calls_present(context: Context) -> None: + """Assert the response contains at least one tool call.""" + tool_calls = _get_tool_calls(context) + _assert_not_empty(tool_calls, "tool_calls") + + +@then('A tool_call has name "{name}"') +def check_tool_call_name(context: Context, name: str) -> None: + """Assert at least one tool call has the expected name.""" + tool_calls = _get_tool_calls(context) + assert tool_calls, "No tool_calls to check" + names = [tc.get("name") for tc in tool_calls] + assert name in names, ( + f"Expected tool_call with name {name!r}, " f"but found names {names!r}" + ) + + +@then('A tool_call has type "{type_name}"') +def check_tool_call_type(context: Context, type_name: str) -> None: + """Assert at least one tool call has the expected type.""" + tool_calls = _get_tool_calls(context) + assert tool_calls, "No tool_calls to check" + types = [tc.get("type") for tc in tool_calls] + matched = any(t in {type_name, f"{type_name}_call"} for t in types) + assert matched, ( + f"Expected tool_call with type {type_name!r}, " f"but found types {types!r}" + ) + + +# ── Then Steps: Content and Results Assertions ── + + +@then('The response contains "{substring}"') +def check_response_contains_substring(context: Context, substring: str) -> None: + """Assert the LLM response contains the expected substring (case-insensitive).""" + response_text = _get_response_text(context) + assert substring.lower() in response_text.lower(), ( + f"Expected substring {substring!r} in response, " + f"but response text: {response_text[:200]}..." + ) + + +@then("The response contains non-empty content") +def check_response_content_present(context: Context) -> None: + """Assert the response contains non-empty content text.""" + content = _get_response_text(context) + assert content, "Expected non-empty response content, but it was empty" + + +@then("The response contains non-empty results") +def check_results_present(context: Context) -> None: + """Assert the Responses API output contains non-empty file_search results.""" + results = _get_file_search_results(context) + _assert_not_empty(results, "file_search results") + + +@then("The number of results returned is {count:d}") +def check_results_count(context: Context, count: int) -> None: + """Assert the number of file_search results matches the expected count.""" + results = _get_file_search_results(context) + _assert_count_matches(results, count, "file_search results") + + +# ── Then Steps: Empty Response Assertions ── + + +@then("The response contains no rag_chunks") +def check_no_rag_chunks(context: Context) -> None: + """Assert the response has no rag_chunks (empty or absent).""" + body = _get_response_body(context) + chunks = body.get("rag_chunks", []) + _assert_empty(chunks, "rag_chunks") + + +@then("The response contains no referenced_documents") +def check_no_referenced_documents(context: Context) -> None: + """Assert the response has no referenced_documents (empty or absent).""" + body = _get_response_body(context) + docs = body.get("referenced_documents", []) + _assert_empty(docs, "referenced_documents") diff --git a/tests/e2e/utils/prow_utils.py b/tests/e2e/utils/prow_utils.py index 7b5f02f10..8ab7c93bb 100644 --- a/tests/e2e/utils/prow_utils.py +++ b/tests/e2e/utils/prow_utils.py @@ -20,6 +20,7 @@ def get_namespace() -> str: _POD_NAME_MAP = { "lightspeed-stack": "lightspeed-stack-service", "llama-stack": "llama-stack-service", + "okp-solr": "okp-solr-service", } @@ -345,3 +346,48 @@ def update_config_configmap( finally: if temp_path and os.path.exists(temp_path): os.remove(temp_path) + + +def disrupt_okp_solr_pod() -> bool: + """Disrupt OKP Solr connection in Prow/OpenShift environment. + + Returns: + True if the pod was running and has been disrupted, False otherwise. + """ + try: + result = run_e2e_ops("disrupt-okp-solr", timeout=60) + print(result.stdout, end="") + + # Exit code 0 = disrupted (was running), exit code 2 = was not running + if result.returncode == 0: + return True + elif result.returncode == 2: + return False + else: + print(result.stderr, end="") + return False + + except subprocess.TimeoutExpired: + print("Warning: Timeout while disrupting OKP Solr connection") + return False + + +def restore_okp_solr_pod() -> None: + """Restore OKP Solr pod in Prow/OpenShift environment. + + Raises: + subprocess.CalledProcessError: If oc/e2e-ops restore fails. + subprocess.TimeoutExpired: If the operation times out. + """ + # restore-okp-solr can spend 180 seconds in wait_for_pod, plus oc apply time. + # This caller also times out at 180 seconds. after_scenario catches that timeout + # and only logs a warning, so later scenarios can run while OKP remains unavailable. + # Use a timeout with margin, such as 240 seconds. + result = run_e2e_ops("restore-okp-solr", timeout=240) + print(result.stdout, end="") + if result.returncode != 0: + print(result.stderr, end="") + raise subprocess.CalledProcessError( + result.returncode, "restore-okp-solr", result.stderr + ) + print("✓ OKP Solr pod restored successfully")