From 6c0c8a46a57ae7391e4eaafcfc0cdf0bc269188d Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Fri, 21 Aug 2026 10:49:19 -0400 Subject: [PATCH 01/18] adding new OKP tests --- tests/e2e/features/okp_rag.feature | 229 +++++++++++++++++++++++++++++ tests/e2e/features/query.feature | 24 +++ tests/e2e/test_list.txt | 1 + 3 files changed, 254 insertions(+) create mode 100644 tests/e2e/features/okp_rag.feature diff --git a/tests/e2e/features/okp_rag.feature b/tests/e2e/features/okp_rag.feature new file mode 100644 index 000000000..989cf1061 --- /dev/null +++ b/tests/e2e/features/okp_rag.feature @@ -0,0 +1,229 @@ +@cfg_okp +Feature: OKP(Solr) RAG retrieval tests + + # Offline Knowledge Portal (OKP) provides a Solr-backed RAG source to LSC. + # Tests verify that Lightspeed Stack can use OKP for both Inline RAG + # (context injected before the LLM request) and Tool RAG (context + # retrieved on demand via file_search), in both offline and online modes. + + Background: + Given The service is started locally + And The system is in default state + And OKP(Solr) server is running + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And REST API service prefix is /v1 + And the Lightspeed stack configuration directory is "tests/e2e/configuration" + + # ── Inline RAG — Query ── + + Scenario Outline: mode query with inline RAG returns rag_chunks and referenced_documents + Given The service uses the configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response contains non-empty rag_chunks + And The response contains non-empty referenced_documents + And The number of rag_chunk returned is + And Each rag_chunk has a non-empty score + And Each rag_chunk source is "okp" + And Each referenced_document has fields doc_url, doc_title, source, and document_id + And Each referenced_document doc_url contains "" + And Each referenced_document doc_title is not empty + And Each referenced_document source is "okp" + And Each referenced_document has a non-empty document_id + + Examples: Offline + | mode | config | max_chunks | doc_url_domain | + | Offline | lightspeed-stack-okp-offline.yaml | 5 | localhost:8081 | + + Examples: Online + | mode | config | max_chunks | doc_url_domain | + | Online | lightspeed-stack-okp-online.yaml | 1 | docs.redhat.com | + + # ── Inline RAG — Streaming Query ── + + Scenario Outline: mode streaming query with inline RAG returns referenced_documents + Given The service uses the configuration + And The service is restarted + When I use "streaming_query" to ask question with authorization header + """ + {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And I wait for the response to be completed + And The response contains non-empty referenced_documents + And Each referenced_document has fields doc_url, doc_title, source, and document_id + And Each referenced_document doc_url contains "" + And Each referenced_document doc_title is not empty + And Each referenced_document source is "okp" + And Each referenced_document has a non-empty document_id + + Examples: Offline + | mode | config | doc_url_domain | + | Offline | lightspeed-stack-okp-offline.yaml | localhost:8081 | + + Examples: Online + | mode | config | doc_url_domain | + | Online | lightspeed-stack-okp-online.yaml | docs.redhat.com | + + # ── Inline RAG — Query with Dynamic Filter ── + + Scenario Outline: Query with inline RAG with dynamic filter returns rag_chunks and referenced_documents + Given The service uses the lightspeed-stack-okp-offline.yaml configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + {"query": "Security best practices", + "solr": { + "mode": "", + "filters": { + "filters": { + "type": "in", + "key": "product", + "value": ["openshift_container_platform", "ansible_automation_platform", "red_hat_enterprise_linux"] + } + } + } + } + """ + Then The status code of the response is 200 + And The response contains non-empty rag_chunks + And The response contains non-empty referenced_documents + And The number of rag_chunk returned is 5 + And Each rag_chunk has a non-empty score + And Each rag_chunk source is "okp" + And Each referenced_document has fields doc_url, doc_title, source, and document_id + And Each referenced_document doc_url contains "" + And Each referenced_document doc_title is not empty + And Each referenced_document source is "okp" + And Each referenced_document has a non-empty document_id + + Examples: + | filter_mode | doc_url_domain | + | semantic | localhost:8081 | + | hybrid | docs.redhat.com | + + # ── Tool RAG — Query API ── + + Scenario Outline: queries API with OKP tool RAG has rag_chunk and referenced_documents returned + Given The service uses the configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + { + "query": "configure remote desktop using gnome", + "model": "{MODEL}", + "provider": "{PROVIDER}", + "system_prompt": "You MUST use the file_search tool to answer." + } + """ + Then The status code of the response is 200 + And The response contains non-empty tool_calls + And A tool_call has name "file_search" + And The response contains non-empty rag_chunks + And The number of rag_chunk returned is + And Each rag_chunk has a non-empty score + And Each rag_chunk source is "okp" + And The response contains non-empty referenced_documents + And Each referenced_document has fields doc_url, doc_title, source, and document_id + And Each referenced_document doc_url contains "" + And Each referenced_document doc_title is not empty + And Each referenced_document source is "okp" + And Each referenced_document has a non-empty document_id + + Examples: Offline + | mode | config | max_chunks | doc_url_domain | + | Offline | lightspeed-stack-okp-tool-offline.yaml | 5 | localhost:8081 | + + Examples: Online + | mode | config | max_chunks | doc_url_domain | + | Online | lightspeed-stack-okp-tool-online.yaml | 1 | access.redhat.com | + + # ── Tool RAG — Streaming Query API ── + + Scenario Outline: streaming query API with OKP tool RAG has rag_chunk and referenced_documents returned + Given The service uses the configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + { + "query": "configure remote desktop using gnome", + "model": "{MODEL}", + "provider": "{PROVIDER}", + "system_prompt": "You MUST use the file_search tool to answer." + } + """ + Then The status code of the response is 200 + And A tool_call has name "file_search" + And The response contains non-empty content + And The response contains non-empty referenced_documents + And Each referenced_document has fields doc_url, doc_title, source, and document_id + And Each referenced_document doc_url contains "" + And Each referenced_document doc_title is not empty + And Each referenced_document source is "okp" + And Each referenced_document has a non-empty document_id + + Examples: Offline + | mode | config | doc_url_domain | + | Offline | lightspeed-stack-okp-tool-offline.yaml | localhost:8081 | + + Examples: Online + | mode | config | doc_url_domain | + | Online | lightspeed-stack-okp-tool-online.yaml | access.redhat.com | + + # ── Tool RAG — Responses API ── + + Scenario Outline: responses API with OKP tool RAG has rag results returned + Given The service uses the configuration + And The service is restarted + When I use "responses" to ask question with authorization header + """ + { + "input": "configure remote desktop using gnome", + "model": "{PROVIDER}/{MODEL}", + "stream": false, + "instructions": "You MUST use the file_search tool to answer." + } + """ + Then The status code of the response is 200 + And The responses output includes an item with type "file_search_call" + And The response contains non-empty tool_calls + And A tool_call has type "file_search" + And The response contains non-empty results + And The number of results returned is + And Each rag_chunk has a non-empty score + And Each rag_chunk source is "okp" + And Each rag_chunk reference_url contains "" + + Examples: Offline + | mode | config | max_chunks | doc_url_domain | + | Offline | lightspeed-stack-okp-tool-offline.yaml | 5 | localhost:8081 | + + Examples: Online + | mode | config | max_chunks | doc_url_domain | + | Online | lightspeed-stack-okp-tool-online.yaml | 1 | access.redhat.com | + + # # ── OKP Server Unavailable — Graceful Error Handling ─────────────── + + Scenario: Query succeeds with empty rag_chunks when OKP server is unavailable + Given The OKP(Solr) server is stopped + When I use "query" to ask question with authorization header + """ + {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response contains no rag_chunks + And The response contains no referenced_documents + + Scenario: Streaming query succeeds with empty referenced_documents when OKP server is unavailable + Given The OKP(Solr) server is stopped + When I use "streaming_query" to ask question with authorization header + """ + {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And I wait for the response to be completed + And The response contains no referenced_documents diff --git a/tests/e2e/features/query.feature b/tests/e2e/features/query.feature index c0207315c..702fc1490 100644 --- a/tests/e2e/features/query.feature +++ b/tests/e2e/features/query.feature @@ -279,3 +279,27 @@ Scenario: Check if LLM responds for query request with error for missing query When I use "query" to ask question with too-long query and authorization header Then The status code of the response is 413 And The body of the response contains Prompt is too long + + # # ── OKP RAG Disabled (okp not in rag.retrieval.inline.sources) ───── + @cfg_okp + Scenario: Query returns no rag_chunks and no reference_documents when OKP OKP is disabled + Given The service uses the lightspeed-stack-okp-disabled.yaml configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response contains no rag_chunks + And The response contains no reference_documents + @cfg_okp + Scenario: Streaming query returns no referenced_documents when OKP is disabled + Given The service uses the lightspeed-stack-okp-disabled.yaml configuration + And The service is restarted + When I use "streaming_query" to ask question with authorization header + """ + {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And I wait for the response to be completed + And The response contains no referenced_documents diff --git a/tests/e2e/test_list.txt b/tests/e2e/test_list.txt index 2a5d97f05..c9ff90ed5 100644 --- a/tests/e2e/test_list.txt +++ b/tests/e2e/test_list.txt @@ -39,3 +39,4 @@ features/unified-mode-legacy.feature features/unified-mode-validation.feature features/unified-mode-migration.feature features/unified-mode-synthesis.feature +features/okp_rag.feature From 8684b24072d32fd571c9153402f69d66b9bfccb4 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Fri, 21 Aug 2026 10:53:48 -0400 Subject: [PATCH 02/18] added skip tags --- tests/e2e/features/okp_rag.feature | 2 +- tests/e2e/features/query.feature | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/e2e/features/okp_rag.feature b/tests/e2e/features/okp_rag.feature index 989cf1061..ba5193cb4 100644 --- a/tests/e2e/features/okp_rag.feature +++ b/tests/e2e/features/okp_rag.feature @@ -1,4 +1,4 @@ -@cfg_okp +@cfg_okp @skip Feature: OKP(Solr) RAG retrieval tests # Offline Knowledge Portal (OKP) provides a Solr-backed RAG source to LSC. diff --git a/tests/e2e/features/query.feature b/tests/e2e/features/query.feature index 702fc1490..555aee0b6 100644 --- a/tests/e2e/features/query.feature +++ b/tests/e2e/features/query.feature @@ -282,6 +282,7 @@ Scenario: Check if LLM responds for query request with error for missing query # # ── OKP RAG Disabled (okp not in rag.retrieval.inline.sources) ───── @cfg_okp + @skip Scenario: Query returns no rag_chunks and no reference_documents when OKP OKP is disabled Given The service uses the lightspeed-stack-okp-disabled.yaml configuration And The service is restarted @@ -293,6 +294,7 @@ Scenario: Check if LLM responds for query request with error for missing query And The response contains no rag_chunks And The response contains no reference_documents @cfg_okp + @skip Scenario: Streaming query returns no referenced_documents when OKP is disabled Given The service uses the lightspeed-stack-okp-disabled.yaml configuration And The service is restarted From 91cf1f9217b0662051df6f1792b27c7318c46018 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Fri, 21 Aug 2026 11:11:14 -0400 Subject: [PATCH 03/18] fixed typo --- tests/e2e/features/okp_rag.feature | 2 +- tests/e2e/features/query.feature | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/features/okp_rag.feature b/tests/e2e/features/okp_rag.feature index ba5193cb4..642393776 100644 --- a/tests/e2e/features/okp_rag.feature +++ b/tests/e2e/features/okp_rag.feature @@ -147,7 +147,7 @@ Feature: OKP(Solr) RAG retrieval tests Scenario Outline: streaming query API with OKP tool RAG has rag_chunk and referenced_documents returned Given The service uses the configuration And The service is restarted - When I use "query" to ask question with authorization header + When I use "streaming_query" to ask question with authorization header """ { "query": "configure remote desktop using gnome", diff --git a/tests/e2e/features/query.feature b/tests/e2e/features/query.feature index 555aee0b6..3660ad290 100644 --- a/tests/e2e/features/query.feature +++ b/tests/e2e/features/query.feature @@ -292,7 +292,7 @@ Scenario: Check if LLM responds for query request with error for missing query """ Then The status code of the response is 200 And The response contains no rag_chunks - And The response contains no reference_documents + And The response contains no referenced_documents @cfg_okp @skip Scenario: Streaming query returns no referenced_documents when OKP is disabled From f8b73f4255669992cac0dcf3ea04793d5bdd03e9 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Fri, 21 Aug 2026 11:22:32 -0400 Subject: [PATCH 04/18] removed extra config calls --- tests/e2e/features/query.feature | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/e2e/features/query.feature b/tests/e2e/features/query.feature index 3660ad290..18e3d155c 100644 --- a/tests/e2e/features/query.feature +++ b/tests/e2e/features/query.feature @@ -284,8 +284,6 @@ Scenario: Check if LLM responds for query request with error for missing query @cfg_okp @skip Scenario: Query returns no rag_chunks and no reference_documents when OKP OKP is disabled - Given The service uses the lightspeed-stack-okp-disabled.yaml configuration - And The service is restarted When I use "query" to ask question with authorization header """ {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} @@ -296,8 +294,6 @@ Scenario: Check if LLM responds for query request with error for missing query @cfg_okp @skip Scenario: Streaming query returns no referenced_documents when OKP is disabled - Given The service uses the lightspeed-stack-okp-disabled.yaml configuration - And The service is restarted When I use "streaming_query" to ask question with authorization header """ {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} From e7c7f3e0e1e0c731d8a0465d02f19338e6e2f6b2 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Mon, 24 Aug 2026 15:20:01 -0400 Subject: [PATCH 05/18] created config yaml and created step definitions --- .../lightspeed-stack-okp-offline.yaml | 39 +++ .../lightspeed-stack-okp-online.yaml | 40 +++ .../lightspeed-stack-okp-tool-offline.yaml | 39 +++ .../lightspeed-stack-okp-tool-online.yaml | 39 +++ .../lightspeed-stack-okp-offline.yaml | 40 +++ .../lightspeed-stack-okp-online.yaml | 40 +++ .../lightspeed-stack-okp-tool-offline.yaml | 39 +++ .../lightspeed-stack-okp-tool-online.yaml | 39 +++ tests/e2e/features/okp_rag.feature | 142 ++------ .../e2e/features/steps/llm_query_response.py | 16 +- tests/e2e/features/steps/okp_rag.py | 307 ++++++++++++++++++ 11 files changed, 671 insertions(+), 109 deletions(-) create mode 100644 tests/e2e/configuration/library-mode/lightspeed-stack-okp-offline.yaml create mode 100644 tests/e2e/configuration/library-mode/lightspeed-stack-okp-online.yaml create mode 100644 tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-offline.yaml create mode 100644 tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-online.yaml create mode 100644 tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml create mode 100644 tests/e2e/configuration/server-mode/lightspeed-stack-okp-online.yaml create mode 100644 tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-offline.yaml create mode 100644 tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-online.yaml create mode 100644 tests/e2e/features/steps/okp_rag.py diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-offline.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-offline.yaml new file mode 100644 index 000000000..0ac491655 --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-offline.yaml @@ -0,0 +1,39 @@ +# @cfg_okp +# OKP inline RAG — offline mode. +# Chunks use parent_id-based URLs (localhost:8081). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: true + config: + profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + inline: + sources: + - okp + max_chunks: 1 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: true + max_chunks: 5 diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-online.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-online.yaml new file mode 100644 index 000000000..78e7e1604 --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-online.yaml @@ -0,0 +1,40 @@ +# @cfg_okp +# OKP inline RAG — online mode. +# Chunks use reference_url-based URLs (docs.redhat.com). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: true + config: + profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + inline: + sources: + - okp + max_chunks: 3 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: false + max_chunks: 3 + chunk_filter_query: "product:openshift_container_platform AND product_version:4.21" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-offline.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-offline.yaml new file mode 100644 index 000000000..e952860a4 --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-offline.yaml @@ -0,0 +1,39 @@ +# @cfg_okp +# OKP tool RAG — offline mode. +# LLM calls file_search on demand; chunks use parent_id-based URLs (localhost:8081). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: true + config: + profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + tool: + sources: + - okp + max_chunks: 2 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: true + max_chunks: 5 diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-online.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-online.yaml new file mode 100644 index 000000000..b058f0fae --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-online.yaml @@ -0,0 +1,39 @@ +# @cfg_okp +# OKP tool RAG — online mode. +# LLM calls file_search on demand; chunks use reference_url-based URLs (access.redhat.com). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: true + config: + profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + tool: + sources: + - okp + max_chunk : 3 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: false + max_chunks: 5 diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml new file mode 100644 index 000000000..e2a3ccd35 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml @@ -0,0 +1,40 @@ +# @cfg_okp +# OKP inline RAG — offline mode. +# Chunks use parent_id-based URLs (localhost:8081). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + inline: + sources: + - okp + max_chunks: 1 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: true + max_chunks: 5 + diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-okp-online.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-online.yaml new file mode 100644 index 000000000..94c9362b9 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-online.yaml @@ -0,0 +1,40 @@ +# @cfg_okp +# OKP inline RAG — online mode. +# Chunks use reference_url-based URLs (docs.redhat.com). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + inline: + sources: + - okp + max_chunks: 3 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: false + max_chunks: 3 + chunk_filter_query: "product:openshift_container_platform AND product_version:4.21" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-offline.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-offline.yaml new file mode 100644 index 000000000..82c981216 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-offline.yaml @@ -0,0 +1,39 @@ +# @cfg_okp +# OKP tool RAG — offline mode. +# LLM calls file_search on demand; chunks use parent_id-based URLs (localhost:8081). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + tool: + sources: + - okp + max_chunks: 2 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: true + max_chunks: 5 diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-online.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-online.yaml new file mode 100644 index 000000000..ec6c258e2 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-online.yaml @@ -0,0 +1,39 @@ +# @cfg_okp +# OKP tool RAG — online mode. +# LLM calls file_search on demand; chunks use reference_url-based URLs (access.redhat.com). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + tool: + sources: + - okp + max_chunks: 3 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: false + max_chunks: 5 diff --git a/tests/e2e/features/okp_rag.feature b/tests/e2e/features/okp_rag.feature index 642393776..6d419224d 100644 --- a/tests/e2e/features/okp_rag.feature +++ b/tests/e2e/features/okp_rag.feature @@ -14,10 +14,10 @@ Feature: OKP(Solr) RAG retrieval tests And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - # ── Inline RAG — Query ── + # ── Inline RAG — Query (offline) ── - Scenario Outline: mode query with inline RAG returns rag_chunks and referenced_documents - Given The service uses the configuration + Scenario: Offline mode query with inline RAG returns rag_chunks and referenced_documents + Given The service uses the lightspeed-stack-okp-offline.yaml configuration And The service is restarted When I use "query" to ask question with authorization header """ @@ -26,27 +26,20 @@ Feature: OKP(Solr) RAG retrieval tests Then The status code of the response is 200 And The response contains non-empty rag_chunks And The response contains non-empty referenced_documents - And The number of rag_chunk returned is + And The number of rag_chunk returned is 1 And Each rag_chunk has a non-empty score And Each rag_chunk source is "okp" And Each referenced_document has fields doc_url, doc_title, source, and document_id - And Each referenced_document doc_url contains "" + And The number of eferenced_document returned is 1 + And Each referenced_document doc_url contains "localhost:8081" And Each referenced_document doc_title is not empty And Each referenced_document source is "okp" And Each referenced_document has a non-empty document_id - Examples: Offline - | mode | config | max_chunks | doc_url_domain | - | Offline | lightspeed-stack-okp-offline.yaml | 5 | localhost:8081 | - - Examples: Online - | mode | config | max_chunks | doc_url_domain | - | Online | lightspeed-stack-okp-online.yaml | 1 | docs.redhat.com | - - # ── Inline RAG — Streaming Query ── + # ── Inline RAG — Streaming Query (online) ── - Scenario Outline: mode streaming query with inline RAG returns referenced_documents - Given The service uses the configuration + Scenario: Online mode streaming query with inline RAG returns referenced_documents + Given The service uses the lightspeed-stack-okp-online.yaml configuration And The service is restarted When I use "streaming_query" to ask question with authorization header """ @@ -56,29 +49,23 @@ Feature: OKP(Solr) RAG retrieval tests And I wait for the response to be completed And The response contains non-empty referenced_documents And Each referenced_document has fields doc_url, doc_title, source, and document_id - And Each referenced_document doc_url contains "" + And The number of eferenced_document returned is 3 + And Each referenced_document doc_url contains "docs.redhat.com" And Each referenced_document doc_title is not empty + And Each referenced_document doc_title contains "openshift container platform 4.21" And Each referenced_document source is "okp" And Each referenced_document has a non-empty document_id - Examples: Offline - | mode | config | doc_url_domain | - | Offline | lightspeed-stack-okp-offline.yaml | localhost:8081 | - - Examples: Online - | mode | config | doc_url_domain | - | Online | lightspeed-stack-okp-online.yaml | docs.redhat.com | - # ── Inline RAG — Query with Dynamic Filter ── - Scenario Outline: Query with inline RAG with dynamic filter returns rag_chunks and referenced_documents + Scenario: Query with inline RAG with dynamic semantic filter returns rag_chunks and referenced_documents Given The service uses the lightspeed-stack-okp-offline.yaml configuration And The service is restarted When I use "query" to ask question with authorization header """ {"query": "Security best practices", "solr": { - "mode": "", + "mode": "semantic", "filters": { "filters": { "type": "in", @@ -90,31 +77,28 @@ Feature: OKP(Solr) RAG retrieval tests } """ Then The status code of the response is 200 + And The response contains "security best practices" And The response contains non-empty rag_chunks And The response contains non-empty referenced_documents - And The number of rag_chunk returned is 5 + And The number of rag_chunk returned is 1 And Each rag_chunk has a non-empty score And Each rag_chunk source is "okp" And Each referenced_document has fields doc_url, doc_title, source, and document_id - And Each referenced_document doc_url contains "" + And The number of eferenced_document returned is 1 + And Each referenced_document doc_url contains "localhost:8081" And Each referenced_document doc_title is not empty And Each referenced_document source is "okp" And Each referenced_document has a non-empty document_id - Examples: - | filter_mode | doc_url_domain | - | semantic | localhost:8081 | - | hybrid | docs.redhat.com | + # ── Tool RAG — Query API (offline) ── - # ── Tool RAG — Query API ── - - Scenario Outline: queries API with OKP tool RAG has rag_chunk and referenced_documents returned - Given The service uses the configuration + Scenario: Offline query API with OKP tool RAG has rag_chunk and referenced_documents returned + Given The service uses the lightspeed-stack-okp-tool-offline.yaml configuration And The service is restarted When I use "query" to ask question with authorization header """ { - "query": "configure remote desktop using gnome", + "query": "Troubleshooting guide", "model": "{MODEL}", "provider": "{PROVIDER}", "system_prompt": "You MUST use the file_search tool to answer." @@ -124,65 +108,25 @@ Feature: OKP(Solr) RAG retrieval tests And The response contains non-empty tool_calls And A tool_call has name "file_search" And The response contains non-empty rag_chunks - And The number of rag_chunk returned is + And The number of rag_chunk returned is 2 And Each rag_chunk has a non-empty score And Each rag_chunk source is "okp" And The response contains non-empty referenced_documents And Each referenced_document has fields doc_url, doc_title, source, and document_id - And Each referenced_document doc_url contains "" - And Each referenced_document doc_title is not empty - And Each referenced_document source is "okp" - And Each referenced_document has a non-empty document_id - - Examples: Offline - | mode | config | max_chunks | doc_url_domain | - | Offline | lightspeed-stack-okp-tool-offline.yaml | 5 | localhost:8081 | - - Examples: Online - | mode | config | max_chunks | doc_url_domain | - | Online | lightspeed-stack-okp-tool-online.yaml | 1 | access.redhat.com | - - # ── Tool RAG — Streaming Query API ── - - Scenario Outline: streaming query API with OKP tool RAG has rag_chunk and referenced_documents returned - Given The service uses the configuration - And The service is restarted - When I use "streaming_query" to ask question with authorization header - """ - { - "query": "configure remote desktop using gnome", - "model": "{MODEL}", - "provider": "{PROVIDER}", - "system_prompt": "You MUST use the file_search tool to answer." - } - """ - Then The status code of the response is 200 - And A tool_call has name "file_search" - And The response contains non-empty content - And The response contains non-empty referenced_documents - And Each referenced_document has fields doc_url, doc_title, source, and document_id - And Each referenced_document doc_url contains "" + And Each referenced_document doc_url contains "localhost:8081" And Each referenced_document doc_title is not empty And Each referenced_document source is "okp" And Each referenced_document has a non-empty document_id - Examples: Offline - | mode | config | doc_url_domain | - | Offline | lightspeed-stack-okp-tool-offline.yaml | localhost:8081 | - - Examples: Online - | mode | config | doc_url_domain | - | Online | lightspeed-stack-okp-tool-online.yaml | access.redhat.com | + # ── Tool RAG — Responses API (online) ── - # ── Tool RAG — Responses API ── - - Scenario Outline: responses API with OKP tool RAG has rag results returned - Given The service uses the configuration + Scenario: Online responses API with OKP tool RAG has rag results returned + Given The service uses the lightspeed-stack-okp-tool-online.yaml configuration And The service is restarted When I use "responses" to ask question with authorization header """ { - "input": "configure remote desktop using gnome", + "input": "Troubleshooting guide", "model": "{PROVIDER}/{MODEL}", "stream": false, "instructions": "You MUST use the file_search tool to answer." @@ -193,37 +137,21 @@ Feature: OKP(Solr) RAG retrieval tests And The response contains non-empty tool_calls And A tool_call has type "file_search" And The response contains non-empty results - And The number of results returned is + And The number of results returned is 3 And Each rag_chunk has a non-empty score And Each rag_chunk source is "okp" - And Each rag_chunk reference_url contains "" - - Examples: Offline - | mode | config | max_chunks | doc_url_domain | - | Offline | lightspeed-stack-okp-tool-offline.yaml | 5 | localhost:8081 | + And Each rag_chunk reference_url contains "access.redhat.com" - Examples: Online - | mode | config | max_chunks | doc_url_domain | - | Online | lightspeed-stack-okp-tool-online.yaml | 1 | access.redhat.com | - - # # ── OKP Server Unavailable — Graceful Error Handling ─────────────── + # ── OKP Server Unavailable — Graceful Error Handling ── Scenario: Query succeeds with empty rag_chunks when OKP server is unavailable - Given The OKP(Solr) server is stopped + Given The service uses the lightspeed-stack-okp-online.yaml configuration + And The service is restarted + And The OKP(Solr) server is stopped When I use "query" to ask question with authorization header """ {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 And The response contains no rag_chunks - And The response contains no referenced_documents - - Scenario: Streaming query succeeds with empty referenced_documents when OKP server is unavailable - Given The OKP(Solr) server is stopped - When I use "streaming_query" to ask question with authorization header - """ - {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 200 - And I wait for the response to be completed - And The response contains no referenced_documents + And The response contains no referenced_documents \ No newline at end of file diff --git a/tests/e2e/features/steps/llm_query_response.py b/tests/e2e/features/steps/llm_query_response.py index 50ff3cbd4..3e308fb21 100644 --- a/tests/e2e/features/steps/llm_query_response.py +++ b/tests/e2e/features/steps/llm_query_response.py @@ -220,7 +220,10 @@ def ask_question_in_same_conversation(context: Context, endpoint: str) -> None: def check_rag_chunks_present(context: Context) -> None: """Check that the response contains non-empty rag_chunks from inline RAG.""" assert context.response is not None - response_json = context.response.json() + if getattr(context, "use_streaming_response_data", False): + response_json = context.response_data + else: + response_json = context.response.json() assert "rag_chunks" in response_json, "rag_chunks field missing from response" assert ( len(response_json["rag_chunks"]) > 0 @@ -231,7 +234,10 @@ def check_rag_chunks_present(context: Context) -> None: def check_referenced_documents_present(context: Context) -> None: """Check that the response contains non-empty referenced_documents.""" assert context.response is not None - response_json = context.response.json() + if getattr(context, "use_streaming_response_data", False): + response_json = context.response_data + else: + response_json = context.response.json() assert ( "referenced_documents" in response_json ), "referenced_documents field missing from response" @@ -379,6 +385,7 @@ def _parse_streaming_response(response_text: str) -> dict: full_response_split: list[str] = [] tool_calls: list[dict[str, Any]] = [] tool_results: list[dict[str, Any]] = [] + referenced_documents: list[dict[str, Any]] = [] finished = False stream_error = ( None # {"status_code": int, "response": str, "cause": str} if event "error" @@ -402,6 +409,10 @@ def _parse_streaming_response(response_text: str) -> dict: full_response = data["data"]["token"] elif event == "end": finished = True + end_data = data.get("data") or {} + referenced_documents = end_data.get( + "referenced_documents", referenced_documents + ) elif event == "error": stream_error = data.get("data") or {} except json.JSONDecodeError: @@ -413,6 +424,7 @@ def _parse_streaming_response(response_text: str) -> dict: "response_complete": full_response, "tool_calls": tool_calls, "tool_results": tool_results, + "referenced_documents": referenced_documents, "finished": finished, "stream_error": stream_error, } diff --git a/tests/e2e/features/steps/okp_rag.py b/tests/e2e/features/steps/okp_rag.py new file mode 100644 index 000000000..38ac57a5e --- /dev/null +++ b/tests/e2e/features/steps/okp_rag.py @@ -0,0 +1,307 @@ +"""Step definitions for OKP(Solr) RAG retrieval tests.""" + +import os +import subprocess +import time +from typing import Any + +import requests +from behave import given, then # pyright: ignore[reportAttributeAccessIssue] +from behave.runner import Context + +# OKP/Solr Docker container name +OKP_CONTAINER_NAME = os.getenv("E2E_OKP_CONTAINER", "okp-solr") + +# Default OKP health check URL +OKP_DEFAULT_URL = os.getenv("E2E_OKP_URL", "http://localhost:8081") + +# Output item types that represent tool invocations +_TOOL_OUTPUT_TYPES = frozenset( + { + "file_search_call", + "function_call", + "mcp_call", + "web_search_call", + "mcp_list_tools", + } +) + + +def _get_response_body(context: Context) -> dict[str, Any]: + """Return the response body dict, handling both JSON and streaming formats.""" + if getattr(context, "use_streaming_response_data", False): + return context.response_data + return context.response.json() + + +def _get_rag_chunks(context: Context) -> list[dict[str, Any]]: + """Extract rag_chunks from query response or results from Responses API output.""" + body = _get_response_body(context) + if "rag_chunks" in body: + return body["rag_chunks"] + # Responses API: collect results from file_search_call output items + results: list[dict[str, Any]] = [] + for item in body.get("output", []): + if item.get("type") == "file_search_call": + results.extend(item.get("results") or []) + return results + + +def _get_referenced_documents(context: Context) -> list[dict[str, Any]]: + """Extract referenced_documents from response body.""" + body = _get_response_body(context) + return body.get("referenced_documents", []) + + +def _get_tool_calls(context: Context) -> list[dict[str, Any]]: + """Extract tool calls from query response or output items from Responses API.""" + body = _get_response_body(context) + if "tool_calls" in body: + return body["tool_calls"] + # Responses API: extract tool-type items from output + return [ + item + for item in body.get("output", []) + if item.get("type") in _TOOL_OUTPUT_TYPES + ] + + +def _get_file_search_results(context: Context) -> list[dict[str, Any]]: + """Extract file_search results from Responses API output items.""" + body = _get_response_body(context) + results: list[dict[str, Any]] = [] + for item in body.get("output", []): + if item.get("type") == "file_search_call": + results.extend(item.get("results") or []) + return results + + +# ── Given steps ── + + +@given("OKP(Solr) server is running") +def okp_server_is_running(context: Context) -> None: + """Verify that the OKP(Solr) server is reachable.""" + url = OKP_DEFAULT_URL + try: + resp = requests.get(url, timeout=10) + assert ( + resp.status_code < 500 + ), f"OKP server at {url} returned status {resp.status_code}" + except requests.ConnectionError as exc: + assert False, f"OKP(Solr) server is not reachable at {url}: {exc}" + + +@given("The OKP(Solr) server is stopped") +def okp_server_is_stopped(context: Context) -> None: + """Stop the OKP(Solr) Docker container to simulate unavailability.""" + context.okp_was_running = False + try: + result = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Running}}", OKP_CONTAINER_NAME], + capture_output=True, + text=True, + check=True, + ) + if "true" in result.stdout.lower(): + context.okp_was_running = True + subprocess.run( + ["docker", "stop", OKP_CONTAINER_NAME], + capture_output=True, + text=True, + check=True, + ) + time.sleep(2) + except subprocess.CalledProcessError as exc: + print(f"Warning: could not stop OKP container: {exc}") + + +# ── Then steps: rag_chunk assertions ── + + +@then("The number of rag_chunk returned is {count:d}") +def check_rag_chunk_count(context: Context, count: int) -> None: + """Assert the number of rag_chunks matches the expected count.""" + chunks = _get_rag_chunks(context) + assert len(chunks) == count, f"Expected {count} rag_chunks, got {len(chunks)}" + + +@then("Each rag_chunk has a non-empty score") +def check_rag_chunk_scores(context: Context) -> None: + """Assert every rag_chunk has a non-empty score.""" + chunks = _get_rag_chunks(context) + assert chunks, "No rag_chunks to check" + for i, chunk in enumerate(chunks): + score = chunk.get("score") + assert score is not None, f"rag_chunk[{i}] has no score" + + +@then('Each rag_chunk source is "{source}"') +def check_rag_chunk_source(context: Context, source: str) -> None: + """Assert every rag_chunk has the expected source.""" + chunks = _get_rag_chunks(context) + assert chunks, "No rag_chunks to check" + for i, chunk in enumerate(chunks): + actual = chunk.get("source") + if actual is None: + attrs = chunk.get("attributes") or {} + actual = attrs.get("source") + assert ( + actual == source + ), f"rag_chunk[{i}] source is {actual!r}, expected {source!r}" + + +@then('Each rag_chunk reference_url contains "{domain}"') +def check_rag_chunk_reference_url(context: Context, domain: str) -> None: + """Assert every rag_chunk's reference_url contains the expected domain.""" + chunks = _get_rag_chunks(context) + assert chunks, "No rag_chunks to check" + for i, chunk in enumerate(chunks): + attrs = chunk.get("attributes") or {} + ref_url = attrs.get("reference_url") or attrs.get("doc_url") or "" + assert domain in str( + ref_url + ), f"rag_chunk[{i}] reference_url {ref_url!r} does not contain {domain!r}" + + +# ── Then steps: referenced_document assertions ── + + +@then("Each referenced_document has fields doc_url, doc_title, source, and document_id") +def check_referenced_document_fields(context: Context) -> None: + """Assert every referenced_document has the required fields.""" + docs = _get_referenced_documents(context) + assert docs, "No referenced_documents to check" + required = {"doc_url", "doc_title", "source", "document_id"} + for i, doc in enumerate(docs): + missing = required - set(doc.keys()) + assert not missing, ( + f"referenced_document[{i}] missing fields: {missing}. " + f"Available: {list(doc.keys())}" + ) + + +@then('Each referenced_document doc_url contains "{domain}"') +def check_referenced_document_doc_url(context: Context, domain: str) -> None: + """Assert every referenced_document doc_url contains the expected domain.""" + docs = _get_referenced_documents(context) + assert docs, "No referenced_documents to check" + for i, doc in enumerate(docs): + doc_url = str(doc.get("doc_url", "")) + assert ( + domain in doc_url + ), f"referenced_document[{i}] doc_url {doc_url!r} does not contain {domain!r}" + + +@then("Each referenced_document doc_title is not empty") +def check_referenced_document_doc_title(context: Context) -> None: + """Assert every referenced_document has a non-empty doc_title.""" + docs = _get_referenced_documents(context) + assert docs, "No referenced_documents to check" + for i, doc in enumerate(docs): + title = doc.get("doc_title") + assert title, f"referenced_document[{i}] has empty or missing doc_title" + + +@then('Each referenced_document source is "{source}"') +def check_referenced_document_source(context: Context, source: str) -> None: + """Assert every referenced_document has the expected source.""" + docs = _get_referenced_documents(context) + assert docs, "No referenced_documents to check" + for i, doc in enumerate(docs): + actual = doc.get("source") + assert ( + actual == source + ), f"referenced_document[{i}] source is {actual!r}, expected {source!r}" + + +@then("Each referenced_document has a non-empty document_id") +def check_referenced_document_id(context: Context) -> None: + """Assert every referenced_document has a non-empty document_id.""" + docs = _get_referenced_documents(context) + assert docs, "No referenced_documents to check" + for i, doc in enumerate(docs): + doc_id = doc.get("document_id") + assert doc_id, f"referenced_document[{i}] has empty or missing document_id" + + +# ── Then steps: tool_calls assertions ── + + +@then("The response contains non-empty tool_calls") +def check_tool_calls_present(context: Context) -> None: + """Assert the response contains at least one tool call.""" + tool_calls = _get_tool_calls(context) + assert len(tool_calls) > 0, "tool_calls is empty — no tool calls were made" + + +@then('A tool_call has name "{name}"') +def check_tool_call_name(context: Context, name: str) -> None: + """Assert at least one tool call has the expected name.""" + tool_calls = _get_tool_calls(context) + assert tool_calls, "No tool_calls to check" + names = [tc.get("name") for tc in tool_calls] + assert ( + name in names + ), f"No tool_call with name {name!r} found. Available names: {names!r}" + + +@then('A tool_call has type "{type_name}"') +def check_tool_call_type(context: Context, type_name: str) -> None: + """Assert at least one tool call has the expected type.""" + tool_calls = _get_tool_calls(context) + assert tool_calls, "No tool_calls to check" + types = [tc.get("type") for tc in tool_calls] + matched = any(t in {type_name, f"{type_name}_call"} for t in types) + assert ( + matched + ), f"No tool_call with type {type_name!r} found. Available types: {types!r}" + + +# ── Then steps: content and results assertions ── + + +@then("The response contains non-empty content") +def check_response_content_present(context: Context) -> None: + """Assert the response contains non-empty content text.""" + body = _get_response_body(context) + if "response" in body: + content = body["response"] + elif "output_text" in body: + content = body["output_text"] + else: + content = body.get("response_complete", "") + assert content, "Response content is empty" + + +@then("The response contains non-empty results") +def check_results_present(context: Context) -> None: + """Assert the Responses API output contains non-empty file_search results.""" + results = _get_file_search_results(context) + assert len(results) > 0, "No file_search results found in response output" + + +@then("The number of results returned is {count:d}") +def check_results_count(context: Context, count: int) -> None: + """Assert the number of file_search results matches the expected count.""" + results = _get_file_search_results(context) + assert len(results) == count, f"Expected {count} results, got {len(results)}" + + +# ── Then steps: empty response assertions ── + + +@then("The response contains no rag_chunks") +def check_no_rag_chunks(context: Context) -> None: + """Assert the response has no rag_chunks (empty or absent).""" + body = _get_response_body(context) + chunks = body.get("rag_chunks", []) + assert len(chunks) == 0, f"Expected no rag_chunks, got {len(chunks)}" + + +@then("The response contains no referenced_documents") +def check_no_referenced_documents(context: Context) -> None: + """Assert the response has no referenced_documents (empty or absent).""" + body = _get_response_body(context) + docs = body.get("referenced_documents", []) + assert len(docs) == 0, f"Expected no referenced_documents, got {len(docs)}" From 335c086bf16e4d483ec10a1fbb5b810e5a7507a9 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Tue, 25 Aug 2026 13:57:36 -0400 Subject: [PATCH 06/18] extended steps --- Makefile | 4 + .../lightspeed-stack-okp-offline.yaml | 3 +- .../lightspeed-stack-okp-online.yaml | 3 +- tests/e2e/features/environment.py | 76 +++++++++ tests/e2e/features/steps/okp_rag.py | 152 ++++++++++++++++-- 5 files changed, 219 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index fcce2c5c6..305dad245 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,7 @@ LLAMA_STACK_CONFIG ?= run.yaml LLAMA_STACK_CONTAINER_NAME ?= lightspeed-llama-stack LLAMA_STACK_IMAGE ?= lightspeed-llama-stack:local LLAMA_STACK_PORT ?= 8321 +LIGHTSPEED_PROVIDERS_DIR ?= $(shell [ -d ../lightspeed-providers ] && cd ../lightspeed-providers && pwd) CONTAINER_RUNTIME ?= $(shell command -v podman 2>/dev/null || command -v docker 2>/dev/null) .PHONY: run \ @@ -85,6 +86,9 @@ start-llama-stack-container: build-llama-stack-image ## Start llama-stack contai -v $(PWD)/$(CONFIG):/opt/app-root/lightspeed-stack.yaml:ro,z \ -v $(PWD)/scripts/llama-stack-entrypoint.sh:/opt/app-root/enrich-entrypoint.sh:ro,z \ -v $(PWD)/src/llama_stack_configuration.py:/opt/app-root/llama_stack_configuration.py:ro,z \ + $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/lightspeed_stack_providers:/opt/app-root/providers/lightspeed_stack_providers:ro) \ + $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/resources/external_providers:/opt/app-root/src/.llama/providers.d:ro) \ + $(if $(LIGHTSPEED_PROVIDERS_DIR),-e EXTERNAL_PROVIDERS_DIR=/opt/app-root/src/.llama/providers.d) \ -e OPENAI_API_KEY \ -e BRAVE_SEARCH_API_KEY \ -e TAVILY_SEARCH_API_KEY \ diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-offline.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-offline.yaml index 0ac491655..1863a7daa 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-offline.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-offline.yaml @@ -12,7 +12,8 @@ service: llama_stack: use_as_library_client: true config: - profile: run.yaml + baseline: default + # profile: run.yaml # Commented out to allow auto-generation with OKP enrichment user_data_collection: feedback_enabled: true feedback_storage: "/tmp/data/feedback" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-online.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-online.yaml index 78e7e1604..2c7936500 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-online.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-online.yaml @@ -12,7 +12,8 @@ service: llama_stack: use_as_library_client: true config: - profile: run.yaml + baseline: default + # profile: run.yaml # Commented out to allow auto-generation with OKP enrichment user_data_collection: feedback_enabled: true feedback_storage: "/tmp/data/feedback" diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index cd05d3bdc..371555409 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -304,6 +304,8 @@ def after_scenario(context: Context, scenario: Scenario) -> None: running before the scenario. - hostname_llama, port_llama (str/int, optional): host and port used for the llama-stack health check. + - okp_was_running (bool, optional): whether OKP server was running + before it was stopped by the scenario. scenario (Scenario): Behave scenario (unused; shield restore uses context flags). """ if is_prow_environment(): @@ -329,6 +331,80 @@ def after_scenario(context: Context, scenario: Scenario) -> None: except Exception as e: # pylint: disable=broad-exception-caught print(f"Warning: Could not re-register shield: {e}") + # Restart OKP server if it was stopped during the scenario + # (only in local/Docker mode - OpenShift manages containers differently) + if getattr(context, "okp_was_running", False) and not is_prow_environment(): + from tests.e2e.features.steps.okp_rag import ( + OKP_DEFAULT_URL, + OKP_IMAGE_NAME, + ) + + container_name = getattr(context, "okp_container_name", None) + if container_name: + # Check if container still exists (may be auto-removed if started with --rm) + check_result = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Running}}", container_name], + capture_output=True, + text=True, + check=False, + ) + + if check_result.returncode != 0: + # Container was removed (likely started with --rm flag) - recreate it + print( + f"OKP container '{container_name}' was removed, recreating from {OKP_IMAGE_NAME}..." + ) + try: + subprocess.run( + [ + "docker", + "run", + "--rm", + "-d", + "-p", + "8081:8080", + OKP_IMAGE_NAME, + ], + capture_output=True, + text=True, + check=True, + ) + except subprocess.CalledProcessError as e: + print(f"Warning: Could not recreate OKP container: {e}") + if e.stderr: + print(f" Error: {e.stderr}") + else: + # Container exists - just start it + try: + subprocess.run( + ["docker", "start", container_name], + capture_output=True, + text=True, + check=True, + ) + except subprocess.CalledProcessError as e: + print( + f"Warning: Could not start OKP container '{container_name}': {e}" + ) + if e.stderr: + print(f" Error: {e.stderr}") + + # Wait for the server to be ready + max_attempts = 30 + for attempt in range(max_attempts): + try: + resp = requests.get(OKP_DEFAULT_URL, timeout=5) + if resp.status_code < 500: + print("✓ OKP server restarted successfully") + break + except requests.ConnectionError: + if attempt < max_attempts - 1: + time.sleep(1) + else: + print( + "Warning: OKP server may not be fully ready after restart" + ) + def _print_llama_stack_diagnostics() -> None: """Print container state, health, and recent logs to diagnose why llama-stack did not recover.""" diff --git a/tests/e2e/features/steps/okp_rag.py b/tests/e2e/features/steps/okp_rag.py index 38ac57a5e..416f8c070 100644 --- a/tests/e2e/features/steps/okp_rag.py +++ b/tests/e2e/features/steps/okp_rag.py @@ -12,6 +12,11 @@ # OKP/Solr Docker container name OKP_CONTAINER_NAME = os.getenv("E2E_OKP_CONTAINER", "okp-solr") +# OKP Docker image name (for finding container by image) +OKP_IMAGE_NAME = os.getenv( + "E2E_OKP_IMAGE", "registry.redhat.io/offline-knowledge-portal/rhokp-rhel9" +) + # Default OKP health check URL OKP_DEFAULT_URL = os.getenv("E2E_OKP_URL", "http://localhost:8081") @@ -76,6 +81,36 @@ def _get_file_search_results(context: Context) -> list[dict[str, Any]]: return results +def _find_okp_container() -> str | None: + """Find OKP container by name or image. + + Returns: + Container name/ID if found, None otherwise. + """ + # Try by configured name first + result = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Running}}", OKP_CONTAINER_NAME], + capture_output=True, + text=True, + check=False, + ) + if result.returncode == 0: + return OKP_CONTAINER_NAME + + # Try by image name + result = subprocess.run( + ["docker", "ps", "-a", "-q", "--filter", f"ancestor={OKP_IMAGE_NAME}"], + capture_output=True, + text=True, + check=False, + ) + if result.returncode == 0 and result.stdout.strip(): + container_id = result.stdout.strip().split()[0] + return container_id + + return None + + # ── Given steps ── @@ -96,24 +131,69 @@ def okp_server_is_running(context: Context) -> None: def okp_server_is_stopped(context: Context) -> None: """Stop the OKP(Solr) Docker container to simulate unavailability.""" context.okp_was_running = False - try: - result = subprocess.run( - ["docker", "inspect", "-f", "{{.State.Running}}", OKP_CONTAINER_NAME], - capture_output=True, - text=True, - check=True, + context.okp_container_name = None + + # Find the OKP container + container_name = _find_okp_container() + if not container_name: + print( + f"✓ OKP container not found (neither '{OKP_CONTAINER_NAME}' nor '{OKP_IMAGE_NAME}') - already unavailable" ) - if "true" in result.stdout.lower(): - context.okp_was_running = True - subprocess.run( - ["docker", "stop", OKP_CONTAINER_NAME], - capture_output=True, - text=True, - check=True, - ) - time.sleep(2) - except subprocess.CalledProcessError as exc: - print(f"Warning: could not stop OKP container: {exc}") + return + + # Check if container is running + result = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Running}}", container_name], + capture_output=True, + text=True, + check=False, + ) + + if result.returncode != 0: + print(f"✓ OKP container '{container_name}' inspection failed - unavailable") + return + + if "true" not in result.stdout.lower(): + print(f"✓ OKP container '{container_name}' already stopped") + return + + # Container is running - stop it + context.okp_was_running = True + context.okp_container_name = container_name + + stop_result = subprocess.run( + ["docker", "stop", container_name], + capture_output=True, + text=True, + check=False, + ) + if stop_result.returncode != 0: + assert ( + False + ), f"Failed to stop OKP container '{container_name}': {stop_result.stderr}" + + # Wait for the container to fully stop + time.sleep(5) + + # Verify the server is actually unreachable + max_attempts = 10 + for attempt in range(max_attempts): + try: + resp = requests.get(OKP_DEFAULT_URL, timeout=2) + if attempt < max_attempts - 1: + time.sleep(1) + else: + assert ( + False + ), f"OKP server still responding after stop: {resp.status_code}" + except requests.ConnectionError: + # Server is unreachable - success + print(f"✓ OKP server stopped and verified unreachable ({container_name})") + break + except requests.Timeout: + # Timeout is also acceptable - server not responding + print(f"✓ OKP server stopped (connection timeout) ({container_name})") + break # ── Then steps: rag_chunk assertions ── @@ -203,6 +283,27 @@ def check_referenced_document_doc_title(context: Context) -> None: assert title, f"referenced_document[{i}] has empty or missing doc_title" +@then('Each referenced_document doc_title contains "{substring}"') +def check_referenced_document_doc_title_contains( + context: Context, substring: str +) -> None: + """Assert every referenced_document doc_title contains the expected substring (case-insensitive).""" + docs = _get_referenced_documents(context) + assert docs, "No referenced_documents to check" + for i, doc in enumerate(docs): + title = str(doc.get("doc_title", "")).lower() + assert ( + substring.lower() in title + ), f"referenced_document[{i}] doc_title {doc.get('doc_title')!r} does not contain {substring!r}" + + +@then("The number of referenced_document returned is {count:d}") +def check_referenced_document_count(context: Context, count: int) -> None: + """Assert the number of referenced_documents matches the expected count.""" + docs = _get_referenced_documents(context) + assert len(docs) == count, f"Expected {count} referenced_documents, got {len(docs)}" + + @then('Each referenced_document source is "{source}"') def check_referenced_document_source(context: Context, source: str) -> None: """Assert every referenced_document has the expected source.""" @@ -261,6 +362,23 @@ def check_tool_call_type(context: Context, type_name: str) -> None: # ── Then steps: content and results assertions ── +@then('The response contains "{substring}"') +def check_response_contains_substring(context: Context, substring: str) -> None: + """Assert the LLM response contains the expected substring (case-insensitive).""" + body = _get_response_body(context) + if "response" in body: + response_text = body["response"] + elif "output_text" in body: + response_text = body["output_text"] + else: + response_text = body.get("response_complete", "") + + assert substring.lower() in response_text.lower(), ( + f"Response does not contain {substring!r}. " + f"Response text: {response_text[:200]}..." + ) + + @then("The response contains non-empty content") def check_response_content_present(context: Context) -> None: """Assert the response contains non-empty content text.""" From b8f0bac51c8b46c864e39bfb1b7622390b081a71 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Wed, 26 Aug 2026 09:06:34 -0400 Subject: [PATCH 07/18] updated parsing logic for streaming --- tests/e2e/features/steps/llm_query_response.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/e2e/features/steps/llm_query_response.py b/tests/e2e/features/steps/llm_query_response.py index 3e308fb21..2b1590250 100644 --- a/tests/e2e/features/steps/llm_query_response.py +++ b/tests/e2e/features/steps/llm_query_response.py @@ -156,6 +156,9 @@ def ask_question_authorized(context: Context, endpoint: str) -> None: body = _read_streamed_response(resp) resp._content = body.encode(resp.encoding or "utf-8") context.response = resp + # Parse SSE events and make data available for assertions + context.response_data = _parse_streaming_response(body) + context.use_streaming_response_data = True else: context.response = request_with_transient_retry( method="POST", From e2faa2a3622e5eb640ca7b7e51c9c6c8c8315a91 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Wed, 26 Aug 2026 09:17:02 -0400 Subject: [PATCH 08/18] updated parameters for stopping docker --- tests/e2e/features/environment.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index 86de7310b..88cccdb60 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -304,6 +304,8 @@ def after_scenario(context: Context, scenario: Scenario) -> None: running before the scenario. - hostname_llama, port_llama (str/int, optional): host and port used for the OGX health check. + - okp_was_running (bool, optional): whether OKP server was running + before it was stopped by the scenario. scenario (Scenario): Behave scenario (unused; shield restore uses context flags). """ if is_prow_environment(): From 065f4a2cde50552ed98e0188998e252c2e677619 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Wed, 26 Aug 2026 14:40:52 -0400 Subject: [PATCH 09/18] added Openshift support for OKP tests --- Makefile | 2 +- .../rhoai/manifests/lightspeed/okp-solr.yaml | 62 ++++ tests/e2e-prow/rhoai/pipeline-konflux.sh | 36 ++ tests/e2e-prow/rhoai/pipeline.sh | 17 + tests/e2e-prow/rhoai/scripts/e2e-ops.sh | 60 ++++ .../lightspeed-stack-okp-tool-offline.yaml | 3 +- .../lightspeed-stack-okp-tool-online.yaml | 5 +- tests/e2e/features/environment.py | 137 +++---- tests/e2e/features/okp_rag.feature | 15 +- tests/e2e/features/steps/okp_rag.py | 340 +++++++++++++----- tests/e2e/utils/prow_utils.py | 42 +++ 11 files changed, 559 insertions(+), 160 deletions(-) create mode 100644 tests/e2e-prow/rhoai/manifests/lightspeed/okp-solr.yaml diff --git a/Makefile b/Makefile index d8b1731bd..bc5598dcb 100644 --- a/Makefile +++ b/Makefile @@ -168,7 +168,7 @@ test-e2e-local: ## Run end to end tests for the service (no script wrapper) # Tag-based subsets (@cfg_* on features/scenarios). Default runs all config groups; override for one shard, e.g. # E2E_BEHAVE_TAG_EXPR='not @skip and @cfg_authorized' make test-e2e-tagged-local -E2E_BEHAVE_TAG_EXPR ?= not @skip and (@cfg_default or @cfg_authorized or @cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth or @cfg_rbac or @cfg_rh_identity or @cfg_negative or @cfg_skills or @cfg_skills_directory or @cfg_byok_pdf or @cfg_tls or @cfg_degraded or @cfg_unified) +E2E_BEHAVE_TAG_EXPR ?= not @skip and (@cfg_default or @cfg_authorized or @cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth or @cfg_rbac or @cfg_rh_identity or @cfg_negative or @cfg_skills or @cfg_skills_directory or @cfg_byok_pdf or @cfg_tls or @cfg_degraded or @cfg_unified or @cfg_okp) test-e2e-tagged: ## Run e2e tests with E2E_BEHAVE_TAG_EXPR (default: all @cfg_*) script -q -e -c "uv run behave --color --format pretty --tags=\"$(E2E_BEHAVE_TAG_EXPR)\" -D dump_errors=true @tests/e2e/test_list.txt" diff --git a/tests/e2e-prow/rhoai/manifests/lightspeed/okp-solr.yaml b/tests/e2e-prow/rhoai/manifests/lightspeed/okp-solr.yaml new file mode 100644 index 000000000..bc730230c --- /dev/null +++ b/tests/e2e-prow/rhoai/manifests/lightspeed/okp-solr.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: Pod +metadata: + name: okp-solr-service + labels: + app: okp-solr +spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + imagePullSecrets: + - name: redhat-registry-pull-secret + containers: + - name: okp-solr + image: registry.redhat.io/offline-knowledge-portal/rhokp-rhel9:latest + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + ports: + - containerPort: 8080 + name: http + readinessProbe: + httpGet: + path: /solr + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /solr + port: 8080 + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "2Gi" + cpu: "1000m" +--- +apiVersion: v1 +kind: Service +metadata: + name: okp-solr-service-svc +spec: + selector: + app: okp-solr + ports: + - port: 8080 + targetPort: 8080 + name: http diff --git a/tests/e2e-prow/rhoai/pipeline-konflux.sh b/tests/e2e-prow/rhoai/pipeline-konflux.sh index 14020dbcf..8633ef8d5 100755 --- a/tests/e2e-prow/rhoai/pipeline-konflux.sh +++ b/tests/e2e-prow/rhoai/pipeline-konflux.sh @@ -110,6 +110,42 @@ oc create secret docker-registry quay-lightspeed-pull-secret \ # Link the secret to default service account for image pulls oc secrets link default quay-lightspeed-pull-secret --for=pull -n "$NAMESPACE" 2>/dev/null || echo "⚠️ Secret already linked to default SA" +# Create Red Hat registry pull secret for OKP images +# Credentials from Konflux secrets (mounted at /var/run/redhat-registry-*) +if [[ -d /var/run/redhat-registry-username ]] && [[ -d /var/run/redhat-registry-password ]]; then + log "Creating Red Hat registry pull secret..." + REDHAT_USERNAME="" + REDHAT_PASSWORD="" + + # Read username + shopt -s nullglob + for _f in /var/run/redhat-registry-username/*; do + [[ -f "$_f" ]] && REDHAT_USERNAME="$(cat "$_f")" && break + done + + # Read password + for _f in /var/run/redhat-registry-password/*; do + [[ -f "$_f" ]] && REDHAT_PASSWORD="$(cat "$_f")" && break + done + shopt -u nullglob + + if [[ -n "$REDHAT_USERNAME" ]] && [[ -n "$REDHAT_PASSWORD" ]]; then + oc create secret docker-registry redhat-registry-pull-secret \ + --docker-server=registry.redhat.io \ + --docker-username="$REDHAT_USERNAME" \ + --docker-password="$REDHAT_PASSWORD" \ + -n "$NAMESPACE" 2>/dev/null && log "✅ Red Hat registry pull secret created" || log "⚠️ Secret exists or creation failed" + + # Link to default service account + oc secrets link default redhat-registry-pull-secret --for=pull -n "$NAMESPACE" 2>/dev/null || echo "⚠️ Secret already linked to default SA" + else + log "⚠️ Red Hat registry credentials not found in /var/run - OKP image pull may fail" + fi +else + log "⚠️ Red Hat registry credential mounts not found - OKP image pull may fail" + log " (This is OK if not testing OKP features)" +fi + #======================================== # 4. DEPLOY MOCK SERVERS (JWKS & MCP) diff --git a/tests/e2e-prow/rhoai/pipeline.sh b/tests/e2e-prow/rhoai/pipeline.sh index c393e2c15..4a86829f5 100755 --- a/tests/e2e-prow/rhoai/pipeline.sh +++ b/tests/e2e-prow/rhoai/pipeline.sh @@ -92,6 +92,23 @@ oc create secret docker-registry quay-lightspeed-pull-secret \ # Link the secret to default service account for image pulls oc secrets link default quay-lightspeed-pull-secret --for=pull -n "$NAMESPACE" 2>/dev/null || echo "⚠️ Secret already linked to default SA" +# Create Red Hat registry pull secret for OKP images +# Credentials from Prow secrets (environment variables or mounted volumes) +if [[ -n "${REDHAT_REGISTRY_USERNAME:-}" ]] && [[ -n "${REDHAT_REGISTRY_PASSWORD:-}" ]]; then + echo "Creating Red Hat registry pull secret from environment..." + oc create secret docker-registry redhat-registry-pull-secret \ + --docker-server=registry.redhat.io \ + --docker-username="$REDHAT_REGISTRY_USERNAME" \ + --docker-password="$REDHAT_REGISTRY_PASSWORD" \ + -n "$NAMESPACE" 2>/dev/null && echo "✅ Red Hat registry pull secret created" || echo "⚠️ Secret exists or creation failed" + + # Link to default service account + oc secrets link default redhat-registry-pull-secret --for=pull -n "$NAMESPACE" 2>/dev/null || echo "⚠️ Secret already linked to default SA" +else + echo "⚠️ REDHAT_REGISTRY_USERNAME/PASSWORD not set - OKP image pull may fail" + echo " (This is OK if not testing OKP features)" +fi + #======================================== # 5. CONFIGMAPS diff --git a/tests/e2e-prow/rhoai/scripts/e2e-ops.sh b/tests/e2e-prow/rhoai/scripts/e2e-ops.sh index ff2139bf5..79b917e4f 100755 --- a/tests/e2e-prow/rhoai/scripts/e2e-ops.sh +++ b/tests/e2e-prow/rhoai/scripts/e2e-ops.sh @@ -33,6 +33,10 @@ # delete-e2e-mock-tls-inference - Remove mock TLS pod + Service (manual cleanup) # restart-e2e-mock-tls-inference - Delete then deploy mock TLS (manual / recovery) # sync-mock-tls-certs-secret - Copy mock /certs into Secret for OGX mount +# deploy-okp-solr - Deploy OKP Solr service +# delete-okp-solr - Delete OKP Solr pod +# disrupt-okp-solr - Delete OKP Solr pod to disrupt connection +# restore-okp-solr - Restore OKP Solr pod set -e @@ -989,6 +993,46 @@ cmd_disrupt_llama_stack() { fi } +cmd_deploy_okp_solr() { + echo "Deploying OKP Solr service in namespace $NAMESPACE..." + oc apply -n "$NAMESPACE" -f "$MANIFEST_DIR/okp-solr.yaml" + wait_for_pod "okp-solr-service" 60 + echo "✓ OKP Solr service deployed and ready" +} + +cmd_delete_okp_solr() { + echo "Deleting OKP Solr pod from namespace $NAMESPACE..." + timeout 60 oc delete pod okp-solr-service -n "$NAMESPACE" --ignore-not-found=true --wait=true 2>/dev/null || { + oc delete pod okp-solr-service -n "$NAMESPACE" --ignore-not-found=true --force --grace-period=0 2>/dev/null || true + sleep 2 + } + echo "✓ OKP Solr pod deleted" +} + +cmd_disrupt_okp_solr() { + local pod_name="okp-solr-service" + + local phase + phase=$(oc get pod "$pod_name" -n "$NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null || echo "NotFound") + + if [[ "$phase" == "Running" ]]; then + oc delete pod "$pod_name" -n "$NAMESPACE" --wait=true + sleep 2 + echo "OKP Solr connection disrupted successfully (pod deleted)" + exit 0 + else + echo "OKP Solr pod was not running (phase: $phase)" + exit 2 + fi +} + +cmd_restore_okp_solr() { + echo "Restoring OKP Solr service in namespace $NAMESPACE..." + oc apply -n "$NAMESPACE" -f "$MANIFEST_DIR/okp-solr.yaml" + wait_for_pod "okp-solr-service" 60 + echo "✓ OKP Solr service restored and ready" +} + # ============================================================================ # Main command dispatcher # ============================================================================ @@ -1060,6 +1104,18 @@ case "$COMMAND" in dump-pod-logs) cmd_dump_pod_logs "$@" ;; + deploy-okp-solr) + cmd_deploy_okp_solr + ;; + delete-okp-solr) + cmd_delete_okp_solr + ;; + disrupt-okp-solr) + cmd_disrupt_okp_solr + ;; + restore-okp-solr) + cmd_restore_okp_solr + ;; *) echo "Usage: $0 [args...]" echo "" @@ -1081,6 +1137,10 @@ case "$COMMAND" in echo " deploy-e2e-interception-proxy - Deploy in-cluster interception proxy pod" echo " deploy-e2e-mock-tls-inference - Deploy mock HTTPS inference (tls-*.feature)" echo " delete-e2e-mock-tls-inference - Remove mock TLS pod + Service" + echo " deploy-okp-solr - Deploy OKP Solr service" + echo " delete-okp-solr - Delete OKP Solr pod" + echo " disrupt-okp-solr - Delete OKP Solr pod to disrupt connection" + echo " restore-okp-solr - Restore OKP Solr pod" echo " restart-e2e-mock-tls-inference - Delete then deploy mock TLS (recovery)" echo " sync-mock-tls-certs-secret - Publish mock TLS /certs to Secret" echo " dump-pod-logs [tail-lines] - Print init + container logs" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-offline.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-offline.yaml index e952860a4..4dd5d37d9 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-offline.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-offline.yaml @@ -12,7 +12,8 @@ service: llama_stack: use_as_library_client: true config: - profile: run.yaml + baseline: default + # profile: run.yaml user_data_collection: feedback_enabled: true feedback_storage: "/tmp/data/feedback" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-online.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-online.yaml index b058f0fae..f39e5aac6 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-online.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-online.yaml @@ -12,7 +12,8 @@ service: llama_stack: use_as_library_client: true config: - profile: run.yaml + baseline: default + # profile: run.yaml user_data_collection: feedback_enabled: true feedback_storage: "/tmp/data/feedback" @@ -32,7 +33,7 @@ rag: tool: sources: - okp - max_chunk : 3 + max_chunks : 3 okp: rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} offline: false diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index 88cccdb60..6434f0621 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -332,78 +332,87 @@ def after_scenario(context: Context, scenario: Scenario) -> None: print(f"Warning: Could not re-register shield: {e}") # Restart OKP server if it was stopped during the scenario - # (only in local/Docker mode - OpenShift manages containers differently) - if getattr(context, "okp_was_running", False) and not is_prow_environment(): - from tests.e2e.features.steps.okp_rag import ( - OKP_DEFAULT_URL, - OKP_IMAGE_NAME, - ) + if getattr(context, "okp_was_running", False): + if is_prow_environment(): + # Prow/OpenShift: restore pod + from tests.e2e.utils.prow_utils import restore_okp_solr_pod - container_name = getattr(context, "okp_container_name", None) - if container_name: - # Check if container still exists (may be auto-removed if started with --rm) - check_result = subprocess.run( - ["docker", "inspect", "-f", "{{.State.Running}}", container_name], - capture_output=True, - text=True, - check=False, + try: + restore_okp_solr_pod() + except Exception as e: # pylint: disable=broad-exception-caught + print(f"Warning: Could not restore OKP Solr pod: {e}") + else: + # Docker mode: recreate or restart container + from tests.e2e.features.steps.okp_rag import ( + OKP_DEFAULT_URL, + OKP_IMAGE_NAME, ) - if check_result.returncode != 0: - # Container was removed (likely started with --rm flag) - recreate it - print( - f"OKP container '{container_name}' was removed, recreating from {OKP_IMAGE_NAME}..." + container_name = getattr(context, "okp_container_name", None) + if container_name: + # Check if container still exists (may be auto-removed if started with --rm) + check_result = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Running}}", container_name], + capture_output=True, + text=True, + check=False, ) - try: - subprocess.run( - [ - "docker", - "run", - "--rm", - "-d", - "-p", - "8081:8080", - OKP_IMAGE_NAME, - ], - capture_output=True, - text=True, - check=True, - ) - except subprocess.CalledProcessError as e: - print(f"Warning: Could not recreate OKP container: {e}") - if e.stderr: - print(f" Error: {e.stderr}") - else: - # Container exists - just start it - try: - subprocess.run( - ["docker", "start", container_name], - capture_output=True, - text=True, - check=True, - ) - except subprocess.CalledProcessError as e: + + if check_result.returncode != 0: + # Container was removed (likely started with --rm flag) - recreate it print( - f"Warning: Could not start OKP container '{container_name}': {e}" + f"OKP container '{container_name}' was removed, recreating from {OKP_IMAGE_NAME}..." ) - if e.stderr: - print(f" Error: {e.stderr}") - - # Wait for the server to be ready - max_attempts = 30 - for attempt in range(max_attempts): - try: - resp = requests.get(OKP_DEFAULT_URL, timeout=5) - if resp.status_code < 500: - print("✓ OKP server restarted successfully") - break - except requests.ConnectionError: - if attempt < max_attempts - 1: - time.sleep(1) - else: + try: + subprocess.run( + [ + "docker", + "run", + "--rm", + "-d", + "-p", + "8081:8080", + OKP_IMAGE_NAME, + ], + capture_output=True, + text=True, + check=True, + ) + except subprocess.CalledProcessError as e: + print(f"Warning: Could not recreate OKP container: {e}") + if e.stderr: + print(f" Error: {e.stderr}") + else: + # Container exists - just start it + try: + subprocess.run( + ["docker", "start", container_name], + capture_output=True, + text=True, + check=True, + ) + except subprocess.CalledProcessError as e: print( - "Warning: OKP server may not be fully ready after restart" + f"Warning: Could not start OKP container '{container_name}': {e}" ) + if e.stderr: + print(f" Error: {e.stderr}") + + # Wait for the server to be ready + max_attempts = 30 + for attempt in range(max_attempts): + try: + resp = requests.get(OKP_DEFAULT_URL, timeout=5) + if resp.status_code < 500: + print("✓ OKP server restarted successfully") + break + except requests.ConnectionError: + if attempt < max_attempts - 1: + time.sleep(1) + else: + print( + "Warning: OKP server may not be fully ready after restart" + ) def _print_llama_stack_diagnostics() -> None: diff --git a/tests/e2e/features/okp_rag.feature b/tests/e2e/features/okp_rag.feature index 6d419224d..309d740f2 100644 --- a/tests/e2e/features/okp_rag.feature +++ b/tests/e2e/features/okp_rag.feature @@ -1,4 +1,4 @@ -@cfg_okp @skip +@cfg_okp Feature: OKP(Solr) RAG retrieval tests # Offline Knowledge Portal (OKP) provides a Solr-backed RAG source to LSC. @@ -94,7 +94,7 @@ Feature: OKP(Solr) RAG retrieval tests Scenario: Offline query API with OKP tool RAG has rag_chunk and referenced_documents returned Given The service uses the lightspeed-stack-okp-tool-offline.yaml configuration - And The service is restarted + # And The service is restarted When I use "query" to ask question with authorization header """ { @@ -154,4 +154,15 @@ Feature: OKP(Solr) RAG retrieval tests """ Then The status code of the response is 200 And The response contains no rag_chunks + And The response contains no referenced_documents + + Scenario: Streaming query succeeds with empty rag_chunks when OKP server is unavailable + Given The service uses the lightspeed-stack-okp-online.yaml configuration + And The service is restarted + And The OKP(Solr) server is stopped + When I use "query" to ask question with authorization header + """ + {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 And The response contains no referenced_documents \ No newline at end of file diff --git a/tests/e2e/features/steps/okp_rag.py b/tests/e2e/features/steps/okp_rag.py index 416f8c070..6911dd2f9 100644 --- a/tests/e2e/features/steps/okp_rag.py +++ b/tests/e2e/features/steps/okp_rag.py @@ -9,6 +9,8 @@ from behave import given, then # pyright: ignore[reportAttributeAccessIssue] from behave.runner import Context +# ── Constants ── + # OKP/Solr Docker container name OKP_CONTAINER_NAME = os.getenv("E2E_OKP_CONTAINER", "okp-solr") @@ -32,6 +34,9 @@ ) +# ── Response Body Extraction ── + + def _get_response_body(context: Context) -> dict[str, Any]: """Return the response body dict, handling both JSON and streaming formats.""" if getattr(context, "use_streaming_response_data", False): @@ -39,6 +44,19 @@ def _get_response_body(context: Context) -> dict[str, Any]: return context.response.json() +def _get_response_text(context: Context) -> str: + """Extract response text from various response formats.""" + body = _get_response_body(context) + return ( + body.get("response") + or body.get("output_text") + or body.get("response_complete", "") + ) + + +# ── Data Extractors ── + + def _get_rag_chunks(context: Context) -> list[dict[str, Any]]: """Extract rag_chunks from query response or results from Responses API output.""" body = _get_response_body(context) @@ -81,6 +99,179 @@ def _get_file_search_results(context: Context) -> list[dict[str, Any]]: return results +# ── Generic Field Accessors ── + + +def _get_nested_field(item: dict[str, Any], field_path: str) -> Any: + """Get a field from item, supporting nested access via dot notation. + + Examples: + _get_nested_field(chunk, "score") -> chunk.get("score") + _get_nested_field(chunk, "attributes.reference_url") + -> chunk.get("attributes", {}).get("reference_url") + + Parameters: + item: Dictionary to extract field from. + field_path: Field path, using dots for nested access. + + Returns: + Field value or None if not found. + """ + keys = field_path.split(".") + value: Any = item + for key in keys: + if isinstance(value, dict): + value = value.get(key) + else: + return None + return value + + +# ── Generic Assertion Helpers ── + + +def _assert_count_matches(items: list, expected_count: int, item_type: str) -> None: + """Assert the number of items matches the expected count. + + Parameters: + items: List of items to check. + expected_count: Expected number of items. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If count doesn't match. + """ + actual_count = len(items) + assert ( + actual_count == expected_count + ), f"Expected {expected_count} {item_type}, but found {actual_count}" + + +def _assert_not_empty(items: list, item_type: str) -> None: + """Assert the collection is not empty. + + Parameters: + items: List of items to check. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If collection is empty. + """ + assert len(items) > 0, f"{item_type} is empty — no items were found" + + +def _assert_empty(items: list, item_type: str) -> None: + """Assert the collection is empty. + + Parameters: + items: List of items to check. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If collection is not empty. + """ + assert len(items) == 0, f"Expected no {item_type}, but found {len(items)}" + + +def _assert_field_not_empty( + items: list[dict[str, Any]], field_path: str, item_type: str +) -> None: + """Assert every item has a non-empty value for the specified field. + + Parameters: + items: List of items to check. + field_path: Field path to check (supports dot notation). + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If any item has empty or missing field. + """ + assert items, f"No {item_type} to check" + for i, item in enumerate(items): + value = _get_nested_field(item, field_path) + assert value is not None and value != "", ( + f"Expected non-empty {field_path} in {item_type}[{i}], " + f"but found {value!r}" + ) + + +def _assert_field_contains( + items: list[dict[str, Any]], field_path: str, substring: str, item_type: str +) -> None: + """Assert every item's field contains the expected substring (case-insensitive). + + Parameters: + items: List of items to check. + field_path: Field path to check (supports dot notation). + substring: Expected substring. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If any item's field doesn't contain substring. + """ + assert items, f"No {item_type} to check" + for i, item in enumerate(items): + value = _get_nested_field(item, field_path) + assert substring.lower() in str(value).lower(), ( + f"Expected {substring!r} in {item_type}[{i}].{field_path}, " + f"but found {value!r}" + ) + + +def _assert_field_matches( + items: list[dict[str, Any]], field_path: str, expected: Any, item_type: str +) -> None: + """Assert every item's field matches the expected value. + + For fields that might be nested (e.g., source in attributes), checks both + the direct field and the attributes.field path. + + Parameters: + items: List of items to check. + field_path: Field path to check (supports dot notation). + expected: Expected value. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If any item's field doesn't match expected value. + """ + assert items, f"No {item_type} to check" + for i, item in enumerate(items): + actual = _get_nested_field(item, field_path) + # Fallback: check if field exists in attributes + if actual is None and "." not in field_path: + actual = _get_nested_field(item, f"attributes.{field_path}") + assert actual == expected, ( + f"Expected {field_path}={expected!r} in {item_type}[{i}], " + f"but found {actual!r}" + ) + + +def _assert_has_fields( + items: list[dict[str, Any]], required_fields: set[str], item_type: str +) -> None: + """Assert every item has all required fields. + + Parameters: + items: List of items to check. + required_fields: Set of required field names. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If any item is missing required fields. + """ + assert items, f"No {item_type} to check" + for i, item in enumerate(items): + missing = required_fields - set(item.keys()) + assert not missing, ( + f"Expected fields {required_fields} in {item_type}[{i}], " + f"but missing {missing}. Available fields: {list(item.keys())}" + ) + + +# ── Docker/OKP Management ── + + def _find_okp_container() -> str | None: """Find OKP container by name or image. @@ -129,15 +320,30 @@ def okp_server_is_running(context: Context) -> None: @given("The OKP(Solr) server is stopped") def okp_server_is_stopped(context: Context) -> None: - """Stop the OKP(Solr) Docker container to simulate unavailability.""" + """Stop the OKP(Solr) Docker container or pod to simulate unavailability.""" + from tests.e2e.utils.utils import is_prow_environment + context.okp_was_running = False context.okp_container_name = None - # Find the OKP container + if is_prow_environment(): + # Prow/OpenShift: use pod disruption + from tests.e2e.utils.prow_utils import disrupt_okp_solr_pod + + was_running = disrupt_okp_solr_pod() + if was_running: + context.okp_was_running = True + print("✓ OKP Solr pod disrupted in Prow environment") + else: + print("✓ OKP Solr pod was not running") + return + + # Docker mode: existing logic container_name = _find_okp_container() if not container_name: print( - f"✓ OKP container not found (neither '{OKP_CONTAINER_NAME}' nor '{OKP_IMAGE_NAME}') - already unavailable" + f"✓ OKP container not found (neither '{OKP_CONTAINER_NAME}' " + f"nor '{OKP_IMAGE_NAME}') - already unavailable" ) return @@ -196,144 +402,111 @@ def okp_server_is_stopped(context: Context) -> None: break -# ── Then steps: rag_chunk assertions ── +# ── Then Steps: rag_chunks Assertions ── @then("The number of rag_chunk returned is {count:d}") def check_rag_chunk_count(context: Context, count: int) -> None: """Assert the number of rag_chunks matches the expected count.""" chunks = _get_rag_chunks(context) - assert len(chunks) == count, f"Expected {count} rag_chunks, got {len(chunks)}" + _assert_count_matches(chunks, count, "rag_chunks") @then("Each rag_chunk has a non-empty score") def check_rag_chunk_scores(context: Context) -> None: """Assert every rag_chunk has a non-empty score.""" chunks = _get_rag_chunks(context) - assert chunks, "No rag_chunks to check" - for i, chunk in enumerate(chunks): - score = chunk.get("score") - assert score is not None, f"rag_chunk[{i}] has no score" + _assert_field_not_empty(chunks, "score", "rag_chunks") @then('Each rag_chunk source is "{source}"') def check_rag_chunk_source(context: Context, source: str) -> None: """Assert every rag_chunk has the expected source.""" chunks = _get_rag_chunks(context) - assert chunks, "No rag_chunks to check" - for i, chunk in enumerate(chunks): - actual = chunk.get("source") - if actual is None: - attrs = chunk.get("attributes") or {} - actual = attrs.get("source") - assert ( - actual == source - ), f"rag_chunk[{i}] source is {actual!r}, expected {source!r}" + _assert_field_matches(chunks, "source", source, "rag_chunks") @then('Each rag_chunk reference_url contains "{domain}"') def check_rag_chunk_reference_url(context: Context, domain: str) -> None: """Assert every rag_chunk's reference_url contains the expected domain.""" chunks = _get_rag_chunks(context) + # Check both possible field paths for reference_url assert chunks, "No rag_chunks to check" for i, chunk in enumerate(chunks): attrs = chunk.get("attributes") or {} ref_url = attrs.get("reference_url") or attrs.get("doc_url") or "" - assert domain in str( - ref_url - ), f"rag_chunk[{i}] reference_url {ref_url!r} does not contain {domain!r}" + assert domain in str(ref_url), ( + f"Expected {domain!r} in rag_chunks[{i}].attributes.reference_url, " + f"but found {ref_url!r}" + ) -# ── Then steps: referenced_document assertions ── +# ── Then Steps: referenced_documents Assertions ── @then("Each referenced_document has fields doc_url, doc_title, source, and document_id") def check_referenced_document_fields(context: Context) -> None: """Assert every referenced_document has the required fields.""" docs = _get_referenced_documents(context) - assert docs, "No referenced_documents to check" - required = {"doc_url", "doc_title", "source", "document_id"} - for i, doc in enumerate(docs): - missing = required - set(doc.keys()) - assert not missing, ( - f"referenced_document[{i}] missing fields: {missing}. " - f"Available: {list(doc.keys())}" - ) + required_fields = {"doc_url", "doc_title", "source", "document_id"} + _assert_has_fields(docs, required_fields, "referenced_documents") @then('Each referenced_document doc_url contains "{domain}"') def check_referenced_document_doc_url(context: Context, domain: str) -> None: """Assert every referenced_document doc_url contains the expected domain.""" docs = _get_referenced_documents(context) - assert docs, "No referenced_documents to check" - for i, doc in enumerate(docs): - doc_url = str(doc.get("doc_url", "")) - assert ( - domain in doc_url - ), f"referenced_document[{i}] doc_url {doc_url!r} does not contain {domain!r}" + _assert_field_contains(docs, "doc_url", domain, "referenced_documents") @then("Each referenced_document doc_title is not empty") def check_referenced_document_doc_title(context: Context) -> None: """Assert every referenced_document has a non-empty doc_title.""" docs = _get_referenced_documents(context) - assert docs, "No referenced_documents to check" - for i, doc in enumerate(docs): - title = doc.get("doc_title") - assert title, f"referenced_document[{i}] has empty or missing doc_title" + _assert_field_not_empty(docs, "doc_title", "referenced_documents") @then('Each referenced_document doc_title contains "{substring}"') def check_referenced_document_doc_title_contains( context: Context, substring: str ) -> None: - """Assert every referenced_document doc_title contains the expected substring (case-insensitive).""" + """Assert every referenced_document doc_title contains the expected substring. + + Matching is case-insensitive. + """ docs = _get_referenced_documents(context) - assert docs, "No referenced_documents to check" - for i, doc in enumerate(docs): - title = str(doc.get("doc_title", "")).lower() - assert ( - substring.lower() in title - ), f"referenced_document[{i}] doc_title {doc.get('doc_title')!r} does not contain {substring!r}" + _assert_field_contains(docs, "doc_title", substring, "referenced_documents") @then("The number of referenced_document returned is {count:d}") def check_referenced_document_count(context: Context, count: int) -> None: """Assert the number of referenced_documents matches the expected count.""" docs = _get_referenced_documents(context) - assert len(docs) == count, f"Expected {count} referenced_documents, got {len(docs)}" + _assert_count_matches(docs, count, "referenced_documents") @then('Each referenced_document source is "{source}"') def check_referenced_document_source(context: Context, source: str) -> None: """Assert every referenced_document has the expected source.""" docs = _get_referenced_documents(context) - assert docs, "No referenced_documents to check" - for i, doc in enumerate(docs): - actual = doc.get("source") - assert ( - actual == source - ), f"referenced_document[{i}] source is {actual!r}, expected {source!r}" + _assert_field_matches(docs, "source", source, "referenced_documents") @then("Each referenced_document has a non-empty document_id") def check_referenced_document_id(context: Context) -> None: """Assert every referenced_document has a non-empty document_id.""" docs = _get_referenced_documents(context) - assert docs, "No referenced_documents to check" - for i, doc in enumerate(docs): - doc_id = doc.get("document_id") - assert doc_id, f"referenced_document[{i}] has empty or missing document_id" + _assert_field_not_empty(docs, "document_id", "referenced_documents") -# ── Then steps: tool_calls assertions ── +# ── Then Steps: tool_calls Assertions ── @then("The response contains non-empty tool_calls") def check_tool_calls_present(context: Context) -> None: """Assert the response contains at least one tool call.""" tool_calls = _get_tool_calls(context) - assert len(tool_calls) > 0, "tool_calls is empty — no tool calls were made" + _assert_not_empty(tool_calls, "tool_calls") @then('A tool_call has name "{name}"') @@ -342,9 +515,9 @@ def check_tool_call_name(context: Context, name: str) -> None: tool_calls = _get_tool_calls(context) assert tool_calls, "No tool_calls to check" names = [tc.get("name") for tc in tool_calls] - assert ( - name in names - ), f"No tool_call with name {name!r} found. Available names: {names!r}" + assert name in names, ( + f"Expected tool_call with name {name!r}, " f"but found names {names!r}" + ) @then('A tool_call has type "{type_name}"') @@ -354,59 +527,46 @@ def check_tool_call_type(context: Context, type_name: str) -> None: assert tool_calls, "No tool_calls to check" types = [tc.get("type") for tc in tool_calls] matched = any(t in {type_name, f"{type_name}_call"} for t in types) - assert ( - matched - ), f"No tool_call with type {type_name!r} found. Available types: {types!r}" + assert matched, ( + f"Expected tool_call with type {type_name!r}, " f"but found types {types!r}" + ) -# ── Then steps: content and results assertions ── +# ── Then Steps: Content and Results Assertions ── @then('The response contains "{substring}"') def check_response_contains_substring(context: Context, substring: str) -> None: """Assert the LLM response contains the expected substring (case-insensitive).""" - body = _get_response_body(context) - if "response" in body: - response_text = body["response"] - elif "output_text" in body: - response_text = body["output_text"] - else: - response_text = body.get("response_complete", "") - + response_text = _get_response_text(context) assert substring.lower() in response_text.lower(), ( - f"Response does not contain {substring!r}. " - f"Response text: {response_text[:200]}..." + f"Expected substring {substring!r} in response, " + f"but response text: {response_text[:200]}..." ) @then("The response contains non-empty content") def check_response_content_present(context: Context) -> None: """Assert the response contains non-empty content text.""" - body = _get_response_body(context) - if "response" in body: - content = body["response"] - elif "output_text" in body: - content = body["output_text"] - else: - content = body.get("response_complete", "") - assert content, "Response content is empty" + content = _get_response_text(context) + assert content, "Expected non-empty response content, but it was empty" @then("The response contains non-empty results") def check_results_present(context: Context) -> None: """Assert the Responses API output contains non-empty file_search results.""" results = _get_file_search_results(context) - assert len(results) > 0, "No file_search results found in response output" + _assert_not_empty(results, "file_search results") @then("The number of results returned is {count:d}") def check_results_count(context: Context, count: int) -> None: """Assert the number of file_search results matches the expected count.""" results = _get_file_search_results(context) - assert len(results) == count, f"Expected {count} results, got {len(results)}" + _assert_count_matches(results, count, "file_search results") -# ── Then steps: empty response assertions ── +# ── Then Steps: Empty Response Assertions ── @then("The response contains no rag_chunks") @@ -414,7 +574,7 @@ def check_no_rag_chunks(context: Context) -> None: """Assert the response has no rag_chunks (empty or absent).""" body = _get_response_body(context) chunks = body.get("rag_chunks", []) - assert len(chunks) == 0, f"Expected no rag_chunks, got {len(chunks)}" + _assert_empty(chunks, "rag_chunks") @then("The response contains no referenced_documents") @@ -422,4 +582,4 @@ def check_no_referenced_documents(context: Context) -> None: """Assert the response has no referenced_documents (empty or absent).""" body = _get_response_body(context) docs = body.get("referenced_documents", []) - assert len(docs) == 0, f"Expected no referenced_documents, got {len(docs)}" + _assert_empty(docs, "referenced_documents") diff --git a/tests/e2e/utils/prow_utils.py b/tests/e2e/utils/prow_utils.py index 7b5f02f10..cc4bf1f35 100644 --- a/tests/e2e/utils/prow_utils.py +++ b/tests/e2e/utils/prow_utils.py @@ -20,6 +20,7 @@ def get_namespace() -> str: _POD_NAME_MAP = { "lightspeed-stack": "lightspeed-stack-service", "llama-stack": "llama-stack-service", + "okp-solr": "okp-solr-service", } @@ -345,3 +346,44 @@ def update_config_configmap( finally: if temp_path and os.path.exists(temp_path): os.remove(temp_path) + + +def disrupt_okp_solr_pod() -> bool: + """Disrupt OKP Solr connection in Prow/OpenShift environment. + + Returns: + True if the pod was running and has been disrupted, False otherwise. + """ + try: + result = run_e2e_ops("disrupt-okp-solr", timeout=60) + print(result.stdout, end="") + + # Exit code 0 = disrupted (was running), exit code 2 = was not running + if result.returncode == 0: + return True + elif result.returncode == 2: + return False + else: + print(result.stderr, end="") + return False + + except subprocess.TimeoutExpired: + print("Warning: Timeout while disrupting OKP Solr connection") + return False + + +def restore_okp_solr_pod() -> None: + """Restore OKP Solr pod in Prow/OpenShift environment. + + Raises: + subprocess.CalledProcessError: If oc/e2e-ops restore fails. + subprocess.TimeoutExpired: If the operation times out. + """ + result = run_e2e_ops("restore-okp-solr", timeout=180) + print(result.stdout, end="") + if result.returncode != 0: + print(result.stderr, end="") + raise subprocess.CalledProcessError( + result.returncode, "restore-okp-solr", result.stderr + ) + print("✓ OKP Solr pod restored successfully") From ae2c13bde4515b5b004634c848d36bd7e05a5140 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Thu, 27 Aug 2026 09:11:20 -0400 Subject: [PATCH 10/18] addressing comments --- Makefile | 4 +- .../lightspeed-stack-okp-offline.yaml | 3 +- tests/e2e/features/environment.py | 73 ++++++++----------- tests/e2e/features/okp_rag.feature | 9 +-- tests/e2e/utils/prow_utils.py | 6 +- 5 files changed, 42 insertions(+), 53 deletions(-) diff --git a/Makefile b/Makefile index bc5598dcb..83567bbf1 100644 --- a/Makefile +++ b/Makefile @@ -86,8 +86,8 @@ start-llama-stack-container: build-llama-stack-image ## Start OGX container -v $(PWD)/$(CONFIG):/opt/app-root/lightspeed-stack.yaml:ro,z \ -v $(PWD)/scripts/llama-stack-entrypoint.sh:/opt/app-root/enrich-entrypoint.sh:ro,z \ -v $(PWD)/src/llama_stack_configuration.py:/opt/app-root/llama_stack_configuration.py:ro,z \ - $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/lightspeed_stack_providers:/opt/app-root/providers/lightspeed_stack_providers:ro) \ - $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/resources/external_providers:/opt/app-root/src/.llama/providers.d:ro) \ + $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/lightspeed_stack_providers:/opt/app-root/providers/lightspeed_stack_providers:ro,z) \ + $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/resources/external_providers:/opt/app-root/src/.llama/providers.d:ro,z) \ $(if $(LIGHTSPEED_PROVIDERS_DIR),-e EXTERNAL_PROVIDERS_DIR=/opt/app-root/src/.llama/providers.d) \ -e OPENAI_API_KEY \ -e BRAVE_SEARCH_API_KEY \ diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml index e2a3ccd35..bc6015887 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml @@ -36,5 +36,4 @@ rag: okp: rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} offline: true - max_chunks: 5 - + max_chunks: 5 \ No newline at end of file diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index 6434f0621..720a493e9 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -334,13 +334,10 @@ def after_scenario(context: Context, scenario: Scenario) -> None: # Restart OKP server if it was stopped during the scenario if getattr(context, "okp_was_running", False): if is_prow_environment(): - # Prow/OpenShift: restore pod + # Prow/OpenShift: restore pod and raise on failure from tests.e2e.utils.prow_utils import restore_okp_solr_pod - try: - restore_okp_solr_pod() - except Exception as e: # pylint: disable=broad-exception-caught - print(f"Warning: Could not restore OKP Solr pod: {e}") + restore_okp_solr_pod() else: # Docker mode: recreate or restart container from tests.e2e.features.steps.okp_rag import ( @@ -363,56 +360,50 @@ def after_scenario(context: Context, scenario: Scenario) -> None: print( f"OKP container '{container_name}' was removed, recreating from {OKP_IMAGE_NAME}..." ) - try: - subprocess.run( - [ - "docker", - "run", - "--rm", - "-d", - "-p", - "8081:8080", - OKP_IMAGE_NAME, - ], - capture_output=True, - text=True, - check=True, - ) - except subprocess.CalledProcessError as e: - print(f"Warning: Could not recreate OKP container: {e}") - if e.stderr: - print(f" Error: {e.stderr}") + subprocess.run( + [ + "docker", + "run", + "--rm", + "-d", + "-p", + "8081:8080", + OKP_IMAGE_NAME, + ], + capture_output=True, + text=True, + check=True, + ) else: # Container exists - just start it - try: - subprocess.run( - ["docker", "start", container_name], - capture_output=True, - text=True, - check=True, - ) - except subprocess.CalledProcessError as e: - print( - f"Warning: Could not start OKP container '{container_name}': {e}" - ) - if e.stderr: - print(f" Error: {e.stderr}") + subprocess.run( + ["docker", "start", container_name], + capture_output=True, + text=True, + check=True, + ) # Wait for the server to be ready max_attempts = 30 + okp_ready = False for attempt in range(max_attempts): try: resp = requests.get(OKP_DEFAULT_URL, timeout=5) if resp.status_code < 500: print("✓ OKP server restarted successfully") + okp_ready = True break + # HTTP 5xx: delay and retry + if attempt < max_attempts - 1: + time.sleep(1) except requests.ConnectionError: if attempt < max_attempts - 1: time.sleep(1) - else: - print( - "Warning: OKP server may not be fully ready after restart" - ) + + if not okp_ready: + raise RuntimeError( + f"OKP server failed to become ready after {max_attempts} attempts" + ) def _print_llama_stack_diagnostics() -> None: diff --git a/tests/e2e/features/okp_rag.feature b/tests/e2e/features/okp_rag.feature index 309d740f2..79659597c 100644 --- a/tests/e2e/features/okp_rag.feature +++ b/tests/e2e/features/okp_rag.feature @@ -24,8 +24,6 @@ Feature: OKP(Solr) RAG retrieval tests {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 - And The response contains non-empty rag_chunks - And The response contains non-empty referenced_documents And The number of rag_chunk returned is 1 And Each rag_chunk has a non-empty score And Each rag_chunk source is "okp" @@ -47,7 +45,6 @@ Feature: OKP(Solr) RAG retrieval tests """ Then The status code of the response is 200 And I wait for the response to be completed - And The response contains non-empty referenced_documents And Each referenced_document has fields doc_url, doc_title, source, and document_id And The number of eferenced_document returned is 3 And Each referenced_document doc_url contains "docs.redhat.com" @@ -78,8 +75,6 @@ Feature: OKP(Solr) RAG retrieval tests """ Then The status code of the response is 200 And The response contains "security best practices" - And The response contains non-empty rag_chunks - And The response contains non-empty referenced_documents And The number of rag_chunk returned is 1 And Each rag_chunk has a non-empty score And Each rag_chunk source is "okp" @@ -94,7 +89,7 @@ Feature: OKP(Solr) RAG retrieval tests Scenario: Offline query API with OKP tool RAG has rag_chunk and referenced_documents returned Given The service uses the lightspeed-stack-okp-tool-offline.yaml configuration - # And The service is restarted + And The service is restarted When I use "query" to ask question with authorization header """ { @@ -160,7 +155,7 @@ Feature: OKP(Solr) RAG retrieval tests Given The service uses the lightspeed-stack-okp-online.yaml configuration And The service is restarted And The OKP(Solr) server is stopped - When I use "query" to ask question with authorization header + When I use "streaming_query" to ask question with authorization header """ {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} """ diff --git a/tests/e2e/utils/prow_utils.py b/tests/e2e/utils/prow_utils.py index cc4bf1f35..8ab7c93bb 100644 --- a/tests/e2e/utils/prow_utils.py +++ b/tests/e2e/utils/prow_utils.py @@ -379,7 +379,11 @@ def restore_okp_solr_pod() -> None: subprocess.CalledProcessError: If oc/e2e-ops restore fails. subprocess.TimeoutExpired: If the operation times out. """ - result = run_e2e_ops("restore-okp-solr", timeout=180) + # restore-okp-solr can spend 180 seconds in wait_for_pod, plus oc apply time. + # This caller also times out at 180 seconds. after_scenario catches that timeout + # and only logs a warning, so later scenarios can run while OKP remains unavailable. + # Use a timeout with margin, such as 240 seconds. + result = run_e2e_ops("restore-okp-solr", timeout=240) print(result.stdout, end="") if result.returncode != 0: print(result.stderr, end="") From 751079ca30b69e9cb5d3ebef6a993e2ee4829719 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Thu, 27 Aug 2026 11:05:04 -0400 Subject: [PATCH 11/18] updated parsing logic in llm_query file --- tests/e2e/features/okp_rag.feature | 2 +- .../e2e/features/steps/llm_query_response.py | 57 +++++++++++++++++-- 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/tests/e2e/features/okp_rag.feature b/tests/e2e/features/okp_rag.feature index fb5c6654f..725417d20 100644 --- a/tests/e2e/features/okp_rag.feature +++ b/tests/e2e/features/okp_rag.feature @@ -1,4 +1,4 @@ -@cfg_okp @skip +@cfg_okp Feature: OKP(Solr) RAG retrieval tests # Offline Knowledge Portal (OKP) provides a Solr-backed RAG source to LSC. diff --git a/tests/e2e/features/steps/llm_query_response.py b/tests/e2e/features/steps/llm_query_response.py index 2b1590250..501adde80 100644 --- a/tests/e2e/features/steps/llm_query_response.py +++ b/tests/e2e/features/steps/llm_query_response.py @@ -95,7 +95,14 @@ def responses_output_should_include_one_of_types(context: Context) -> None: @step("I wait for the response to be completed") def wait_for_complete_response(context: Context) -> None: """Wait for the response to be complete.""" - context.response_data = _parse_streaming_response(context.response.text) + # Reuse cached parse result if it was created for the current response + if not ( + hasattr(context, "response_data") + and hasattr(context, "_response_data_source") + and context._response_data_source is context.response + ): + context.response_data = _parse_streaming_response(context.response.text) + context._response_data_source = context.response context.response.raise_for_status() assert context.response_data["finished"] is True context.use_streaming_response_data = True @@ -157,7 +164,29 @@ def ask_question_authorized(context: Context, endpoint: str) -> None: resp._content = body.encode(resp.encoding or "utf-8") context.response = resp # Parse SSE events and make data available for assertions + # Preserve existing conversation_id if the new response doesn't have one + # (e.g., 403 errors won't have a 'start' event with conversation_id) + old_conversation_id = ( + context.response_data.get("conversation_id") + if hasattr(context, "response_data") + else None + ) context.response_data = _parse_streaming_response(body) + # Extract conversation from terminal event if not found in start event (Responses API) + if not context.response_data.get("conversation_id"): + if old_conversation_id: + # Preserve from previous response (for 403 error scenarios) + context.response_data["conversation_id"] = old_conversation_id + else: + # Try to extract from terminal event (Responses API streaming format) + try: + terminal = parse_responses_sse_final_response_object(body) + context.response_data["conversation"] = terminal.get("conversation") + context.response_data["conversation_id"] = terminal.get("conversation") + except AssertionError: + pass # No terminal event found (e.g., error responses) + # Mark that this parsed data is from the current response (for cache reuse) + context._response_data_source = resp context.use_streaming_response_data = True else: context.response = request_with_transient_retry( @@ -189,15 +218,33 @@ def ask_question_too_long_authorized(context: Context, endpoint: str) -> None: @step("I store conversation details") def store_conversation_details(context: Context) -> None: - """Store details about the conversation.""" + """Store details about the conversation. + + Reuses cached parse result if it was created for the current response. + """ + # Reuse if already parsed for this response + if ( + hasattr(context, "response_data") + and hasattr(context, "_response_data_source") + and context._response_data_source is context.response + ): + return + try: context.response_data = json.loads(context.response.text) + context._response_data_source = context.response except json.JSONDecodeError: + # Streaming response - parse it context.response_data = _parse_streaming_response(context.response.text) + # Extract conversation from terminal event if not in start event (Responses API) if not context.response_data.get("conversation_id"): - terminal = parse_responses_sse_final_response_object(context.response.text) - context.response_data["conversation"] = terminal.get("conversation") - context.response_data["conversation_id"] = terminal.get("conversation") + try: + terminal = parse_responses_sse_final_response_object(context.response.text) + context.response_data["conversation"] = terminal.get("conversation") + context.response_data["conversation_id"] = terminal.get("conversation") + except AssertionError: + pass # No terminal event found + context._response_data_source = context.response @step('I use "{endpoint}" to ask question with same conversation_id') From 1b73eb1298627caf9237eb01d18677cf706224f4 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Thu, 27 Aug 2026 11:07:24 -0400 Subject: [PATCH 12/18] linting --- tests/e2e/features/steps/llm_query_response.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/e2e/features/steps/llm_query_response.py b/tests/e2e/features/steps/llm_query_response.py index 501adde80..cf1dbe88f 100644 --- a/tests/e2e/features/steps/llm_query_response.py +++ b/tests/e2e/features/steps/llm_query_response.py @@ -182,7 +182,9 @@ def ask_question_authorized(context: Context, endpoint: str) -> None: try: terminal = parse_responses_sse_final_response_object(body) context.response_data["conversation"] = terminal.get("conversation") - context.response_data["conversation_id"] = terminal.get("conversation") + context.response_data["conversation_id"] = terminal.get( + "conversation" + ) except AssertionError: pass # No terminal event found (e.g., error responses) # Mark that this parsed data is from the current response (for cache reuse) @@ -239,7 +241,9 @@ def store_conversation_details(context: Context) -> None: # Extract conversation from terminal event if not in start event (Responses API) if not context.response_data.get("conversation_id"): try: - terminal = parse_responses_sse_final_response_object(context.response.text) + terminal = parse_responses_sse_final_response_object( + context.response.text + ) context.response_data["conversation"] = terminal.get("conversation") context.response_data["conversation_id"] = terminal.get("conversation") except AssertionError: From 2e60615df5833a7929014ce6007f7089dc78ba92 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Thu, 27 Aug 2026 13:49:31 -0400 Subject: [PATCH 13/18] Reset use_streaming_response_data for non-SSE responses --- tests/e2e/features/environment.py | 2 +- tests/e2e/features/steps/llm_query_response.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index 720a493e9..3c7b5bc45 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -396,7 +396,7 @@ def after_scenario(context: Context, scenario: Scenario) -> None: # HTTP 5xx: delay and retry if attempt < max_attempts - 1: time.sleep(1) - except requests.ConnectionError: + except (requests.ConnectionError, requests.Timeout): if attempt < max_attempts - 1: time.sleep(1) diff --git a/tests/e2e/features/steps/llm_query_response.py b/tests/e2e/features/steps/llm_query_response.py index cf1dbe88f..3a4b811e8 100644 --- a/tests/e2e/features/steps/llm_query_response.py +++ b/tests/e2e/features/steps/llm_query_response.py @@ -198,6 +198,8 @@ def ask_question_authorized(context: Context, endpoint: str) -> None: headers=context.auth_headers, timeout=DEFAULT_LLM_TIMEOUT, ) + # Reset flag so assertions read from response.json(), not stale response_data + context.use_streaming_response_data = False # Query length chosen to exceed typical model context windows (e.g. 128k tokens) From 9ba8056a64747a50e5ddaad52a5474226f1ea6e1 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Thu, 27 Aug 2026 13:50:22 -0400 Subject: [PATCH 14/18] updated konflu runner for OKP deploy/cleanup and port forwarding --- tests/e2e-prow/rhoai/pipeline-konflux.sh | 58 +++++++++++++++++++++++- tests/e2e-prow/rhoai/pipeline.sh | 58 ++++++++++++++++++++++-- 2 files changed, 112 insertions(+), 4 deletions(-) diff --git a/tests/e2e-prow/rhoai/pipeline-konflux.sh b/tests/e2e-prow/rhoai/pipeline-konflux.sh index 8633ef8d5..f82bec6fc 100755 --- a/tests/e2e-prow/rhoai/pipeline-konflux.sh +++ b/tests/e2e-prow/rhoai/pipeline-konflux.sh @@ -182,6 +182,22 @@ oc wait pod/mock-jwks pod/mock-mcp \ } log "✅ Mock servers deployed" +# Deploy OKP Solr server for RAG tests +log "Deploying OKP Solr server..." +oc apply -n "$NAMESPACE" -f "$PIPELINE_DIR/manifests/lightspeed/okp-solr.yaml" + +# Wait for OKP Solr to be ready +log "Waiting for OKP Solr to be ready..." +oc wait pod/okp-solr-service \ + -n "$NAMESPACE" --for=condition=Ready --timeout=180s || { + echo "⚠️ OKP Solr not ready, checking status..." + oc get pods -n "$NAMESPACE" | grep okp-solr || true + oc describe pod okp-solr-service -n "$NAMESPACE" 2>/dev/null | tail -30 || true + echo "❌ OKP Solr failed to become ready" + exit 1 +} +log "✅ OKP Solr deployed" + # e2e-tunnel-proxy and e2e-interception-proxy are deployed from proxy.feature steps # (see tests/e2e/features/steps/proxy.py + e2e-ops deploy-e2e-*-proxy). @@ -333,8 +349,10 @@ fi # Debug hook/port churn: export E2E_OPS_VERBOSE=1 before running pipeline.sh export E2E_LSC_PORT_FORWARD_PID_FILE="${E2E_LSC_PORT_FORWARD_PID_FILE:-/tmp/e2e-lightspeed-port-forward.pid}" export E2E_LLAMA_PORT_FORWARD_PID_FILE="${E2E_LLAMA_PORT_FORWARD_PID_FILE:-/tmp/e2e-llama-port-forward.pid}" +export E2E_OKP_PORT_FORWARD_PID_FILE="${E2E_OKP_PORT_FORWARD_PID_FILE:-/tmp/e2e-okp-port-forward.pid}" rm -f "$E2E_LSC_PORT_FORWARD_PID_FILE" rm -f "$E2E_LLAMA_PORT_FORWARD_PID_FILE" +rm -f "$E2E_OKP_PORT_FORWARD_PID_FILE" oc label pod lightspeed-stack-service pod=lightspeed-stack-service -n $NAMESPACE @@ -355,7 +373,7 @@ kill_listeners_on_ports() { fi done } -kill_listeners_on_ports 8080 8000 8321 +kill_listeners_on_ports 8080 8000 8321 8081 # Start port-forward for lightspeed-stack progress "Starting port-forward, then E2E tests" @@ -375,6 +393,12 @@ oc port-forward svc/llama-stack-service-svc 8321:8321 -n $NAMESPACE & PF_LLAMA_PID=$! echo "$PF_LLAMA_PID" >"$E2E_LLAMA_PORT_FORWARD_PID_FILE" +# Start port-forward for OKP Solr (RAG tests) +log "Starting port-forward for OKP Solr..." +oc port-forward svc/okp-solr-service-svc 8081:8080 -n $NAMESPACE & +PF_OKP_PID=$! +echo "$PF_OKP_PID" >"$E2E_OKP_PORT_FORWARD_PID_FILE" + # Wait for port-forward to be usable (app may not be listening immediately; port-forward can drop) log "Waiting for port-forward to lightspeed-stack to be ready..." for i in $(seq 1 36); do @@ -395,6 +419,7 @@ for i in $(seq 1 36); do kill $PF_LCS_PID 2>/dev/null || true kill $PF_JWKS_PID 2>/dev/null || true kill $PF_LLAMA_PID 2>/dev/null || true + kill $PF_OKP_PID 2>/dev/null || true exit 1 fi # If port-forward process died, restart it (e.g. "connection refused" / "lost connection to pod") @@ -421,6 +446,7 @@ for i in $(seq 1 36); do kill $PF_LCS_PID 2>/dev/null || true kill $PF_JWKS_PID 2>/dev/null || true kill $PF_LLAMA_PID 2>/dev/null || true + kill $PF_OKP_PID 2>/dev/null || true exit 1 fi if ! kill -0 $PF_LLAMA_PID 2>/dev/null; then @@ -432,10 +458,30 @@ for i in $(seq 1 36); do sleep 5 done +log "Waiting for OKP Solr port-forward (localhost:8081 /solr)..." +for i in $(seq 1 24); do + if curl -sf http://localhost:8081/solr > /dev/null 2>&1; then + log "✅ OKP Solr port-forward ready after $(( i * 5 ))s" + break + fi + if [ $i -eq 24 ]; then + echo "⚠️ Port-forward to OKP Solr never became healthy (2 min) - OKP RAG tests may fail" | tee /dev/stderr + # Don't exit - OKP is optional, other tests can still run + fi + if ! kill -0 $PF_OKP_PID 2>/dev/null; then + log "OKP port-forward died, restarting (attempt $i)..." + oc port-forward svc/okp-solr-service-svc 8081:8080 -n $NAMESPACE & + PF_OKP_PID=$! + echo "$PF_OKP_PID" >"$E2E_OKP_PORT_FORWARD_PID_FILE" + fi + sleep 5 +done + export E2E_LSC_HOSTNAME="localhost" export E2E_JWKS_HOSTNAME="localhost" export E2E_LLAMA_HOSTNAME="localhost" export E2E_LLAMA_PORT="8321" +export E2E_OKP_URL="http://localhost:8081" # Same pattern as tests/e2e-prow/rhoai/pipeline.sh and .github/workflows/e2e_tests_*.yaml: # Behave {MODEL}/{PROVIDER} use these when set; avoids wrong fallbacks if /v1/models # discovery in before_all is empty (matches run-ci.yaml openai + E2E_OPENAI_MODEL). @@ -450,6 +496,7 @@ export E2E_DEFAULT_PROVIDER_OVERRIDE E2E_DEFAULT_MODEL_OVERRIDE log "LCS accessible at: http://$E2E_LSC_HOSTNAME:8080" log "Mock JWKS accessible at: http://$E2E_JWKS_HOSTNAME:8000" log "OGX (e2e client hooks) at: http://$E2E_LLAMA_HOSTNAME:$E2E_LLAMA_PORT" +log "OKP Solr (RAG tests) at: $E2E_OKP_URL" #======================================== # 7. RUN TESTS @@ -484,13 +531,22 @@ if [[ -n "${E2E_LLAMA_PORT_FORWARD_PID_FILE:-}" && -f "$E2E_LLAMA_PORT_FORWARD_P fi rm -f "$E2E_LLAMA_PORT_FORWARD_PID_FILE" fi +if [[ -n "${E2E_OKP_PORT_FORWARD_PID_FILE:-}" && -f "$E2E_OKP_PORT_FORWARD_PID_FILE" ]]; then + read -r _okp_pf <"$E2E_OKP_PORT_FORWARD_PID_FILE" 2>/dev/null || true + if [[ "${_okp_pf:-}" =~ ^[0-9]+$ ]]; then + kill -9 "$_okp_pf" 2>/dev/null || true + fi + rm -f "$E2E_OKP_PORT_FORWARD_PID_FILE" +fi kill $PF_LCS_PID 2>/dev/null || true kill $PF_JWKS_PID 2>/dev/null || true kill $PF_LLAMA_PID 2>/dev/null || true +kill $PF_OKP_PID 2>/dev/null || true wait $PF_LCS_PID 2>/dev/null || true wait $PF_JWKS_PID 2>/dev/null || true wait $PF_LLAMA_PID 2>/dev/null || true +wait $PF_OKP_PID 2>/dev/null || true set -e trap 'echo "❌ Pipeline failed at line $LINENO"; exit 1' ERR diff --git a/tests/e2e-prow/rhoai/pipeline.sh b/tests/e2e-prow/rhoai/pipeline.sh index 4a86829f5..906ab2773 100755 --- a/tests/e2e-prow/rhoai/pipeline.sh +++ b/tests/e2e-prow/rhoai/pipeline.sh @@ -232,6 +232,22 @@ oc wait pod/mock-jwks pod/mock-mcp \ } echo "✅ Mock servers deployed" +# Deploy OKP Solr server for RAG tests +echo "Deploying OKP Solr server..." +oc apply -n "$NAMESPACE" -f "$PIPELINE_DIR/manifests/lightspeed/okp-solr.yaml" + +# Wait for OKP Solr to be ready +echo "Waiting for OKP Solr to be ready..." +oc wait pod/okp-solr-service \ + -n "$NAMESPACE" --for=condition=Ready --timeout=180s || { + echo "⚠️ OKP Solr not ready, checking status..." + oc get pods -n "$NAMESPACE" | grep okp-solr || true + oc describe pod okp-solr-service -n "$NAMESPACE" 2>/dev/null | tail -30 || true + echo "❌ OKP Solr failed to become ready" + exit 1 +} +echo "✅ OKP Solr deployed" + #======================================== # 8. BUILD OGX IMAGE #======================================== @@ -375,9 +391,11 @@ oc describe pod llama-stack-service -n "$NAMESPACE" || true export E2E_LSC_PORT_FORWARD_PID_FILE="${E2E_LSC_PORT_FORWARD_PID_FILE:-/tmp/e2e-lightspeed-port-forward.pid}" export E2E_LLAMA_PORT_FORWARD_PID_FILE="${E2E_LLAMA_PORT_FORWARD_PID_FILE:-/tmp/e2e-llama-port-forward.pid}" export E2E_JWKS_PORT_FORWARD_PID_FILE="${E2E_JWKS_PORT_FORWARD_PID_FILE:-/tmp/e2e-jwks-port-forward.pid}" +export E2E_OKP_PORT_FORWARD_PID_FILE="${E2E_OKP_PORT_FORWARD_PID_FILE:-/tmp/e2e-okp-port-forward.pid}" rm -f "$E2E_LSC_PORT_FORWARD_PID_FILE" rm -f "$E2E_LLAMA_PORT_FORWARD_PID_FILE" rm -f "$E2E_JWKS_PORT_FORWARD_PID_FILE" +rm -f "$E2E_OKP_PORT_FORWARD_PID_FILE" oc label pod lightspeed-stack-service pod=lightspeed-stack-service -n $NAMESPACE @@ -387,16 +405,18 @@ oc expose pod lightspeed-stack-service \ --type=ClusterIP \ -n $NAMESPACE -# Kill any existing processes on ports 8080, 8000, and 8321 (lsof may be missing in minimal images) -echo "Checking for existing processes on ports 8080, 8000, and 8321..." +# Kill any existing processes on ports 8080, 8000, 8321, and 8081 (lsof may be missing in minimal images) +echo "Checking for existing processes on ports 8080, 8000, 8321, and 8081..." if command -v lsof >/dev/null 2>&1; then lsof -ti:8080 | xargs kill -9 2>/dev/null || true lsof -ti:8000 | xargs kill -9 2>/dev/null || true lsof -ti:8321 | xargs kill -9 2>/dev/null || true + lsof -ti:8081 | xargs kill -9 2>/dev/null || true elif command -v fuser >/dev/null 2>&1; then fuser -k 8080/tcp 2>/dev/null || true fuser -k 8000/tcp 2>/dev/null || true fuser -k 8321/tcp 2>/dev/null || true + fuser -k 8081/tcp 2>/dev/null || true fi # Start port-forward for lightspeed-stack @@ -418,6 +438,12 @@ oc port-forward svc/llama-stack-service-svc 8321:8321 -n $NAMESPACE & PF_LLAMA_PID=$! echo "$PF_LLAMA_PID" >"$E2E_LLAMA_PORT_FORWARD_PID_FILE" +# Start port-forward for OKP Solr (RAG tests) +echo "Starting port-forward for OKP Solr..." +oc port-forward svc/okp-solr-service-svc 8081:8080 -n $NAMESPACE & +PF_OKP_PID=$! +echo "$PF_OKP_PID" >"$E2E_OKP_PORT_FORWARD_PID_FILE" + # Wait for port-forward to be usable (app may not be listening immediately; port-forward can drop) echo "Waiting for port-forward to lightspeed-stack to be ready..." for i in $(seq 1 36); do @@ -439,6 +465,7 @@ for i in $(seq 1 36); do kill $PF_LCS_PID 2>/dev/null || true kill $PF_JWKS_PID 2>/dev/null || true kill $PF_LLAMA_PID 2>/dev/null || true + kill $PF_OKP_PID 2>/dev/null || true exit 1 fi # If port-forward process died, restart it (e.g. "connection refused" / "lost connection to pod") @@ -464,6 +491,7 @@ for i in $(seq 1 36); do kill $PF_LCS_PID 2>/dev/null || true kill $PF_JWKS_PID 2>/dev/null || true kill $PF_LLAMA_PID 2>/dev/null || true + kill $PF_OKP_PID 2>/dev/null || true exit 1 fi if ! kill -0 $PF_LLAMA_PID 2>/dev/null; then @@ -475,13 +503,35 @@ for i in $(seq 1 36); do sleep 5 done +# Wait for OKP Solr port-forward to be usable (non-fatal - OKP is optional) +echo "Waiting for OKP Solr port-forward (localhost:8081 /solr)..." +for i in $(seq 1 24); do + if curl -sf http://localhost:8081/solr > /dev/null 2>&1; then + echo "✅ OKP Solr port-forward ready after $(( i * 5 ))s" + break + fi + if [ $i -eq 24 ]; then + echo "⚠️ Port-forward to OKP Solr never became healthy (2 min) - OKP RAG tests may fail" + # Don't exit - OKP is optional, other tests can still run + fi + if ! kill -0 $PF_OKP_PID 2>/dev/null; then + echo "OKP port-forward died, restarting (attempt $i)..." + oc port-forward svc/okp-solr-service-svc 8081:8080 -n $NAMESPACE & + PF_OKP_PID=$! + echo "$PF_OKP_PID" >"$E2E_OKP_PORT_FORWARD_PID_FILE" + fi + sleep 5 +done + export E2E_LSC_HOSTNAME="localhost" export E2E_JWKS_HOSTNAME="localhost" export E2E_DEFAULT_MODEL_OVERRIDE="$MODEL_NAME" export E2E_DEFAULT_PROVIDER_OVERRIDE="vllm" +export E2E_OKP_URL="http://localhost:8081" echo "LCS accessible at: http://$E2E_LSC_HOSTNAME:8080" echo "Mock JWKS accessible at: http://$E2E_JWKS_HOSTNAME:8000" echo "OGX accessible at: http://localhost:8321" +echo "OKP Solr (RAG tests) at: $E2E_OKP_URL" @@ -505,10 +555,12 @@ TEST_EXIT_CODE=$(cat "$E2E_EXIT_CODE_FILE" 2>/dev/null || echo 1) kill $PF_LCS_PID 2>/dev/null || true kill $PF_JWKS_PID 2>/dev/null || true kill $PF_LLAMA_PID 2>/dev/null || true +kill $PF_OKP_PID 2>/dev/null || true wait $PF_LCS_PID 2>/dev/null || true wait $PF_JWKS_PID 2>/dev/null || true wait $PF_LLAMA_PID 2>/dev/null || true -rm -f "$E2E_LSC_PORT_FORWARD_PID_FILE" "$E2E_LLAMA_PORT_FORWARD_PID_FILE" "$E2E_JWKS_PORT_FORWARD_PID_FILE" +wait $PF_OKP_PID 2>/dev/null || true +rm -f "$E2E_LSC_PORT_FORWARD_PID_FILE" "$E2E_LLAMA_PORT_FORWARD_PID_FILE" "$E2E_JWKS_PORT_FORWARD_PID_FILE" "$E2E_OKP_PORT_FORWARD_PID_FILE" set -e trap 'echo "❌ Pipeline failed at line $LINENO"; exit 1' ERR From 9bb8ac77018f01ca1ef4068124cbcc1005649e68 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Thu, 27 Aug 2026 14:06:44 -0400 Subject: [PATCH 15/18] Add an OwnerReference to redhat-registry-pull-secret --- tests/e2e-prow/rhoai/pipeline-konflux.sh | 38 ++++++++++++++++++++--- tests/e2e-prow/rhoai/pipeline.sh | 39 +++++++++++++++++++++--- 2 files changed, 67 insertions(+), 10 deletions(-) diff --git a/tests/e2e-prow/rhoai/pipeline-konflux.sh b/tests/e2e-prow/rhoai/pipeline-konflux.sh index f82bec6fc..e8c32c18f 100755 --- a/tests/e2e-prow/rhoai/pipeline-konflux.sh +++ b/tests/e2e-prow/rhoai/pipeline-konflux.sh @@ -130,11 +130,39 @@ if [[ -d /var/run/redhat-registry-username ]] && [[ -d /var/run/redhat-registry- shopt -u nullglob if [[ -n "$REDHAT_USERNAME" ]] && [[ -n "$REDHAT_PASSWORD" ]]; then - oc create secret docker-registry redhat-registry-pull-secret \ - --docker-server=registry.redhat.io \ - --docker-username="$REDHAT_USERNAME" \ - --docker-password="$REDHAT_PASSWORD" \ - -n "$NAMESPACE" 2>/dev/null && log "✅ Red Hat registry pull secret created" || log "⚠️ Secret exists or creation failed" + # Get current pipeline Pod metadata for ownerReference (ensures cleanup after job completes) + PIPELINE_POD_NAME="${HOSTNAME}" + if PIPELINE_POD_UID=$(oc get pod "$PIPELINE_POD_NAME" -n "$NAMESPACE" -o jsonpath='{.metadata.uid}' 2>/dev/null); then + log "Setting ownerReference to pipeline Pod: $PIPELINE_POD_NAME" + + # Create secret with ownerReference using YAML (ensures automatic cleanup) + cat </dev/null && log "✅ Red Hat registry pull secret created" || log "⚠️ Secret exists or creation failed" + fi # Link to default service account oc secrets link default redhat-registry-pull-secret --for=pull -n "$NAMESPACE" 2>/dev/null || echo "⚠️ Secret already linked to default SA" diff --git a/tests/e2e-prow/rhoai/pipeline.sh b/tests/e2e-prow/rhoai/pipeline.sh index 906ab2773..662d45fb5 100755 --- a/tests/e2e-prow/rhoai/pipeline.sh +++ b/tests/e2e-prow/rhoai/pipeline.sh @@ -96,11 +96,40 @@ oc secrets link default quay-lightspeed-pull-secret --for=pull -n "$NAMESPACE" 2 # Credentials from Prow secrets (environment variables or mounted volumes) if [[ -n "${REDHAT_REGISTRY_USERNAME:-}" ]] && [[ -n "${REDHAT_REGISTRY_PASSWORD:-}" ]]; then echo "Creating Red Hat registry pull secret from environment..." - oc create secret docker-registry redhat-registry-pull-secret \ - --docker-server=registry.redhat.io \ - --docker-username="$REDHAT_REGISTRY_USERNAME" \ - --docker-password="$REDHAT_REGISTRY_PASSWORD" \ - -n "$NAMESPACE" 2>/dev/null && echo "✅ Red Hat registry pull secret created" || echo "⚠️ Secret exists or creation failed" + + # Get current pipeline Pod metadata for ownerReference (ensures cleanup after job completes) + PIPELINE_POD_NAME="${HOSTNAME}" + if PIPELINE_POD_UID=$(oc get pod "$PIPELINE_POD_NAME" -n "$NAMESPACE" -o jsonpath='{.metadata.uid}' 2>/dev/null); then + echo "Setting ownerReference to pipeline Pod: $PIPELINE_POD_NAME" + + # Create secret with ownerReference using YAML (ensures automatic cleanup) + cat </dev/null && echo "✅ Red Hat registry pull secret created" || echo "⚠️ Secret exists or creation failed" + fi # Link to default service account oc secrets link default redhat-registry-pull-secret --for=pull -n "$NAMESPACE" 2>/dev/null || echo "⚠️ Secret already linked to default SA" From 0877f661f7ec4d9115b17d0d0ca5767c8e31de7f Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Thu, 27 Aug 2026 14:07:23 -0400 Subject: [PATCH 16/18] fixing syntax error --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c031a27ec..f817358a5 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ SHELL := /bin/bash #TODO: We need to rename all those python and config files as well +# Define comma for use in $(if) expressions (where commas are argument separators) +COMMA := , + ARTIFACT_DIR := $(if $(ARTIFACT_DIR),$(ARTIFACT_DIR),tests/test_results) PATH_TO_PLANTUML := ~/bin @@ -87,8 +90,8 @@ start-ogx-container: build-ogx-image ## Start OGX container -v $(PWD)/$(CONFIG):/opt/app-root/lightspeed-stack.yaml:ro,z \ -v $(PWD)/scripts/llama-stack-entrypoint.sh:/opt/app-root/enrich-entrypoint.sh:ro,z \ -v $(PWD)/src/llama_stack_configuration.py:/opt/app-root/llama_stack_configuration.py:ro,z \ - $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/lightspeed_stack_providers:/opt/app-root/providers/lightspeed_stack_providers:ro,z) \ - $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/resources/external_providers:/opt/app-root/src/.llama/providers.d:ro,z) \ + $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/lightspeed_stack_providers:/opt/app-root/providers/lightspeed_stack_providers:ro$(COMMA)z) \ + $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/resources/external_providers:/opt/app-root/src/.llama/providers.d:ro$(COMMA)z) \ $(if $(LIGHTSPEED_PROVIDERS_DIR),-e EXTERNAL_PROVIDERS_DIR=/opt/app-root/src/.llama/providers.d) \ -e OPENAI_API_KEY \ -e BRAVE_SEARCH_API_KEY \ From 9e5aa1e04d6fe2caa340be2d06b9a182eb631d09 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Thu, 27 Aug 2026 14:11:45 -0400 Subject: [PATCH 17/18] port-forward restart issue fix --- tests/e2e-prow/rhoai/scripts/e2e-ops.sh | 109 +++++++++++++++++++++++- 1 file changed, 108 insertions(+), 1 deletion(-) diff --git a/tests/e2e-prow/rhoai/scripts/e2e-ops.sh b/tests/e2e-prow/rhoai/scripts/e2e-ops.sh index 79b917e4f..9df97fb3a 100755 --- a/tests/e2e-prow/rhoai/scripts/e2e-ops.sh +++ b/tests/e2e-prow/rhoai/scripts/e2e-ops.sh @@ -23,6 +23,7 @@ # restart-both-services - Full OGX then lightspeed-stack restart (explicit only) # restart-port-forward - Re-establish port-forward for lightspeed # restart-llama-port-forward - Re-establish port-forward for OGX (8321) +# restart-okp-port-forward - Re-establish port-forward for OKP Solr (8081) # wait-for-pod [attempts] - Wait for a pod to be ready # update-configmap - Update ConfigMap from file # get-configmap-content - Get ConfigMap content (outputs to stdout) @@ -47,6 +48,7 @@ MANIFEST_DIR="$SCRIPT_DIR/../manifests/lightspeed" E2E_LSC_PORT_FORWARD_PID_FILE="${E2E_LSC_PORT_FORWARD_PID_FILE:-/tmp/e2e-lightspeed-port-forward.pid}" E2E_LLAMA_PORT_FORWARD_PID_FILE="${E2E_LLAMA_PORT_FORWARD_PID_FILE:-/tmp/e2e-llama-port-forward.pid}" E2E_JWKS_PORT_FORWARD_PID_FILE="${E2E_JWKS_PORT_FORWARD_PID_FILE:-/tmp/e2e-jwks-port-forward.pid}" +E2E_OKP_PORT_FORWARD_PID_FILE="${E2E_OKP_PORT_FORWARD_PID_FILE:-/tmp/e2e-okp-port-forward.pid}" # ============================================================================ # Helper functions @@ -212,6 +214,24 @@ kill_stale_jwks_forward() { free_local_tcp_port "$port" } +# Kill anything likely to hold the OKP Solr local forward (localhost:8081). +kill_stale_okp_forward() { + local port="${1:-8081}" + local saved_pf + if [[ -f "$E2E_OKP_PORT_FORWARD_PID_FILE" ]]; then + read -r saved_pf <"$E2E_OKP_PORT_FORWARD_PID_FILE" 2>/dev/null || true + if [[ "$saved_pf" =~ ^[0-9]+$ ]]; then + kill -9 "$saved_pf" 2>/dev/null || true + fi + fi + pkill -9 -f "port-forward.*okp-solr-service-svc.*8081:8080" 2>/dev/null || true + pkill -9 -f "oc port-forward svc/okp-solr-service-svc 8081:8080" 2>/dev/null || true + pkill -9 -f "port-forward pod/okp-solr-service.*8081:8080" 2>/dev/null || true + free_local_tcp_port "$port" + sleep 1 + free_local_tcp_port "$port" +} + # After oc port-forward dies in <2s, show recent oc stderr from the log file. e2e_ops_emit_port_forward_immediate_failure_diag() { echo "[e2e-ops] /tmp/port-forward.log (tail 25):" @@ -558,6 +578,25 @@ verify_llama_local_forward() { return 1 } +verify_okp_connectivity() { + local max_attempts="${1:-15}" + local http_code="" + local attempt + + for ((attempt=1; attempt<=max_attempts; attempt++)); do + http_code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 "http://localhost:8081/solr" 2>/dev/null) || http_code="000" + # OKP Solr returns various 200-399 codes for /solr endpoint + if [[ "$http_code" =~ ^[23][0-9][0-9]$ ]]; then + return 0 + fi + if [[ $attempt -lt $max_attempts ]]; then + sleep 2 + fi + done + echo "OKP Solr localhost:8081 connectivity check failed (HTTP: ${http_code:-unknown})" + return 1 +} + cmd_restart_llama_port_forward() { local local_port="${LOCAL_LLAMA_PORT:-8321}" local remote_port="${REMOTE_LLAMA_PORT:-8321}" @@ -680,6 +719,67 @@ cmd_restart_jwks_port_forward() { return 1 } +cmd_restart_okp_port_forward() { + local local_port="${LOCAL_OKP_PORT:-8081}" + local remote_port="${REMOTE_OKP_PORT:-8080}" + local max_attempts=6 + local pf_pid + local pf_resource + local okp_pf_log="/tmp/port-forward-okp.log" + + echo "Re-establishing OKP Solr port-forward on $local_port:$remote_port..." + + for ((attempt=1; attempt<=max_attempts; attempt++)); do + kill_stale_okp_forward "$local_port" + sleep 3 + + if [[ $attempt -le 2 ]]; then + pf_resource="svc/okp-solr-service-svc" + else + pf_resource="pod/okp-solr-service" + fi + echo "OKP port-forward attempt $attempt/$max_attempts -> $pf_resource" + + : >"$okp_pf_log" + nohup oc port-forward "$pf_resource" "$local_port:$remote_port" -n "$NAMESPACE" \ + "$okp_pf_log" 2>&1 & + pf_pid=$! + disown "$pf_pid" 2>/dev/null || true + sleep 3 + + if ! kill -0 "$pf_pid" 2>/dev/null; then + echo "OKP port-forward process exited immediately:" + if [[ -s "$okp_pf_log" ]]; then + tail -25 "$okp_pf_log" 2>/dev/null | sed 's/^/[e2e-ops] /' || true + fi + kill_stale_okp_forward "$local_port" + sleep 2 + continue + fi + sleep 4 + + if verify_okp_connectivity 12; then + echo "$pf_pid" >"$E2E_OKP_PORT_FORWARD_PID_FILE" + echo "✓ OKP Solr port-forward established (PID: $pf_pid)" + return 0 + fi + + if [[ $attempt -lt $max_attempts ]]; then + echo "OKP forward attempt $attempt failed (connectivity check failed), retrying..." + kill -9 "$pf_pid" 2>/dev/null || true + kill_stale_okp_forward "$local_port" + sleep 3 + fi + done + + echo "Failed to establish OKP Solr port-forward after $max_attempts attempts" + if [[ -s "$okp_pf_log" ]]; then + echo "Port-forward log (tail 30):" + tail -30 "$okp_pf_log" 2>/dev/null | sed 's/^/[e2e-ops] /' || true + fi + return 1 +} + cmd_wait_for_pod() { local pod_name="${1:?Pod name required}" local max_attempts="${2:-24}" @@ -1030,7 +1130,10 @@ cmd_restore_okp_solr() { echo "Restoring OKP Solr service in namespace $NAMESPACE..." oc apply -n "$NAMESPACE" -f "$MANIFEST_DIR/okp-solr.yaml" wait_for_pod "okp-solr-service" 60 - echo "✓ OKP Solr service restored and ready" + echo "✓ OKP Solr pod restored and ready" + + # Restart port-forward since pod was replaced + cmd_restart_okp_port_forward } # ============================================================================ @@ -1056,6 +1159,9 @@ case "$COMMAND" in restart-jwks-port-forward) cmd_restart_jwks_port_forward ;; + restart-okp-port-forward) + cmd_restart_okp_port_forward + ;; restart-port-forward) cmd_restart_port_forward ;; @@ -1124,6 +1230,7 @@ case "$COMMAND" in echo " restart-llama-stack - Restart/restore llama-stack pod" echo " restart-both-services - Full llama-stack + lightspeed-stack restart (explicit)" echo " restart-llama-port-forward - Re-establish port-forward for OGX (8321)" + echo " restart-okp-port-forward - Re-establish port-forward for OKP Solr (8081)" echo " restart-port-forward - Re-establish port-forward for lightspeed" echo " wait-for-pod [attempts] - Wait for a pod to be ready" echo " update-configmap - Update ConfigMap from file" From 14fc382cd3e6d88912a2af5b9c646866c9f676b6 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Thu, 27 Aug 2026 14:14:41 -0400 Subject: [PATCH 18/18] Added a timeout to both OKP readiness --- tests/e2e-prow/rhoai/pipeline-konflux.sh | 2 +- tests/e2e-prow/rhoai/pipeline.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e-prow/rhoai/pipeline-konflux.sh b/tests/e2e-prow/rhoai/pipeline-konflux.sh index e8c32c18f..f391dc898 100755 --- a/tests/e2e-prow/rhoai/pipeline-konflux.sh +++ b/tests/e2e-prow/rhoai/pipeline-konflux.sh @@ -488,7 +488,7 @@ done log "Waiting for OKP Solr port-forward (localhost:8081 /solr)..." for i in $(seq 1 24); do - if curl -sf http://localhost:8081/solr > /dev/null 2>&1; then + if curl -sf --max-time 5 http://localhost:8081/solr > /dev/null 2>&1; then log "✅ OKP Solr port-forward ready after $(( i * 5 ))s" break fi diff --git a/tests/e2e-prow/rhoai/pipeline.sh b/tests/e2e-prow/rhoai/pipeline.sh index 662d45fb5..cb93e3695 100755 --- a/tests/e2e-prow/rhoai/pipeline.sh +++ b/tests/e2e-prow/rhoai/pipeline.sh @@ -535,7 +535,7 @@ done # Wait for OKP Solr port-forward to be usable (non-fatal - OKP is optional) echo "Waiting for OKP Solr port-forward (localhost:8081 /solr)..." for i in $(seq 1 24); do - if curl -sf http://localhost:8081/solr > /dev/null 2>&1; then + if curl -sf --max-time 5 http://localhost:8081/solr > /dev/null 2>&1; then echo "✅ OKP Solr port-forward ready after $(( i * 5 ))s" break fi