From 6c0c8a46a57ae7391e4eaafcfc0cdf0bc269188d Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Fri, 21 Aug 2026 10:49:19 -0400 Subject: [PATCH 01/25] adding new OKP tests --- tests/e2e/features/okp_rag.feature | 229 +++++++++++++++++++++++++++++ tests/e2e/features/query.feature | 24 +++ tests/e2e/test_list.txt | 1 + 3 files changed, 254 insertions(+) create mode 100644 tests/e2e/features/okp_rag.feature diff --git a/tests/e2e/features/okp_rag.feature b/tests/e2e/features/okp_rag.feature new file mode 100644 index 000000000..989cf1061 --- /dev/null +++ b/tests/e2e/features/okp_rag.feature @@ -0,0 +1,229 @@ +@cfg_okp +Feature: OKP(Solr) RAG retrieval tests + + # Offline Knowledge Portal (OKP) provides a Solr-backed RAG source to LSC. + # Tests verify that Lightspeed Stack can use OKP for both Inline RAG + # (context injected before the LLM request) and Tool RAG (context + # retrieved on demand via file_search), in both offline and online modes. + + Background: + Given The service is started locally + And The system is in default state + And OKP(Solr) server is running + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And REST API service prefix is /v1 + And the Lightspeed stack configuration directory is "tests/e2e/configuration" + + # ── Inline RAG — Query ── + + Scenario Outline: mode query with inline RAG returns rag_chunks and referenced_documents + Given The service uses the configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response contains non-empty rag_chunks + And The response contains non-empty referenced_documents + And The number of rag_chunk returned is + And Each rag_chunk has a non-empty score + And Each rag_chunk source is "okp" + And Each referenced_document has fields doc_url, doc_title, source, and document_id + And Each referenced_document doc_url contains "" + And Each referenced_document doc_title is not empty + And Each referenced_document source is "okp" + And Each referenced_document has a non-empty document_id + + Examples: Offline + | mode | config | max_chunks | doc_url_domain | + | Offline | lightspeed-stack-okp-offline.yaml | 5 | localhost:8081 | + + Examples: Online + | mode | config | max_chunks | doc_url_domain | + | Online | lightspeed-stack-okp-online.yaml | 1 | docs.redhat.com | + + # ── Inline RAG — Streaming Query ── + + Scenario Outline: mode streaming query with inline RAG returns referenced_documents + Given The service uses the configuration + And The service is restarted + When I use "streaming_query" to ask question with authorization header + """ + {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And I wait for the response to be completed + And The response contains non-empty referenced_documents + And Each referenced_document has fields doc_url, doc_title, source, and document_id + And Each referenced_document doc_url contains "" + And Each referenced_document doc_title is not empty + And Each referenced_document source is "okp" + And Each referenced_document has a non-empty document_id + + Examples: Offline + | mode | config | doc_url_domain | + | Offline | lightspeed-stack-okp-offline.yaml | localhost:8081 | + + Examples: Online + | mode | config | doc_url_domain | + | Online | lightspeed-stack-okp-online.yaml | docs.redhat.com | + + # ── Inline RAG — Query with Dynamic Filter ── + + Scenario Outline: Query with inline RAG with dynamic filter returns rag_chunks and referenced_documents + Given The service uses the lightspeed-stack-okp-offline.yaml configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + {"query": "Security best practices", + "solr": { + "mode": "", + "filters": { + "filters": { + "type": "in", + "key": "product", + "value": ["openshift_container_platform", "ansible_automation_platform", "red_hat_enterprise_linux"] + } + } + } + } + """ + Then The status code of the response is 200 + And The response contains non-empty rag_chunks + And The response contains non-empty referenced_documents + And The number of rag_chunk returned is 5 + And Each rag_chunk has a non-empty score + And Each rag_chunk source is "okp" + And Each referenced_document has fields doc_url, doc_title, source, and document_id + And Each referenced_document doc_url contains "" + And Each referenced_document doc_title is not empty + And Each referenced_document source is "okp" + And Each referenced_document has a non-empty document_id + + Examples: + | filter_mode | doc_url_domain | + | semantic | localhost:8081 | + | hybrid | docs.redhat.com | + + # ── Tool RAG — Query API ── + + Scenario Outline: queries API with OKP tool RAG has rag_chunk and referenced_documents returned + Given The service uses the configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + { + "query": "configure remote desktop using gnome", + "model": "{MODEL}", + "provider": "{PROVIDER}", + "system_prompt": "You MUST use the file_search tool to answer." + } + """ + Then The status code of the response is 200 + And The response contains non-empty tool_calls + And A tool_call has name "file_search" + And The response contains non-empty rag_chunks + And The number of rag_chunk returned is + And Each rag_chunk has a non-empty score + And Each rag_chunk source is "okp" + And The response contains non-empty referenced_documents + And Each referenced_document has fields doc_url, doc_title, source, and document_id + And Each referenced_document doc_url contains "" + And Each referenced_document doc_title is not empty + And Each referenced_document source is "okp" + And Each referenced_document has a non-empty document_id + + Examples: Offline + | mode | config | max_chunks | doc_url_domain | + | Offline | lightspeed-stack-okp-tool-offline.yaml | 5 | localhost:8081 | + + Examples: Online + | mode | config | max_chunks | doc_url_domain | + | Online | lightspeed-stack-okp-tool-online.yaml | 1 | access.redhat.com | + + # ── Tool RAG — Streaming Query API ── + + Scenario Outline: streaming query API with OKP tool RAG has rag_chunk and referenced_documents returned + Given The service uses the configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + { + "query": "configure remote desktop using gnome", + "model": "{MODEL}", + "provider": "{PROVIDER}", + "system_prompt": "You MUST use the file_search tool to answer." + } + """ + Then The status code of the response is 200 + And A tool_call has name "file_search" + And The response contains non-empty content + And The response contains non-empty referenced_documents + And Each referenced_document has fields doc_url, doc_title, source, and document_id + And Each referenced_document doc_url contains "" + And Each referenced_document doc_title is not empty + And Each referenced_document source is "okp" + And Each referenced_document has a non-empty document_id + + Examples: Offline + | mode | config | doc_url_domain | + | Offline | lightspeed-stack-okp-tool-offline.yaml | localhost:8081 | + + Examples: Online + | mode | config | doc_url_domain | + | Online | lightspeed-stack-okp-tool-online.yaml | access.redhat.com | + + # ── Tool RAG — Responses API ── + + Scenario Outline: responses API with OKP tool RAG has rag results returned + Given The service uses the configuration + And The service is restarted + When I use "responses" to ask question with authorization header + """ + { + "input": "configure remote desktop using gnome", + "model": "{PROVIDER}/{MODEL}", + "stream": false, + "instructions": "You MUST use the file_search tool to answer." + } + """ + Then The status code of the response is 200 + And The responses output includes an item with type "file_search_call" + And The response contains non-empty tool_calls + And A tool_call has type "file_search" + And The response contains non-empty results + And The number of results returned is + And Each rag_chunk has a non-empty score + And Each rag_chunk source is "okp" + And Each rag_chunk reference_url contains "" + + Examples: Offline + | mode | config | max_chunks | doc_url_domain | + | Offline | lightspeed-stack-okp-tool-offline.yaml | 5 | localhost:8081 | + + Examples: Online + | mode | config | max_chunks | doc_url_domain | + | Online | lightspeed-stack-okp-tool-online.yaml | 1 | access.redhat.com | + + # # ── OKP Server Unavailable — Graceful Error Handling ─────────────── + + Scenario: Query succeeds with empty rag_chunks when OKP server is unavailable + Given The OKP(Solr) server is stopped + When I use "query" to ask question with authorization header + """ + {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response contains no rag_chunks + And The response contains no referenced_documents + + Scenario: Streaming query succeeds with empty referenced_documents when OKP server is unavailable + Given The OKP(Solr) server is stopped + When I use "streaming_query" to ask question with authorization header + """ + {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And I wait for the response to be completed + And The response contains no referenced_documents diff --git a/tests/e2e/features/query.feature b/tests/e2e/features/query.feature index c0207315c..702fc1490 100644 --- a/tests/e2e/features/query.feature +++ b/tests/e2e/features/query.feature @@ -279,3 +279,27 @@ Scenario: Check if LLM responds for query request with error for missing query When I use "query" to ask question with too-long query and authorization header Then The status code of the response is 413 And The body of the response contains Prompt is too long + + # # ── OKP RAG Disabled (okp not in rag.retrieval.inline.sources) ───── + @cfg_okp + Scenario: Query returns no rag_chunks and no reference_documents when OKP OKP is disabled + Given The service uses the lightspeed-stack-okp-disabled.yaml configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response contains no rag_chunks + And The response contains no reference_documents + @cfg_okp + Scenario: Streaming query returns no referenced_documents when OKP is disabled + Given The service uses the lightspeed-stack-okp-disabled.yaml configuration + And The service is restarted + When I use "streaming_query" to ask question with authorization header + """ + {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And I wait for the response to be completed + And The response contains no referenced_documents diff --git a/tests/e2e/test_list.txt b/tests/e2e/test_list.txt index 2a5d97f05..c9ff90ed5 100644 --- a/tests/e2e/test_list.txt +++ b/tests/e2e/test_list.txt @@ -39,3 +39,4 @@ features/unified-mode-legacy.feature features/unified-mode-validation.feature features/unified-mode-migration.feature features/unified-mode-synthesis.feature +features/okp_rag.feature From 8684b24072d32fd571c9153402f69d66b9bfccb4 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Fri, 21 Aug 2026 10:53:48 -0400 Subject: [PATCH 02/25] added skip tags --- tests/e2e/features/okp_rag.feature | 2 +- tests/e2e/features/query.feature | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/e2e/features/okp_rag.feature b/tests/e2e/features/okp_rag.feature index 989cf1061..ba5193cb4 100644 --- a/tests/e2e/features/okp_rag.feature +++ b/tests/e2e/features/okp_rag.feature @@ -1,4 +1,4 @@ -@cfg_okp +@cfg_okp @skip Feature: OKP(Solr) RAG retrieval tests # Offline Knowledge Portal (OKP) provides a Solr-backed RAG source to LSC. diff --git a/tests/e2e/features/query.feature b/tests/e2e/features/query.feature index 702fc1490..555aee0b6 100644 --- a/tests/e2e/features/query.feature +++ b/tests/e2e/features/query.feature @@ -282,6 +282,7 @@ Scenario: Check if LLM responds for query request with error for missing query # # ── OKP RAG Disabled (okp not in rag.retrieval.inline.sources) ───── @cfg_okp + @skip Scenario: Query returns no rag_chunks and no reference_documents when OKP OKP is disabled Given The service uses the lightspeed-stack-okp-disabled.yaml configuration And The service is restarted @@ -293,6 +294,7 @@ Scenario: Check if LLM responds for query request with error for missing query And The response contains no rag_chunks And The response contains no reference_documents @cfg_okp + @skip Scenario: Streaming query returns no referenced_documents when OKP is disabled Given The service uses the lightspeed-stack-okp-disabled.yaml configuration And The service is restarted From 91cf1f9217b0662051df6f1792b27c7318c46018 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Fri, 21 Aug 2026 11:11:14 -0400 Subject: [PATCH 03/25] fixed typo --- tests/e2e/features/okp_rag.feature | 2 +- tests/e2e/features/query.feature | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/features/okp_rag.feature b/tests/e2e/features/okp_rag.feature index ba5193cb4..642393776 100644 --- a/tests/e2e/features/okp_rag.feature +++ b/tests/e2e/features/okp_rag.feature @@ -147,7 +147,7 @@ Feature: OKP(Solr) RAG retrieval tests Scenario Outline: streaming query API with OKP tool RAG has rag_chunk and referenced_documents returned Given The service uses the configuration And The service is restarted - When I use "query" to ask question with authorization header + When I use "streaming_query" to ask question with authorization header """ { "query": "configure remote desktop using gnome", diff --git a/tests/e2e/features/query.feature b/tests/e2e/features/query.feature index 555aee0b6..3660ad290 100644 --- a/tests/e2e/features/query.feature +++ b/tests/e2e/features/query.feature @@ -292,7 +292,7 @@ Scenario: Check if LLM responds for query request with error for missing query """ Then The status code of the response is 200 And The response contains no rag_chunks - And The response contains no reference_documents + And The response contains no referenced_documents @cfg_okp @skip Scenario: Streaming query returns no referenced_documents when OKP is disabled From f8b73f4255669992cac0dcf3ea04793d5bdd03e9 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Fri, 21 Aug 2026 11:22:32 -0400 Subject: [PATCH 04/25] removed extra config calls --- tests/e2e/features/query.feature | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/e2e/features/query.feature b/tests/e2e/features/query.feature index 3660ad290..18e3d155c 100644 --- a/tests/e2e/features/query.feature +++ b/tests/e2e/features/query.feature @@ -284,8 +284,6 @@ Scenario: Check if LLM responds for query request with error for missing query @cfg_okp @skip Scenario: Query returns no rag_chunks and no reference_documents when OKP OKP is disabled - Given The service uses the lightspeed-stack-okp-disabled.yaml configuration - And The service is restarted When I use "query" to ask question with authorization header """ {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} @@ -296,8 +294,6 @@ Scenario: Check if LLM responds for query request with error for missing query @cfg_okp @skip Scenario: Streaming query returns no referenced_documents when OKP is disabled - Given The service uses the lightspeed-stack-okp-disabled.yaml configuration - And The service is restarted When I use "streaming_query" to ask question with authorization header """ {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} From e7c7f3e0e1e0c731d8a0465d02f19338e6e2f6b2 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Mon, 24 Aug 2026 15:20:01 -0400 Subject: [PATCH 05/25] created config yaml and created step definitions --- .../lightspeed-stack-okp-offline.yaml | 39 +++ .../lightspeed-stack-okp-online.yaml | 40 +++ .../lightspeed-stack-okp-tool-offline.yaml | 39 +++ .../lightspeed-stack-okp-tool-online.yaml | 39 +++ .../lightspeed-stack-okp-offline.yaml | 40 +++ .../lightspeed-stack-okp-online.yaml | 40 +++ .../lightspeed-stack-okp-tool-offline.yaml | 39 +++ .../lightspeed-stack-okp-tool-online.yaml | 39 +++ tests/e2e/features/okp_rag.feature | 142 ++------ .../e2e/features/steps/llm_query_response.py | 16 +- tests/e2e/features/steps/okp_rag.py | 307 ++++++++++++++++++ 11 files changed, 671 insertions(+), 109 deletions(-) create mode 100644 tests/e2e/configuration/library-mode/lightspeed-stack-okp-offline.yaml create mode 100644 tests/e2e/configuration/library-mode/lightspeed-stack-okp-online.yaml create mode 100644 tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-offline.yaml create mode 100644 tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-online.yaml create mode 100644 tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml create mode 100644 tests/e2e/configuration/server-mode/lightspeed-stack-okp-online.yaml create mode 100644 tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-offline.yaml create mode 100644 tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-online.yaml create mode 100644 tests/e2e/features/steps/okp_rag.py diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-offline.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-offline.yaml new file mode 100644 index 000000000..0ac491655 --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-offline.yaml @@ -0,0 +1,39 @@ +# @cfg_okp +# OKP inline RAG — offline mode. +# Chunks use parent_id-based URLs (localhost:8081). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: true + config: + profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + inline: + sources: + - okp + max_chunks: 1 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: true + max_chunks: 5 diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-online.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-online.yaml new file mode 100644 index 000000000..78e7e1604 --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-online.yaml @@ -0,0 +1,40 @@ +# @cfg_okp +# OKP inline RAG — online mode. +# Chunks use reference_url-based URLs (docs.redhat.com). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: true + config: + profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + inline: + sources: + - okp + max_chunks: 3 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: false + max_chunks: 3 + chunk_filter_query: "product:openshift_container_platform AND product_version:4.21" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-offline.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-offline.yaml new file mode 100644 index 000000000..e952860a4 --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-offline.yaml @@ -0,0 +1,39 @@ +# @cfg_okp +# OKP tool RAG — offline mode. +# LLM calls file_search on demand; chunks use parent_id-based URLs (localhost:8081). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: true + config: + profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + tool: + sources: + - okp + max_chunks: 2 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: true + max_chunks: 5 diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-online.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-online.yaml new file mode 100644 index 000000000..b058f0fae --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-online.yaml @@ -0,0 +1,39 @@ +# @cfg_okp +# OKP tool RAG — online mode. +# LLM calls file_search on demand; chunks use reference_url-based URLs (access.redhat.com). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: true + config: + profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + tool: + sources: + - okp + max_chunk : 3 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: false + max_chunks: 5 diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml new file mode 100644 index 000000000..e2a3ccd35 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml @@ -0,0 +1,40 @@ +# @cfg_okp +# OKP inline RAG — offline mode. +# Chunks use parent_id-based URLs (localhost:8081). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + inline: + sources: + - okp + max_chunks: 1 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: true + max_chunks: 5 + diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-okp-online.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-online.yaml new file mode 100644 index 000000000..94c9362b9 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-online.yaml @@ -0,0 +1,40 @@ +# @cfg_okp +# OKP inline RAG — online mode. +# Chunks use reference_url-based URLs (docs.redhat.com). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + inline: + sources: + - okp + max_chunks: 3 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: false + max_chunks: 3 + chunk_filter_query: "product:openshift_container_platform AND product_version:4.21" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-offline.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-offline.yaml new file mode 100644 index 000000000..82c981216 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-offline.yaml @@ -0,0 +1,39 @@ +# @cfg_okp +# OKP tool RAG — offline mode. +# LLM calls file_search on demand; chunks use parent_id-based URLs (localhost:8081). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + tool: + sources: + - okp + max_chunks: 2 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: true + max_chunks: 5 diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-online.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-online.yaml new file mode 100644 index 000000000..ec6c258e2 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-online.yaml @@ -0,0 +1,39 @@ +# @cfg_okp +# OKP tool RAG — online mode. +# LLM calls file_search on demand; chunks use reference_url-based URLs (access.redhat.com). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + tool: + sources: + - okp + max_chunks: 3 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: false + max_chunks: 5 diff --git a/tests/e2e/features/okp_rag.feature b/tests/e2e/features/okp_rag.feature index 642393776..6d419224d 100644 --- a/tests/e2e/features/okp_rag.feature +++ b/tests/e2e/features/okp_rag.feature @@ -14,10 +14,10 @@ Feature: OKP(Solr) RAG retrieval tests And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - # ── Inline RAG — Query ── + # ── Inline RAG — Query (offline) ── - Scenario Outline: mode query with inline RAG returns rag_chunks and referenced_documents - Given The service uses the configuration + Scenario: Offline mode query with inline RAG returns rag_chunks and referenced_documents + Given The service uses the lightspeed-stack-okp-offline.yaml configuration And The service is restarted When I use "query" to ask question with authorization header """ @@ -26,27 +26,20 @@ Feature: OKP(Solr) RAG retrieval tests Then The status code of the response is 200 And The response contains non-empty rag_chunks And The response contains non-empty referenced_documents - And The number of rag_chunk returned is + And The number of rag_chunk returned is 1 And Each rag_chunk has a non-empty score And Each rag_chunk source is "okp" And Each referenced_document has fields doc_url, doc_title, source, and document_id - And Each referenced_document doc_url contains "" + And The number of eferenced_document returned is 1 + And Each referenced_document doc_url contains "localhost:8081" And Each referenced_document doc_title is not empty And Each referenced_document source is "okp" And Each referenced_document has a non-empty document_id - Examples: Offline - | mode | config | max_chunks | doc_url_domain | - | Offline | lightspeed-stack-okp-offline.yaml | 5 | localhost:8081 | - - Examples: Online - | mode | config | max_chunks | doc_url_domain | - | Online | lightspeed-stack-okp-online.yaml | 1 | docs.redhat.com | - - # ── Inline RAG — Streaming Query ── + # ── Inline RAG — Streaming Query (online) ── - Scenario Outline: mode streaming query with inline RAG returns referenced_documents - Given The service uses the configuration + Scenario: Online mode streaming query with inline RAG returns referenced_documents + Given The service uses the lightspeed-stack-okp-online.yaml configuration And The service is restarted When I use "streaming_query" to ask question with authorization header """ @@ -56,29 +49,23 @@ Feature: OKP(Solr) RAG retrieval tests And I wait for the response to be completed And The response contains non-empty referenced_documents And Each referenced_document has fields doc_url, doc_title, source, and document_id - And Each referenced_document doc_url contains "" + And The number of eferenced_document returned is 3 + And Each referenced_document doc_url contains "docs.redhat.com" And Each referenced_document doc_title is not empty + And Each referenced_document doc_title contains "openshift container platform 4.21" And Each referenced_document source is "okp" And Each referenced_document has a non-empty document_id - Examples: Offline - | mode | config | doc_url_domain | - | Offline | lightspeed-stack-okp-offline.yaml | localhost:8081 | - - Examples: Online - | mode | config | doc_url_domain | - | Online | lightspeed-stack-okp-online.yaml | docs.redhat.com | - # ── Inline RAG — Query with Dynamic Filter ── - Scenario Outline: Query with inline RAG with dynamic filter returns rag_chunks and referenced_documents + Scenario: Query with inline RAG with dynamic semantic filter returns rag_chunks and referenced_documents Given The service uses the lightspeed-stack-okp-offline.yaml configuration And The service is restarted When I use "query" to ask question with authorization header """ {"query": "Security best practices", "solr": { - "mode": "", + "mode": "semantic", "filters": { "filters": { "type": "in", @@ -90,31 +77,28 @@ Feature: OKP(Solr) RAG retrieval tests } """ Then The status code of the response is 200 + And The response contains "security best practices" And The response contains non-empty rag_chunks And The response contains non-empty referenced_documents - And The number of rag_chunk returned is 5 + And The number of rag_chunk returned is 1 And Each rag_chunk has a non-empty score And Each rag_chunk source is "okp" And Each referenced_document has fields doc_url, doc_title, source, and document_id - And Each referenced_document doc_url contains "" + And The number of eferenced_document returned is 1 + And Each referenced_document doc_url contains "localhost:8081" And Each referenced_document doc_title is not empty And Each referenced_document source is "okp" And Each referenced_document has a non-empty document_id - Examples: - | filter_mode | doc_url_domain | - | semantic | localhost:8081 | - | hybrid | docs.redhat.com | + # ── Tool RAG — Query API (offline) ── - # ── Tool RAG — Query API ── - - Scenario Outline: queries API with OKP tool RAG has rag_chunk and referenced_documents returned - Given The service uses the configuration + Scenario: Offline query API with OKP tool RAG has rag_chunk and referenced_documents returned + Given The service uses the lightspeed-stack-okp-tool-offline.yaml configuration And The service is restarted When I use "query" to ask question with authorization header """ { - "query": "configure remote desktop using gnome", + "query": "Troubleshooting guide", "model": "{MODEL}", "provider": "{PROVIDER}", "system_prompt": "You MUST use the file_search tool to answer." @@ -124,65 +108,25 @@ Feature: OKP(Solr) RAG retrieval tests And The response contains non-empty tool_calls And A tool_call has name "file_search" And The response contains non-empty rag_chunks - And The number of rag_chunk returned is + And The number of rag_chunk returned is 2 And Each rag_chunk has a non-empty score And Each rag_chunk source is "okp" And The response contains non-empty referenced_documents And Each referenced_document has fields doc_url, doc_title, source, and document_id - And Each referenced_document doc_url contains "" - And Each referenced_document doc_title is not empty - And Each referenced_document source is "okp" - And Each referenced_document has a non-empty document_id - - Examples: Offline - | mode | config | max_chunks | doc_url_domain | - | Offline | lightspeed-stack-okp-tool-offline.yaml | 5 | localhost:8081 | - - Examples: Online - | mode | config | max_chunks | doc_url_domain | - | Online | lightspeed-stack-okp-tool-online.yaml | 1 | access.redhat.com | - - # ── Tool RAG — Streaming Query API ── - - Scenario Outline: streaming query API with OKP tool RAG has rag_chunk and referenced_documents returned - Given The service uses the configuration - And The service is restarted - When I use "streaming_query" to ask question with authorization header - """ - { - "query": "configure remote desktop using gnome", - "model": "{MODEL}", - "provider": "{PROVIDER}", - "system_prompt": "You MUST use the file_search tool to answer." - } - """ - Then The status code of the response is 200 - And A tool_call has name "file_search" - And The response contains non-empty content - And The response contains non-empty referenced_documents - And Each referenced_document has fields doc_url, doc_title, source, and document_id - And Each referenced_document doc_url contains "" + And Each referenced_document doc_url contains "localhost:8081" And Each referenced_document doc_title is not empty And Each referenced_document source is "okp" And Each referenced_document has a non-empty document_id - Examples: Offline - | mode | config | doc_url_domain | - | Offline | lightspeed-stack-okp-tool-offline.yaml | localhost:8081 | - - Examples: Online - | mode | config | doc_url_domain | - | Online | lightspeed-stack-okp-tool-online.yaml | access.redhat.com | + # ── Tool RAG — Responses API (online) ── - # ── Tool RAG — Responses API ── - - Scenario Outline: responses API with OKP tool RAG has rag results returned - Given The service uses the configuration + Scenario: Online responses API with OKP tool RAG has rag results returned + Given The service uses the lightspeed-stack-okp-tool-online.yaml configuration And The service is restarted When I use "responses" to ask question with authorization header """ { - "input": "configure remote desktop using gnome", + "input": "Troubleshooting guide", "model": "{PROVIDER}/{MODEL}", "stream": false, "instructions": "You MUST use the file_search tool to answer." @@ -193,37 +137,21 @@ Feature: OKP(Solr) RAG retrieval tests And The response contains non-empty tool_calls And A tool_call has type "file_search" And The response contains non-empty results - And The number of results returned is + And The number of results returned is 3 And Each rag_chunk has a non-empty score And Each rag_chunk source is "okp" - And Each rag_chunk reference_url contains "" - - Examples: Offline - | mode | config | max_chunks | doc_url_domain | - | Offline | lightspeed-stack-okp-tool-offline.yaml | 5 | localhost:8081 | + And Each rag_chunk reference_url contains "access.redhat.com" - Examples: Online - | mode | config | max_chunks | doc_url_domain | - | Online | lightspeed-stack-okp-tool-online.yaml | 1 | access.redhat.com | - - # # ── OKP Server Unavailable — Graceful Error Handling ─────────────── + # ── OKP Server Unavailable — Graceful Error Handling ── Scenario: Query succeeds with empty rag_chunks when OKP server is unavailable - Given The OKP(Solr) server is stopped + Given The service uses the lightspeed-stack-okp-online.yaml configuration + And The service is restarted + And The OKP(Solr) server is stopped When I use "query" to ask question with authorization header """ {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 And The response contains no rag_chunks - And The response contains no referenced_documents - - Scenario: Streaming query succeeds with empty referenced_documents when OKP server is unavailable - Given The OKP(Solr) server is stopped - When I use "streaming_query" to ask question with authorization header - """ - {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 200 - And I wait for the response to be completed - And The response contains no referenced_documents + And The response contains no referenced_documents \ No newline at end of file diff --git a/tests/e2e/features/steps/llm_query_response.py b/tests/e2e/features/steps/llm_query_response.py index 50ff3cbd4..3e308fb21 100644 --- a/tests/e2e/features/steps/llm_query_response.py +++ b/tests/e2e/features/steps/llm_query_response.py @@ -220,7 +220,10 @@ def ask_question_in_same_conversation(context: Context, endpoint: str) -> None: def check_rag_chunks_present(context: Context) -> None: """Check that the response contains non-empty rag_chunks from inline RAG.""" assert context.response is not None - response_json = context.response.json() + if getattr(context, "use_streaming_response_data", False): + response_json = context.response_data + else: + response_json = context.response.json() assert "rag_chunks" in response_json, "rag_chunks field missing from response" assert ( len(response_json["rag_chunks"]) > 0 @@ -231,7 +234,10 @@ def check_rag_chunks_present(context: Context) -> None: def check_referenced_documents_present(context: Context) -> None: """Check that the response contains non-empty referenced_documents.""" assert context.response is not None - response_json = context.response.json() + if getattr(context, "use_streaming_response_data", False): + response_json = context.response_data + else: + response_json = context.response.json() assert ( "referenced_documents" in response_json ), "referenced_documents field missing from response" @@ -379,6 +385,7 @@ def _parse_streaming_response(response_text: str) -> dict: full_response_split: list[str] = [] tool_calls: list[dict[str, Any]] = [] tool_results: list[dict[str, Any]] = [] + referenced_documents: list[dict[str, Any]] = [] finished = False stream_error = ( None # {"status_code": int, "response": str, "cause": str} if event "error" @@ -402,6 +409,10 @@ def _parse_streaming_response(response_text: str) -> dict: full_response = data["data"]["token"] elif event == "end": finished = True + end_data = data.get("data") or {} + referenced_documents = end_data.get( + "referenced_documents", referenced_documents + ) elif event == "error": stream_error = data.get("data") or {} except json.JSONDecodeError: @@ -413,6 +424,7 @@ def _parse_streaming_response(response_text: str) -> dict: "response_complete": full_response, "tool_calls": tool_calls, "tool_results": tool_results, + "referenced_documents": referenced_documents, "finished": finished, "stream_error": stream_error, } diff --git a/tests/e2e/features/steps/okp_rag.py b/tests/e2e/features/steps/okp_rag.py new file mode 100644 index 000000000..38ac57a5e --- /dev/null +++ b/tests/e2e/features/steps/okp_rag.py @@ -0,0 +1,307 @@ +"""Step definitions for OKP(Solr) RAG retrieval tests.""" + +import os +import subprocess +import time +from typing import Any + +import requests +from behave import given, then # pyright: ignore[reportAttributeAccessIssue] +from behave.runner import Context + +# OKP/Solr Docker container name +OKP_CONTAINER_NAME = os.getenv("E2E_OKP_CONTAINER", "okp-solr") + +# Default OKP health check URL +OKP_DEFAULT_URL = os.getenv("E2E_OKP_URL", "http://localhost:8081") + +# Output item types that represent tool invocations +_TOOL_OUTPUT_TYPES = frozenset( + { + "file_search_call", + "function_call", + "mcp_call", + "web_search_call", + "mcp_list_tools", + } +) + + +def _get_response_body(context: Context) -> dict[str, Any]: + """Return the response body dict, handling both JSON and streaming formats.""" + if getattr(context, "use_streaming_response_data", False): + return context.response_data + return context.response.json() + + +def _get_rag_chunks(context: Context) -> list[dict[str, Any]]: + """Extract rag_chunks from query response or results from Responses API output.""" + body = _get_response_body(context) + if "rag_chunks" in body: + return body["rag_chunks"] + # Responses API: collect results from file_search_call output items + results: list[dict[str, Any]] = [] + for item in body.get("output", []): + if item.get("type") == "file_search_call": + results.extend(item.get("results") or []) + return results + + +def _get_referenced_documents(context: Context) -> list[dict[str, Any]]: + """Extract referenced_documents from response body.""" + body = _get_response_body(context) + return body.get("referenced_documents", []) + + +def _get_tool_calls(context: Context) -> list[dict[str, Any]]: + """Extract tool calls from query response or output items from Responses API.""" + body = _get_response_body(context) + if "tool_calls" in body: + return body["tool_calls"] + # Responses API: extract tool-type items from output + return [ + item + for item in body.get("output", []) + if item.get("type") in _TOOL_OUTPUT_TYPES + ] + + +def _get_file_search_results(context: Context) -> list[dict[str, Any]]: + """Extract file_search results from Responses API output items.""" + body = _get_response_body(context) + results: list[dict[str, Any]] = [] + for item in body.get("output", []): + if item.get("type") == "file_search_call": + results.extend(item.get("results") or []) + return results + + +# ── Given steps ── + + +@given("OKP(Solr) server is running") +def okp_server_is_running(context: Context) -> None: + """Verify that the OKP(Solr) server is reachable.""" + url = OKP_DEFAULT_URL + try: + resp = requests.get(url, timeout=10) + assert ( + resp.status_code < 500 + ), f"OKP server at {url} returned status {resp.status_code}" + except requests.ConnectionError as exc: + assert False, f"OKP(Solr) server is not reachable at {url}: {exc}" + + +@given("The OKP(Solr) server is stopped") +def okp_server_is_stopped(context: Context) -> None: + """Stop the OKP(Solr) Docker container to simulate unavailability.""" + context.okp_was_running = False + try: + result = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Running}}", OKP_CONTAINER_NAME], + capture_output=True, + text=True, + check=True, + ) + if "true" in result.stdout.lower(): + context.okp_was_running = True + subprocess.run( + ["docker", "stop", OKP_CONTAINER_NAME], + capture_output=True, + text=True, + check=True, + ) + time.sleep(2) + except subprocess.CalledProcessError as exc: + print(f"Warning: could not stop OKP container: {exc}") + + +# ── Then steps: rag_chunk assertions ── + + +@then("The number of rag_chunk returned is {count:d}") +def check_rag_chunk_count(context: Context, count: int) -> None: + """Assert the number of rag_chunks matches the expected count.""" + chunks = _get_rag_chunks(context) + assert len(chunks) == count, f"Expected {count} rag_chunks, got {len(chunks)}" + + +@then("Each rag_chunk has a non-empty score") +def check_rag_chunk_scores(context: Context) -> None: + """Assert every rag_chunk has a non-empty score.""" + chunks = _get_rag_chunks(context) + assert chunks, "No rag_chunks to check" + for i, chunk in enumerate(chunks): + score = chunk.get("score") + assert score is not None, f"rag_chunk[{i}] has no score" + + +@then('Each rag_chunk source is "{source}"') +def check_rag_chunk_source(context: Context, source: str) -> None: + """Assert every rag_chunk has the expected source.""" + chunks = _get_rag_chunks(context) + assert chunks, "No rag_chunks to check" + for i, chunk in enumerate(chunks): + actual = chunk.get("source") + if actual is None: + attrs = chunk.get("attributes") or {} + actual = attrs.get("source") + assert ( + actual == source + ), f"rag_chunk[{i}] source is {actual!r}, expected {source!r}" + + +@then('Each rag_chunk reference_url contains "{domain}"') +def check_rag_chunk_reference_url(context: Context, domain: str) -> None: + """Assert every rag_chunk's reference_url contains the expected domain.""" + chunks = _get_rag_chunks(context) + assert chunks, "No rag_chunks to check" + for i, chunk in enumerate(chunks): + attrs = chunk.get("attributes") or {} + ref_url = attrs.get("reference_url") or attrs.get("doc_url") or "" + assert domain in str( + ref_url + ), f"rag_chunk[{i}] reference_url {ref_url!r} does not contain {domain!r}" + + +# ── Then steps: referenced_document assertions ── + + +@then("Each referenced_document has fields doc_url, doc_title, source, and document_id") +def check_referenced_document_fields(context: Context) -> None: + """Assert every referenced_document has the required fields.""" + docs = _get_referenced_documents(context) + assert docs, "No referenced_documents to check" + required = {"doc_url", "doc_title", "source", "document_id"} + for i, doc in enumerate(docs): + missing = required - set(doc.keys()) + assert not missing, ( + f"referenced_document[{i}] missing fields: {missing}. " + f"Available: {list(doc.keys())}" + ) + + +@then('Each referenced_document doc_url contains "{domain}"') +def check_referenced_document_doc_url(context: Context, domain: str) -> None: + """Assert every referenced_document doc_url contains the expected domain.""" + docs = _get_referenced_documents(context) + assert docs, "No referenced_documents to check" + for i, doc in enumerate(docs): + doc_url = str(doc.get("doc_url", "")) + assert ( + domain in doc_url + ), f"referenced_document[{i}] doc_url {doc_url!r} does not contain {domain!r}" + + +@then("Each referenced_document doc_title is not empty") +def check_referenced_document_doc_title(context: Context) -> None: + """Assert every referenced_document has a non-empty doc_title.""" + docs = _get_referenced_documents(context) + assert docs, "No referenced_documents to check" + for i, doc in enumerate(docs): + title = doc.get("doc_title") + assert title, f"referenced_document[{i}] has empty or missing doc_title" + + +@then('Each referenced_document source is "{source}"') +def check_referenced_document_source(context: Context, source: str) -> None: + """Assert every referenced_document has the expected source.""" + docs = _get_referenced_documents(context) + assert docs, "No referenced_documents to check" + for i, doc in enumerate(docs): + actual = doc.get("source") + assert ( + actual == source + ), f"referenced_document[{i}] source is {actual!r}, expected {source!r}" + + +@then("Each referenced_document has a non-empty document_id") +def check_referenced_document_id(context: Context) -> None: + """Assert every referenced_document has a non-empty document_id.""" + docs = _get_referenced_documents(context) + assert docs, "No referenced_documents to check" + for i, doc in enumerate(docs): + doc_id = doc.get("document_id") + assert doc_id, f"referenced_document[{i}] has empty or missing document_id" + + +# ── Then steps: tool_calls assertions ── + + +@then("The response contains non-empty tool_calls") +def check_tool_calls_present(context: Context) -> None: + """Assert the response contains at least one tool call.""" + tool_calls = _get_tool_calls(context) + assert len(tool_calls) > 0, "tool_calls is empty — no tool calls were made" + + +@then('A tool_call has name "{name}"') +def check_tool_call_name(context: Context, name: str) -> None: + """Assert at least one tool call has the expected name.""" + tool_calls = _get_tool_calls(context) + assert tool_calls, "No tool_calls to check" + names = [tc.get("name") for tc in tool_calls] + assert ( + name in names + ), f"No tool_call with name {name!r} found. Available names: {names!r}" + + +@then('A tool_call has type "{type_name}"') +def check_tool_call_type(context: Context, type_name: str) -> None: + """Assert at least one tool call has the expected type.""" + tool_calls = _get_tool_calls(context) + assert tool_calls, "No tool_calls to check" + types = [tc.get("type") for tc in tool_calls] + matched = any(t in {type_name, f"{type_name}_call"} for t in types) + assert ( + matched + ), f"No tool_call with type {type_name!r} found. Available types: {types!r}" + + +# ── Then steps: content and results assertions ── + + +@then("The response contains non-empty content") +def check_response_content_present(context: Context) -> None: + """Assert the response contains non-empty content text.""" + body = _get_response_body(context) + if "response" in body: + content = body["response"] + elif "output_text" in body: + content = body["output_text"] + else: + content = body.get("response_complete", "") + assert content, "Response content is empty" + + +@then("The response contains non-empty results") +def check_results_present(context: Context) -> None: + """Assert the Responses API output contains non-empty file_search results.""" + results = _get_file_search_results(context) + assert len(results) > 0, "No file_search results found in response output" + + +@then("The number of results returned is {count:d}") +def check_results_count(context: Context, count: int) -> None: + """Assert the number of file_search results matches the expected count.""" + results = _get_file_search_results(context) + assert len(results) == count, f"Expected {count} results, got {len(results)}" + + +# ── Then steps: empty response assertions ── + + +@then("The response contains no rag_chunks") +def check_no_rag_chunks(context: Context) -> None: + """Assert the response has no rag_chunks (empty or absent).""" + body = _get_response_body(context) + chunks = body.get("rag_chunks", []) + assert len(chunks) == 0, f"Expected no rag_chunks, got {len(chunks)}" + + +@then("The response contains no referenced_documents") +def check_no_referenced_documents(context: Context) -> None: + """Assert the response has no referenced_documents (empty or absent).""" + body = _get_response_body(context) + docs = body.get("referenced_documents", []) + assert len(docs) == 0, f"Expected no referenced_documents, got {len(docs)}" From 335c086bf16e4d483ec10a1fbb5b810e5a7507a9 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Tue, 25 Aug 2026 13:57:36 -0400 Subject: [PATCH 06/25] extended steps --- Makefile | 4 + .../lightspeed-stack-okp-offline.yaml | 3 +- .../lightspeed-stack-okp-online.yaml | 3 +- tests/e2e/features/environment.py | 76 +++++++++ tests/e2e/features/steps/okp_rag.py | 152 ++++++++++++++++-- 5 files changed, 219 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index fcce2c5c6..305dad245 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,7 @@ LLAMA_STACK_CONFIG ?= run.yaml LLAMA_STACK_CONTAINER_NAME ?= lightspeed-llama-stack LLAMA_STACK_IMAGE ?= lightspeed-llama-stack:local LLAMA_STACK_PORT ?= 8321 +LIGHTSPEED_PROVIDERS_DIR ?= $(shell [ -d ../lightspeed-providers ] && cd ../lightspeed-providers && pwd) CONTAINER_RUNTIME ?= $(shell command -v podman 2>/dev/null || command -v docker 2>/dev/null) .PHONY: run \ @@ -85,6 +86,9 @@ start-llama-stack-container: build-llama-stack-image ## Start llama-stack contai -v $(PWD)/$(CONFIG):/opt/app-root/lightspeed-stack.yaml:ro,z \ -v $(PWD)/scripts/llama-stack-entrypoint.sh:/opt/app-root/enrich-entrypoint.sh:ro,z \ -v $(PWD)/src/llama_stack_configuration.py:/opt/app-root/llama_stack_configuration.py:ro,z \ + $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/lightspeed_stack_providers:/opt/app-root/providers/lightspeed_stack_providers:ro) \ + $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/resources/external_providers:/opt/app-root/src/.llama/providers.d:ro) \ + $(if $(LIGHTSPEED_PROVIDERS_DIR),-e EXTERNAL_PROVIDERS_DIR=/opt/app-root/src/.llama/providers.d) \ -e OPENAI_API_KEY \ -e BRAVE_SEARCH_API_KEY \ -e TAVILY_SEARCH_API_KEY \ diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-offline.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-offline.yaml index 0ac491655..1863a7daa 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-offline.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-offline.yaml @@ -12,7 +12,8 @@ service: llama_stack: use_as_library_client: true config: - profile: run.yaml + baseline: default + # profile: run.yaml # Commented out to allow auto-generation with OKP enrichment user_data_collection: feedback_enabled: true feedback_storage: "/tmp/data/feedback" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-online.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-online.yaml index 78e7e1604..2c7936500 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-online.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-online.yaml @@ -12,7 +12,8 @@ service: llama_stack: use_as_library_client: true config: - profile: run.yaml + baseline: default + # profile: run.yaml # Commented out to allow auto-generation with OKP enrichment user_data_collection: feedback_enabled: true feedback_storage: "/tmp/data/feedback" diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index cd05d3bdc..371555409 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -304,6 +304,8 @@ def after_scenario(context: Context, scenario: Scenario) -> None: running before the scenario. - hostname_llama, port_llama (str/int, optional): host and port used for the llama-stack health check. + - okp_was_running (bool, optional): whether OKP server was running + before it was stopped by the scenario. scenario (Scenario): Behave scenario (unused; shield restore uses context flags). """ if is_prow_environment(): @@ -329,6 +331,80 @@ def after_scenario(context: Context, scenario: Scenario) -> None: except Exception as e: # pylint: disable=broad-exception-caught print(f"Warning: Could not re-register shield: {e}") + # Restart OKP server if it was stopped during the scenario + # (only in local/Docker mode - OpenShift manages containers differently) + if getattr(context, "okp_was_running", False) and not is_prow_environment(): + from tests.e2e.features.steps.okp_rag import ( + OKP_DEFAULT_URL, + OKP_IMAGE_NAME, + ) + + container_name = getattr(context, "okp_container_name", None) + if container_name: + # Check if container still exists (may be auto-removed if started with --rm) + check_result = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Running}}", container_name], + capture_output=True, + text=True, + check=False, + ) + + if check_result.returncode != 0: + # Container was removed (likely started with --rm flag) - recreate it + print( + f"OKP container '{container_name}' was removed, recreating from {OKP_IMAGE_NAME}..." + ) + try: + subprocess.run( + [ + "docker", + "run", + "--rm", + "-d", + "-p", + "8081:8080", + OKP_IMAGE_NAME, + ], + capture_output=True, + text=True, + check=True, + ) + except subprocess.CalledProcessError as e: + print(f"Warning: Could not recreate OKP container: {e}") + if e.stderr: + print(f" Error: {e.stderr}") + else: + # Container exists - just start it + try: + subprocess.run( + ["docker", "start", container_name], + capture_output=True, + text=True, + check=True, + ) + except subprocess.CalledProcessError as e: + print( + f"Warning: Could not start OKP container '{container_name}': {e}" + ) + if e.stderr: + print(f" Error: {e.stderr}") + + # Wait for the server to be ready + max_attempts = 30 + for attempt in range(max_attempts): + try: + resp = requests.get(OKP_DEFAULT_URL, timeout=5) + if resp.status_code < 500: + print("✓ OKP server restarted successfully") + break + except requests.ConnectionError: + if attempt < max_attempts - 1: + time.sleep(1) + else: + print( + "Warning: OKP server may not be fully ready after restart" + ) + def _print_llama_stack_diagnostics() -> None: """Print container state, health, and recent logs to diagnose why llama-stack did not recover.""" diff --git a/tests/e2e/features/steps/okp_rag.py b/tests/e2e/features/steps/okp_rag.py index 38ac57a5e..416f8c070 100644 --- a/tests/e2e/features/steps/okp_rag.py +++ b/tests/e2e/features/steps/okp_rag.py @@ -12,6 +12,11 @@ # OKP/Solr Docker container name OKP_CONTAINER_NAME = os.getenv("E2E_OKP_CONTAINER", "okp-solr") +# OKP Docker image name (for finding container by image) +OKP_IMAGE_NAME = os.getenv( + "E2E_OKP_IMAGE", "registry.redhat.io/offline-knowledge-portal/rhokp-rhel9" +) + # Default OKP health check URL OKP_DEFAULT_URL = os.getenv("E2E_OKP_URL", "http://localhost:8081") @@ -76,6 +81,36 @@ def _get_file_search_results(context: Context) -> list[dict[str, Any]]: return results +def _find_okp_container() -> str | None: + """Find OKP container by name or image. + + Returns: + Container name/ID if found, None otherwise. + """ + # Try by configured name first + result = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Running}}", OKP_CONTAINER_NAME], + capture_output=True, + text=True, + check=False, + ) + if result.returncode == 0: + return OKP_CONTAINER_NAME + + # Try by image name + result = subprocess.run( + ["docker", "ps", "-a", "-q", "--filter", f"ancestor={OKP_IMAGE_NAME}"], + capture_output=True, + text=True, + check=False, + ) + if result.returncode == 0 and result.stdout.strip(): + container_id = result.stdout.strip().split()[0] + return container_id + + return None + + # ── Given steps ── @@ -96,24 +131,69 @@ def okp_server_is_running(context: Context) -> None: def okp_server_is_stopped(context: Context) -> None: """Stop the OKP(Solr) Docker container to simulate unavailability.""" context.okp_was_running = False - try: - result = subprocess.run( - ["docker", "inspect", "-f", "{{.State.Running}}", OKP_CONTAINER_NAME], - capture_output=True, - text=True, - check=True, + context.okp_container_name = None + + # Find the OKP container + container_name = _find_okp_container() + if not container_name: + print( + f"✓ OKP container not found (neither '{OKP_CONTAINER_NAME}' nor '{OKP_IMAGE_NAME}') - already unavailable" ) - if "true" in result.stdout.lower(): - context.okp_was_running = True - subprocess.run( - ["docker", "stop", OKP_CONTAINER_NAME], - capture_output=True, - text=True, - check=True, - ) - time.sleep(2) - except subprocess.CalledProcessError as exc: - print(f"Warning: could not stop OKP container: {exc}") + return + + # Check if container is running + result = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Running}}", container_name], + capture_output=True, + text=True, + check=False, + ) + + if result.returncode != 0: + print(f"✓ OKP container '{container_name}' inspection failed - unavailable") + return + + if "true" not in result.stdout.lower(): + print(f"✓ OKP container '{container_name}' already stopped") + return + + # Container is running - stop it + context.okp_was_running = True + context.okp_container_name = container_name + + stop_result = subprocess.run( + ["docker", "stop", container_name], + capture_output=True, + text=True, + check=False, + ) + if stop_result.returncode != 0: + assert ( + False + ), f"Failed to stop OKP container '{container_name}': {stop_result.stderr}" + + # Wait for the container to fully stop + time.sleep(5) + + # Verify the server is actually unreachable + max_attempts = 10 + for attempt in range(max_attempts): + try: + resp = requests.get(OKP_DEFAULT_URL, timeout=2) + if attempt < max_attempts - 1: + time.sleep(1) + else: + assert ( + False + ), f"OKP server still responding after stop: {resp.status_code}" + except requests.ConnectionError: + # Server is unreachable - success + print(f"✓ OKP server stopped and verified unreachable ({container_name})") + break + except requests.Timeout: + # Timeout is also acceptable - server not responding + print(f"✓ OKP server stopped (connection timeout) ({container_name})") + break # ── Then steps: rag_chunk assertions ── @@ -203,6 +283,27 @@ def check_referenced_document_doc_title(context: Context) -> None: assert title, f"referenced_document[{i}] has empty or missing doc_title" +@then('Each referenced_document doc_title contains "{substring}"') +def check_referenced_document_doc_title_contains( + context: Context, substring: str +) -> None: + """Assert every referenced_document doc_title contains the expected substring (case-insensitive).""" + docs = _get_referenced_documents(context) + assert docs, "No referenced_documents to check" + for i, doc in enumerate(docs): + title = str(doc.get("doc_title", "")).lower() + assert ( + substring.lower() in title + ), f"referenced_document[{i}] doc_title {doc.get('doc_title')!r} does not contain {substring!r}" + + +@then("The number of referenced_document returned is {count:d}") +def check_referenced_document_count(context: Context, count: int) -> None: + """Assert the number of referenced_documents matches the expected count.""" + docs = _get_referenced_documents(context) + assert len(docs) == count, f"Expected {count} referenced_documents, got {len(docs)}" + + @then('Each referenced_document source is "{source}"') def check_referenced_document_source(context: Context, source: str) -> None: """Assert every referenced_document has the expected source.""" @@ -261,6 +362,23 @@ def check_tool_call_type(context: Context, type_name: str) -> None: # ── Then steps: content and results assertions ── +@then('The response contains "{substring}"') +def check_response_contains_substring(context: Context, substring: str) -> None: + """Assert the LLM response contains the expected substring (case-insensitive).""" + body = _get_response_body(context) + if "response" in body: + response_text = body["response"] + elif "output_text" in body: + response_text = body["output_text"] + else: + response_text = body.get("response_complete", "") + + assert substring.lower() in response_text.lower(), ( + f"Response does not contain {substring!r}. " + f"Response text: {response_text[:200]}..." + ) + + @then("The response contains non-empty content") def check_response_content_present(context: Context) -> None: """Assert the response contains non-empty content text.""" From b8f0bac51c8b46c864e39bfb1b7622390b081a71 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Wed, 26 Aug 2026 09:06:34 -0400 Subject: [PATCH 07/25] updated parsing logic for streaming --- tests/e2e/features/steps/llm_query_response.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/e2e/features/steps/llm_query_response.py b/tests/e2e/features/steps/llm_query_response.py index 3e308fb21..2b1590250 100644 --- a/tests/e2e/features/steps/llm_query_response.py +++ b/tests/e2e/features/steps/llm_query_response.py @@ -156,6 +156,9 @@ def ask_question_authorized(context: Context, endpoint: str) -> None: body = _read_streamed_response(resp) resp._content = body.encode(resp.encoding or "utf-8") context.response = resp + # Parse SSE events and make data available for assertions + context.response_data = _parse_streaming_response(body) + context.use_streaming_response_data = True else: context.response = request_with_transient_retry( method="POST", From e2faa2a3622e5eb640ca7b7e51c9c6c8c8315a91 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Wed, 26 Aug 2026 09:17:02 -0400 Subject: [PATCH 08/25] updated parameters for stopping docker --- tests/e2e/features/environment.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index 86de7310b..88cccdb60 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -304,6 +304,8 @@ def after_scenario(context: Context, scenario: Scenario) -> None: running before the scenario. - hostname_llama, port_llama (str/int, optional): host and port used for the OGX health check. + - okp_was_running (bool, optional): whether OKP server was running + before it was stopped by the scenario. scenario (Scenario): Behave scenario (unused; shield restore uses context flags). """ if is_prow_environment(): From 065f4a2cde50552ed98e0188998e252c2e677619 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Wed, 26 Aug 2026 14:40:52 -0400 Subject: [PATCH 09/25] added Openshift support for OKP tests --- Makefile | 2 +- .../rhoai/manifests/lightspeed/okp-solr.yaml | 62 ++++ tests/e2e-prow/rhoai/pipeline-konflux.sh | 36 ++ tests/e2e-prow/rhoai/pipeline.sh | 17 + tests/e2e-prow/rhoai/scripts/e2e-ops.sh | 60 ++++ .../lightspeed-stack-okp-tool-offline.yaml | 3 +- .../lightspeed-stack-okp-tool-online.yaml | 5 +- tests/e2e/features/environment.py | 137 +++---- tests/e2e/features/okp_rag.feature | 15 +- tests/e2e/features/steps/okp_rag.py | 340 +++++++++++++----- tests/e2e/utils/prow_utils.py | 42 +++ 11 files changed, 559 insertions(+), 160 deletions(-) create mode 100644 tests/e2e-prow/rhoai/manifests/lightspeed/okp-solr.yaml diff --git a/Makefile b/Makefile index d8b1731bd..bc5598dcb 100644 --- a/Makefile +++ b/Makefile @@ -168,7 +168,7 @@ test-e2e-local: ## Run end to end tests for the service (no script wrapper) # Tag-based subsets (@cfg_* on features/scenarios). Default runs all config groups; override for one shard, e.g. # E2E_BEHAVE_TAG_EXPR='not @skip and @cfg_authorized' make test-e2e-tagged-local -E2E_BEHAVE_TAG_EXPR ?= not @skip and (@cfg_default or @cfg_authorized or @cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth or @cfg_rbac or @cfg_rh_identity or @cfg_negative or @cfg_skills or @cfg_skills_directory or @cfg_byok_pdf or @cfg_tls or @cfg_degraded or @cfg_unified) +E2E_BEHAVE_TAG_EXPR ?= not @skip and (@cfg_default or @cfg_authorized or @cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth or @cfg_rbac or @cfg_rh_identity or @cfg_negative or @cfg_skills or @cfg_skills_directory or @cfg_byok_pdf or @cfg_tls or @cfg_degraded or @cfg_unified or @cfg_okp) test-e2e-tagged: ## Run e2e tests with E2E_BEHAVE_TAG_EXPR (default: all @cfg_*) script -q -e -c "uv run behave --color --format pretty --tags=\"$(E2E_BEHAVE_TAG_EXPR)\" -D dump_errors=true @tests/e2e/test_list.txt" diff --git a/tests/e2e-prow/rhoai/manifests/lightspeed/okp-solr.yaml b/tests/e2e-prow/rhoai/manifests/lightspeed/okp-solr.yaml new file mode 100644 index 000000000..bc730230c --- /dev/null +++ b/tests/e2e-prow/rhoai/manifests/lightspeed/okp-solr.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: Pod +metadata: + name: okp-solr-service + labels: + app: okp-solr +spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + imagePullSecrets: + - name: redhat-registry-pull-secret + containers: + - name: okp-solr + image: registry.redhat.io/offline-knowledge-portal/rhokp-rhel9:latest + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + ports: + - containerPort: 8080 + name: http + readinessProbe: + httpGet: + path: /solr + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /solr + port: 8080 + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "2Gi" + cpu: "1000m" +--- +apiVersion: v1 +kind: Service +metadata: + name: okp-solr-service-svc +spec: + selector: + app: okp-solr + ports: + - port: 8080 + targetPort: 8080 + name: http diff --git a/tests/e2e-prow/rhoai/pipeline-konflux.sh b/tests/e2e-prow/rhoai/pipeline-konflux.sh index 14020dbcf..8633ef8d5 100755 --- a/tests/e2e-prow/rhoai/pipeline-konflux.sh +++ b/tests/e2e-prow/rhoai/pipeline-konflux.sh @@ -110,6 +110,42 @@ oc create secret docker-registry quay-lightspeed-pull-secret \ # Link the secret to default service account for image pulls oc secrets link default quay-lightspeed-pull-secret --for=pull -n "$NAMESPACE" 2>/dev/null || echo "⚠️ Secret already linked to default SA" +# Create Red Hat registry pull secret for OKP images +# Credentials from Konflux secrets (mounted at /var/run/redhat-registry-*) +if [[ -d /var/run/redhat-registry-username ]] && [[ -d /var/run/redhat-registry-password ]]; then + log "Creating Red Hat registry pull secret..." + REDHAT_USERNAME="" + REDHAT_PASSWORD="" + + # Read username + shopt -s nullglob + for _f in /var/run/redhat-registry-username/*; do + [[ -f "$_f" ]] && REDHAT_USERNAME="$(cat "$_f")" && break + done + + # Read password + for _f in /var/run/redhat-registry-password/*; do + [[ -f "$_f" ]] && REDHAT_PASSWORD="$(cat "$_f")" && break + done + shopt -u nullglob + + if [[ -n "$REDHAT_USERNAME" ]] && [[ -n "$REDHAT_PASSWORD" ]]; then + oc create secret docker-registry redhat-registry-pull-secret \ + --docker-server=registry.redhat.io \ + --docker-username="$REDHAT_USERNAME" \ + --docker-password="$REDHAT_PASSWORD" \ + -n "$NAMESPACE" 2>/dev/null && log "✅ Red Hat registry pull secret created" || log "⚠️ Secret exists or creation failed" + + # Link to default service account + oc secrets link default redhat-registry-pull-secret --for=pull -n "$NAMESPACE" 2>/dev/null || echo "⚠️ Secret already linked to default SA" + else + log "⚠️ Red Hat registry credentials not found in /var/run - OKP image pull may fail" + fi +else + log "⚠️ Red Hat registry credential mounts not found - OKP image pull may fail" + log " (This is OK if not testing OKP features)" +fi + #======================================== # 4. DEPLOY MOCK SERVERS (JWKS & MCP) diff --git a/tests/e2e-prow/rhoai/pipeline.sh b/tests/e2e-prow/rhoai/pipeline.sh index c393e2c15..4a86829f5 100755 --- a/tests/e2e-prow/rhoai/pipeline.sh +++ b/tests/e2e-prow/rhoai/pipeline.sh @@ -92,6 +92,23 @@ oc create secret docker-registry quay-lightspeed-pull-secret \ # Link the secret to default service account for image pulls oc secrets link default quay-lightspeed-pull-secret --for=pull -n "$NAMESPACE" 2>/dev/null || echo "⚠️ Secret already linked to default SA" +# Create Red Hat registry pull secret for OKP images +# Credentials from Prow secrets (environment variables or mounted volumes) +if [[ -n "${REDHAT_REGISTRY_USERNAME:-}" ]] && [[ -n "${REDHAT_REGISTRY_PASSWORD:-}" ]]; then + echo "Creating Red Hat registry pull secret from environment..." + oc create secret docker-registry redhat-registry-pull-secret \ + --docker-server=registry.redhat.io \ + --docker-username="$REDHAT_REGISTRY_USERNAME" \ + --docker-password="$REDHAT_REGISTRY_PASSWORD" \ + -n "$NAMESPACE" 2>/dev/null && echo "✅ Red Hat registry pull secret created" || echo "⚠️ Secret exists or creation failed" + + # Link to default service account + oc secrets link default redhat-registry-pull-secret --for=pull -n "$NAMESPACE" 2>/dev/null || echo "⚠️ Secret already linked to default SA" +else + echo "⚠️ REDHAT_REGISTRY_USERNAME/PASSWORD not set - OKP image pull may fail" + echo " (This is OK if not testing OKP features)" +fi + #======================================== # 5. CONFIGMAPS diff --git a/tests/e2e-prow/rhoai/scripts/e2e-ops.sh b/tests/e2e-prow/rhoai/scripts/e2e-ops.sh index ff2139bf5..79b917e4f 100755 --- a/tests/e2e-prow/rhoai/scripts/e2e-ops.sh +++ b/tests/e2e-prow/rhoai/scripts/e2e-ops.sh @@ -33,6 +33,10 @@ # delete-e2e-mock-tls-inference - Remove mock TLS pod + Service (manual cleanup) # restart-e2e-mock-tls-inference - Delete then deploy mock TLS (manual / recovery) # sync-mock-tls-certs-secret - Copy mock /certs into Secret for OGX mount +# deploy-okp-solr - Deploy OKP Solr service +# delete-okp-solr - Delete OKP Solr pod +# disrupt-okp-solr - Delete OKP Solr pod to disrupt connection +# restore-okp-solr - Restore OKP Solr pod set -e @@ -989,6 +993,46 @@ cmd_disrupt_llama_stack() { fi } +cmd_deploy_okp_solr() { + echo "Deploying OKP Solr service in namespace $NAMESPACE..." + oc apply -n "$NAMESPACE" -f "$MANIFEST_DIR/okp-solr.yaml" + wait_for_pod "okp-solr-service" 60 + echo "✓ OKP Solr service deployed and ready" +} + +cmd_delete_okp_solr() { + echo "Deleting OKP Solr pod from namespace $NAMESPACE..." + timeout 60 oc delete pod okp-solr-service -n "$NAMESPACE" --ignore-not-found=true --wait=true 2>/dev/null || { + oc delete pod okp-solr-service -n "$NAMESPACE" --ignore-not-found=true --force --grace-period=0 2>/dev/null || true + sleep 2 + } + echo "✓ OKP Solr pod deleted" +} + +cmd_disrupt_okp_solr() { + local pod_name="okp-solr-service" + + local phase + phase=$(oc get pod "$pod_name" -n "$NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null || echo "NotFound") + + if [[ "$phase" == "Running" ]]; then + oc delete pod "$pod_name" -n "$NAMESPACE" --wait=true + sleep 2 + echo "OKP Solr connection disrupted successfully (pod deleted)" + exit 0 + else + echo "OKP Solr pod was not running (phase: $phase)" + exit 2 + fi +} + +cmd_restore_okp_solr() { + echo "Restoring OKP Solr service in namespace $NAMESPACE..." + oc apply -n "$NAMESPACE" -f "$MANIFEST_DIR/okp-solr.yaml" + wait_for_pod "okp-solr-service" 60 + echo "✓ OKP Solr service restored and ready" +} + # ============================================================================ # Main command dispatcher # ============================================================================ @@ -1060,6 +1104,18 @@ case "$COMMAND" in dump-pod-logs) cmd_dump_pod_logs "$@" ;; + deploy-okp-solr) + cmd_deploy_okp_solr + ;; + delete-okp-solr) + cmd_delete_okp_solr + ;; + disrupt-okp-solr) + cmd_disrupt_okp_solr + ;; + restore-okp-solr) + cmd_restore_okp_solr + ;; *) echo "Usage: $0 [args...]" echo "" @@ -1081,6 +1137,10 @@ case "$COMMAND" in echo " deploy-e2e-interception-proxy - Deploy in-cluster interception proxy pod" echo " deploy-e2e-mock-tls-inference - Deploy mock HTTPS inference (tls-*.feature)" echo " delete-e2e-mock-tls-inference - Remove mock TLS pod + Service" + echo " deploy-okp-solr - Deploy OKP Solr service" + echo " delete-okp-solr - Delete OKP Solr pod" + echo " disrupt-okp-solr - Delete OKP Solr pod to disrupt connection" + echo " restore-okp-solr - Restore OKP Solr pod" echo " restart-e2e-mock-tls-inference - Delete then deploy mock TLS (recovery)" echo " sync-mock-tls-certs-secret - Publish mock TLS /certs to Secret" echo " dump-pod-logs [tail-lines] - Print init + container logs" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-offline.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-offline.yaml index e952860a4..4dd5d37d9 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-offline.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-offline.yaml @@ -12,7 +12,8 @@ service: llama_stack: use_as_library_client: true config: - profile: run.yaml + baseline: default + # profile: run.yaml user_data_collection: feedback_enabled: true feedback_storage: "/tmp/data/feedback" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-online.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-online.yaml index b058f0fae..f39e5aac6 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-online.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-okp-tool-online.yaml @@ -12,7 +12,8 @@ service: llama_stack: use_as_library_client: true config: - profile: run.yaml + baseline: default + # profile: run.yaml user_data_collection: feedback_enabled: true feedback_storage: "/tmp/data/feedback" @@ -32,7 +33,7 @@ rag: tool: sources: - okp - max_chunk : 3 + max_chunks : 3 okp: rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} offline: false diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index 88cccdb60..6434f0621 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -332,78 +332,87 @@ def after_scenario(context: Context, scenario: Scenario) -> None: print(f"Warning: Could not re-register shield: {e}") # Restart OKP server if it was stopped during the scenario - # (only in local/Docker mode - OpenShift manages containers differently) - if getattr(context, "okp_was_running", False) and not is_prow_environment(): - from tests.e2e.features.steps.okp_rag import ( - OKP_DEFAULT_URL, - OKP_IMAGE_NAME, - ) + if getattr(context, "okp_was_running", False): + if is_prow_environment(): + # Prow/OpenShift: restore pod + from tests.e2e.utils.prow_utils import restore_okp_solr_pod - container_name = getattr(context, "okp_container_name", None) - if container_name: - # Check if container still exists (may be auto-removed if started with --rm) - check_result = subprocess.run( - ["docker", "inspect", "-f", "{{.State.Running}}", container_name], - capture_output=True, - text=True, - check=False, + try: + restore_okp_solr_pod() + except Exception as e: # pylint: disable=broad-exception-caught + print(f"Warning: Could not restore OKP Solr pod: {e}") + else: + # Docker mode: recreate or restart container + from tests.e2e.features.steps.okp_rag import ( + OKP_DEFAULT_URL, + OKP_IMAGE_NAME, ) - if check_result.returncode != 0: - # Container was removed (likely started with --rm flag) - recreate it - print( - f"OKP container '{container_name}' was removed, recreating from {OKP_IMAGE_NAME}..." + container_name = getattr(context, "okp_container_name", None) + if container_name: + # Check if container still exists (may be auto-removed if started with --rm) + check_result = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Running}}", container_name], + capture_output=True, + text=True, + check=False, ) - try: - subprocess.run( - [ - "docker", - "run", - "--rm", - "-d", - "-p", - "8081:8080", - OKP_IMAGE_NAME, - ], - capture_output=True, - text=True, - check=True, - ) - except subprocess.CalledProcessError as e: - print(f"Warning: Could not recreate OKP container: {e}") - if e.stderr: - print(f" Error: {e.stderr}") - else: - # Container exists - just start it - try: - subprocess.run( - ["docker", "start", container_name], - capture_output=True, - text=True, - check=True, - ) - except subprocess.CalledProcessError as e: + + if check_result.returncode != 0: + # Container was removed (likely started with --rm flag) - recreate it print( - f"Warning: Could not start OKP container '{container_name}': {e}" + f"OKP container '{container_name}' was removed, recreating from {OKP_IMAGE_NAME}..." ) - if e.stderr: - print(f" Error: {e.stderr}") - - # Wait for the server to be ready - max_attempts = 30 - for attempt in range(max_attempts): - try: - resp = requests.get(OKP_DEFAULT_URL, timeout=5) - if resp.status_code < 500: - print("✓ OKP server restarted successfully") - break - except requests.ConnectionError: - if attempt < max_attempts - 1: - time.sleep(1) - else: + try: + subprocess.run( + [ + "docker", + "run", + "--rm", + "-d", + "-p", + "8081:8080", + OKP_IMAGE_NAME, + ], + capture_output=True, + text=True, + check=True, + ) + except subprocess.CalledProcessError as e: + print(f"Warning: Could not recreate OKP container: {e}") + if e.stderr: + print(f" Error: {e.stderr}") + else: + # Container exists - just start it + try: + subprocess.run( + ["docker", "start", container_name], + capture_output=True, + text=True, + check=True, + ) + except subprocess.CalledProcessError as e: print( - "Warning: OKP server may not be fully ready after restart" + f"Warning: Could not start OKP container '{container_name}': {e}" ) + if e.stderr: + print(f" Error: {e.stderr}") + + # Wait for the server to be ready + max_attempts = 30 + for attempt in range(max_attempts): + try: + resp = requests.get(OKP_DEFAULT_URL, timeout=5) + if resp.status_code < 500: + print("✓ OKP server restarted successfully") + break + except requests.ConnectionError: + if attempt < max_attempts - 1: + time.sleep(1) + else: + print( + "Warning: OKP server may not be fully ready after restart" + ) def _print_llama_stack_diagnostics() -> None: diff --git a/tests/e2e/features/okp_rag.feature b/tests/e2e/features/okp_rag.feature index 6d419224d..309d740f2 100644 --- a/tests/e2e/features/okp_rag.feature +++ b/tests/e2e/features/okp_rag.feature @@ -1,4 +1,4 @@ -@cfg_okp @skip +@cfg_okp Feature: OKP(Solr) RAG retrieval tests # Offline Knowledge Portal (OKP) provides a Solr-backed RAG source to LSC. @@ -94,7 +94,7 @@ Feature: OKP(Solr) RAG retrieval tests Scenario: Offline query API with OKP tool RAG has rag_chunk and referenced_documents returned Given The service uses the lightspeed-stack-okp-tool-offline.yaml configuration - And The service is restarted + # And The service is restarted When I use "query" to ask question with authorization header """ { @@ -154,4 +154,15 @@ Feature: OKP(Solr) RAG retrieval tests """ Then The status code of the response is 200 And The response contains no rag_chunks + And The response contains no referenced_documents + + Scenario: Streaming query succeeds with empty rag_chunks when OKP server is unavailable + Given The service uses the lightspeed-stack-okp-online.yaml configuration + And The service is restarted + And The OKP(Solr) server is stopped + When I use "query" to ask question with authorization header + """ + {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 And The response contains no referenced_documents \ No newline at end of file diff --git a/tests/e2e/features/steps/okp_rag.py b/tests/e2e/features/steps/okp_rag.py index 416f8c070..6911dd2f9 100644 --- a/tests/e2e/features/steps/okp_rag.py +++ b/tests/e2e/features/steps/okp_rag.py @@ -9,6 +9,8 @@ from behave import given, then # pyright: ignore[reportAttributeAccessIssue] from behave.runner import Context +# ── Constants ── + # OKP/Solr Docker container name OKP_CONTAINER_NAME = os.getenv("E2E_OKP_CONTAINER", "okp-solr") @@ -32,6 +34,9 @@ ) +# ── Response Body Extraction ── + + def _get_response_body(context: Context) -> dict[str, Any]: """Return the response body dict, handling both JSON and streaming formats.""" if getattr(context, "use_streaming_response_data", False): @@ -39,6 +44,19 @@ def _get_response_body(context: Context) -> dict[str, Any]: return context.response.json() +def _get_response_text(context: Context) -> str: + """Extract response text from various response formats.""" + body = _get_response_body(context) + return ( + body.get("response") + or body.get("output_text") + or body.get("response_complete", "") + ) + + +# ── Data Extractors ── + + def _get_rag_chunks(context: Context) -> list[dict[str, Any]]: """Extract rag_chunks from query response or results from Responses API output.""" body = _get_response_body(context) @@ -81,6 +99,179 @@ def _get_file_search_results(context: Context) -> list[dict[str, Any]]: return results +# ── Generic Field Accessors ── + + +def _get_nested_field(item: dict[str, Any], field_path: str) -> Any: + """Get a field from item, supporting nested access via dot notation. + + Examples: + _get_nested_field(chunk, "score") -> chunk.get("score") + _get_nested_field(chunk, "attributes.reference_url") + -> chunk.get("attributes", {}).get("reference_url") + + Parameters: + item: Dictionary to extract field from. + field_path: Field path, using dots for nested access. + + Returns: + Field value or None if not found. + """ + keys = field_path.split(".") + value: Any = item + for key in keys: + if isinstance(value, dict): + value = value.get(key) + else: + return None + return value + + +# ── Generic Assertion Helpers ── + + +def _assert_count_matches(items: list, expected_count: int, item_type: str) -> None: + """Assert the number of items matches the expected count. + + Parameters: + items: List of items to check. + expected_count: Expected number of items. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If count doesn't match. + """ + actual_count = len(items) + assert ( + actual_count == expected_count + ), f"Expected {expected_count} {item_type}, but found {actual_count}" + + +def _assert_not_empty(items: list, item_type: str) -> None: + """Assert the collection is not empty. + + Parameters: + items: List of items to check. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If collection is empty. + """ + assert len(items) > 0, f"{item_type} is empty — no items were found" + + +def _assert_empty(items: list, item_type: str) -> None: + """Assert the collection is empty. + + Parameters: + items: List of items to check. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If collection is not empty. + """ + assert len(items) == 0, f"Expected no {item_type}, but found {len(items)}" + + +def _assert_field_not_empty( + items: list[dict[str, Any]], field_path: str, item_type: str +) -> None: + """Assert every item has a non-empty value for the specified field. + + Parameters: + items: List of items to check. + field_path: Field path to check (supports dot notation). + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If any item has empty or missing field. + """ + assert items, f"No {item_type} to check" + for i, item in enumerate(items): + value = _get_nested_field(item, field_path) + assert value is not None and value != "", ( + f"Expected non-empty {field_path} in {item_type}[{i}], " + f"but found {value!r}" + ) + + +def _assert_field_contains( + items: list[dict[str, Any]], field_path: str, substring: str, item_type: str +) -> None: + """Assert every item's field contains the expected substring (case-insensitive). + + Parameters: + items: List of items to check. + field_path: Field path to check (supports dot notation). + substring: Expected substring. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If any item's field doesn't contain substring. + """ + assert items, f"No {item_type} to check" + for i, item in enumerate(items): + value = _get_nested_field(item, field_path) + assert substring.lower() in str(value).lower(), ( + f"Expected {substring!r} in {item_type}[{i}].{field_path}, " + f"but found {value!r}" + ) + + +def _assert_field_matches( + items: list[dict[str, Any]], field_path: str, expected: Any, item_type: str +) -> None: + """Assert every item's field matches the expected value. + + For fields that might be nested (e.g., source in attributes), checks both + the direct field and the attributes.field path. + + Parameters: + items: List of items to check. + field_path: Field path to check (supports dot notation). + expected: Expected value. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If any item's field doesn't match expected value. + """ + assert items, f"No {item_type} to check" + for i, item in enumerate(items): + actual = _get_nested_field(item, field_path) + # Fallback: check if field exists in attributes + if actual is None and "." not in field_path: + actual = _get_nested_field(item, f"attributes.{field_path}") + assert actual == expected, ( + f"Expected {field_path}={expected!r} in {item_type}[{i}], " + f"but found {actual!r}" + ) + + +def _assert_has_fields( + items: list[dict[str, Any]], required_fields: set[str], item_type: str +) -> None: + """Assert every item has all required fields. + + Parameters: + items: List of items to check. + required_fields: Set of required field names. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If any item is missing required fields. + """ + assert items, f"No {item_type} to check" + for i, item in enumerate(items): + missing = required_fields - set(item.keys()) + assert not missing, ( + f"Expected fields {required_fields} in {item_type}[{i}], " + f"but missing {missing}. Available fields: {list(item.keys())}" + ) + + +# ── Docker/OKP Management ── + + def _find_okp_container() -> str | None: """Find OKP container by name or image. @@ -129,15 +320,30 @@ def okp_server_is_running(context: Context) -> None: @given("The OKP(Solr) server is stopped") def okp_server_is_stopped(context: Context) -> None: - """Stop the OKP(Solr) Docker container to simulate unavailability.""" + """Stop the OKP(Solr) Docker container or pod to simulate unavailability.""" + from tests.e2e.utils.utils import is_prow_environment + context.okp_was_running = False context.okp_container_name = None - # Find the OKP container + if is_prow_environment(): + # Prow/OpenShift: use pod disruption + from tests.e2e.utils.prow_utils import disrupt_okp_solr_pod + + was_running = disrupt_okp_solr_pod() + if was_running: + context.okp_was_running = True + print("✓ OKP Solr pod disrupted in Prow environment") + else: + print("✓ OKP Solr pod was not running") + return + + # Docker mode: existing logic container_name = _find_okp_container() if not container_name: print( - f"✓ OKP container not found (neither '{OKP_CONTAINER_NAME}' nor '{OKP_IMAGE_NAME}') - already unavailable" + f"✓ OKP container not found (neither '{OKP_CONTAINER_NAME}' " + f"nor '{OKP_IMAGE_NAME}') - already unavailable" ) return @@ -196,144 +402,111 @@ def okp_server_is_stopped(context: Context) -> None: break -# ── Then steps: rag_chunk assertions ── +# ── Then Steps: rag_chunks Assertions ── @then("The number of rag_chunk returned is {count:d}") def check_rag_chunk_count(context: Context, count: int) -> None: """Assert the number of rag_chunks matches the expected count.""" chunks = _get_rag_chunks(context) - assert len(chunks) == count, f"Expected {count} rag_chunks, got {len(chunks)}" + _assert_count_matches(chunks, count, "rag_chunks") @then("Each rag_chunk has a non-empty score") def check_rag_chunk_scores(context: Context) -> None: """Assert every rag_chunk has a non-empty score.""" chunks = _get_rag_chunks(context) - assert chunks, "No rag_chunks to check" - for i, chunk in enumerate(chunks): - score = chunk.get("score") - assert score is not None, f"rag_chunk[{i}] has no score" + _assert_field_not_empty(chunks, "score", "rag_chunks") @then('Each rag_chunk source is "{source}"') def check_rag_chunk_source(context: Context, source: str) -> None: """Assert every rag_chunk has the expected source.""" chunks = _get_rag_chunks(context) - assert chunks, "No rag_chunks to check" - for i, chunk in enumerate(chunks): - actual = chunk.get("source") - if actual is None: - attrs = chunk.get("attributes") or {} - actual = attrs.get("source") - assert ( - actual == source - ), f"rag_chunk[{i}] source is {actual!r}, expected {source!r}" + _assert_field_matches(chunks, "source", source, "rag_chunks") @then('Each rag_chunk reference_url contains "{domain}"') def check_rag_chunk_reference_url(context: Context, domain: str) -> None: """Assert every rag_chunk's reference_url contains the expected domain.""" chunks = _get_rag_chunks(context) + # Check both possible field paths for reference_url assert chunks, "No rag_chunks to check" for i, chunk in enumerate(chunks): attrs = chunk.get("attributes") or {} ref_url = attrs.get("reference_url") or attrs.get("doc_url") or "" - assert domain in str( - ref_url - ), f"rag_chunk[{i}] reference_url {ref_url!r} does not contain {domain!r}" + assert domain in str(ref_url), ( + f"Expected {domain!r} in rag_chunks[{i}].attributes.reference_url, " + f"but found {ref_url!r}" + ) -# ── Then steps: referenced_document assertions ── +# ── Then Steps: referenced_documents Assertions ── @then("Each referenced_document has fields doc_url, doc_title, source, and document_id") def check_referenced_document_fields(context: Context) -> None: """Assert every referenced_document has the required fields.""" docs = _get_referenced_documents(context) - assert docs, "No referenced_documents to check" - required = {"doc_url", "doc_title", "source", "document_id"} - for i, doc in enumerate(docs): - missing = required - set(doc.keys()) - assert not missing, ( - f"referenced_document[{i}] missing fields: {missing}. " - f"Available: {list(doc.keys())}" - ) + required_fields = {"doc_url", "doc_title", "source", "document_id"} + _assert_has_fields(docs, required_fields, "referenced_documents") @then('Each referenced_document doc_url contains "{domain}"') def check_referenced_document_doc_url(context: Context, domain: str) -> None: """Assert every referenced_document doc_url contains the expected domain.""" docs = _get_referenced_documents(context) - assert docs, "No referenced_documents to check" - for i, doc in enumerate(docs): - doc_url = str(doc.get("doc_url", "")) - assert ( - domain in doc_url - ), f"referenced_document[{i}] doc_url {doc_url!r} does not contain {domain!r}" + _assert_field_contains(docs, "doc_url", domain, "referenced_documents") @then("Each referenced_document doc_title is not empty") def check_referenced_document_doc_title(context: Context) -> None: """Assert every referenced_document has a non-empty doc_title.""" docs = _get_referenced_documents(context) - assert docs, "No referenced_documents to check" - for i, doc in enumerate(docs): - title = doc.get("doc_title") - assert title, f"referenced_document[{i}] has empty or missing doc_title" + _assert_field_not_empty(docs, "doc_title", "referenced_documents") @then('Each referenced_document doc_title contains "{substring}"') def check_referenced_document_doc_title_contains( context: Context, substring: str ) -> None: - """Assert every referenced_document doc_title contains the expected substring (case-insensitive).""" + """Assert every referenced_document doc_title contains the expected substring. + + Matching is case-insensitive. + """ docs = _get_referenced_documents(context) - assert docs, "No referenced_documents to check" - for i, doc in enumerate(docs): - title = str(doc.get("doc_title", "")).lower() - assert ( - substring.lower() in title - ), f"referenced_document[{i}] doc_title {doc.get('doc_title')!r} does not contain {substring!r}" + _assert_field_contains(docs, "doc_title", substring, "referenced_documents") @then("The number of referenced_document returned is {count:d}") def check_referenced_document_count(context: Context, count: int) -> None: """Assert the number of referenced_documents matches the expected count.""" docs = _get_referenced_documents(context) - assert len(docs) == count, f"Expected {count} referenced_documents, got {len(docs)}" + _assert_count_matches(docs, count, "referenced_documents") @then('Each referenced_document source is "{source}"') def check_referenced_document_source(context: Context, source: str) -> None: """Assert every referenced_document has the expected source.""" docs = _get_referenced_documents(context) - assert docs, "No referenced_documents to check" - for i, doc in enumerate(docs): - actual = doc.get("source") - assert ( - actual == source - ), f"referenced_document[{i}] source is {actual!r}, expected {source!r}" + _assert_field_matches(docs, "source", source, "referenced_documents") @then("Each referenced_document has a non-empty document_id") def check_referenced_document_id(context: Context) -> None: """Assert every referenced_document has a non-empty document_id.""" docs = _get_referenced_documents(context) - assert docs, "No referenced_documents to check" - for i, doc in enumerate(docs): - doc_id = doc.get("document_id") - assert doc_id, f"referenced_document[{i}] has empty or missing document_id" + _assert_field_not_empty(docs, "document_id", "referenced_documents") -# ── Then steps: tool_calls assertions ── +# ── Then Steps: tool_calls Assertions ── @then("The response contains non-empty tool_calls") def check_tool_calls_present(context: Context) -> None: """Assert the response contains at least one tool call.""" tool_calls = _get_tool_calls(context) - assert len(tool_calls) > 0, "tool_calls is empty — no tool calls were made" + _assert_not_empty(tool_calls, "tool_calls") @then('A tool_call has name "{name}"') @@ -342,9 +515,9 @@ def check_tool_call_name(context: Context, name: str) -> None: tool_calls = _get_tool_calls(context) assert tool_calls, "No tool_calls to check" names = [tc.get("name") for tc in tool_calls] - assert ( - name in names - ), f"No tool_call with name {name!r} found. Available names: {names!r}" + assert name in names, ( + f"Expected tool_call with name {name!r}, " f"but found names {names!r}" + ) @then('A tool_call has type "{type_name}"') @@ -354,59 +527,46 @@ def check_tool_call_type(context: Context, type_name: str) -> None: assert tool_calls, "No tool_calls to check" types = [tc.get("type") for tc in tool_calls] matched = any(t in {type_name, f"{type_name}_call"} for t in types) - assert ( - matched - ), f"No tool_call with type {type_name!r} found. Available types: {types!r}" + assert matched, ( + f"Expected tool_call with type {type_name!r}, " f"but found types {types!r}" + ) -# ── Then steps: content and results assertions ── +# ── Then Steps: Content and Results Assertions ── @then('The response contains "{substring}"') def check_response_contains_substring(context: Context, substring: str) -> None: """Assert the LLM response contains the expected substring (case-insensitive).""" - body = _get_response_body(context) - if "response" in body: - response_text = body["response"] - elif "output_text" in body: - response_text = body["output_text"] - else: - response_text = body.get("response_complete", "") - + response_text = _get_response_text(context) assert substring.lower() in response_text.lower(), ( - f"Response does not contain {substring!r}. " - f"Response text: {response_text[:200]}..." + f"Expected substring {substring!r} in response, " + f"but response text: {response_text[:200]}..." ) @then("The response contains non-empty content") def check_response_content_present(context: Context) -> None: """Assert the response contains non-empty content text.""" - body = _get_response_body(context) - if "response" in body: - content = body["response"] - elif "output_text" in body: - content = body["output_text"] - else: - content = body.get("response_complete", "") - assert content, "Response content is empty" + content = _get_response_text(context) + assert content, "Expected non-empty response content, but it was empty" @then("The response contains non-empty results") def check_results_present(context: Context) -> None: """Assert the Responses API output contains non-empty file_search results.""" results = _get_file_search_results(context) - assert len(results) > 0, "No file_search results found in response output" + _assert_not_empty(results, "file_search results") @then("The number of results returned is {count:d}") def check_results_count(context: Context, count: int) -> None: """Assert the number of file_search results matches the expected count.""" results = _get_file_search_results(context) - assert len(results) == count, f"Expected {count} results, got {len(results)}" + _assert_count_matches(results, count, "file_search results") -# ── Then steps: empty response assertions ── +# ── Then Steps: Empty Response Assertions ── @then("The response contains no rag_chunks") @@ -414,7 +574,7 @@ def check_no_rag_chunks(context: Context) -> None: """Assert the response has no rag_chunks (empty or absent).""" body = _get_response_body(context) chunks = body.get("rag_chunks", []) - assert len(chunks) == 0, f"Expected no rag_chunks, got {len(chunks)}" + _assert_empty(chunks, "rag_chunks") @then("The response contains no referenced_documents") @@ -422,4 +582,4 @@ def check_no_referenced_documents(context: Context) -> None: """Assert the response has no referenced_documents (empty or absent).""" body = _get_response_body(context) docs = body.get("referenced_documents", []) - assert len(docs) == 0, f"Expected no referenced_documents, got {len(docs)}" + _assert_empty(docs, "referenced_documents") diff --git a/tests/e2e/utils/prow_utils.py b/tests/e2e/utils/prow_utils.py index 7b5f02f10..cc4bf1f35 100644 --- a/tests/e2e/utils/prow_utils.py +++ b/tests/e2e/utils/prow_utils.py @@ -20,6 +20,7 @@ def get_namespace() -> str: _POD_NAME_MAP = { "lightspeed-stack": "lightspeed-stack-service", "llama-stack": "llama-stack-service", + "okp-solr": "okp-solr-service", } @@ -345,3 +346,44 @@ def update_config_configmap( finally: if temp_path and os.path.exists(temp_path): os.remove(temp_path) + + +def disrupt_okp_solr_pod() -> bool: + """Disrupt OKP Solr connection in Prow/OpenShift environment. + + Returns: + True if the pod was running and has been disrupted, False otherwise. + """ + try: + result = run_e2e_ops("disrupt-okp-solr", timeout=60) + print(result.stdout, end="") + + # Exit code 0 = disrupted (was running), exit code 2 = was not running + if result.returncode == 0: + return True + elif result.returncode == 2: + return False + else: + print(result.stderr, end="") + return False + + except subprocess.TimeoutExpired: + print("Warning: Timeout while disrupting OKP Solr connection") + return False + + +def restore_okp_solr_pod() -> None: + """Restore OKP Solr pod in Prow/OpenShift environment. + + Raises: + subprocess.CalledProcessError: If oc/e2e-ops restore fails. + subprocess.TimeoutExpired: If the operation times out. + """ + result = run_e2e_ops("restore-okp-solr", timeout=180) + print(result.stdout, end="") + if result.returncode != 0: + print(result.stderr, end="") + raise subprocess.CalledProcessError( + result.returncode, "restore-okp-solr", result.stderr + ) + print("✓ OKP Solr pod restored successfully") From ae2c13bde4515b5b004634c848d36bd7e05a5140 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Thu, 27 Aug 2026 09:11:20 -0400 Subject: [PATCH 10/25] addressing comments --- Makefile | 4 +- .../lightspeed-stack-okp-offline.yaml | 3 +- tests/e2e/features/environment.py | 73 ++++++++----------- tests/e2e/features/okp_rag.feature | 9 +-- tests/e2e/utils/prow_utils.py | 6 +- 5 files changed, 42 insertions(+), 53 deletions(-) diff --git a/Makefile b/Makefile index bc5598dcb..83567bbf1 100644 --- a/Makefile +++ b/Makefile @@ -86,8 +86,8 @@ start-llama-stack-container: build-llama-stack-image ## Start OGX container -v $(PWD)/$(CONFIG):/opt/app-root/lightspeed-stack.yaml:ro,z \ -v $(PWD)/scripts/llama-stack-entrypoint.sh:/opt/app-root/enrich-entrypoint.sh:ro,z \ -v $(PWD)/src/llama_stack_configuration.py:/opt/app-root/llama_stack_configuration.py:ro,z \ - $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/lightspeed_stack_providers:/opt/app-root/providers/lightspeed_stack_providers:ro) \ - $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/resources/external_providers:/opt/app-root/src/.llama/providers.d:ro) \ + $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/lightspeed_stack_providers:/opt/app-root/providers/lightspeed_stack_providers:ro,z) \ + $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/resources/external_providers:/opt/app-root/src/.llama/providers.d:ro,z) \ $(if $(LIGHTSPEED_PROVIDERS_DIR),-e EXTERNAL_PROVIDERS_DIR=/opt/app-root/src/.llama/providers.d) \ -e OPENAI_API_KEY \ -e BRAVE_SEARCH_API_KEY \ diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml index e2a3ccd35..bc6015887 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml @@ -36,5 +36,4 @@ rag: okp: rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} offline: true - max_chunks: 5 - + max_chunks: 5 \ No newline at end of file diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index 6434f0621..720a493e9 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -334,13 +334,10 @@ def after_scenario(context: Context, scenario: Scenario) -> None: # Restart OKP server if it was stopped during the scenario if getattr(context, "okp_was_running", False): if is_prow_environment(): - # Prow/OpenShift: restore pod + # Prow/OpenShift: restore pod and raise on failure from tests.e2e.utils.prow_utils import restore_okp_solr_pod - try: - restore_okp_solr_pod() - except Exception as e: # pylint: disable=broad-exception-caught - print(f"Warning: Could not restore OKP Solr pod: {e}") + restore_okp_solr_pod() else: # Docker mode: recreate or restart container from tests.e2e.features.steps.okp_rag import ( @@ -363,56 +360,50 @@ def after_scenario(context: Context, scenario: Scenario) -> None: print( f"OKP container '{container_name}' was removed, recreating from {OKP_IMAGE_NAME}..." ) - try: - subprocess.run( - [ - "docker", - "run", - "--rm", - "-d", - "-p", - "8081:8080", - OKP_IMAGE_NAME, - ], - capture_output=True, - text=True, - check=True, - ) - except subprocess.CalledProcessError as e: - print(f"Warning: Could not recreate OKP container: {e}") - if e.stderr: - print(f" Error: {e.stderr}") + subprocess.run( + [ + "docker", + "run", + "--rm", + "-d", + "-p", + "8081:8080", + OKP_IMAGE_NAME, + ], + capture_output=True, + text=True, + check=True, + ) else: # Container exists - just start it - try: - subprocess.run( - ["docker", "start", container_name], - capture_output=True, - text=True, - check=True, - ) - except subprocess.CalledProcessError as e: - print( - f"Warning: Could not start OKP container '{container_name}': {e}" - ) - if e.stderr: - print(f" Error: {e.stderr}") + subprocess.run( + ["docker", "start", container_name], + capture_output=True, + text=True, + check=True, + ) # Wait for the server to be ready max_attempts = 30 + okp_ready = False for attempt in range(max_attempts): try: resp = requests.get(OKP_DEFAULT_URL, timeout=5) if resp.status_code < 500: print("✓ OKP server restarted successfully") + okp_ready = True break + # HTTP 5xx: delay and retry + if attempt < max_attempts - 1: + time.sleep(1) except requests.ConnectionError: if attempt < max_attempts - 1: time.sleep(1) - else: - print( - "Warning: OKP server may not be fully ready after restart" - ) + + if not okp_ready: + raise RuntimeError( + f"OKP server failed to become ready after {max_attempts} attempts" + ) def _print_llama_stack_diagnostics() -> None: diff --git a/tests/e2e/features/okp_rag.feature b/tests/e2e/features/okp_rag.feature index 309d740f2..79659597c 100644 --- a/tests/e2e/features/okp_rag.feature +++ b/tests/e2e/features/okp_rag.feature @@ -24,8 +24,6 @@ Feature: OKP(Solr) RAG retrieval tests {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 - And The response contains non-empty rag_chunks - And The response contains non-empty referenced_documents And The number of rag_chunk returned is 1 And Each rag_chunk has a non-empty score And Each rag_chunk source is "okp" @@ -47,7 +45,6 @@ Feature: OKP(Solr) RAG retrieval tests """ Then The status code of the response is 200 And I wait for the response to be completed - And The response contains non-empty referenced_documents And Each referenced_document has fields doc_url, doc_title, source, and document_id And The number of eferenced_document returned is 3 And Each referenced_document doc_url contains "docs.redhat.com" @@ -78,8 +75,6 @@ Feature: OKP(Solr) RAG retrieval tests """ Then The status code of the response is 200 And The response contains "security best practices" - And The response contains non-empty rag_chunks - And The response contains non-empty referenced_documents And The number of rag_chunk returned is 1 And Each rag_chunk has a non-empty score And Each rag_chunk source is "okp" @@ -94,7 +89,7 @@ Feature: OKP(Solr) RAG retrieval tests Scenario: Offline query API with OKP tool RAG has rag_chunk and referenced_documents returned Given The service uses the lightspeed-stack-okp-tool-offline.yaml configuration - # And The service is restarted + And The service is restarted When I use "query" to ask question with authorization header """ { @@ -160,7 +155,7 @@ Feature: OKP(Solr) RAG retrieval tests Given The service uses the lightspeed-stack-okp-online.yaml configuration And The service is restarted And The OKP(Solr) server is stopped - When I use "query" to ask question with authorization header + When I use "streaming_query" to ask question with authorization header """ {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} """ diff --git a/tests/e2e/utils/prow_utils.py b/tests/e2e/utils/prow_utils.py index cc4bf1f35..8ab7c93bb 100644 --- a/tests/e2e/utils/prow_utils.py +++ b/tests/e2e/utils/prow_utils.py @@ -379,7 +379,11 @@ def restore_okp_solr_pod() -> None: subprocess.CalledProcessError: If oc/e2e-ops restore fails. subprocess.TimeoutExpired: If the operation times out. """ - result = run_e2e_ops("restore-okp-solr", timeout=180) + # restore-okp-solr can spend 180 seconds in wait_for_pod, plus oc apply time. + # This caller also times out at 180 seconds. after_scenario catches that timeout + # and only logs a warning, so later scenarios can run while OKP remains unavailable. + # Use a timeout with margin, such as 240 seconds. + result = run_e2e_ops("restore-okp-solr", timeout=240) print(result.stdout, end="") if result.returncode != 0: print(result.stderr, end="") From 751079ca30b69e9cb5d3ebef6a993e2ee4829719 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Thu, 27 Aug 2026 11:05:04 -0400 Subject: [PATCH 11/25] updated parsing logic in llm_query file --- tests/e2e/features/okp_rag.feature | 2 +- .../e2e/features/steps/llm_query_response.py | 57 +++++++++++++++++-- 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/tests/e2e/features/okp_rag.feature b/tests/e2e/features/okp_rag.feature index fb5c6654f..725417d20 100644 --- a/tests/e2e/features/okp_rag.feature +++ b/tests/e2e/features/okp_rag.feature @@ -1,4 +1,4 @@ -@cfg_okp @skip +@cfg_okp Feature: OKP(Solr) RAG retrieval tests # Offline Knowledge Portal (OKP) provides a Solr-backed RAG source to LSC. diff --git a/tests/e2e/features/steps/llm_query_response.py b/tests/e2e/features/steps/llm_query_response.py index 2b1590250..501adde80 100644 --- a/tests/e2e/features/steps/llm_query_response.py +++ b/tests/e2e/features/steps/llm_query_response.py @@ -95,7 +95,14 @@ def responses_output_should_include_one_of_types(context: Context) -> None: @step("I wait for the response to be completed") def wait_for_complete_response(context: Context) -> None: """Wait for the response to be complete.""" - context.response_data = _parse_streaming_response(context.response.text) + # Reuse cached parse result if it was created for the current response + if not ( + hasattr(context, "response_data") + and hasattr(context, "_response_data_source") + and context._response_data_source is context.response + ): + context.response_data = _parse_streaming_response(context.response.text) + context._response_data_source = context.response context.response.raise_for_status() assert context.response_data["finished"] is True context.use_streaming_response_data = True @@ -157,7 +164,29 @@ def ask_question_authorized(context: Context, endpoint: str) -> None: resp._content = body.encode(resp.encoding or "utf-8") context.response = resp # Parse SSE events and make data available for assertions + # Preserve existing conversation_id if the new response doesn't have one + # (e.g., 403 errors won't have a 'start' event with conversation_id) + old_conversation_id = ( + context.response_data.get("conversation_id") + if hasattr(context, "response_data") + else None + ) context.response_data = _parse_streaming_response(body) + # Extract conversation from terminal event if not found in start event (Responses API) + if not context.response_data.get("conversation_id"): + if old_conversation_id: + # Preserve from previous response (for 403 error scenarios) + context.response_data["conversation_id"] = old_conversation_id + else: + # Try to extract from terminal event (Responses API streaming format) + try: + terminal = parse_responses_sse_final_response_object(body) + context.response_data["conversation"] = terminal.get("conversation") + context.response_data["conversation_id"] = terminal.get("conversation") + except AssertionError: + pass # No terminal event found (e.g., error responses) + # Mark that this parsed data is from the current response (for cache reuse) + context._response_data_source = resp context.use_streaming_response_data = True else: context.response = request_with_transient_retry( @@ -189,15 +218,33 @@ def ask_question_too_long_authorized(context: Context, endpoint: str) -> None: @step("I store conversation details") def store_conversation_details(context: Context) -> None: - """Store details about the conversation.""" + """Store details about the conversation. + + Reuses cached parse result if it was created for the current response. + """ + # Reuse if already parsed for this response + if ( + hasattr(context, "response_data") + and hasattr(context, "_response_data_source") + and context._response_data_source is context.response + ): + return + try: context.response_data = json.loads(context.response.text) + context._response_data_source = context.response except json.JSONDecodeError: + # Streaming response - parse it context.response_data = _parse_streaming_response(context.response.text) + # Extract conversation from terminal event if not in start event (Responses API) if not context.response_data.get("conversation_id"): - terminal = parse_responses_sse_final_response_object(context.response.text) - context.response_data["conversation"] = terminal.get("conversation") - context.response_data["conversation_id"] = terminal.get("conversation") + try: + terminal = parse_responses_sse_final_response_object(context.response.text) + context.response_data["conversation"] = terminal.get("conversation") + context.response_data["conversation_id"] = terminal.get("conversation") + except AssertionError: + pass # No terminal event found + context._response_data_source = context.response @step('I use "{endpoint}" to ask question with same conversation_id') From 1b73eb1298627caf9237eb01d18677cf706224f4 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Thu, 27 Aug 2026 11:07:24 -0400 Subject: [PATCH 12/25] linting --- tests/e2e/features/steps/llm_query_response.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/e2e/features/steps/llm_query_response.py b/tests/e2e/features/steps/llm_query_response.py index 501adde80..cf1dbe88f 100644 --- a/tests/e2e/features/steps/llm_query_response.py +++ b/tests/e2e/features/steps/llm_query_response.py @@ -182,7 +182,9 @@ def ask_question_authorized(context: Context, endpoint: str) -> None: try: terminal = parse_responses_sse_final_response_object(body) context.response_data["conversation"] = terminal.get("conversation") - context.response_data["conversation_id"] = terminal.get("conversation") + context.response_data["conversation_id"] = terminal.get( + "conversation" + ) except AssertionError: pass # No terminal event found (e.g., error responses) # Mark that this parsed data is from the current response (for cache reuse) @@ -239,7 +241,9 @@ def store_conversation_details(context: Context) -> None: # Extract conversation from terminal event if not in start event (Responses API) if not context.response_data.get("conversation_id"): try: - terminal = parse_responses_sse_final_response_object(context.response.text) + terminal = parse_responses_sse_final_response_object( + context.response.text + ) context.response_data["conversation"] = terminal.get("conversation") context.response_data["conversation_id"] = terminal.get("conversation") except AssertionError: From 2e60615df5833a7929014ce6007f7089dc78ba92 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Thu, 27 Aug 2026 13:49:31 -0400 Subject: [PATCH 13/25] Reset use_streaming_response_data for non-SSE responses --- tests/e2e/features/environment.py | 2 +- tests/e2e/features/steps/llm_query_response.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index 720a493e9..3c7b5bc45 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -396,7 +396,7 @@ def after_scenario(context: Context, scenario: Scenario) -> None: # HTTP 5xx: delay and retry if attempt < max_attempts - 1: time.sleep(1) - except requests.ConnectionError: + except (requests.ConnectionError, requests.Timeout): if attempt < max_attempts - 1: time.sleep(1) diff --git a/tests/e2e/features/steps/llm_query_response.py b/tests/e2e/features/steps/llm_query_response.py index cf1dbe88f..3a4b811e8 100644 --- a/tests/e2e/features/steps/llm_query_response.py +++ b/tests/e2e/features/steps/llm_query_response.py @@ -198,6 +198,8 @@ def ask_question_authorized(context: Context, endpoint: str) -> None: headers=context.auth_headers, timeout=DEFAULT_LLM_TIMEOUT, ) + # Reset flag so assertions read from response.json(), not stale response_data + context.use_streaming_response_data = False # Query length chosen to exceed typical model context windows (e.g. 128k tokens) From 9ba8056a64747a50e5ddaad52a5474226f1ea6e1 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Thu, 27 Aug 2026 13:50:22 -0400 Subject: [PATCH 14/25] updated konflu runner for OKP deploy/cleanup and port forwarding --- tests/e2e-prow/rhoai/pipeline-konflux.sh | 58 +++++++++++++++++++++++- tests/e2e-prow/rhoai/pipeline.sh | 58 ++++++++++++++++++++++-- 2 files changed, 112 insertions(+), 4 deletions(-) diff --git a/tests/e2e-prow/rhoai/pipeline-konflux.sh b/tests/e2e-prow/rhoai/pipeline-konflux.sh index 8633ef8d5..f82bec6fc 100755 --- a/tests/e2e-prow/rhoai/pipeline-konflux.sh +++ b/tests/e2e-prow/rhoai/pipeline-konflux.sh @@ -182,6 +182,22 @@ oc wait pod/mock-jwks pod/mock-mcp \ } log "✅ Mock servers deployed" +# Deploy OKP Solr server for RAG tests +log "Deploying OKP Solr server..." +oc apply -n "$NAMESPACE" -f "$PIPELINE_DIR/manifests/lightspeed/okp-solr.yaml" + +# Wait for OKP Solr to be ready +log "Waiting for OKP Solr to be ready..." +oc wait pod/okp-solr-service \ + -n "$NAMESPACE" --for=condition=Ready --timeout=180s || { + echo "⚠️ OKP Solr not ready, checking status..." + oc get pods -n "$NAMESPACE" | grep okp-solr || true + oc describe pod okp-solr-service -n "$NAMESPACE" 2>/dev/null | tail -30 || true + echo "❌ OKP Solr failed to become ready" + exit 1 +} +log "✅ OKP Solr deployed" + # e2e-tunnel-proxy and e2e-interception-proxy are deployed from proxy.feature steps # (see tests/e2e/features/steps/proxy.py + e2e-ops deploy-e2e-*-proxy). @@ -333,8 +349,10 @@ fi # Debug hook/port churn: export E2E_OPS_VERBOSE=1 before running pipeline.sh export E2E_LSC_PORT_FORWARD_PID_FILE="${E2E_LSC_PORT_FORWARD_PID_FILE:-/tmp/e2e-lightspeed-port-forward.pid}" export E2E_LLAMA_PORT_FORWARD_PID_FILE="${E2E_LLAMA_PORT_FORWARD_PID_FILE:-/tmp/e2e-llama-port-forward.pid}" +export E2E_OKP_PORT_FORWARD_PID_FILE="${E2E_OKP_PORT_FORWARD_PID_FILE:-/tmp/e2e-okp-port-forward.pid}" rm -f "$E2E_LSC_PORT_FORWARD_PID_FILE" rm -f "$E2E_LLAMA_PORT_FORWARD_PID_FILE" +rm -f "$E2E_OKP_PORT_FORWARD_PID_FILE" oc label pod lightspeed-stack-service pod=lightspeed-stack-service -n $NAMESPACE @@ -355,7 +373,7 @@ kill_listeners_on_ports() { fi done } -kill_listeners_on_ports 8080 8000 8321 +kill_listeners_on_ports 8080 8000 8321 8081 # Start port-forward for lightspeed-stack progress "Starting port-forward, then E2E tests" @@ -375,6 +393,12 @@ oc port-forward svc/llama-stack-service-svc 8321:8321 -n $NAMESPACE & PF_LLAMA_PID=$! echo "$PF_LLAMA_PID" >"$E2E_LLAMA_PORT_FORWARD_PID_FILE" +# Start port-forward for OKP Solr (RAG tests) +log "Starting port-forward for OKP Solr..." +oc port-forward svc/okp-solr-service-svc 8081:8080 -n $NAMESPACE & +PF_OKP_PID=$! +echo "$PF_OKP_PID" >"$E2E_OKP_PORT_FORWARD_PID_FILE" + # Wait for port-forward to be usable (app may not be listening immediately; port-forward can drop) log "Waiting for port-forward to lightspeed-stack to be ready..." for i in $(seq 1 36); do @@ -395,6 +419,7 @@ for i in $(seq 1 36); do kill $PF_LCS_PID 2>/dev/null || true kill $PF_JWKS_PID 2>/dev/null || true kill $PF_LLAMA_PID 2>/dev/null || true + kill $PF_OKP_PID 2>/dev/null || true exit 1 fi # If port-forward process died, restart it (e.g. "connection refused" / "lost connection to pod") @@ -421,6 +446,7 @@ for i in $(seq 1 36); do kill $PF_LCS_PID 2>/dev/null || true kill $PF_JWKS_PID 2>/dev/null || true kill $PF_LLAMA_PID 2>/dev/null || true + kill $PF_OKP_PID 2>/dev/null || true exit 1 fi if ! kill -0 $PF_LLAMA_PID 2>/dev/null; then @@ -432,10 +458,30 @@ for i in $(seq 1 36); do sleep 5 done +log "Waiting for OKP Solr port-forward (localhost:8081 /solr)..." +for i in $(seq 1 24); do + if curl -sf http://localhost:8081/solr > /dev/null 2>&1; then + log "✅ OKP Solr port-forward ready after $(( i * 5 ))s" + break + fi + if [ $i -eq 24 ]; then + echo "⚠️ Port-forward to OKP Solr never became healthy (2 min) - OKP RAG tests may fail" | tee /dev/stderr + # Don't exit - OKP is optional, other tests can still run + fi + if ! kill -0 $PF_OKP_PID 2>/dev/null; then + log "OKP port-forward died, restarting (attempt $i)..." + oc port-forward svc/okp-solr-service-svc 8081:8080 -n $NAMESPACE & + PF_OKP_PID=$! + echo "$PF_OKP_PID" >"$E2E_OKP_PORT_FORWARD_PID_FILE" + fi + sleep 5 +done + export E2E_LSC_HOSTNAME="localhost" export E2E_JWKS_HOSTNAME="localhost" export E2E_LLAMA_HOSTNAME="localhost" export E2E_LLAMA_PORT="8321" +export E2E_OKP_URL="http://localhost:8081" # Same pattern as tests/e2e-prow/rhoai/pipeline.sh and .github/workflows/e2e_tests_*.yaml: # Behave {MODEL}/{PROVIDER} use these when set; avoids wrong fallbacks if /v1/models # discovery in before_all is empty (matches run-ci.yaml openai + E2E_OPENAI_MODEL). @@ -450,6 +496,7 @@ export E2E_DEFAULT_PROVIDER_OVERRIDE E2E_DEFAULT_MODEL_OVERRIDE log "LCS accessible at: http://$E2E_LSC_HOSTNAME:8080" log "Mock JWKS accessible at: http://$E2E_JWKS_HOSTNAME:8000" log "OGX (e2e client hooks) at: http://$E2E_LLAMA_HOSTNAME:$E2E_LLAMA_PORT" +log "OKP Solr (RAG tests) at: $E2E_OKP_URL" #======================================== # 7. RUN TESTS @@ -484,13 +531,22 @@ if [[ -n "${E2E_LLAMA_PORT_FORWARD_PID_FILE:-}" && -f "$E2E_LLAMA_PORT_FORWARD_P fi rm -f "$E2E_LLAMA_PORT_FORWARD_PID_FILE" fi +if [[ -n "${E2E_OKP_PORT_FORWARD_PID_FILE:-}" && -f "$E2E_OKP_PORT_FORWARD_PID_FILE" ]]; then + read -r _okp_pf <"$E2E_OKP_PORT_FORWARD_PID_FILE" 2>/dev/null || true + if [[ "${_okp_pf:-}" =~ ^[0-9]+$ ]]; then + kill -9 "$_okp_pf" 2>/dev/null || true + fi + rm -f "$E2E_OKP_PORT_FORWARD_PID_FILE" +fi kill $PF_LCS_PID 2>/dev/null || true kill $PF_JWKS_PID 2>/dev/null || true kill $PF_LLAMA_PID 2>/dev/null || true +kill $PF_OKP_PID 2>/dev/null || true wait $PF_LCS_PID 2>/dev/null || true wait $PF_JWKS_PID 2>/dev/null || true wait $PF_LLAMA_PID 2>/dev/null || true +wait $PF_OKP_PID 2>/dev/null || true set -e trap 'echo "❌ Pipeline failed at line $LINENO"; exit 1' ERR diff --git a/tests/e2e-prow/rhoai/pipeline.sh b/tests/e2e-prow/rhoai/pipeline.sh index 4a86829f5..906ab2773 100755 --- a/tests/e2e-prow/rhoai/pipeline.sh +++ b/tests/e2e-prow/rhoai/pipeline.sh @@ -232,6 +232,22 @@ oc wait pod/mock-jwks pod/mock-mcp \ } echo "✅ Mock servers deployed" +# Deploy OKP Solr server for RAG tests +echo "Deploying OKP Solr server..." +oc apply -n "$NAMESPACE" -f "$PIPELINE_DIR/manifests/lightspeed/okp-solr.yaml" + +# Wait for OKP Solr to be ready +echo "Waiting for OKP Solr to be ready..." +oc wait pod/okp-solr-service \ + -n "$NAMESPACE" --for=condition=Ready --timeout=180s || { + echo "⚠️ OKP Solr not ready, checking status..." + oc get pods -n "$NAMESPACE" | grep okp-solr || true + oc describe pod okp-solr-service -n "$NAMESPACE" 2>/dev/null | tail -30 || true + echo "❌ OKP Solr failed to become ready" + exit 1 +} +echo "✅ OKP Solr deployed" + #======================================== # 8. BUILD OGX IMAGE #======================================== @@ -375,9 +391,11 @@ oc describe pod llama-stack-service -n "$NAMESPACE" || true export E2E_LSC_PORT_FORWARD_PID_FILE="${E2E_LSC_PORT_FORWARD_PID_FILE:-/tmp/e2e-lightspeed-port-forward.pid}" export E2E_LLAMA_PORT_FORWARD_PID_FILE="${E2E_LLAMA_PORT_FORWARD_PID_FILE:-/tmp/e2e-llama-port-forward.pid}" export E2E_JWKS_PORT_FORWARD_PID_FILE="${E2E_JWKS_PORT_FORWARD_PID_FILE:-/tmp/e2e-jwks-port-forward.pid}" +export E2E_OKP_PORT_FORWARD_PID_FILE="${E2E_OKP_PORT_FORWARD_PID_FILE:-/tmp/e2e-okp-port-forward.pid}" rm -f "$E2E_LSC_PORT_FORWARD_PID_FILE" rm -f "$E2E_LLAMA_PORT_FORWARD_PID_FILE" rm -f "$E2E_JWKS_PORT_FORWARD_PID_FILE" +rm -f "$E2E_OKP_PORT_FORWARD_PID_FILE" oc label pod lightspeed-stack-service pod=lightspeed-stack-service -n $NAMESPACE @@ -387,16 +405,18 @@ oc expose pod lightspeed-stack-service \ --type=ClusterIP \ -n $NAMESPACE -# Kill any existing processes on ports 8080, 8000, and 8321 (lsof may be missing in minimal images) -echo "Checking for existing processes on ports 8080, 8000, and 8321..." +# Kill any existing processes on ports 8080, 8000, 8321, and 8081 (lsof may be missing in minimal images) +echo "Checking for existing processes on ports 8080, 8000, 8321, and 8081..." if command -v lsof >/dev/null 2>&1; then lsof -ti:8080 | xargs kill -9 2>/dev/null || true lsof -ti:8000 | xargs kill -9 2>/dev/null || true lsof -ti:8321 | xargs kill -9 2>/dev/null || true + lsof -ti:8081 | xargs kill -9 2>/dev/null || true elif command -v fuser >/dev/null 2>&1; then fuser -k 8080/tcp 2>/dev/null || true fuser -k 8000/tcp 2>/dev/null || true fuser -k 8321/tcp 2>/dev/null || true + fuser -k 8081/tcp 2>/dev/null || true fi # Start port-forward for lightspeed-stack @@ -418,6 +438,12 @@ oc port-forward svc/llama-stack-service-svc 8321:8321 -n $NAMESPACE & PF_LLAMA_PID=$! echo "$PF_LLAMA_PID" >"$E2E_LLAMA_PORT_FORWARD_PID_FILE" +# Start port-forward for OKP Solr (RAG tests) +echo "Starting port-forward for OKP Solr..." +oc port-forward svc/okp-solr-service-svc 8081:8080 -n $NAMESPACE & +PF_OKP_PID=$! +echo "$PF_OKP_PID" >"$E2E_OKP_PORT_FORWARD_PID_FILE" + # Wait for port-forward to be usable (app may not be listening immediately; port-forward can drop) echo "Waiting for port-forward to lightspeed-stack to be ready..." for i in $(seq 1 36); do @@ -439,6 +465,7 @@ for i in $(seq 1 36); do kill $PF_LCS_PID 2>/dev/null || true kill $PF_JWKS_PID 2>/dev/null || true kill $PF_LLAMA_PID 2>/dev/null || true + kill $PF_OKP_PID 2>/dev/null || true exit 1 fi # If port-forward process died, restart it (e.g. "connection refused" / "lost connection to pod") @@ -464,6 +491,7 @@ for i in $(seq 1 36); do kill $PF_LCS_PID 2>/dev/null || true kill $PF_JWKS_PID 2>/dev/null || true kill $PF_LLAMA_PID 2>/dev/null || true + kill $PF_OKP_PID 2>/dev/null || true exit 1 fi if ! kill -0 $PF_LLAMA_PID 2>/dev/null; then @@ -475,13 +503,35 @@ for i in $(seq 1 36); do sleep 5 done +# Wait for OKP Solr port-forward to be usable (non-fatal - OKP is optional) +echo "Waiting for OKP Solr port-forward (localhost:8081 /solr)..." +for i in $(seq 1 24); do + if curl -sf http://localhost:8081/solr > /dev/null 2>&1; then + echo "✅ OKP Solr port-forward ready after $(( i * 5 ))s" + break + fi + if [ $i -eq 24 ]; then + echo "⚠️ Port-forward to OKP Solr never became healthy (2 min) - OKP RAG tests may fail" + # Don't exit - OKP is optional, other tests can still run + fi + if ! kill -0 $PF_OKP_PID 2>/dev/null; then + echo "OKP port-forward died, restarting (attempt $i)..." + oc port-forward svc/okp-solr-service-svc 8081:8080 -n $NAMESPACE & + PF_OKP_PID=$! + echo "$PF_OKP_PID" >"$E2E_OKP_PORT_FORWARD_PID_FILE" + fi + sleep 5 +done + export E2E_LSC_HOSTNAME="localhost" export E2E_JWKS_HOSTNAME="localhost" export E2E_DEFAULT_MODEL_OVERRIDE="$MODEL_NAME" export E2E_DEFAULT_PROVIDER_OVERRIDE="vllm" +export E2E_OKP_URL="http://localhost:8081" echo "LCS accessible at: http://$E2E_LSC_HOSTNAME:8080" echo "Mock JWKS accessible at: http://$E2E_JWKS_HOSTNAME:8000" echo "OGX accessible at: http://localhost:8321" +echo "OKP Solr (RAG tests) at: $E2E_OKP_URL" @@ -505,10 +555,12 @@ TEST_EXIT_CODE=$(cat "$E2E_EXIT_CODE_FILE" 2>/dev/null || echo 1) kill $PF_LCS_PID 2>/dev/null || true kill $PF_JWKS_PID 2>/dev/null || true kill $PF_LLAMA_PID 2>/dev/null || true +kill $PF_OKP_PID 2>/dev/null || true wait $PF_LCS_PID 2>/dev/null || true wait $PF_JWKS_PID 2>/dev/null || true wait $PF_LLAMA_PID 2>/dev/null || true -rm -f "$E2E_LSC_PORT_FORWARD_PID_FILE" "$E2E_LLAMA_PORT_FORWARD_PID_FILE" "$E2E_JWKS_PORT_FORWARD_PID_FILE" +wait $PF_OKP_PID 2>/dev/null || true +rm -f "$E2E_LSC_PORT_FORWARD_PID_FILE" "$E2E_LLAMA_PORT_FORWARD_PID_FILE" "$E2E_JWKS_PORT_FORWARD_PID_FILE" "$E2E_OKP_PORT_FORWARD_PID_FILE" set -e trap 'echo "❌ Pipeline failed at line $LINENO"; exit 1' ERR From 9bb8ac77018f01ca1ef4068124cbcc1005649e68 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Thu, 27 Aug 2026 14:06:44 -0400 Subject: [PATCH 15/25] Add an OwnerReference to redhat-registry-pull-secret --- tests/e2e-prow/rhoai/pipeline-konflux.sh | 38 ++++++++++++++++++++--- tests/e2e-prow/rhoai/pipeline.sh | 39 +++++++++++++++++++++--- 2 files changed, 67 insertions(+), 10 deletions(-) diff --git a/tests/e2e-prow/rhoai/pipeline-konflux.sh b/tests/e2e-prow/rhoai/pipeline-konflux.sh index f82bec6fc..e8c32c18f 100755 --- a/tests/e2e-prow/rhoai/pipeline-konflux.sh +++ b/tests/e2e-prow/rhoai/pipeline-konflux.sh @@ -130,11 +130,39 @@ if [[ -d /var/run/redhat-registry-username ]] && [[ -d /var/run/redhat-registry- shopt -u nullglob if [[ -n "$REDHAT_USERNAME" ]] && [[ -n "$REDHAT_PASSWORD" ]]; then - oc create secret docker-registry redhat-registry-pull-secret \ - --docker-server=registry.redhat.io \ - --docker-username="$REDHAT_USERNAME" \ - --docker-password="$REDHAT_PASSWORD" \ - -n "$NAMESPACE" 2>/dev/null && log "✅ Red Hat registry pull secret created" || log "⚠️ Secret exists or creation failed" + # Get current pipeline Pod metadata for ownerReference (ensures cleanup after job completes) + PIPELINE_POD_NAME="${HOSTNAME}" + if PIPELINE_POD_UID=$(oc get pod "$PIPELINE_POD_NAME" -n "$NAMESPACE" -o jsonpath='{.metadata.uid}' 2>/dev/null); then + log "Setting ownerReference to pipeline Pod: $PIPELINE_POD_NAME" + + # Create secret with ownerReference using YAML (ensures automatic cleanup) + cat </dev/null && log "✅ Red Hat registry pull secret created" || log "⚠️ Secret exists or creation failed" + fi # Link to default service account oc secrets link default redhat-registry-pull-secret --for=pull -n "$NAMESPACE" 2>/dev/null || echo "⚠️ Secret already linked to default SA" diff --git a/tests/e2e-prow/rhoai/pipeline.sh b/tests/e2e-prow/rhoai/pipeline.sh index 906ab2773..662d45fb5 100755 --- a/tests/e2e-prow/rhoai/pipeline.sh +++ b/tests/e2e-prow/rhoai/pipeline.sh @@ -96,11 +96,40 @@ oc secrets link default quay-lightspeed-pull-secret --for=pull -n "$NAMESPACE" 2 # Credentials from Prow secrets (environment variables or mounted volumes) if [[ -n "${REDHAT_REGISTRY_USERNAME:-}" ]] && [[ -n "${REDHAT_REGISTRY_PASSWORD:-}" ]]; then echo "Creating Red Hat registry pull secret from environment..." - oc create secret docker-registry redhat-registry-pull-secret \ - --docker-server=registry.redhat.io \ - --docker-username="$REDHAT_REGISTRY_USERNAME" \ - --docker-password="$REDHAT_REGISTRY_PASSWORD" \ - -n "$NAMESPACE" 2>/dev/null && echo "✅ Red Hat registry pull secret created" || echo "⚠️ Secret exists or creation failed" + + # Get current pipeline Pod metadata for ownerReference (ensures cleanup after job completes) + PIPELINE_POD_NAME="${HOSTNAME}" + if PIPELINE_POD_UID=$(oc get pod "$PIPELINE_POD_NAME" -n "$NAMESPACE" -o jsonpath='{.metadata.uid}' 2>/dev/null); then + echo "Setting ownerReference to pipeline Pod: $PIPELINE_POD_NAME" + + # Create secret with ownerReference using YAML (ensures automatic cleanup) + cat </dev/null && echo "✅ Red Hat registry pull secret created" || echo "⚠️ Secret exists or creation failed" + fi # Link to default service account oc secrets link default redhat-registry-pull-secret --for=pull -n "$NAMESPACE" 2>/dev/null || echo "⚠️ Secret already linked to default SA" From 0877f661f7ec4d9115b17d0d0ca5767c8e31de7f Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Thu, 27 Aug 2026 14:07:23 -0400 Subject: [PATCH 16/25] fixing syntax error --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c031a27ec..f817358a5 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ SHELL := /bin/bash #TODO: We need to rename all those python and config files as well +# Define comma for use in $(if) expressions (where commas are argument separators) +COMMA := , + ARTIFACT_DIR := $(if $(ARTIFACT_DIR),$(ARTIFACT_DIR),tests/test_results) PATH_TO_PLANTUML := ~/bin @@ -87,8 +90,8 @@ start-ogx-container: build-ogx-image ## Start OGX container -v $(PWD)/$(CONFIG):/opt/app-root/lightspeed-stack.yaml:ro,z \ -v $(PWD)/scripts/llama-stack-entrypoint.sh:/opt/app-root/enrich-entrypoint.sh:ro,z \ -v $(PWD)/src/llama_stack_configuration.py:/opt/app-root/llama_stack_configuration.py:ro,z \ - $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/lightspeed_stack_providers:/opt/app-root/providers/lightspeed_stack_providers:ro,z) \ - $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/resources/external_providers:/opt/app-root/src/.llama/providers.d:ro,z) \ + $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/lightspeed_stack_providers:/opt/app-root/providers/lightspeed_stack_providers:ro$(COMMA)z) \ + $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/resources/external_providers:/opt/app-root/src/.llama/providers.d:ro$(COMMA)z) \ $(if $(LIGHTSPEED_PROVIDERS_DIR),-e EXTERNAL_PROVIDERS_DIR=/opt/app-root/src/.llama/providers.d) \ -e OPENAI_API_KEY \ -e BRAVE_SEARCH_API_KEY \ From 9e5aa1e04d6fe2caa340be2d06b9a182eb631d09 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Thu, 27 Aug 2026 14:11:45 -0400 Subject: [PATCH 17/25] port-forward restart issue fix --- tests/e2e-prow/rhoai/scripts/e2e-ops.sh | 109 +++++++++++++++++++++++- 1 file changed, 108 insertions(+), 1 deletion(-) diff --git a/tests/e2e-prow/rhoai/scripts/e2e-ops.sh b/tests/e2e-prow/rhoai/scripts/e2e-ops.sh index 79b917e4f..9df97fb3a 100755 --- a/tests/e2e-prow/rhoai/scripts/e2e-ops.sh +++ b/tests/e2e-prow/rhoai/scripts/e2e-ops.sh @@ -23,6 +23,7 @@ # restart-both-services - Full OGX then lightspeed-stack restart (explicit only) # restart-port-forward - Re-establish port-forward for lightspeed # restart-llama-port-forward - Re-establish port-forward for OGX (8321) +# restart-okp-port-forward - Re-establish port-forward for OKP Solr (8081) # wait-for-pod [attempts] - Wait for a pod to be ready # update-configmap - Update ConfigMap from file # get-configmap-content - Get ConfigMap content (outputs to stdout) @@ -47,6 +48,7 @@ MANIFEST_DIR="$SCRIPT_DIR/../manifests/lightspeed" E2E_LSC_PORT_FORWARD_PID_FILE="${E2E_LSC_PORT_FORWARD_PID_FILE:-/tmp/e2e-lightspeed-port-forward.pid}" E2E_LLAMA_PORT_FORWARD_PID_FILE="${E2E_LLAMA_PORT_FORWARD_PID_FILE:-/tmp/e2e-llama-port-forward.pid}" E2E_JWKS_PORT_FORWARD_PID_FILE="${E2E_JWKS_PORT_FORWARD_PID_FILE:-/tmp/e2e-jwks-port-forward.pid}" +E2E_OKP_PORT_FORWARD_PID_FILE="${E2E_OKP_PORT_FORWARD_PID_FILE:-/tmp/e2e-okp-port-forward.pid}" # ============================================================================ # Helper functions @@ -212,6 +214,24 @@ kill_stale_jwks_forward() { free_local_tcp_port "$port" } +# Kill anything likely to hold the OKP Solr local forward (localhost:8081). +kill_stale_okp_forward() { + local port="${1:-8081}" + local saved_pf + if [[ -f "$E2E_OKP_PORT_FORWARD_PID_FILE" ]]; then + read -r saved_pf <"$E2E_OKP_PORT_FORWARD_PID_FILE" 2>/dev/null || true + if [[ "$saved_pf" =~ ^[0-9]+$ ]]; then + kill -9 "$saved_pf" 2>/dev/null || true + fi + fi + pkill -9 -f "port-forward.*okp-solr-service-svc.*8081:8080" 2>/dev/null || true + pkill -9 -f "oc port-forward svc/okp-solr-service-svc 8081:8080" 2>/dev/null || true + pkill -9 -f "port-forward pod/okp-solr-service.*8081:8080" 2>/dev/null || true + free_local_tcp_port "$port" + sleep 1 + free_local_tcp_port "$port" +} + # After oc port-forward dies in <2s, show recent oc stderr from the log file. e2e_ops_emit_port_forward_immediate_failure_diag() { echo "[e2e-ops] /tmp/port-forward.log (tail 25):" @@ -558,6 +578,25 @@ verify_llama_local_forward() { return 1 } +verify_okp_connectivity() { + local max_attempts="${1:-15}" + local http_code="" + local attempt + + for ((attempt=1; attempt<=max_attempts; attempt++)); do + http_code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 "http://localhost:8081/solr" 2>/dev/null) || http_code="000" + # OKP Solr returns various 200-399 codes for /solr endpoint + if [[ "$http_code" =~ ^[23][0-9][0-9]$ ]]; then + return 0 + fi + if [[ $attempt -lt $max_attempts ]]; then + sleep 2 + fi + done + echo "OKP Solr localhost:8081 connectivity check failed (HTTP: ${http_code:-unknown})" + return 1 +} + cmd_restart_llama_port_forward() { local local_port="${LOCAL_LLAMA_PORT:-8321}" local remote_port="${REMOTE_LLAMA_PORT:-8321}" @@ -680,6 +719,67 @@ cmd_restart_jwks_port_forward() { return 1 } +cmd_restart_okp_port_forward() { + local local_port="${LOCAL_OKP_PORT:-8081}" + local remote_port="${REMOTE_OKP_PORT:-8080}" + local max_attempts=6 + local pf_pid + local pf_resource + local okp_pf_log="/tmp/port-forward-okp.log" + + echo "Re-establishing OKP Solr port-forward on $local_port:$remote_port..." + + for ((attempt=1; attempt<=max_attempts; attempt++)); do + kill_stale_okp_forward "$local_port" + sleep 3 + + if [[ $attempt -le 2 ]]; then + pf_resource="svc/okp-solr-service-svc" + else + pf_resource="pod/okp-solr-service" + fi + echo "OKP port-forward attempt $attempt/$max_attempts -> $pf_resource" + + : >"$okp_pf_log" + nohup oc port-forward "$pf_resource" "$local_port:$remote_port" -n "$NAMESPACE" \ + "$okp_pf_log" 2>&1 & + pf_pid=$! + disown "$pf_pid" 2>/dev/null || true + sleep 3 + + if ! kill -0 "$pf_pid" 2>/dev/null; then + echo "OKP port-forward process exited immediately:" + if [[ -s "$okp_pf_log" ]]; then + tail -25 "$okp_pf_log" 2>/dev/null | sed 's/^/[e2e-ops] /' || true + fi + kill_stale_okp_forward "$local_port" + sleep 2 + continue + fi + sleep 4 + + if verify_okp_connectivity 12; then + echo "$pf_pid" >"$E2E_OKP_PORT_FORWARD_PID_FILE" + echo "✓ OKP Solr port-forward established (PID: $pf_pid)" + return 0 + fi + + if [[ $attempt -lt $max_attempts ]]; then + echo "OKP forward attempt $attempt failed (connectivity check failed), retrying..." + kill -9 "$pf_pid" 2>/dev/null || true + kill_stale_okp_forward "$local_port" + sleep 3 + fi + done + + echo "Failed to establish OKP Solr port-forward after $max_attempts attempts" + if [[ -s "$okp_pf_log" ]]; then + echo "Port-forward log (tail 30):" + tail -30 "$okp_pf_log" 2>/dev/null | sed 's/^/[e2e-ops] /' || true + fi + return 1 +} + cmd_wait_for_pod() { local pod_name="${1:?Pod name required}" local max_attempts="${2:-24}" @@ -1030,7 +1130,10 @@ cmd_restore_okp_solr() { echo "Restoring OKP Solr service in namespace $NAMESPACE..." oc apply -n "$NAMESPACE" -f "$MANIFEST_DIR/okp-solr.yaml" wait_for_pod "okp-solr-service" 60 - echo "✓ OKP Solr service restored and ready" + echo "✓ OKP Solr pod restored and ready" + + # Restart port-forward since pod was replaced + cmd_restart_okp_port_forward } # ============================================================================ @@ -1056,6 +1159,9 @@ case "$COMMAND" in restart-jwks-port-forward) cmd_restart_jwks_port_forward ;; + restart-okp-port-forward) + cmd_restart_okp_port_forward + ;; restart-port-forward) cmd_restart_port_forward ;; @@ -1124,6 +1230,7 @@ case "$COMMAND" in echo " restart-llama-stack - Restart/restore llama-stack pod" echo " restart-both-services - Full llama-stack + lightspeed-stack restart (explicit)" echo " restart-llama-port-forward - Re-establish port-forward for OGX (8321)" + echo " restart-okp-port-forward - Re-establish port-forward for OKP Solr (8081)" echo " restart-port-forward - Re-establish port-forward for lightspeed" echo " wait-for-pod [attempts] - Wait for a pod to be ready" echo " update-configmap - Update ConfigMap from file" From 14fc382cd3e6d88912a2af5b9c646866c9f676b6 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Thu, 27 Aug 2026 14:14:41 -0400 Subject: [PATCH 18/25] Added a timeout to both OKP readiness --- tests/e2e-prow/rhoai/pipeline-konflux.sh | 2 +- tests/e2e-prow/rhoai/pipeline.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e-prow/rhoai/pipeline-konflux.sh b/tests/e2e-prow/rhoai/pipeline-konflux.sh index e8c32c18f..f391dc898 100755 --- a/tests/e2e-prow/rhoai/pipeline-konflux.sh +++ b/tests/e2e-prow/rhoai/pipeline-konflux.sh @@ -488,7 +488,7 @@ done log "Waiting for OKP Solr port-forward (localhost:8081 /solr)..." for i in $(seq 1 24); do - if curl -sf http://localhost:8081/solr > /dev/null 2>&1; then + if curl -sf --max-time 5 http://localhost:8081/solr > /dev/null 2>&1; then log "✅ OKP Solr port-forward ready after $(( i * 5 ))s" break fi diff --git a/tests/e2e-prow/rhoai/pipeline.sh b/tests/e2e-prow/rhoai/pipeline.sh index 662d45fb5..cb93e3695 100755 --- a/tests/e2e-prow/rhoai/pipeline.sh +++ b/tests/e2e-prow/rhoai/pipeline.sh @@ -535,7 +535,7 @@ done # Wait for OKP Solr port-forward to be usable (non-fatal - OKP is optional) echo "Waiting for OKP Solr port-forward (localhost:8081 /solr)..." for i in $(seq 1 24); do - if curl -sf http://localhost:8081/solr > /dev/null 2>&1; then + if curl -sf --max-time 5 http://localhost:8081/solr > /dev/null 2>&1; then echo "✅ OKP Solr port-forward ready after $(( i * 5 ))s" break fi From a3b8be592807862b603c79f3bb631f6f83610cd7 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Mon, 31 Aug 2026 12:04:40 -0400 Subject: [PATCH 19/25] adding OKP deploy logs --- .../lightspeed-stack-integration-test.yaml | 10 +++ tests/e2e-prow/rhoai/pipeline-konflux.sh | 68 ++++++++++++++++--- tests/e2e-prow/rhoai/pipeline.sh | 68 ++++++++++++++++--- 3 files changed, 130 insertions(+), 16 deletions(-) diff --git a/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml b/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml index c3060e079..a0e909375 100644 --- a/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml +++ b/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml @@ -212,6 +212,12 @@ spec: - name: quay-aipcc-password secret: secretName: quay-aipcc-password + - name: redhat-registry-username + secret: + secretName: redhat-registry-username + - name: redhat-registry-password + secret: + secretName: redhat-registry-password - name: credentials emptyDir: {} steps: @@ -251,6 +257,10 @@ spec: mountPath: /var/run/quay-aipcc-name - name: quay-aipcc-password mountPath: /var/run/quay-aipcc-password + - name: redhat-registry-username + mountPath: /var/run/redhat-registry-username + - name: redhat-registry-password + mountPath: /var/run/redhat-registry-password - name: credentials mountPath: /credentials env: diff --git a/tests/e2e-prow/rhoai/pipeline-konflux.sh b/tests/e2e-prow/rhoai/pipeline-konflux.sh index f391dc898..bf564b3d5 100755 --- a/tests/e2e-prow/rhoai/pipeline-konflux.sh +++ b/tests/e2e-prow/rhoai/pipeline-konflux.sh @@ -214,16 +214,68 @@ log "✅ Mock servers deployed" log "Deploying OKP Solr server..." oc apply -n "$NAMESPACE" -f "$PIPELINE_DIR/manifests/lightspeed/okp-solr.yaml" +# Check if redhat-registry-pull-secret exists before waiting +if oc get secret redhat-registry-pull-secret -n "$NAMESPACE" &>/dev/null; then + log "✅ redhat-registry-pull-secret exists" +else + echo "❌ WARNING: redhat-registry-pull-secret NOT found - image pull will fail" + echo "Available secrets in namespace $NAMESPACE:" + oc get secrets -n "$NAMESPACE" | grep -E "NAME|pull-secret" || true +fi + # Wait for OKP Solr to be ready -log "Waiting for OKP Solr to be ready..." -oc wait pod/okp-solr-service \ - -n "$NAMESPACE" --for=condition=Ready --timeout=180s || { - echo "⚠️ OKP Solr not ready, checking status..." - oc get pods -n "$NAMESPACE" | grep okp-solr || true - oc describe pod okp-solr-service -n "$NAMESPACE" 2>/dev/null | tail -30 || true - echo "❌ OKP Solr failed to become ready" +log "Waiting for OKP Solr to be ready (180s timeout)..." +log "Initial pod status:" +oc get pod okp-solr-service -n "$NAMESPACE" || true + +if ! oc wait pod/okp-solr-service \ + -n "$NAMESPACE" --for=condition=Ready --timeout=180s; then + + echo "==========================================" + echo "⚠️ OKP Solr not ready - DETAILED DIAGNOSTICS" + echo "==========================================" + + echo "" + echo "=== Pod Status ===" + oc get pod okp-solr-service -n "$NAMESPACE" -o wide || true + + echo "" + echo "=== Container State ===" + oc get pod okp-solr-service -n "$NAMESPACE" \ + -o jsonpath='{.status.containerStatuses[*].state}' | jq '.' || echo "No container status available" + + echo "" + echo "=== Image Pull Status ===" + oc get pod okp-solr-service -n "$NAMESPACE" \ + -o jsonpath='{.status.containerStatuses[*].image}' && echo "" || true + oc get pod okp-solr-service -n "$NAMESPACE" \ + -o jsonpath='{.status.containerStatuses[*].imageID}' && echo "" || true + + echo "" + echo "=== Pod Events (last 30) ===" + oc get events -n "$NAMESPACE" --sort-by='.lastTimestamp' \ + --field-selector involvedObject.name=okp-solr-service 2>/dev/null | tail -30 || \ + oc get events -n "$NAMESPACE" --sort-by='.lastTimestamp' | grep okp-solr || true + + echo "" + echo "=== Full Pod Description ===" + oc describe pod okp-solr-service -n "$NAMESPACE" || true + + echo "" + echo "=== Red Hat Registry Secret Status ===" + if oc get secret redhat-registry-pull-secret -n "$NAMESPACE" &>/dev/null; then + oc get secret redhat-registry-pull-secret -n "$NAMESPACE" -o yaml | grep -A 2 "type:" + else + echo "❌ redhat-registry-pull-secret NOT FOUND" + fi + + echo "" + echo "==========================================" + echo "❌ OKP Solr failed to become ready within 180s" + echo "==========================================" exit 1 -} +fi + log "✅ OKP Solr deployed" # e2e-tunnel-proxy and e2e-interception-proxy are deployed from proxy.feature steps diff --git a/tests/e2e-prow/rhoai/pipeline.sh b/tests/e2e-prow/rhoai/pipeline.sh index cb93e3695..e3e62a005 100755 --- a/tests/e2e-prow/rhoai/pipeline.sh +++ b/tests/e2e-prow/rhoai/pipeline.sh @@ -265,16 +265,68 @@ echo "✅ Mock servers deployed" echo "Deploying OKP Solr server..." oc apply -n "$NAMESPACE" -f "$PIPELINE_DIR/manifests/lightspeed/okp-solr.yaml" +# Check if redhat-registry-pull-secret exists before waiting +if oc get secret redhat-registry-pull-secret -n "$NAMESPACE" &>/dev/null; then + echo "✅ redhat-registry-pull-secret exists" +else + echo "❌ WARNING: redhat-registry-pull-secret NOT found - image pull will fail" + echo "Available secrets in namespace $NAMESPACE:" + oc get secrets -n "$NAMESPACE" | grep -E "NAME|pull-secret" || true +fi + # Wait for OKP Solr to be ready -echo "Waiting for OKP Solr to be ready..." -oc wait pod/okp-solr-service \ - -n "$NAMESPACE" --for=condition=Ready --timeout=180s || { - echo "⚠️ OKP Solr not ready, checking status..." - oc get pods -n "$NAMESPACE" | grep okp-solr || true - oc describe pod okp-solr-service -n "$NAMESPACE" 2>/dev/null | tail -30 || true - echo "❌ OKP Solr failed to become ready" +echo "Waiting for OKP Solr to be ready (180s timeout)..." +echo "Initial pod status:" +oc get pod okp-solr-service -n "$NAMESPACE" || true + +if ! oc wait pod/okp-solr-service \ + -n "$NAMESPACE" --for=condition=Ready --timeout=180s; then + + echo "==========================================" + echo "⚠️ OKP Solr not ready - DETAILED DIAGNOSTICS" + echo "==========================================" + + echo "" + echo "=== Pod Status ===" + oc get pod okp-solr-service -n "$NAMESPACE" -o wide || true + + echo "" + echo "=== Container State ===" + oc get pod okp-solr-service -n "$NAMESPACE" \ + -o jsonpath='{.status.containerStatuses[*].state}' | jq '.' || echo "No container status available" + + echo "" + echo "=== Image Pull Status ===" + oc get pod okp-solr-service -n "$NAMESPACE" \ + -o jsonpath='{.status.containerStatuses[*].image}' && echo "" || true + oc get pod okp-solr-service -n "$NAMESPACE" \ + -o jsonpath='{.status.containerStatuses[*].imageID}' && echo "" || true + + echo "" + echo "=== Pod Events (last 30) ===" + oc get events -n "$NAMESPACE" --sort-by='.lastTimestamp' \ + --field-selector involvedObject.name=okp-solr-service 2>/dev/null | tail -30 || \ + oc get events -n "$NAMESPACE" --sort-by='.lastTimestamp' | grep okp-solr || true + + echo "" + echo "=== Full Pod Description ===" + oc describe pod okp-solr-service -n "$NAMESPACE" || true + + echo "" + echo "=== Red Hat Registry Secret Status ===" + if oc get secret redhat-registry-pull-secret -n "$NAMESPACE" &>/dev/null; then + oc get secret redhat-registry-pull-secret -n "$NAMESPACE" -o yaml | grep -A 2 "type:" + else + echo "❌ redhat-registry-pull-secret NOT FOUND" + fi + + echo "" + echo "==========================================" + echo "❌ OKP Solr failed to become ready within 180s" + echo "==========================================" exit 1 -} +fi + echo "✅ OKP Solr deployed" #======================================== From c1a00d2add498d8bef8d3204d38551a170d08820 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Mon, 31 Aug 2026 13:26:55 -0400 Subject: [PATCH 20/25] adding a fix for silent long waits --- tests/e2e-prow/rhoai/pipeline-konflux.sh | 50 ++++++++--- .../rhoai/pipeline-services-konflux.sh | 22 ++++- tests/e2e-prow/rhoai/pipeline.sh | 86 ++++++++++++------- 3 files changed, 112 insertions(+), 46 deletions(-) diff --git a/tests/e2e-prow/rhoai/pipeline-konflux.sh b/tests/e2e-prow/rhoai/pipeline-konflux.sh index bf564b3d5..fe68299ab 100755 --- a/tests/e2e-prow/rhoai/pipeline-konflux.sh +++ b/tests/e2e-prow/rhoai/pipeline-konflux.sh @@ -132,8 +132,10 @@ if [[ -d /var/run/redhat-registry-username ]] && [[ -d /var/run/redhat-registry- if [[ -n "$REDHAT_USERNAME" ]] && [[ -n "$REDHAT_PASSWORD" ]]; then # Get current pipeline Pod metadata for ownerReference (ensures cleanup after job completes) PIPELINE_POD_NAME="${HOSTNAME}" - if PIPELINE_POD_UID=$(oc get pod "$PIPELINE_POD_NAME" -n "$NAMESPACE" -o jsonpath='{.metadata.uid}' 2>/dev/null); then - log "Setting ownerReference to pipeline Pod: $PIPELINE_POD_NAME" + # Pipeline pod runs in its own namespace, not $NAMESPACE - don't specify -n + if PIPELINE_POD_UID=$(oc get pod "$PIPELINE_POD_NAME" -o jsonpath='{.metadata.uid}' 2>/dev/null) && \ + PIPELINE_POD_NAMESPACE=$(oc get pod "$PIPELINE_POD_NAME" -o jsonpath='{.metadata.namespace}' 2>/dev/null); then + log "Setting ownerReference to pipeline Pod: $PIPELINE_POD_NAME (namespace: $PIPELINE_POD_NAMESPACE)" # Create secret with ownerReference using YAML (ensures automatic cleanup) cat </dev/null || true + if oc create secret docker-registry redhat-registry-pull-secret \ --docker-server=registry.redhat.io \ --docker-username="$REDHAT_USERNAME" \ --docker-password="$REDHAT_PASSWORD" \ - -n "$NAMESPACE" 2>/dev/null && log "✅ Red Hat registry pull secret created" || log "⚠️ Secret exists or creation failed" + -n "$NAMESPACE" 2>&1; then + log "✅ Red Hat registry pull secret created" + else + echo "❌ FATAL: Failed to create redhat-registry-pull-secret" + exit 1 + fi fi # Link to default service account @@ -401,14 +410,29 @@ e2e_echo_pod_logs() { done < <(oc logs llama-stack-service -n "$NAMESPACE" --tail="$n" 2>&1) || true } -progress "Waiting for lightspeed-stack and llama-stack pods" -if ! oc wait pod/lightspeed-stack-service pod/llama-stack-service \ - -n "$NAMESPACE" --for=condition=Ready --timeout=600s; then - progress "❌ One or both service pods failed to become ready within timeout" - e2e_echo_pod_logs 200 - exit 1 -fi -log "✅ Both service pods are ready" +progress "Waiting for lightspeed-stack and llama-stack pods (up to 10 min)" +for i in $(seq 1 60); do + lcs_ready=$(oc get pod lightspeed-stack-service -n "$NAMESPACE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "False") + llama_ready=$(oc get pod llama-stack-service -n "$NAMESPACE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "False") + + if [[ "$lcs_ready" == "True" ]] && [[ "$llama_ready" == "True" ]]; then + log "✅ Both service pods are ready after $(( i * 10 ))s" + break + fi + + if [ $((i % 6)) -eq 0 ]; then + lcs_status=$(oc get pod lightspeed-stack-service -n "$NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null || echo "unknown") + llama_status=$(oc get pod llama-stack-service -n "$NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null || echo "unknown") + progress "[$(( i * 10 ))s] lightspeed-stack: $lcs_status ($lcs_ready), llama-stack: $llama_status ($llama_ready)" + fi + + if [ $i -eq 60 ]; then + progress "❌ One or both service pods failed to become ready within 600s timeout" + e2e_echo_pod_logs 200 + exit 1 + fi + sleep 10 +done if [ "$QUIET" = "1" ]; then e2e_echo_pod_logs 80 diff --git a/tests/e2e-prow/rhoai/pipeline-services-konflux.sh b/tests/e2e-prow/rhoai/pipeline-services-konflux.sh index 585ab0123..ffbc51ded 100755 --- a/tests/e2e-prow/rhoai/pipeline-services-konflux.sh +++ b/tests/e2e-prow/rhoai/pipeline-services-konflux.sh @@ -43,8 +43,26 @@ PVCEOF timeout 120 oc delete pod llama-stack-service -n "$NAMESPACE" --ignore-not-found=true --wait=true 2>/dev/null || true oc apply -n "$NAMESPACE" -f "$BASE_DIR/manifests/lightspeed/llama-stack-openai.yaml" -# First boot runs the full init (dnf + git clone + uv sync ≈ 6-15 min); use a generous timeout. -oc wait pod/llama-stack-service -n "$NAMESPACE" --for=condition=Ready --timeout=900s + +# First boot runs the full init (dnf + git clone + uv sync ≈ 6-15 min); poll with progress updates +echo "Waiting for llama-stack-service to be ready (up to 15 min for first boot)..." +for i in $(seq 1 90); do + if oc get pod llama-stack-service -n "$NAMESPACE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null | grep -q "True"; then + echo "✅ llama-stack-service ready after $(( i * 10 ))s" + break + fi + if [ $((i % 6)) -eq 0 ]; then + echo "[$(( i * 10 ))s] Still waiting for llama-stack-service... (pod status: $(oc get pod llama-stack-service -n "$NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null || echo 'unknown'))" + fi + if [ $i -eq 90 ]; then + echo "❌ llama-stack-service not ready after 900s" + oc get pod llama-stack-service -n "$NAMESPACE" -o wide 2>/dev/null || true + oc describe pod llama-stack-service -n "$NAMESPACE" 2>/dev/null | tail -50 || true + exit 1 + fi + sleep 10 +done + oc label pod llama-stack-service pod=llama-stack-service -n "$NAMESPACE" oc expose pod llama-stack-service --name=llama-stack-service-svc --port=8321 --type=ClusterIP -n "$NAMESPACE" diff --git a/tests/e2e-prow/rhoai/pipeline.sh b/tests/e2e-prow/rhoai/pipeline.sh index e3e62a005..cc2e839a7 100755 --- a/tests/e2e-prow/rhoai/pipeline.sh +++ b/tests/e2e-prow/rhoai/pipeline.sh @@ -99,8 +99,10 @@ if [[ -n "${REDHAT_REGISTRY_USERNAME:-}" ]] && [[ -n "${REDHAT_REGISTRY_PASSWORD # Get current pipeline Pod metadata for ownerReference (ensures cleanup after job completes) PIPELINE_POD_NAME="${HOSTNAME}" - if PIPELINE_POD_UID=$(oc get pod "$PIPELINE_POD_NAME" -n "$NAMESPACE" -o jsonpath='{.metadata.uid}' 2>/dev/null); then - echo "Setting ownerReference to pipeline Pod: $PIPELINE_POD_NAME" + # Pipeline pod runs in its own namespace, not $NAMESPACE - don't specify -n + if PIPELINE_POD_UID=$(oc get pod "$PIPELINE_POD_NAME" -o jsonpath='{.metadata.uid}' 2>/dev/null) && \ + PIPELINE_POD_NAMESPACE=$(oc get pod "$PIPELINE_POD_NAME" -o jsonpath='{.metadata.namespace}' 2>/dev/null); then + echo "Setting ownerReference to pipeline Pod: $PIPELINE_POD_NAME (namespace: $PIPELINE_POD_NAMESPACE)" # Create secret with ownerReference using YAML (ensures automatic cleanup) cat </dev/null || true + if oc create secret docker-registry redhat-registry-pull-secret \ --docker-server=registry.redhat.io \ --docker-username="$REDHAT_REGISTRY_USERNAME" \ --docker-password="$REDHAT_REGISTRY_PASSWORD" \ - -n "$NAMESPACE" 2>/dev/null && echo "✅ Red Hat registry pull secret created" || echo "⚠️ Secret exists or creation failed" + -n "$NAMESPACE" 2>&1; then + echo "✅ Red Hat registry pull secret created" + else + echo "❌ FATAL: Failed to create redhat-registry-pull-secret" + exit 1 + fi fi # Link to default service account @@ -426,32 +435,47 @@ fi ./pipeline-services.sh -echo "--> Final wait for both lightspeed-stack-service and llama-stack-service pods..." -if ! oc wait pod/lightspeed-stack-service pod/llama-stack-service \ - -n "$NAMESPACE" --for=condition=Ready --timeout=600s; then - echo "" - echo "❌ One or both service pods failed to become ready within timeout" - echo "" - echo "DEBUG: Pod status:" - oc get pods -n "$NAMESPACE" -o wide || true - echo "" - echo "DEBUG: lightspeed-stack-service description:" - oc describe pod lightspeed-stack-service -n "$NAMESPACE" || true - echo "" - echo "DEBUG: llama-stack-service description:" - oc describe pod llama-stack-service -n "$NAMESPACE" || true - echo "" - echo "DEBUG: lightspeed-stack-service logs:" - oc logs lightspeed-stack-service -n "$NAMESPACE" --tail=100 || true - echo "" - echo "DEBUG: llama-stack-service logs:" - oc logs llama-stack-service -n "$NAMESPACE" --tail=100 || true - echo "" - echo "DEBUG: Recent events in namespace:" - oc get events -n "$NAMESPACE" --sort-by='.lastTimestamp' | tail -20 || true - exit 1 -fi -echo "✅ Both service pods are ready" +echo "--> Waiting for both lightspeed-stack-service and llama-stack-service pods (up to 10 min)..." +for i in $(seq 1 60); do + lcs_ready=$(oc get pod lightspeed-stack-service -n "$NAMESPACE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "False") + llama_ready=$(oc get pod llama-stack-service -n "$NAMESPACE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "False") + + if [[ "$lcs_ready" == "True" ]] && [[ "$llama_ready" == "True" ]]; then + echo "✅ Both service pods are ready after $(( i * 10 ))s" + break + fi + + if [ $((i % 6)) -eq 0 ]; then + lcs_status=$(oc get pod lightspeed-stack-service -n "$NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null || echo "unknown") + llama_status=$(oc get pod llama-stack-service -n "$NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null || echo "unknown") + echo "[$(( i * 10 ))s] lightspeed-stack: $lcs_status ($lcs_ready), llama-stack: $llama_status ($llama_ready)" + fi + + if [ $i -eq 60 ]; then + echo "" + echo "❌ One or both service pods failed to become ready within 600s timeout" + echo "" + echo "DEBUG: Pod status:" + oc get pods -n "$NAMESPACE" -o wide || true + echo "" + echo "DEBUG: lightspeed-stack-service description:" + oc describe pod lightspeed-stack-service -n "$NAMESPACE" || true + echo "" + echo "DEBUG: llama-stack-service description:" + oc describe pod llama-stack-service -n "$NAMESPACE" || true + echo "" + echo "DEBUG: lightspeed-stack-service logs:" + oc logs lightspeed-stack-service -n "$NAMESPACE" --tail=100 || true + echo "" + echo "DEBUG: llama-stack-service logs:" + oc logs llama-stack-service -n "$NAMESPACE" --tail=100 || true + echo "" + echo "DEBUG: Recent events in namespace:" + oc get events -n "$NAMESPACE" --sort-by='.lastTimestamp' | tail -20 || true + exit 1 + fi + sleep 10 +done oc get pods -n "$NAMESPACE" From 05f701ef82bdf5d927ab57e541efdfcccee7fede Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Mon, 31 Aug 2026 14:45:53 -0400 Subject: [PATCH 21/25] removing the namespace-wide secret listing --- tests/e2e-prow/rhoai/pipeline-konflux.sh | 30 ++++++++++-------------- tests/e2e-prow/rhoai/pipeline.sh | 30 ++++++++++-------------- 2 files changed, 24 insertions(+), 36 deletions(-) diff --git a/tests/e2e-prow/rhoai/pipeline-konflux.sh b/tests/e2e-prow/rhoai/pipeline-konflux.sh index fe68299ab..fdad238d9 100755 --- a/tests/e2e-prow/rhoai/pipeline-konflux.sh +++ b/tests/e2e-prow/rhoai/pipeline-konflux.sh @@ -157,20 +157,12 @@ data: EOF log "✅ Red Hat registry pull secret created with ownerReference" else - # Fallback: create without ownerReference if Pod metadata unavailable - log "⚠️ Could not get pipeline Pod metadata - creating secret without ownerReference" - # Delete existing secret if any (may be stale from previous run) - oc delete secret redhat-registry-pull-secret -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true - if oc create secret docker-registry redhat-registry-pull-secret \ - --docker-server=registry.redhat.io \ - --docker-username="$REDHAT_USERNAME" \ - --docker-password="$REDHAT_PASSWORD" \ - -n "$NAMESPACE" 2>&1; then - log "✅ Red Hat registry pull secret created" - else - echo "❌ FATAL: Failed to create redhat-registry-pull-secret" - exit 1 - fi + # SECURITY: Fail closed if we cannot set ownerReference + # Creating secrets without ownerReference leads to credential sprawl + echo "❌ FATAL: Cannot get pipeline Pod metadata for ownerReference" + echo " Secret creation requires valid owner UID to ensure cleanup" + echo " Pipeline Pod: $PIPELINE_POD_NAME" + exit 1 fi # Link to default service account @@ -228,8 +220,9 @@ if oc get secret redhat-registry-pull-secret -n "$NAMESPACE" &>/dev/null; then log "✅ redhat-registry-pull-secret exists" else echo "❌ WARNING: redhat-registry-pull-secret NOT found - image pull will fail" - echo "Available secrets in namespace $NAMESPACE:" - oc get secrets -n "$NAMESPACE" | grep -E "NAME|pull-secret" || true + echo "Checking for pull secrets in namespace $NAMESPACE:" + oc get secrets -n "$NAMESPACE" --field-selector type=kubernetes.io/dockerconfigjson -o name 2>/dev/null || \ + echo "No dockerconfigjson secrets found" fi # Wait for OKP Solr to be ready @@ -262,9 +255,10 @@ if ! oc wait pod/okp-solr-service \ echo "" echo "=== Pod Events (last 30) ===" + # Use server-side filtering with a limit to avoid unbounded list calls oc get events -n "$NAMESPACE" --sort-by='.lastTimestamp' \ - --field-selector involvedObject.name=okp-solr-service 2>/dev/null | tail -30 || \ - oc get events -n "$NAMESPACE" --sort-by='.lastTimestamp' | grep okp-solr || true + --field-selector involvedObject.name=okp-solr-service \ + --limit=30 2>/dev/null || echo "No events found for okp-solr-service" echo "" echo "=== Full Pod Description ===" diff --git a/tests/e2e-prow/rhoai/pipeline.sh b/tests/e2e-prow/rhoai/pipeline.sh index cc2e839a7..44bee2952 100755 --- a/tests/e2e-prow/rhoai/pipeline.sh +++ b/tests/e2e-prow/rhoai/pipeline.sh @@ -124,20 +124,12 @@ data: EOF echo "✅ Red Hat registry pull secret created with ownerReference" else - # Fallback: create without ownerReference if Pod metadata unavailable - echo "⚠️ Could not get pipeline Pod metadata - creating secret without ownerReference" - # Delete existing secret if any (may be stale from previous run) - oc delete secret redhat-registry-pull-secret -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true - if oc create secret docker-registry redhat-registry-pull-secret \ - --docker-server=registry.redhat.io \ - --docker-username="$REDHAT_REGISTRY_USERNAME" \ - --docker-password="$REDHAT_REGISTRY_PASSWORD" \ - -n "$NAMESPACE" 2>&1; then - echo "✅ Red Hat registry pull secret created" - else - echo "❌ FATAL: Failed to create redhat-registry-pull-secret" - exit 1 - fi + # SECURITY: Fail closed if we cannot set ownerReference + # Creating secrets without ownerReference leads to credential sprawl + echo "❌ FATAL: Cannot get pipeline Pod metadata for ownerReference" + echo " Secret creation requires valid owner UID to ensure cleanup" + echo " Pipeline Pod: $PIPELINE_POD_NAME" + exit 1 fi # Link to default service account @@ -279,8 +271,9 @@ if oc get secret redhat-registry-pull-secret -n "$NAMESPACE" &>/dev/null; then echo "✅ redhat-registry-pull-secret exists" else echo "❌ WARNING: redhat-registry-pull-secret NOT found - image pull will fail" - echo "Available secrets in namespace $NAMESPACE:" - oc get secrets -n "$NAMESPACE" | grep -E "NAME|pull-secret" || true + echo "Checking for pull secrets in namespace $NAMESPACE:" + oc get secrets -n "$NAMESPACE" --field-selector type=kubernetes.io/dockerconfigjson -o name 2>/dev/null || \ + echo "No dockerconfigjson secrets found" fi # Wait for OKP Solr to be ready @@ -313,9 +306,10 @@ if ! oc wait pod/okp-solr-service \ echo "" echo "=== Pod Events (last 30) ===" + # Use server-side filtering with a limit to avoid unbounded list calls oc get events -n "$NAMESPACE" --sort-by='.lastTimestamp' \ - --field-selector involvedObject.name=okp-solr-service 2>/dev/null | tail -30 || \ - oc get events -n "$NAMESPACE" --sort-by='.lastTimestamp' | grep okp-solr || true + --field-selector involvedObject.name=okp-solr-service \ + --limit=30 2>/dev/null || echo "No events found for okp-solr-service" echo "" echo "=== Full Pod Description ===" From c664fe74e488af47ef01cb4d4760120bd1d72c64 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Mon, 31 Aug 2026 14:46:21 -0400 Subject: [PATCH 22/25] commenting out registry username and pass volume mounts --- .../lightspeed-stack-integration-test.yaml | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml b/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml index a0e909375..d7033726e 100644 --- a/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml +++ b/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml @@ -212,12 +212,13 @@ spec: - name: quay-aipcc-password secret: secretName: quay-aipcc-password - - name: redhat-registry-username - secret: - secretName: redhat-registry-username - - name: redhat-registry-password - secret: - secretName: redhat-registry-password + # TEMPORARILY DISABLED: Uncomment when redhat-registry-username/password secrets are created in lightspeed-core-tenant + # - name: redhat-registry-username + # secret: + # secretName: redhat-registry-username + # - name: redhat-registry-password + # secret: + # secretName: redhat-registry-password - name: credentials emptyDir: {} steps: @@ -257,10 +258,11 @@ spec: mountPath: /var/run/quay-aipcc-name - name: quay-aipcc-password mountPath: /var/run/quay-aipcc-password - - name: redhat-registry-username - mountPath: /var/run/redhat-registry-username - - name: redhat-registry-password - mountPath: /var/run/redhat-registry-password + # TEMPORARILY DISABLED: Uncomment when redhat-registry-username/password secrets are created in lightspeed-core-tenant + # - name: redhat-registry-username + # mountPath: /var/run/redhat-registry-username + # - name: redhat-registry-password + # mountPath: /var/run/redhat-registry-password - name: credentials mountPath: /credentials env: From e133fe2b9fdc8e2a04cc4e15bd9d226a854a4c0e Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Mon, 31 Aug 2026 15:42:09 -0400 Subject: [PATCH 23/25] updating volume mounts to use existing Konflux secrets set for pulling image --- .../lightspeed-stack-integration-test.yaml | 19 +++---- tests/e2e-prow/rhoai/pipeline-konflux.sh | 50 +++++++++++++++++-- tests/e2e-prow/rhoai/pipeline.sh | 50 +++++++++++++++++-- 3 files changed, 101 insertions(+), 18 deletions(-) diff --git a/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml b/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml index d7033726e..ed47669eb 100644 --- a/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml +++ b/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml @@ -212,13 +212,10 @@ spec: - name: quay-aipcc-password secret: secretName: quay-aipcc-password - # TEMPORARILY DISABLED: Uncomment when redhat-registry-username/password secrets are created in lightspeed-core-tenant - # - name: redhat-registry-username - # secret: - # secretName: redhat-registry-username - # - name: redhat-registry-password - # secret: - # secretName: redhat-registry-password + # Red Hat registry pull secret (docker-registry type) - will be copied to test namespace + - name: redhat-registry-pull-secret-source + secret: + secretName: redhat-registry-io-pull-secret - name: credentials emptyDir: {} steps: @@ -258,11 +255,9 @@ spec: mountPath: /var/run/quay-aipcc-name - name: quay-aipcc-password mountPath: /var/run/quay-aipcc-password - # TEMPORARILY DISABLED: Uncomment when redhat-registry-username/password secrets are created in lightspeed-core-tenant - # - name: redhat-registry-username - # mountPath: /var/run/redhat-registry-username - # - name: redhat-registry-password - # mountPath: /var/run/redhat-registry-password + # Red Hat registry pull secret (docker-registry type) + - name: redhat-registry-pull-secret-source + mountPath: /var/run/redhat-registry-pull-secret - name: credentials mountPath: /credentials env: diff --git a/tests/e2e-prow/rhoai/pipeline-konflux.sh b/tests/e2e-prow/rhoai/pipeline-konflux.sh index fdad238d9..80d4a189d 100755 --- a/tests/e2e-prow/rhoai/pipeline-konflux.sh +++ b/tests/e2e-prow/rhoai/pipeline-konflux.sh @@ -111,9 +111,53 @@ oc create secret docker-registry quay-lightspeed-pull-secret \ oc secrets link default quay-lightspeed-pull-secret --for=pull -n "$NAMESPACE" 2>/dev/null || echo "⚠️ Secret already linked to default SA" # Create Red Hat registry pull secret for OKP images -# Credentials from Konflux secrets (mounted at /var/run/redhat-registry-*) -if [[ -d /var/run/redhat-registry-username ]] && [[ -d /var/run/redhat-registry-password ]]; then - log "Creating Red Hat registry pull secret..." +# Option 1: Use mounted docker-registry secret (preferred - simpler) +if [[ -f /var/run/redhat-registry-pull-secret/.dockerconfigjson ]]; then + log "Creating Red Hat registry pull secret from mounted docker-registry secret..." + + DOCKERCONFIG_BASE64=$(cat /var/run/redhat-registry-pull-secret/.dockerconfigjson | base64 -w0) + + # Get current pipeline Pod metadata for ownerReference (ensures cleanup after job completes) + PIPELINE_POD_NAME="${HOSTNAME}" + # Pipeline pod runs in its own namespace, not $NAMESPACE - don't specify -n + if PIPELINE_POD_UID=$(oc get pod "$PIPELINE_POD_NAME" -o jsonpath='{.metadata.uid}' 2>/dev/null) && \ + PIPELINE_POD_NAMESPACE=$(oc get pod "$PIPELINE_POD_NAME" -o jsonpath='{.metadata.namespace}' 2>/dev/null); then + log "Setting ownerReference to pipeline Pod: $PIPELINE_POD_NAME (namespace: $PIPELINE_POD_NAMESPACE)" + + # Create secret with ownerReference using YAML (ensures automatic cleanup) + cat </dev/null || echo "⚠️ Secret already linked to default SA" + else + # SECURITY: Fail closed if we cannot set ownerReference + # Creating secrets without ownerReference leads to credential sprawl + echo "❌ FATAL: Cannot get pipeline Pod metadata for ownerReference" + echo " Secret creation requires valid owner UID to ensure cleanup" + echo " Pipeline Pod: $PIPELINE_POD_NAME" + exit 1 + fi + +# Option 2: Fallback to username/password mounted separately (legacy approach) +elif [[ -d /var/run/redhat-registry-username ]] && [[ -d /var/run/redhat-registry-password ]]; then + log "Creating Red Hat registry pull secret from username/password..." REDHAT_USERNAME="" REDHAT_PASSWORD="" diff --git a/tests/e2e-prow/rhoai/pipeline.sh b/tests/e2e-prow/rhoai/pipeline.sh index f96bb83a7..ea56fcfd5 100755 --- a/tests/e2e-prow/rhoai/pipeline.sh +++ b/tests/e2e-prow/rhoai/pipeline.sh @@ -93,9 +93,53 @@ oc create secret docker-registry quay-lightspeed-pull-secret \ oc secrets link default quay-lightspeed-pull-secret --for=pull -n "$NAMESPACE" 2>/dev/null || echo "⚠️ Secret already linked to default SA" # Create Red Hat registry pull secret for OKP images -# Credentials from Prow secrets (environment variables or mounted volumes) -if [[ -n "${REDHAT_REGISTRY_USERNAME:-}" ]] && [[ -n "${REDHAT_REGISTRY_PASSWORD:-}" ]]; then - echo "Creating Red Hat registry pull secret from environment..." +# Option 1: Use mounted docker-registry secret (preferred - simpler) +if [[ -f /var/run/redhat-registry-pull-secret/.dockerconfigjson ]]; then + echo "Creating Red Hat registry pull secret from mounted docker-registry secret..." + + DOCKERCONFIG_BASE64=$(cat /var/run/redhat-registry-pull-secret/.dockerconfigjson | base64 -w0) + + # Get current pipeline Pod metadata for ownerReference (ensures cleanup after job completes) + PIPELINE_POD_NAME="${HOSTNAME}" + # Pipeline pod runs in its own namespace, not $NAMESPACE - don't specify -n + if PIPELINE_POD_UID=$(oc get pod "$PIPELINE_POD_NAME" -o jsonpath='{.metadata.uid}' 2>/dev/null) && \ + PIPELINE_POD_NAMESPACE=$(oc get pod "$PIPELINE_POD_NAME" -o jsonpath='{.metadata.namespace}' 2>/dev/null); then + echo "Setting ownerReference to pipeline Pod: $PIPELINE_POD_NAME (namespace: $PIPELINE_POD_NAMESPACE)" + + # Create secret with ownerReference using YAML (ensures automatic cleanup) + cat </dev/null || echo "⚠️ Secret already linked to default SA" + else + # SECURITY: Fail closed if we cannot set ownerReference + # Creating secrets without ownerReference leads to credential sprawl + echo "❌ FATAL: Cannot get pipeline Pod metadata for ownerReference" + echo " Secret creation requires valid owner UID to ensure cleanup" + echo " Pipeline Pod: $PIPELINE_POD_NAME" + exit 1 + fi + +# Option 2: Fallback to environment variables (legacy Prow approach) +elif [[ -n "${REDHAT_REGISTRY_USERNAME:-}" ]] && [[ -n "${REDHAT_REGISTRY_PASSWORD:-}" ]]; then + echo "Creating Red Hat registry pull secret from environment variables..." # Get current pipeline Pod metadata for ownerReference (ensures cleanup after job completes) PIPELINE_POD_NAME="${HOSTNAME}" From c5bd67282e0aedbc607bbe7df4b8e9589d6c386f Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Mon, 31 Aug 2026 16:29:22 -0400 Subject: [PATCH 24/25] updating the okp_guide --- docs/user_doc/okp_guide.md | 49 +++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/docs/user_doc/okp_guide.md b/docs/user_doc/okp_guide.md index 60136d892..349b1ee11 100644 --- a/docs/user_doc/okp_guide.md +++ b/docs/user_doc/okp_guide.md @@ -51,12 +51,20 @@ validating that queries return referenced chunks. ## Step 1: Launch OKP -Start the OKP RAG service with Podman: +Start the OKP RAG service with Podman or Docker: +**Using Podman:** ```bash +podman login registry.redhat.io podman run --rm -d -p 8081:8080 registry.redhat.io/offline-knowledge-portal/rhokp-rhel9:latest ``` +**Using Docker:** +```bash +docker login registry.redhat.io +docker run --rm -d -p 8081:8080 registry.redhat.io/offline-knowledge-portal/rhokp-rhel9:latest +``` + > **Note:** Remove `-d` to run in the foreground. * The service listens on **port 8081** on the host (mapped from 8080 in the container). Lightspeed Stack itself listens on `8080`, so this avoids port conflicts. @@ -105,32 +113,43 @@ Edit your Lightspeed Stack config file (e.g. `lightspeed-stack.yaml`) and add the following top-level sections so that OKP is used for either inline or tool RAG: -Inline RAG: +**Inline RAG:** ```yaml # RAG configuration rag: - inline: - - okp -okp: - rhokp_url: ${env.RH_SERVER_OKP} - offline: true + retrieval: + inline: + sources: + - okp + max_chunks: 3 # Final tight cap on chunks returned to user + okp: + rhokp_url: ${env.RH_SERVER_OKP} + offline: true + max_chunks: 5 # Initial generous fetch from OKP ``` -Tool RAG: +**Tool RAG:** ```yaml # RAG configuration rag: - tool: - - okp -okp: - rhokp_url: ${env.RH_SERVER_OKP} - offline: true + retrieval: + tool: + sources: + - okp + max_chunks: 10 # Tool RAG limit + okp: + rhokp_url: ${env.RH_SERVER_OKP} + offline: true + max_chunks: 5 ``` -* **`rag.inline`** and **`rag.tool`**: Enable OKP as the RAG source for inline context injection and for the RAG tool. Tool rag means the LLM will be provided a search tool it can choose to invoke to find relevant content and augment the user prompt. The tool may or may not be invoked. Inline means a rag search and prompt augmentation will always occur. +* **`rag.retrieval.inline`** and **`rag.retrieval.tool`**: Enable OKP as the RAG source for inline context injection and for the RAG tool. Tool RAG means the LLM will be provided a search tool it can choose to invoke to find relevant content and augment the user prompt. The tool may or may not be invoked. Inline RAG means a RAG search and prompt augmentation will always occur. * **`okp.offline`**: When `true`, source URLs use `parent_id` (offline/Mimir-style). When `false`, use `reference_url` (online). +* **`max_chunks` behavior**: + - **Inline RAG**: `rag.retrieval.inline.max_chunks` is the final tight cap on chunks returned in the response, while `rag.okp.max_chunks` is the initial generous fetch from OKP before filtering and ranking. + - **Tool RAG**: `rag.retrieval.tool.max_chunks` sets the limit on chunks available to the tool. If you want to filter the docs to a specific product, you can include a static query filter such as: @@ -140,6 +159,8 @@ okp: chunk_filter_query: "product:*openshift* AND product_version:4.21" ``` +> **Important:** When running OGX in **Server mode**, changes to static filters like `chunk_filter_query` require an OGX restart to take effect. The static query configuration is loaded at startup and will not propagate automatically during runtime. + When you launch Lightspeed Stack it will augment the OGX configuration (the synthesized run.yaml in unified mode, or your external run.yaml in the deprecated legacy mode) with configuration for OKP. From d4ab395b4379a38013a3b5d41aa50c3571d2cf78 Mon Sep 17 00:00:00 2001 From: Sofiya Nuryyeva Date: Mon, 31 Aug 2026 16:32:38 -0400 Subject: [PATCH 25/25] fixing ownerReference issue --- tests/e2e-prow/rhoai/pipeline-konflux.sh | 56 ++++++++++++++---------- tests/e2e-prow/rhoai/pipeline.sh | 56 ++++++++++++++---------- tests/e2e-prow/rhoai/scripts/e2e-ops.sh | 7 +-- 3 files changed, 68 insertions(+), 51 deletions(-) diff --git a/tests/e2e-prow/rhoai/pipeline-konflux.sh b/tests/e2e-prow/rhoai/pipeline-konflux.sh index 80d4a189d..4f6a39923 100755 --- a/tests/e2e-prow/rhoai/pipeline-konflux.sh +++ b/tests/e2e-prow/rhoai/pipeline-konflux.sh @@ -119,9 +119,17 @@ if [[ -f /var/run/redhat-registry-pull-secret/.dockerconfigjson ]]; then # Get current pipeline Pod metadata for ownerReference (ensures cleanup after job completes) PIPELINE_POD_NAME="${HOSTNAME}" - # Pipeline pod runs in its own namespace, not $NAMESPACE - don't specify -n - if PIPELINE_POD_UID=$(oc get pod "$PIPELINE_POD_NAME" -o jsonpath='{.metadata.uid}' 2>/dev/null) && \ - PIPELINE_POD_NAMESPACE=$(oc get pod "$PIPELINE_POD_NAME" -o jsonpath='{.metadata.namespace}' 2>/dev/null); then + # Pipeline pod runs in its own namespace - read from service account mount + PIPELINE_POD_NAMESPACE="" + if [[ -f /var/run/secrets/kubernetes.io/serviceaccount/namespace ]]; then + PIPELINE_POD_NAMESPACE="$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)" + fi + if [[ -z "$PIPELINE_POD_NAMESPACE" ]]; then + echo "❌ FATAL: Cannot determine pipeline pod namespace" + echo " Service account namespace file not found or empty" + exit 1 + fi + if PIPELINE_POD_UID=$(oc get pod "$PIPELINE_POD_NAME" -n "$PIPELINE_POD_NAMESPACE" -o jsonpath='{.metadata.uid}' 2>/dev/null); then log "Setting ownerReference to pipeline Pod: $PIPELINE_POD_NAME (namespace: $PIPELINE_POD_NAMESPACE)" # Create secret with ownerReference using YAML (ensures automatic cleanup) @@ -176,29 +184,29 @@ elif [[ -d /var/run/redhat-registry-username ]] && [[ -d /var/run/redhat-registr if [[ -n "$REDHAT_USERNAME" ]] && [[ -n "$REDHAT_PASSWORD" ]]; then # Get current pipeline Pod metadata for ownerReference (ensures cleanup after job completes) PIPELINE_POD_NAME="${HOSTNAME}" - # Pipeline pod runs in its own namespace, not $NAMESPACE - don't specify -n - if PIPELINE_POD_UID=$(oc get pod "$PIPELINE_POD_NAME" -o jsonpath='{.metadata.uid}' 2>/dev/null) && \ - PIPELINE_POD_NAMESPACE=$(oc get pod "$PIPELINE_POD_NAME" -o jsonpath='{.metadata.namespace}' 2>/dev/null); then + # Pipeline pod runs in its own namespace - read from service account mount + PIPELINE_POD_NAMESPACE="" + if [[ -f /var/run/secrets/kubernetes.io/serviceaccount/namespace ]]; then + PIPELINE_POD_NAMESPACE="$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)" + fi + if [[ -z "$PIPELINE_POD_NAMESPACE" ]]; then + echo "❌ FATAL: Cannot determine pipeline pod namespace" + echo " Service account namespace file not found or empty" + exit 1 + fi + if PIPELINE_POD_UID=$(oc get pod "$PIPELINE_POD_NAME" -n "$PIPELINE_POD_NAMESPACE" -o jsonpath='{.metadata.uid}' 2>/dev/null); then log "Setting ownerReference to pipeline Pod: $PIPELINE_POD_NAME (namespace: $PIPELINE_POD_NAMESPACE)" - # Create secret with ownerReference using YAML (ensures automatic cleanup) - cat </dev/null) && \ - PIPELINE_POD_NAMESPACE=$(oc get pod "$PIPELINE_POD_NAME" -o jsonpath='{.metadata.namespace}' 2>/dev/null); then + # Pipeline pod runs in its own namespace - read from service account mount + PIPELINE_POD_NAMESPACE="" + if [[ -f /var/run/secrets/kubernetes.io/serviceaccount/namespace ]]; then + PIPELINE_POD_NAMESPACE="$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)" + fi + if [[ -z "$PIPELINE_POD_NAMESPACE" ]]; then + echo "❌ FATAL: Cannot determine pipeline pod namespace" + echo " Service account namespace file not found or empty" + exit 1 + fi + if PIPELINE_POD_UID=$(oc get pod "$PIPELINE_POD_NAME" -n "$PIPELINE_POD_NAMESPACE" -o jsonpath='{.metadata.uid}' 2>/dev/null); then echo "Setting ownerReference to pipeline Pod: $PIPELINE_POD_NAME (namespace: $PIPELINE_POD_NAMESPACE)" # Create secret with ownerReference using YAML (ensures automatic cleanup) @@ -143,29 +151,29 @@ elif [[ -n "${REDHAT_REGISTRY_USERNAME:-}" ]] && [[ -n "${REDHAT_REGISTRY_PASSWO # Get current pipeline Pod metadata for ownerReference (ensures cleanup after job completes) PIPELINE_POD_NAME="${HOSTNAME}" - # Pipeline pod runs in its own namespace, not $NAMESPACE - don't specify -n - if PIPELINE_POD_UID=$(oc get pod "$PIPELINE_POD_NAME" -o jsonpath='{.metadata.uid}' 2>/dev/null) && \ - PIPELINE_POD_NAMESPACE=$(oc get pod "$PIPELINE_POD_NAME" -o jsonpath='{.metadata.namespace}' 2>/dev/null); then + # Pipeline pod runs in its own namespace - read from service account mount + PIPELINE_POD_NAMESPACE="" + if [[ -f /var/run/secrets/kubernetes.io/serviceaccount/namespace ]]; then + PIPELINE_POD_NAMESPACE="$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)" + fi + if [[ -z "$PIPELINE_POD_NAMESPACE" ]]; then + echo "❌ FATAL: Cannot determine pipeline pod namespace" + echo " Service account namespace file not found or empty" + exit 1 + fi + if PIPELINE_POD_UID=$(oc get pod "$PIPELINE_POD_NAME" -n "$PIPELINE_POD_NAMESPACE" -o jsonpath='{.metadata.uid}' 2>/dev/null); then echo "Setting ownerReference to pipeline Pod: $PIPELINE_POD_NAME (namespace: $PIPELINE_POD_NAMESPACE)" - # Create secret with ownerReference using YAML (ensures automatic cleanup) - cat </dev/null) || http_code="000" + http_code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 "http://localhost:${local_port}/solr" 2>/dev/null) || http_code="000" # OKP Solr returns various 200-399 codes for /solr endpoint if [[ "$http_code" =~ ^[23][0-9][0-9]$ ]]; then return 0 @@ -593,7 +594,7 @@ verify_okp_connectivity() { sleep 2 fi done - echo "OKP Solr localhost:8081 connectivity check failed (HTTP: ${http_code:-unknown})" + echo "OKP Solr localhost:${local_port} connectivity check failed (HTTP: ${http_code:-unknown})" return 1 } @@ -758,7 +759,7 @@ cmd_restart_okp_port_forward() { fi sleep 4 - if verify_okp_connectivity 12; then + if verify_okp_connectivity 12 "$local_port"; then echo "$pf_pid" >"$E2E_OKP_PORT_FORWARD_PID_FILE" echo "✓ OKP Solr port-forward established (PID: $pf_pid)" return 0