diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 6a1f464e5..bdb841ad5 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -33,6 +33,8 @@ jobs: tags: "not @skip and @cfg_rbac" - name: skills tags: "not @skip and (@cfg_skills or @cfg_skills_directory)" + - name: shields + tags: "not @skip and @cfg_shields" - name: other tags: "not @skip and (@cfg_rh_identity or @cfg_negative or @cfg_byok_pdf or @cfg_degraded or @cfg_unified)" # Server-only; listed in shard (not matrix.include) so it expands with diff --git a/.github/workflows/e2e_tests_lightspeed_evaluation.yaml b/.github/workflows/e2e_tests_lightspeed_evaluation.yaml index df8b6399c..38b1d1e6b 100644 --- a/.github/workflows/e2e_tests_lightspeed_evaluation.yaml +++ b/.github/workflows/e2e_tests_lightspeed_evaluation.yaml @@ -1,9 +1,7 @@ name: E2E Tests for Lightspeed Evaluation on: - push: - pull_request_target: - types: [opened, synchronize, reopened, labeled] + [] jobs: # Own job so repository secrets are never sent to a runner unless this passes. diff --git a/Makefile b/Makefile index 4a9cfa2e9..172435aa7 100644 --- a/Makefile +++ b/Makefile @@ -164,7 +164,7 @@ test-e2e-local: ## Run end to end tests for the service (no script wrapper) # Tag-based subsets (@cfg_* on features/scenarios). Default runs all config groups; override for one shard, e.g. # E2E_BEHAVE_TAG_EXPR='not @skip and @cfg_authorized' make test-e2e-tagged-local -E2E_BEHAVE_TAG_EXPR ?= not @skip and (@cfg_default or @cfg_authorized or @cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth or @cfg_rbac or @cfg_rh_identity or @cfg_negative or @cfg_skills or @cfg_skills_directory or @cfg_byok_pdf or @cfg_tls or @cfg_degraded or @cfg_unified) +E2E_BEHAVE_TAG_EXPR ?= not @skip and (@cfg_default or @cfg_authorized or @cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth or @cfg_rbac or @cfg_rh_identity or @cfg_negative or @cfg_skills or @cfg_skills_directory or @cfg_shields or @cfg_byok_pdf or @cfg_tls or @cfg_degraded or @cfg_unified) test-e2e-tagged: ## Run e2e tests with E2E_BEHAVE_TAG_EXPR (default: all @cfg_*) script -q -e -c "uv run behave --color --format pretty --tags=\"$(E2E_BEHAVE_TAG_EXPR)\" -D dump_errors=true @tests/e2e/test_list.txt" diff --git a/examples/lightspeed-stack-shields.yaml b/examples/lightspeed-stack-shields.yaml index fd671fc55..1e7f503f0 100644 --- a/examples/lightspeed-stack-shields.yaml +++ b/examples/lightspeed-stack-shields.yaml @@ -19,14 +19,16 @@ authentication: # LCORE-owned safety shields (not OGX / OGX Safety API resources). # Listed via GET /v1/shields; selected per request with optional shield_ids. shields: - - identifier: topic-guard + - name: topic-guard provider_id: question_validity config: model_id: openai/gpt-4o-mini - # Optional; omit to use built-in defaults: - model_prompt: "Classify whether the question is about OpenShift. Reply ALLOWED or REJECTED." + # Optional; omit to use built-in defaults. Must reference ${message} + # (string.Template substitution) or the classifier never sees the + # question it's meant to classify. + model_prompt: "Classify whether the following question is about OpenShift or Kubernetes. Reply with exactly one word: ${allowed} if it is about OpenShift or Kubernetes, or ${rejected} if it is not. Do not explain your answer or add any other text.\n\nQuestion: ${message}\nAnswer:" invalid_question_response: "I can only answer questions about OpenShift." - - identifier: pii-redaction + - name: pii-redaction provider_id: redaction config: rules: diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-shields-empty.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-shields-empty.yaml new file mode 100644 index 000000000..0ccb9b2d5 --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-shields-empty.yaml @@ -0,0 +1,28 @@ +# @cfg_shields +# No `shields` key at all: GET /v1/shields must return an empty list. +# See tests/e2e/features/shields.feature. +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds llama-stack as library + use_as_library_client: true + # Unified mode: run.yaml (materialized per provider by CI/the harness) + # is consumed as the synthesis profile instead of the legacy two-file path. + config: + profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-shields-override-disabled.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-shields-override-disabled.yaml new file mode 100644 index 000000000..d93431032 --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-shields-override-disabled.yaml @@ -0,0 +1,46 @@ +# @cfg_shields +# Same shield setup as lightspeed-stack-shields.yaml, but with +# customization.disable_shield_ids_override enabled so that any client +# supplied `shield_ids` field on /query or /streaming_query is rejected +# with a 422. See tests/e2e/features/shields_overrides.feature. +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds llama-stack as library + use_as_library_client: true + # Unified mode: run.yaml (materialized per provider by CI/the harness) + # is consumed as the synthesis profile instead of the legacy two-file path. + config: + profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +customization: + disable_shield_ids_override: true +shields: + - name: topic-guard + provider_id: question_validity + config: + model_id: openai/gpt-4o-mini + model_prompt: "Classify whether the following question is about OpenShift or Kubernetes. Reply with exactly one word: ${allowed} if it is about OpenShift or Kubernetes, or ${rejected} if it is not. Do not explain your answer or add any other text.\n\nQuestion: ${message}\nAnswer:" + invalid_question_response: "I can only answer questions about OpenShift." + - name: pii-redaction + provider_id: redaction + config: + rules: + - pattern: '\d+' + replacement: '[NUM]' + case_sensitive: false diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-shields.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-shields.yaml new file mode 100644 index 000000000..715f254c6 --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-shields.yaml @@ -0,0 +1,46 @@ +# @cfg_shields +# LCORE-owned safety shields (not Llama Stack / OGX Safety API resources). +# Configures one shield of each supported type so GET /v1/shields can be +# asserted against both `question_validity` and `redaction` shapes. See +# tests/e2e/features/shields.feature. +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds llama-stack as library + use_as_library_client: true + # Unified mode: run.yaml (materialized per provider by CI/the harness) + # is consumed as the synthesis profile instead of the legacy two-file path. + config: + profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +shields: + - name: topic-guard + provider_id: question_validity + config: + model_id: openai/gpt-4o-mini + # Must reference ${message}; string.Template silently drops the question + # otherwise (the classifier would never see what it's classifying). + model_prompt: "Classify whether the following question is about OpenShift or Kubernetes. Reply with exactly one word: ${allowed} if it is about OpenShift or Kubernetes, or ${rejected} if it is not. Do not explain your answer or add any other text.\n\nQuestion: ${message}\nAnswer:" + invalid_question_response: "I can only answer questions about OpenShift." + - name: pii-redaction + provider_id: redaction + config: + rules: + - pattern: '\d+' + replacement: '[NUM]' + case_sensitive: false diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-shields-empty.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-shields-empty.yaml new file mode 100644 index 000000000..5df05d93a --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-shields-empty.yaml @@ -0,0 +1,26 @@ +# @cfg_shields +# No `shields` key at all: GET /v1/shields must return an empty list. +# See tests/e2e/features/shields.feature. +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-shields-override-disabled.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-shields-override-disabled.yaml new file mode 100644 index 000000000..b51d86570 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-shields-override-disabled.yaml @@ -0,0 +1,44 @@ +# @cfg_shields +# Same shield setup as lightspeed-stack-shields.yaml, but with +# customization.disable_shield_ids_override enabled so that any client +# supplied `shield_ids` field on /query or /streaming_query is rejected +# with a 422. See tests/e2e/features/shields_overrides.feature. +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +customization: + disable_shield_ids_override: true +shields: + - name: topic-guard + provider_id: question_validity + config: + model_id: openai/gpt-4o-mini + model_prompt: "Classify whether the following question is about OpenShift or Kubernetes. Reply with exactly one word: ${allowed} if it is about OpenShift or Kubernetes, or ${rejected} if it is not. Do not explain your answer or add any other text.\n\nQuestion: ${message}\nAnswer:" + invalid_question_response: "I can only answer questions about OpenShift." + - name: pii-redaction + provider_id: redaction + config: + rules: + - pattern: '\d+' + replacement: '[NUM]' + case_sensitive: false diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-shields.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-shields.yaml new file mode 100644 index 000000000..17e011c0a --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-shields.yaml @@ -0,0 +1,44 @@ +# @cfg_shields +# LCORE-owned safety shields (not Llama Stack / OGX Safety API resources). +# Configures one shield of each supported type so GET /v1/shields can be +# asserted against both `question_validity` and `redaction` shapes. See +# tests/e2e/features/shields.feature. +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +shields: + - name: topic-guard + provider_id: question_validity + config: + model_id: openai/gpt-4o-mini + # Must reference ${message}; string.Template silently drops the question + # otherwise (the classifier would never see what it's classifying). + model_prompt: "Classify whether the following question is about OpenShift or Kubernetes. Reply with exactly one word: ${allowed} if it is about OpenShift or Kubernetes, or ${rejected} if it is not. Do not explain your answer or add any other text.\n\nQuestion: ${message}\nAnswer:" + invalid_question_response: "I can only answer questions about OpenShift." + - name: pii-redaction + provider_id: redaction + config: + rules: + - pattern: '\d+' + replacement: '[NUM]' + case_sensitive: false diff --git a/tests/e2e/features/info.feature b/tests/e2e/features/info.feature index d04923e95..3fd062ff1 100644 --- a/tests/e2e/features/info.feature +++ b/tests/e2e/features/info.feature @@ -21,11 +21,6 @@ Feature: Info tests And The body of the response has proper name Lightspeed Core Service (LCS) and version 0.6.0rc2 And The body of the response has llama-stack version 1.0.2 - Scenario: Check if shields endpoint is working - When I access REST API endpoint "shields" using HTTP GET method - Then The status code of the response is 200 - And The body of the response has proper shield structure - Scenario: Check if tools endpoint is working When I access REST API endpoint "tools" using HTTP GET method diff --git a/tests/e2e/features/shields.feature b/tests/e2e/features/shields.feature new file mode 100644 index 000000000..5914b9012 --- /dev/null +++ b/tests/e2e/features/shields.feature @@ -0,0 +1,65 @@ +Feature: Shields endpoint tests + + Tests for the LCORE-owned GET /v1/shields catalog endpoint. These shields + (question_validity, redaction) are configured directly in + lightspeed-stack.yaml; they are not Llama Stack / OGX Safety API resources. + See docs/user_doc/shields_guide.md for the full shield configuration and + runtime-behavior reference. + + Background: + Given The service is started locally + And The system is in default state + And REST API service prefix is /v1 + And the Lightspeed stack configuration directory is "tests/e2e/configuration" + + @cfg_shields + Scenario: Shields endpoint returns every configured shield type + Given The service uses the lightspeed-stack-shields.yaml configuration + And The service is restarted + When I access REST API endpoint "shields" using HTTP GET method + Then The status code of the response is 200 + And The body of the response is the following + """ + { + "shields": [ + { + "name": "topic-guard", + "provider_id": "question_validity", + "type": "shield", + "config": { + "model_id": "openai/gpt-4o-mini", + "model_prompt": "Classify whether the following question is about OpenShift or Kubernetes. Reply with exactly one word: ${allowed} if it is about OpenShift or Kubernetes, or ${rejected} if it is not. Do not explain your answer or add any other text.\n\nQuestion: ${message}\nAnswer:", + "invalid_question_response": "I can only answer questions about OpenShift." + } + }, + { + "name": "pii-redaction", + "provider_id": "redaction", + "type": "shield", + "config": { + "rules": [ + { + "pattern": "\\d+", + "replacement": "[NUM]", + "case_sensitive": null + } + ], + "case_sensitive": false + } + } + ] + } + """ + + @cfg_shields + Scenario: Shields endpoint returns an empty list when no shields are configured + Given The service uses the lightspeed-stack-shields-empty.yaml configuration + And The service is restarted + When I access REST API endpoint "shields" using HTTP GET method + Then The status code of the response is 200 + And The body of the response is the following + """ + { + "shields": [] + } + """ diff --git a/tests/e2e/features/shields_overrides.feature b/tests/e2e/features/shields_overrides.feature new file mode 100644 index 000000000..21aaec998 --- /dev/null +++ b/tests/e2e/features/shields_overrides.feature @@ -0,0 +1,45 @@ +Feature: shield_ids override tests + + Functional tests for the `shield_ids` request field that lets a client + override which configured shields run for a single request + + Background: + Given The service is started locally + And The system is in default state + And REST API service prefix is /v1 + And the Lightspeed stack configuration directory is "tests/e2e/configuration" + + @cfg_shields @flaky + Scenario Outline: shield_ids overrides change which shields apply to the request + Given The service uses the lightspeed-stack-shields.yaml configuration + And The service is restarted + When I use "query" to ask question + """ + + """ + Then The status code of the response is 200 + And The body of the response contains + + Examples: + | request_body | expected_fragment | + | {"query": "What is the best topping for a pizza?", "shield_ids": [], "model": "{MODEL}", "provider": "{PROVIDER}"} | topping | + | {"query": "My lucky number is 7042. Repeat that exact sentence back to me and nothing else.", "system_prompt": "You are a strict echo assistant. Repeat the user's message back exactly, character for character, with no other commentary.", "shield_ids": ["pii-redaction"], "model": "{MODEL}", "provider": "{PROVIDER}"} | My lucky number is [NUM]. | + + @cfg_shields + Scenario Outline: shield_ids validation failures return the expected error response + Given The service uses the configuration + And The service is restarted + When I use "query" to ask question + """ + + """ + Then The status code of the response is + And The body of the response is the following + """ + + """ + + Examples: + | config | request_body | status | expected_body | + | lightspeed-stack-shields.yaml | {"query": "Say hello.", "shield_ids": ["no-such-shield"], "model": "{MODEL}", "provider": "{PROVIDER}"} | 404 | {"detail": {"response": "Shield not found", "cause": "Shield with ID no-such-shield does not exist"}} | + | lightspeed-stack-shields-override-disabled.yaml | {"query": "Say hello.", "shield_ids": [], "model": "{MODEL}", "provider": "{PROVIDER}"} | 422 | {"detail": {"response": "Shield IDs customization is disabled", "cause": "This instance does not support customizing shield IDs in the query request (disable_shield_ids_override is set). Please remove the shield_ids field from your request."}} | diff --git a/tests/e2e/features/shields_question_validity.feature b/tests/e2e/features/shields_question_validity.feature new file mode 100644 index 000000000..5224ab816 --- /dev/null +++ b/tests/e2e/features/shields_question_validity.feature @@ -0,0 +1,49 @@ +Feature: question_validity shield functional tests + + Functional tests for the LCORE-owned `question_validity` shield + + Background: + Given The service is started locally + And The system is in default state + And REST API service prefix is /v1 + And the Lightspeed stack configuration directory is "tests/e2e/configuration" + And The service uses the lightspeed-stack-shields.yaml configuration + And The service is restarted + + @cfg_shields @flaky + Scenario Outline: question_validity allows in-topic and rejects off-topic questions + When I use "" to ask question + """ + + """ + Then The status code of the response is 200 + And The body of the response contains + + Examples: + | endpoint | request_body | expected_fragment | + | query | {"query": "What is OpenShift and how do I deploy an application on it?", "model": "{MODEL}", "provider": "{PROVIDER}"} | deploy | + | query | {"query": "What is the best topping for a pizza?", "model": "{MODEL}", "provider": "{PROVIDER}"} | I can only answer questions about OpenShift. | + | responses | {"input": "What is the best topping for a pizza?", "model": "{PROVIDER}/{MODEL}", "stream": false} | I can only answer questions about OpenShift. | + | infer | {"question": "What is the best topping for a pizza?"} | I can only answer questions about OpenShift. | + + # disabled due to https://redhat.atlassian.net/browse/LCORE-3756 + @skip + Examples: + | endpoint | request_body | expected_fragment | + | responses | {"input": "What is OpenShift and how do I deploy an application on it?", "model": "{PROVIDER}/{MODEL}", "stream": false} | deploy | + | infer | {"question": "What is OpenShift and how do I deploy an application on it?"} | deploy | + + @cfg_shields @flaky + Scenario Outline: question_validity allows in-topic and rejects off-topic questions via streaming_query + When I use "streaming_query" to ask question + """ + + """ + When I wait for the response to be completed + Then The status code of the response is 200 + And The body of the response contains + + Examples: + | request_body | expected_fragment | + | {"query": "What is OpenShift and how do I deploy an application on it?", "model": "{MODEL}", "provider": "{PROVIDER}"} | deploy | + | {"query": "What is the best topping for a pizza?", "model": "{MODEL}", "provider": "{PROVIDER}"} | I can only answer questions about OpenShift. | diff --git a/tests/e2e/features/shields_redaction.feature b/tests/e2e/features/shields_redaction.feature new file mode 100644 index 000000000..171fa2022 --- /dev/null +++ b/tests/e2e/features/shields_redaction.feature @@ -0,0 +1,50 @@ +Feature: redaction shield functional tests + + Functional tests for the LCORE-owned `redaction` shield + + Background: + Given The service is started locally + And The system is in default state + And REST API service prefix is /v1 + And the Lightspeed stack configuration directory is "tests/e2e/configuration" + And The service uses the lightspeed-stack-shields.yaml configuration + And The service is restarted + + @cfg_shields @flaky + Scenario Outline: redaction shield redacts matching PII and leaves non-matching text unchanged + When I use "query" to ask question + """ + { + "query": "", + "system_prompt": "You are a strict echo assistant. Repeat the user's message back exactly, character for character, with no other commentary.", + "model": "{MODEL}", + "provider": "{PROVIDER}" + } + """ + Then The status code of the response is 200 + And The body of the response contains + + Examples: + | query | expected_fragment | + | My OpenShift support ticket number is 48213. Repeat that exact sentence back to me and nothing else. | My OpenShift support ticket number is [NUM]. | + | My OpenShift cluster is healthy and stable. Repeat that exact sentence back to me and nothing else. | My OpenShift cluster is healthy and stable. | + + @cfg_shields @flaky + Scenario Outline: redaction shield redacts matching PII and leaves non-matching text unchanged via streaming_query + When I use "streaming_query" to ask question + """ + { + "query": "", + "system_prompt": "You are a strict echo assistant. Repeat the user's message back exactly, character for character, with no other commentary.", + "model": "{MODEL}", + "provider": "{PROVIDER}" + } + """ + When I wait for the response to be completed + Then The status code of the response is 200 + And The body of the response contains + + Examples: + | query | expected_fragment | + | My OpenShift support ticket number is 48213. Repeat that exact sentence back to me and nothing else. | My OpenShift support ticket number is [NUM]. | + | My OpenShift cluster is healthy and stable. Repeat that exact sentence back to me and nothing else. | My OpenShift cluster is healthy and stable. | diff --git a/tests/e2e/features/steps/info.py b/tests/e2e/features/steps/info.py index 08cec90db..1c8a6edf8 100644 --- a/tests/e2e/features/steps/info.py +++ b/tests/e2e/features/steps/info.py @@ -36,35 +36,6 @@ def check_llama_version(context: Context, llama_version: str) -> None: ), f"llama-stack version is {extracted_version}, expected {llama_version}" -@then("The body of the response has proper shield structure") -def check_shield_structure(context: Context) -> None: - """Check that the first shield has the correct structure and required fields.""" - response_json = context.response.json() - assert response_json is not None, "Response is not valid JSON" - - assert "shields" in response_json, "Response missing 'shields' field" - shields = response_json["shields"] - assert len(shields) > 0, "Response has empty list of shields" - - # Find first shield - found_shield = None - for shield in shields: - if shield.get("type") == "shield": - found_shield = shield - break - - assert found_shield is not None, "No shield found in response" - - # Validate structure and values - assert found_shield["type"] == "shield", "type should be 'shield'" - assert ( - found_shield["provider_id"] == "redaction" - ), "provider_id should be 'redaction'" - assert found_shield["name"] == "pii-redaction", ( - f"name should be 'pii-redaction', " f"but is '{found_shield['name']}'" - ) - - @then("The response contains {count:d} tools listed for provider {provider_name}") def check_tool_count(context: Context, count: int, provider_name: str) -> None: """Check that the number of tools for defined provider is correct.""" diff --git a/tests/e2e/test_list.txt b/tests/e2e/test_list.txt index c9ff90ed5..684585d66 100644 --- a/tests/e2e/test_list.txt +++ b/tests/e2e/test_list.txt @@ -3,6 +3,10 @@ features/health.feature features/info.feature features/models.feature features/rest_api.feature +features/shields.feature +features/shields_question_validity.feature +features/shields_redaction.feature +features/shields_overrides.feature features/smoketests.feature features/inline_rag.feature features/proxy.feature