LCORE-3365: shields e2e

.github/workflows/e2e_tests.yaml

@@ -33,6 +33,8 @@ jobs:
             tags: "not @skip and @cfg_rbac"
           - name: skills
             tags: "not @skip and (@cfg_skills or @cfg_skills_directory)"
+          - name: shields
+            tags: "not @skip and @cfg_shields"
           - name: other
             tags: "not @skip and (@cfg_rh_identity or @cfg_negative or @cfg_byok_pdf or @cfg_degraded or @cfg_unified)"
           # Server-only; listed in shard (not matrix.include) so it expands with

.github/workflows/e2e_tests_lightspeed_evaluation.yaml

@@ -1,9 +1,7 @@
 name: E2E Tests for Lightspeed Evaluation
 
 on:
-  push:
-  pull_request_target:
-    types: [opened, synchronize, reopened, labeled]
+  []
 
 jobs:
   # Own job so repository secrets are never sent to a runner unless this passes.

Makefile

@@ -164,7 +164,7 @@ test-e2e-local: ## Run end to end tests for the service (no script wrapper)
 
 # Tag-based subsets (@cfg_* on features/scenarios). Default runs all config groups; override for one shard, e.g.
 #   E2E_BEHAVE_TAG_EXPR='not @skip and @cfg_authorized' make test-e2e-tagged-local
-E2E_BEHAVE_TAG_EXPR ?= not @skip and (@cfg_default or @cfg_authorized or @cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth or @cfg_rbac or @cfg_rh_identity or @cfg_negative or @cfg_skills or @cfg_skills_directory or @cfg_byok_pdf or @cfg_tls or @cfg_degraded or @cfg_unified)
+E2E_BEHAVE_TAG_EXPR ?= not @skip and (@cfg_default or @cfg_authorized or @cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth or @cfg_rbac or @cfg_rh_identity or @cfg_negative or @cfg_skills or @cfg_skills_directory or @cfg_shields or @cfg_byok_pdf or @cfg_tls or @cfg_degraded or @cfg_unified)
 
 test-e2e-tagged: ## Run e2e tests with E2E_BEHAVE_TAG_EXPR (default: all @cfg_*)
 	script -q -e -c "uv run behave --color --format pretty --tags=\"$(E2E_BEHAVE_TAG_EXPR)\" -D dump_errors=true @tests/e2e/test_list.txt"

examples/lightspeed-stack-shields.yaml

@@ -19,14 +19,16 @@ authentication:
 # LCORE-owned safety shields (not OGX / OGX Safety API resources).
 # Listed via GET /v1/shields; selected per request with optional shield_ids.
 shields:
-  - identifier: topic-guard
+  - name: topic-guard
     provider_id: question_validity
     config:
       model_id: openai/gpt-4o-mini
-      # Optional; omit to use built-in defaults:
-      model_prompt: "Classify whether the question is about OpenShift. Reply ALLOWED or REJECTED."
+      # Optional; omit to use built-in defaults. Must reference ${message}
+      # (string.Template substitution) or the classifier never sees the
+      # question it's meant to classify.
+      model_prompt: "Classify whether the following question is about OpenShift or Kubernetes. Reply with exactly one word: ${allowed} if it is about OpenShift or Kubernetes, or ${rejected} if it is not. Do not explain your answer or add any other text.\n\nQuestion: ${message}\nAnswer:"
       invalid_question_response: "I can only answer questions about OpenShift."
-  - identifier: pii-redaction
+  - name: pii-redaction
     provider_id: redaction
     config:
       rules:

tests/e2e/configuration/library-mode/lightspeed-stack-shields-empty.yaml

@@ -0,0 +1,28 @@
+# @cfg_shields
+# No `shields` key at all: GET /v1/shields must return an empty list.
+# See tests/e2e/features/shields.feature.
+name: Lightspeed Core Service (LCS)
+service:
+  host: 0.0.0.0
+  port: 8080
+  auth_enabled: false
+  workers: 1
+  color_log: true
+  access_log: true
+llama_stack:
+  # Library mode - embeds llama-stack as library
+  use_as_library_client: true
+  # Unified mode: run.yaml (materialized per provider by CI/the harness)
+  # is consumed as the synthesis profile instead of the legacy two-file path.
+  config:
+    profile: run.yaml
+user_data_collection:
+  feedback_enabled: true
+  feedback_storage: "/tmp/data/feedback"
+  transcripts_enabled: true
+  transcripts_storage: "/tmp/data/transcripts"
+authentication:
+  module: "noop"
+inference:
+  default_provider: openai
+  default_model: gpt-4o-mini

tests/e2e/configuration/library-mode/lightspeed-stack-shields-override-disabled.yaml

@@ -0,0 +1,46 @@
+# @cfg_shields
+# Same shield setup as lightspeed-stack-shields.yaml, but with
+# customization.disable_shield_ids_override enabled so that any client
+# supplied `shield_ids` field on /query or /streaming_query is rejected
+# with a 422. See tests/e2e/features/shields_overrides.feature.
+name: Lightspeed Core Service (LCS)
+service:
+  host: 0.0.0.0
+  port: 8080
+  auth_enabled: false
+  workers: 1
+  color_log: true
+  access_log: true
+llama_stack:
+  # Library mode - embeds llama-stack as library
+  use_as_library_client: true
+  # Unified mode: run.yaml (materialized per provider by CI/the harness)
+  # is consumed as the synthesis profile instead of the legacy two-file path.
+  config:
+    profile: run.yaml
+user_data_collection:
+  feedback_enabled: true
+  feedback_storage: "/tmp/data/feedback"
+  transcripts_enabled: true
+  transcripts_storage: "/tmp/data/transcripts"
+authentication:
+  module: "noop"
+inference:
+  default_provider: openai
+  default_model: gpt-4o-mini
+customization:
+  disable_shield_ids_override: true
+shields:
+  - name: topic-guard
+    provider_id: question_validity
+    config:
+      model_id: openai/gpt-4o-mini
+      model_prompt: "Classify whether the following question is about OpenShift or Kubernetes. Reply with exactly one word: ${allowed} if it is about OpenShift or Kubernetes, or ${rejected} if it is not. Do not explain your answer or add any other text.\n\nQuestion: ${message}\nAnswer:"
+      invalid_question_response: "I can only answer questions about OpenShift."
+  - name: pii-redaction
+    provider_id: redaction
+    config:
+      rules:
+        - pattern: '\d+'
+          replacement: '[NUM]'
+      case_sensitive: false

tests/e2e/configuration/library-mode/lightspeed-stack-shields.yaml

@@ -0,0 +1,46 @@
+# @cfg_shields
+# LCORE-owned safety shields (not Llama Stack / OGX Safety API resources).
+# Configures one shield of each supported type so GET /v1/shields can be
+# asserted against both `question_validity` and `redaction` shapes. See
+# tests/e2e/features/shields.feature.
+name: Lightspeed Core Service (LCS)
+service:
+  host: 0.0.0.0
+  port: 8080
+  auth_enabled: false
+  workers: 1
+  color_log: true
+  access_log: true
+llama_stack:
+  # Library mode - embeds llama-stack as library
+  use_as_library_client: true
+  # Unified mode: run.yaml (materialized per provider by CI/the harness)
+  # is consumed as the synthesis profile instead of the legacy two-file path.
+  config:
+    profile: run.yaml
+user_data_collection:
+  feedback_enabled: true
+  feedback_storage: "/tmp/data/feedback"
+  transcripts_enabled: true
+  transcripts_storage: "/tmp/data/transcripts"
+authentication:
+  module: "noop"
+inference:
+  default_provider: openai
+  default_model: gpt-4o-mini
+shields:
+  - name: topic-guard
+    provider_id: question_validity
+    config:
+      model_id: openai/gpt-4o-mini
+      # Must reference ${message}; string.Template silently drops the question
+      # otherwise (the classifier would never see what it's classifying).
+      model_prompt: "Classify whether the following question is about OpenShift or Kubernetes. Reply with exactly one word: ${allowed} if it is about OpenShift or Kubernetes, or ${rejected} if it is not. Do not explain your answer or add any other text.\n\nQuestion: ${message}\nAnswer:"
+      invalid_question_response: "I can only answer questions about OpenShift."
+  - name: pii-redaction
+    provider_id: redaction
+    config:
+      rules:
+        - pattern: '\d+'
+          replacement: '[NUM]'
+      case_sensitive: false

tests/e2e/configuration/server-mode/lightspeed-stack-shields-empty.yaml

@@ -0,0 +1,26 @@
+# @cfg_shields
+# No `shields` key at all: GET /v1/shields must return an empty list.
+# See tests/e2e/features/shields.feature.
+name: Lightspeed Core Service (LCS)
+service:
+  host: 0.0.0.0
+  port: 8080
+  auth_enabled: false
+  workers: 1
+  color_log: true
+  access_log: true
+llama_stack:
+  # Server mode - connects to separate llama-stack service
+  use_as_library_client: false
+  url: http://${env.E2E_LLAMA_HOSTNAME}:8321
+  api_key: xyzzy
+user_data_collection:
+  feedback_enabled: true
+  feedback_storage: "/tmp/data/feedback"
+  transcripts_enabled: true
+  transcripts_storage: "/tmp/data/transcripts"
+authentication:
+  module: "noop"
+inference:
+  default_provider: openai
+  default_model: gpt-4o-mini

tests/e2e/configuration/server-mode/lightspeed-stack-shields-override-disabled.yaml

@@ -0,0 +1,44 @@
+# @cfg_shields
+# Same shield setup as lightspeed-stack-shields.yaml, but with
+# customization.disable_shield_ids_override enabled so that any client
+# supplied `shield_ids` field on /query or /streaming_query is rejected
+# with a 422. See tests/e2e/features/shields_overrides.feature.
+name: Lightspeed Core Service (LCS)
+service:
+  host: 0.0.0.0
+  port: 8080
+  auth_enabled: false
+  workers: 1
+  color_log: true
+  access_log: true
+llama_stack:
+  # Server mode - connects to separate llama-stack service
+  use_as_library_client: false
+  url: http://${env.E2E_LLAMA_HOSTNAME}:8321
+  api_key: xyzzy
+user_data_collection:
+  feedback_enabled: true
+  feedback_storage: "/tmp/data/feedback"
+  transcripts_enabled: true
+  transcripts_storage: "/tmp/data/transcripts"
+authentication:
+  module: "noop"
+inference:
+  default_provider: openai
+  default_model: gpt-4o-mini
+customization:
+  disable_shield_ids_override: true
+shields:
+  - name: topic-guard
+    provider_id: question_validity
+    config:
+      model_id: openai/gpt-4o-mini
+      model_prompt: "Classify whether the following question is about OpenShift or Kubernetes. Reply with exactly one word: ${allowed} if it is about OpenShift or Kubernetes, or ${rejected} if it is not. Do not explain your answer or add any other text.\n\nQuestion: ${message}\nAnswer:"
+      invalid_question_response: "I can only answer questions about OpenShift."
+  - name: pii-redaction
+    provider_id: redaction
+    config:
+      rules:
+        - pattern: '\d+'
+          replacement: '[NUM]'
+      case_sensitive: false

tests/e2e/configuration/server-mode/lightspeed-stack-shields.yaml

@@ -0,0 +1,44 @@
+# @cfg_shields
+# LCORE-owned safety shields (not Llama Stack / OGX Safety API resources).
+# Configures one shield of each supported type so GET /v1/shields can be
+# asserted against both `question_validity` and `redaction` shapes. See
+# tests/e2e/features/shields.feature.
+name: Lightspeed Core Service (LCS)
+service:
+  host: 0.0.0.0
+  port: 8080
+  auth_enabled: false
+  workers: 1
+  color_log: true
+  access_log: true
+llama_stack:
+  # Server mode - connects to separate llama-stack service
+  use_as_library_client: false
+  url: http://${env.E2E_LLAMA_HOSTNAME}:8321
+  api_key: xyzzy
+user_data_collection:
+  feedback_enabled: true
+  feedback_storage: "/tmp/data/feedback"
+  transcripts_enabled: true
+  transcripts_storage: "/tmp/data/transcripts"
+authentication:
+  module: "noop"
+inference:
+  default_provider: openai
+  default_model: gpt-4o-mini
+shields:
+  - name: topic-guard
+    provider_id: question_validity
+    config:
+      model_id: openai/gpt-4o-mini
+      # Must reference ${message}; string.Template silently drops the question
+      # otherwise (the classifier would never see what it's classifying).
+      model_prompt: "Classify whether the following question is about OpenShift or Kubernetes. Reply with exactly one word: ${allowed} if it is about OpenShift or Kubernetes, or ${rejected} if it is not. Do not explain your answer or add any other text.\n\nQuestion: ${message}\nAnswer:"
+      invalid_question_response: "I can only answer questions about OpenShift."
+  - name: pii-redaction
+    provider_id: redaction
+    config:
+      rules:
+        - pattern: '\d+'
+          replacement: '[NUM]'
+      case_sensitive: false

tests/e2e/features/info.feature

@@ -21,11 +21,6 @@ Feature: Info tests
       And The body of the response has proper name Lightspeed Core Service (LCS) and version 0.6.0rc2
       And The body of the response has llama-stack version 1.0.2
 
-  Scenario: Check if shields endpoint is working
-     When I access REST API endpoint "shields" using HTTP GET method
-     Then The status code of the response is 200
-      And The body of the response has proper shield structure
-
 
   Scenario: Check if tools endpoint is working
      When I access REST API endpoint "tools" using HTTP GET method

tests/e2e/features/shields.feature

@@ -0,0 +1,65 @@
+Feature: Shields endpoint tests
+
+  Tests for the LCORE-owned GET /v1/shields catalog endpoint. These shields
+  (question_validity, redaction) are configured directly in
+  lightspeed-stack.yaml; they are not Llama Stack / OGX Safety API resources.
+  See docs/user_doc/shields_guide.md for the full shield configuration and
+  runtime-behavior reference.
+
+  Background:
+    Given The service is started locally
+      And The system is in default state
+      And REST API service prefix is /v1
+      And the Lightspeed stack configuration directory is "tests/e2e/configuration"
+
+  @cfg_shields
+  Scenario: Shields endpoint returns every configured shield type
+    Given The service uses the lightspeed-stack-shields.yaml configuration
+      And The service is restarted
+     When I access REST API endpoint "shields" using HTTP GET method
+     Then The status code of the response is 200
+      And The body of the response is the following
+      """
+      {
+        "shields": [
+          {
+            "name": "topic-guard",
+            "provider_id": "question_validity",
+            "type": "shield",
+            "config": {
+              "model_id": "openai/gpt-4o-mini",
+              "model_prompt": "Classify whether the following question is about OpenShift or Kubernetes. Reply with exactly one word: ${allowed} if it is about OpenShift or Kubernetes, or ${rejected} if it is not. Do not explain your answer or add any other text.\n\nQuestion: ${message}\nAnswer:",
+              "invalid_question_response": "I can only answer questions about OpenShift."
+            }
+          },
+          {
+            "name": "pii-redaction",
+            "provider_id": "redaction",
+            "type": "shield",
+            "config": {
+              "rules": [
+                {
+                  "pattern": "\\d+",
+                  "replacement": "[NUM]",
+                  "case_sensitive": null
+                }
+              ],
+              "case_sensitive": false
+            }
+          }
+        ]
+      }
+      """
+
+  @cfg_shields
+  Scenario: Shields endpoint returns an empty list when no shields are configured
+    Given The service uses the lightspeed-stack-shields-empty.yaml configuration
+      And The service is restarted
+     When I access REST API endpoint "shields" using HTTP GET method
+     Then The status code of the response is 200
+      And The body of the response is the following
+      """
+      {
+        "shields": []
+      }
+      """