Add new E2E tests for shields endpoint

Author: radofuchs

run.yaml

@@ -131,7 +131,9 @@ server:
   tls_cafile: null
   tls_certfile: null
   tls_keyfile: null
-shields: []
+shields: 
+  - shield_id: llama-guard-shield
+    provider_id: llama-guard
 vector_dbs:
   - vector_db_id: my_knowledge_base
     embedding_model: sentence-transformers/all-mpnet-base-v2

tests/e2e/configs/run-azure.yaml

@@ -120,7 +120,9 @@ server:
   tls_cafile: null
   tls_certfile: null
   tls_keyfile: null
-shields: []
+shields: 
+  - shield_id: llama-guard-shield
+    provider_id: llama-guard
 models:
   - model_id: gpt-4o-mini
     model_type: llm

tests/e2e/configs/run-ci.yaml

@@ -131,7 +131,9 @@ server:
   tls_cafile: null
   tls_certfile: null
   tls_keyfile: null
-shields: []
+shields:
+  - shield_id: llama-guard-shield
+    provider_id: llama-guard
 vector_dbs:
   - vector_db_id: my_knowledge_base
     embedding_model: sentence-transformers/all-mpnet-base-v2

tests/e2e/configs/run-rhaiis.yaml

@@ -123,7 +123,9 @@ server:
   tls_cafile: null
   tls_certfile: null
   tls_keyfile: null
-shields: []
+shields: 
+  - shield_id: llama-guard-shield
+    provider_id: llama-guard
 models:
   - metadata:
       embedding_dimension: 768 # Depends on chosen model

tests/e2e/features/info.feature

@@ -37,7 +37,7 @@ Feature: Info tests
       And The body of the response for model gpt-4o-mini has proper structure
 
 
-  Scenario: Check if models endpoint is working
+  Scenario: Check if models endpoint reports error when llama-stack in unreachable
     Given The system is in default state
     And  The llama-stack connection is disrupted
      When I access REST API endpoint "models" using HTTP GET method
@@ -47,6 +47,22 @@ Feature: Info tests
          {"detail": {"response": "Unable to connect to Llama Stack", "cause": "Connection error."}}
       """
 
+  Scenario: Check if shields endpoint is working
+    Given The system is in default state
+     When I access REST API endpoint "shields" using HTTP GET method
+     Then The status code of the response is 200
+      And The body of the response has proper shield structure
+
+
+  Scenario: Check if shields endpoint reports error when llama-stack in unreachable
+    Given The system is in default state
+    And  The llama-stack connection is disrupted
+     When I access REST API endpoint "shields" using HTTP GET method
+     Then The status code of the response is 500
+      And The body of the response is the following
+      """
+         {"detail": {"response": "Unable to connect to Llama Stack", "cause": "Connection error."}}
+      """
 
   Scenario: Check if metrics endpoint is working
     Given The system is in default state

tests/e2e/features/steps/info.py

@@ -37,21 +37,53 @@ def check_model_structure(context: Context, model: str) -> None:
     models = response_json["models"]
     assert len(models) > 0, "Models list should not be empty"
 
-    gpt_model = None
+    llm_model = None
     for model_id in models:
         if "gpt-4o-mini" in model_id.get("identifier", ""):
-            gpt_model = model_id
+            llm_model = model_id
             break
 
-    assert gpt_model is not None
+    assert llm_model is not None
 
-    assert gpt_model["type"] == "model", "type should be 'model'"
-    assert gpt_model["api_model_type"] == "llm", "api_model_type should be 'llm'"
-    assert gpt_model["model_type"] == "llm", "model_type should be 'llm'"
-    assert gpt_model["provider_id"] == "openai", "provider_id should be 'openai'"
+    assert llm_model["type"] == "model", "type should be 'model'"
+    assert llm_model["api_model_type"] == "llm", "api_model_type should be 'llm'"
+    assert llm_model["model_type"] == "llm", "model_type should be 'llm'"
+    assert llm_model["provider_id"] == "openai", "provider_id should be 'openai'"
     assert (
-        gpt_model["provider_resource_id"] == model
+        llm_model["provider_resource_id"] == model
     ), "provider_resource_id should be 'gpt-4o-mini'"
     assert (
-        gpt_model["identifier"] == f"openai/{model}"
+        llm_model["identifier"] == f"openai/{model}"
     ), "identifier should be 'openai/gpt-4o-mini'"
+
+
+@then("The body of the response has proper shield structure")
+def check_shield_structure(context: Context) -> None:
+    """Check that the first shield has the correct structure and required fields."""
+    response_json = context.response.json()
+    assert response_json is not None, "Response is not valid JSON"
+
+    assert "shields" in response_json, "Response missing 'shields' field"
+    shields = response_json["shields"]
+    assert len(shields) > 0, "Response has empty list of shields"
+
+    # Find first shield
+    found_shield = None
+    for shield in shields:
+        if shield.get("type") == "shield":
+            found_shield = shield
+            break
+
+    assert found_shield is not None, "No LLM model found in response"
+
+    # Validate structure and values
+    assert found_shield["type"] == "shield", "type should be 'model'"
+    assert (
+        found_shield["provider_id"] == "llama-guard"
+    ), "provider_id should be 'llama-guard'"
+    assert (
+        found_shield["provider_resource_id"] == "llama-guard-shield"
+    ), "provider_resource_id should be 'llama-guard-shield'"
+    assert (
+        found_shield["identifier"] == "llama-guard-shield"
+    ), "identifier should be 'llama-guard-shield'"