diff --git a/tests/e2e/features/conversations.feature b/tests/e2e/features/conversations.feature index 8119771f..2fa41ed1 100644 --- a/tests/e2e/features/conversations.feature +++ b/tests/e2e/features/conversations.feature @@ -22,7 +22,7 @@ Feature: conversations endpoint API tests And The conversation with conversation_id from above is returned And The conversation details are following """ - {"last_used_model": "gpt-4-turbo", "last_used_provider": "openai", "message_count": 1} + {"last_used_model": "{MODEL}", "last_used_provider": "{PROVIDER}", "message_count": 1} """ Scenario: Check if conversations endpoint fails when the auth header is not present diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index c4631160..f3d49f19 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -20,14 +20,47 @@ create_config_backup, ) -try: - import os # noqa: F401 -except ImportError as e: - print("Warning: unable to import module:", e) + +def _fetch_models_from_service(hostname: str = "localhost", port: int = 8080) -> dict: + """Query /v1/models endpoint and return first LLM model. + + Returns: + Dict with model_id and provider_id, or empty dict if unavailable + """ + try: + url = f"http://{hostname}:{port}/v1/models" + response = requests.get(url, timeout=5) + response.raise_for_status() + data = response.json() + + # Find first LLM model + for model in data.get("models", []): + if model.get("api_model_type") == "llm": + provider_id = model.get("provider_id") + model_id = model.get("provider_resource_id") + if provider_id and model_id: + return {"model_id": model_id, "provider_id": provider_id} + return {} + except (requests.RequestException, ValueError, KeyError): + return {} def before_all(context: Context) -> None: """Run before and after the whole shooting match.""" + # Get first LLM model from running service + llm_model = _fetch_models_from_service() + + if llm_model: + context.default_model = llm_model["model_id"] + context.default_provider = llm_model["provider_id"] + print( + f"Detected LLM: {context.default_model} (provider: {context.default_provider})" + ) + else: + # Fallback for development + context.default_model = "gpt-4-turbo" + context.default_provider = "openai" + print("⚠ Could not detect models, using fallback: gpt-4-turbo/openai") def before_scenario(context: Context, scenario: Scenario) -> None: diff --git a/tests/e2e/features/info.feature b/tests/e2e/features/info.feature index 2c211a6d..2f67e53a 100644 --- a/tests/e2e/features/info.feature +++ b/tests/e2e/features/info.feature @@ -34,7 +34,7 @@ Feature: Info tests Given The system is in default state When I access REST API endpoint "models" using HTTP GET method Then The status code of the response is 200 - And The body of the response for model gpt-4o-mini has proper structure + And The body of the response has proper model structure Scenario: Check if models endpoint is working diff --git a/tests/e2e/features/query.feature b/tests/e2e/features/query.feature index fb89a04e..c8c9394c 100644 --- a/tests/e2e/features/query.feature +++ b/tests/e2e/features/query.feature @@ -42,7 +42,7 @@ Feature: Query endpoint API tests And I store conversation details And I use "query" to ask question with same conversation_id """ - {"query": "Write a simple code for reversing string", "system_prompt": "provide coding assistance", "model": "gpt-4-turbo", "provider": "openai"} + {"query": "Write a simple code for reversing string", "system_prompt": "provide coding assistance", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 And The response should contain following fragments @@ -76,12 +76,12 @@ Scenario: Check if LLM responds for query request with error for missing query And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva When I use "query" to ask question with authorization header """ - {"provider": "openai"} + {"provider": "{PROVIDER}"} """ Then The status code of the response is 422 And The body of the response is the following """ - { "detail": [{"type": "missing", "loc": [ "body", "query" ], "msg": "Field required", "input": {"provider": "openai"}}] } + { "detail": [{"type": "missing", "loc": [ "body", "query" ], "msg": "Field required", "input": {"provider": "{PROVIDER}"}}] } """ Scenario: Check if LLM responds for query request with error for missing model @@ -89,7 +89,7 @@ Scenario: Check if LLM responds for query request with error for missing query And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva When I use "query" to ask question with authorization header """ - {"query": "Say hello", "provider": "openai"} + {"query": "Say hello", "provider": "{PROVIDER}"} """ Then The status code of the response is 422 And The body of the response contains Value error, Model must be specified if provider is specified @@ -99,7 +99,7 @@ Scenario: Check if LLM responds for query request with error for missing query And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva When I use "query" to ask question with authorization header """ - {"query": "Say hello", "model": "gpt-4-turbo"} + {"query": "Say hello", "model": "{MODEL}"} """ Then The status code of the response is 422 And The body of the response contains Value error, Provider must be specified if model is specified diff --git a/tests/e2e/features/steps/common_http.py b/tests/e2e/features/steps/common_http.py index 78490b7c..2f244933 100644 --- a/tests/e2e/features/steps/common_http.py +++ b/tests/e2e/features/steps/common_http.py @@ -7,6 +7,7 @@ from behave.runner import Context from tests.e2e.utils.utils import ( normalize_endpoint, + replace_placeholders, validate_json, validate_json_partially, ) @@ -170,7 +171,10 @@ def check_prediction_result(context: Context) -> None: assert context.response is not None, "Request needs to be performed first" assert context.text is not None, "Response does not contain any payload" - expected_body = json.loads(context.text) + # Replace {MODEL} and {PROVIDER} placeholders with actual values + json_str = replace_placeholders(context, context.text or "{}") + + expected_body = json.loads(json_str) result = context.response.json() # compare both JSONs and print actual result in case of any difference diff --git a/tests/e2e/features/steps/conversation.py b/tests/e2e/features/steps/conversation.py index a4d94f7e..efc39db0 100644 --- a/tests/e2e/features/steps/conversation.py +++ b/tests/e2e/features/steps/conversation.py @@ -4,6 +4,7 @@ from behave import step, when, then # pyright: ignore[reportAttributeAccessIssue] from behave.runner import Context import requests +from tests.e2e.utils.utils import replace_placeholders # default timeout for HTTP operations DEFAULT_TIMEOUT = 10 @@ -109,7 +110,10 @@ def check_returned_conversation_id(context: Context) -> None: @then("The conversation details are following") def check_returned_conversation_content(context: Context) -> None: """Check the conversation content in response.""" - expected_data = json.loads(context.text) + # Replace {MODEL} and {PROVIDER} placeholders with actual values + json_str = replace_placeholders(context, context.text or "{}") + + expected_data = json.loads(json_str) found_conversation = context.found_conversation assert ( diff --git a/tests/e2e/features/steps/info.py b/tests/e2e/features/steps/info.py index d9777e26..2dbd1e6c 100644 --- a/tests/e2e/features/steps/info.py +++ b/tests/e2e/features/steps/info.py @@ -27,31 +27,39 @@ def check_llama_version(context: Context, llama_version: str) -> None: ), f"llama-stack version is {response_json["llama_stack_version"]}" -@then("The body of the response for model {model} has proper structure") -def check_model_structure(context: Context, model: str) -> None: - """Check that the gpt-4o-mini model has the correct structure and required fields.""" +@then("The body of the response has proper model structure") +def check_model_structure(context: Context) -> None: + """Check that the first LLM model has the correct structure and required fields.""" response_json = context.response.json() assert response_json is not None, "Response is not valid JSON" assert "models" in response_json, "Response missing 'models' field" models = response_json["models"] - assert len(models) > 0, "Models list should not be empty" + assert len(models) > 0, "Response has empty list of models" - gpt_model = None - for model_id in models: - if "gpt-4o-mini" in model_id.get("identifier", ""): - gpt_model = model_id + # Find first LLM model (same logic as environment.py) + llm_model = None + for model in models: + if model.get("api_model_type") == "llm": + llm_model = model break - assert gpt_model is not None + assert llm_model is not None, "No LLM model found in response" - assert gpt_model["type"] == "model", "type should be 'model'" - assert gpt_model["api_model_type"] == "llm", "api_model_type should be 'llm'" - assert gpt_model["model_type"] == "llm", "model_type should be 'llm'" - assert gpt_model["provider_id"] == "openai", "provider_id should be 'openai'" + # Get expected values from context + expected_model = context.default_model + expected_provider = context.default_provider + + # Validate structure and values + assert llm_model["type"] == "model", "type should be 'model'" + assert llm_model["api_model_type"] == "llm", "api_model_type should be 'llm'" + assert llm_model["model_type"] == "llm", "model_type should be 'llm'" + assert ( + llm_model["provider_id"] == expected_provider + ), f"provider_id should be '{expected_provider}'" assert ( - gpt_model["provider_resource_id"] == model - ), "provider_resource_id should be 'gpt-4o-mini'" + llm_model["provider_resource_id"] == expected_model + ), f"provider_resource_id should be '{expected_model}'" assert ( - gpt_model["identifier"] == f"openai/{model}" - ), "identifier should be 'openai/gpt-4o-mini'" + llm_model["identifier"] == f"{expected_provider}/{expected_model}" + ), f"identifier should be '{expected_provider}/{expected_model}'" diff --git a/tests/e2e/features/steps/llm_query_response.py b/tests/e2e/features/steps/llm_query_response.py index f1e67e0d..b0f61845 100644 --- a/tests/e2e/features/steps/llm_query_response.py +++ b/tests/e2e/features/steps/llm_query_response.py @@ -4,6 +4,7 @@ import requests from behave import then, step # pyright: ignore[reportAttributeAccessIssue] from behave.runner import Context +from tests.e2e.utils.utils import replace_placeholders DEFAULT_LLM_TIMEOUT = 60 @@ -24,9 +25,11 @@ def ask_question(context: Context, endpoint: str) -> None: path = f"{context.api_prefix}/{endpoint}".replace("//", "/") url = base + path - # Use context.text if available, otherwise use empty query - data = json.loads(context.text or "{}") - print(data) + # Replace {MODEL} and {PROVIDER} placeholders with actual values + json_str = replace_placeholders(context, context.text or "{}") + + data = json.loads(json_str) + print(f"Request data: {data}") context.response = requests.post(url, json=data, timeout=DEFAULT_LLM_TIMEOUT) @@ -37,9 +40,11 @@ def ask_question_authorized(context: Context, endpoint: str) -> None: path = f"{context.api_prefix}/{endpoint}".replace("//", "/") url = base + path - # Use context.text if available, otherwise use empty query - data = json.loads(context.text or "{}") - print(data) + # Replace {MODEL} and {PROVIDER} placeholders with actual values + json_str = replace_placeholders(context, context.text or "{}") + + data = json.loads(json_str) + print(f"Request data: {data}") context.response = requests.post( url, json=data, headers=context.auth_headers, timeout=DEFAULT_LLM_TIMEOUT ) @@ -58,12 +63,14 @@ def ask_question_in_same_conversation(context: Context, endpoint: str) -> None: path = f"{context.api_prefix}/{endpoint}".replace("//", "/") url = base + path - # Use context.text if available, otherwise use empty query - data = json.loads(context.text or "{}") + # Replace {MODEL} and {PROVIDER} placeholders with actual values + json_str = replace_placeholders(context, context.text or "{}") + + data = json.loads(json_str) headers = context.auth_headers if hasattr(context, "auth_headers") else {} data["conversation_id"] = context.response_data["conversation_id"] - print(data) + print(f"Request data: {data}") context.response = requests.post( url, json=data, headers=headers, timeout=DEFAULT_LLM_TIMEOUT ) diff --git a/tests/e2e/features/streaming_query.feature b/tests/e2e/features/streaming_query.feature index ea5a3b57..17f9c978 100644 --- a/tests/e2e/features/streaming_query.feature +++ b/tests/e2e/features/streaming_query.feature @@ -51,7 +51,7 @@ Feature: streaming_query endpoint API tests Then The status code of the response is 200 And I use "streaming_query" to ask question with same conversation_id """ - {"query": "Write a simple code for reversing string", "system_prompt": "provide coding assistance", "model": "gpt-4-turbo", "provider": "openai"} + {"query": "Write a simple code for reversing string", "system_prompt": "provide coding assistance", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 When I wait for the response to be completed @@ -64,19 +64,19 @@ Feature: streaming_query endpoint API tests Given The system is in default state When I use "streaming_query" to ask question """ - {"provider": "openai"} + {"provider": "{PROVIDER}"} """ Then The status code of the response is 422 And The body of the response is the following """ - { "detail": [{"type": "missing", "loc": [ "body", "query" ], "msg": "Field required", "input": {"provider": "openai"}}] } + { "detail": [{"type": "missing", "loc": [ "body", "query" ], "msg": "Field required", "input": {"provider": "{PROVIDER}"}}] } """ Scenario: Check if LLM responds for streaming_query request with error for missing model Given The system is in default state When I use "streaming_query" to ask question """ - {"query": "Say hello", "provider": "openai"} + {"query": "Say hello", "provider": "{PROVIDER}"} """ Then The status code of the response is 422 And The body of the response contains Value error, Model must be specified if provider is specified @@ -85,7 +85,7 @@ Feature: streaming_query endpoint API tests Given The system is in default state When I use "streaming_query" to ask question """ - {"query": "Say hello", "model": "gpt-4-turbo"} + {"query": "Say hello", "model": "{MODEL}"} """ Then The status code of the response is 422 And The body of the response contains Value error, Provider must be specified if model is specified \ No newline at end of file diff --git a/tests/e2e/utils/utils.py b/tests/e2e/utils/utils.py index 54407350..e58a0a2b 100644 --- a/tests/e2e/utils/utils.py +++ b/tests/e2e/utils/utils.py @@ -6,6 +6,7 @@ import time import jsonschema from typing import Any +from behave.runner import Context def normalize_endpoint(endpoint: str) -> str: @@ -141,3 +142,19 @@ def restart_container(container_name: str) -> None: # Wait for container to be healthy wait_for_container_health(container_name) + + +def replace_placeholders(context: Context, text: str) -> str: + """Replace {MODEL} and {PROVIDER} placeholders with actual values from context. + + Args: + context: Behave context containing default_model and default_provider + text: String that may contain {MODEL} and {PROVIDER} placeholders + + Returns: + String with placeholders replaced by actual values + + """ + result = text.replace("{MODEL}", context.default_model) + result = result.replace("{PROVIDER}", context.default_provider) + return result