From 1e21612687a3b3408c8e18ab787776efb0bef8c5 Mon Sep 17 00:00:00 2001 From: Alexandre Marques Date: Fri, 6 Dec 2024 21:54:52 +0000 Subject: [PATCH 01/43] Adds aiohttp backend --- src/guidellm/backend/aiohttp.py | 160 ++++++++++++++++++++++++++++++++ src/guidellm/config.py | 4 + 2 files changed, 164 insertions(+) create mode 100644 src/guidellm/backend/aiohttp.py diff --git a/src/guidellm/backend/aiohttp.py b/src/guidellm/backend/aiohttp.py new file mode 100644 index 00000000..138f45a2 --- /dev/null +++ b/src/guidellm/backend/aiohttp.py @@ -0,0 +1,160 @@ +from typing import AsyncGenerator, Dict, List, Optional +from loguru import logger + +import aiohttp +import json + +from guidellm.backend.base import Backend, GenerativeResponse +from guidellm.config import settings +from guidellm.core import TextGenerationRequest + +__all__ = ["AiohttpBackend"] + +@Backend.register("aiohttp_server") +class AiohttpBackend(Backend): + """ + An aiohttp-based backend implementation for LLM requests. + + This class provides an interface to communicate with a server hosting + an LLM API using aiohttp for asynchronous requests. + """ + + def __init__( + self, + openai_api_key: Optional[str] = None, + target: Optional[str] = None, + model: Optional[str] = None, + timeout: Optional[float] = None, + **request_args, + ): + self._request_args: Dict = request_args + self._api_key: str = openai_api_key or settings.aiohttp.api_key + + if not self._api_key: + err = ValueError( + "`GUIDELLM__AIOHTTP__API_KEY` environment variable or " + "--openai-api-key CLI parameter must be specified for the " + "aiohttp backend." + ) + logger.error("{}", err) + raise err + + base_url = target or settings.aiohttp.base_url + self._api_url = f"{base_url}/chat/completions" + + if not base_url: + err = ValueError( + "`GUIDELLM__AIOHTTP__BASE_URL` environment variable or " + "target parameter must be specified for the OpenAI backend." + ) + logger.error("{}", err) + raise err + + self._timeout = aiohttp.ClientTimeout(total=timeout or settings.request_timeout) + self._model = model + + super().__init__(type_="aiohttp_backend", target=base_url, model=self._model) + logger.info("aiohttp {} Backend listening on {}", self._model, base_url) + + async def make_request( + self, + request: TextGenerationRequest, + ) -> AsyncGenerator[GenerativeResponse, None]: + """ + Make a request to the aiohttp backend. + + Sends a prompt to the LLM server and streams the response tokens. + + :param request: The text generation request to submit. + :type request: TextGenerationRequest + :yield: A stream of GenerativeResponse objects. + :rtype: AsyncGenerator[GenerativeResponse, None] + """ + + async with aiohttp.ClientSession(timeout=self._timeout) as session: + logger.debug("Making request to aiohttp backend with prompt: {}", request.prompt) + + request_args = {} + if request.output_token_count is not None: + request_args.update( + { + "max_completion_tokens": request.output_token_count, + "stop": None, + "ignore_eos": True, + } + ) + elif settings.aiohttp.max_gen_tokens and settings.aiohttp.max_gen_tokens > 0: + request_args.update( + { + "max_tokens": settings.aiohttp.max_gen_tokens, + } + ) + + request_args.update(self._request_args) + + payload = { + "model": self._model, + "messages": [ + {"role": "user", "content": request.prompt}, + ], + "stream": True, + **request_args, + } + + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {self._api_key}", + } + + try: + async with session.post(url=self._api_url, json=payload, headers=headers) as response: + if response.status != 200: + error_message = await response.text() + logger.error("Request failed: {} - {}", response.status, error_message) + raise Exception(f"Failed to generate response: {error_message}") + + token_count = 0 + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = chunk_bytes.decode("utf-8").removeprefix("data: ") + if chunk == "[DONE]": + # Final response + yield GenerativeResponse( + type_="final", + prompt=request.prompt, + output_token_count=token_count, + prompt_token_count=request.prompt_token_count, + ) + else: + # Intermediate token response + token_count += 1 + data = json.loads(chunk) + delta = data["choices"][0]["delta"] + token = delta["content"] + yield GenerativeResponse( + type_="token_iter", + add_token=token, + prompt=request.prompt, + output_token_count=token_count, + prompt_token_count=request.prompt_token_count, + ) + except Exception as e: + logger.error("Error while making request: {}", e) + raise + + def available_models(self) -> List[str]: + """ + Retrieve a list of available models from the server. + """ + # This could include an API call to `self._api_url/models` if the server supports it. + logger.warning("Fetching available models is not implemented for aiohttp backend.") + return [] + + def validate_connection(self): + """ + Validate the connection to the backend server. + """ + logger.info("Connection validation is not explicitly implemented for aiohttp backend.") diff --git a/src/guidellm/config.py b/src/guidellm/config.py index 2d4e102a..98dfb8f4 100644 --- a/src/guidellm/config.py +++ b/src/guidellm/config.py @@ -106,6 +106,9 @@ class OpenAISettings(BaseModel): max_output_tokens: int = 16384 +class AiohttpSettings(OpenAISettings): + pass + class ReportGenerationSettings(BaseModel): """ Report generation settings for the application @@ -154,6 +157,7 @@ class Settings(BaseSettings): preferred_output_tokens_source: Optional[Literal["backend", "local"]] = None preferred_backend: Literal["openai"] = "openai" openai: OpenAISettings = OpenAISettings() + aiohttp: AiohttpSettings = AiohttpSettings() # Report settings report_generation: ReportGenerationSettings = ReportGenerationSettings() From ce0c3c5ca68bd25874fb34200b618f4f5de3e0cb Mon Sep 17 00:00:00 2001 From: Alexandre Marques Date: Thu, 5 Sep 2024 01:19:42 +0000 Subject: [PATCH 02/43] quality fixes --- src/guidellm/core/result.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/guidellm/core/result.py b/src/guidellm/core/result.py index 2670c105..4d5a45e8 100644 --- a/src/guidellm/core/result.py +++ b/src/guidellm/core/result.py @@ -369,6 +369,7 @@ def inter_token_latency(self) -> float: """ return self.itl_distribution.mean + @computed_field # type: ignore[misc] @property def inter_token_latency_percentiles(self) -> Dict[str, float]: From 8cb98766882b78fe56d72196ecb8ad61e4611aa8 Mon Sep 17 00:00:00 2001 From: Alexandre Marques Date: Fri, 6 Sep 2024 14:28:54 +0000 Subject: [PATCH 03/43] Quality fixes --- src/guidellm/core/result.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/guidellm/core/result.py b/src/guidellm/core/result.py index 4d5a45e8..7bc91506 100644 --- a/src/guidellm/core/result.py +++ b/src/guidellm/core/result.py @@ -183,7 +183,7 @@ def __iter__(self): """ return iter(self.results) - @computed_field # type: ignore[misc] + @computed_field # type: ignore @property def request_count(self) -> int: """ @@ -369,7 +369,6 @@ def inter_token_latency(self) -> float: """ return self.itl_distribution.mean - @computed_field # type: ignore[misc] @property def inter_token_latency_percentiles(self) -> Dict[str, float]: From 039900aa089446042e10ab903776cb1f1678a047 Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Thu, 6 Mar 2025 13:46:31 +0000 Subject: [PATCH 04/43] Rework for OpenAI backend to use native http requests with httpx and http/2 support. Additionally, add in support for multi modal requests (needs further enablement in the rest of the system for future TODO). Still needs testing and test fixes --- src/guidellm/backend/aiohttp.py | 160 ---------------------- src/guidellm/config.py | 4 - src/guidellm/core/result.py | 2 +- tests/unit/backend/test_openai_backend.py | 1 + 4 files changed, 2 insertions(+), 165 deletions(-) delete mode 100644 src/guidellm/backend/aiohttp.py diff --git a/src/guidellm/backend/aiohttp.py b/src/guidellm/backend/aiohttp.py deleted file mode 100644 index 138f45a2..00000000 --- a/src/guidellm/backend/aiohttp.py +++ /dev/null @@ -1,160 +0,0 @@ -from typing import AsyncGenerator, Dict, List, Optional -from loguru import logger - -import aiohttp -import json - -from guidellm.backend.base import Backend, GenerativeResponse -from guidellm.config import settings -from guidellm.core import TextGenerationRequest - -__all__ = ["AiohttpBackend"] - -@Backend.register("aiohttp_server") -class AiohttpBackend(Backend): - """ - An aiohttp-based backend implementation for LLM requests. - - This class provides an interface to communicate with a server hosting - an LLM API using aiohttp for asynchronous requests. - """ - - def __init__( - self, - openai_api_key: Optional[str] = None, - target: Optional[str] = None, - model: Optional[str] = None, - timeout: Optional[float] = None, - **request_args, - ): - self._request_args: Dict = request_args - self._api_key: str = openai_api_key or settings.aiohttp.api_key - - if not self._api_key: - err = ValueError( - "`GUIDELLM__AIOHTTP__API_KEY` environment variable or " - "--openai-api-key CLI parameter must be specified for the " - "aiohttp backend." - ) - logger.error("{}", err) - raise err - - base_url = target or settings.aiohttp.base_url - self._api_url = f"{base_url}/chat/completions" - - if not base_url: - err = ValueError( - "`GUIDELLM__AIOHTTP__BASE_URL` environment variable or " - "target parameter must be specified for the OpenAI backend." - ) - logger.error("{}", err) - raise err - - self._timeout = aiohttp.ClientTimeout(total=timeout or settings.request_timeout) - self._model = model - - super().__init__(type_="aiohttp_backend", target=base_url, model=self._model) - logger.info("aiohttp {} Backend listening on {}", self._model, base_url) - - async def make_request( - self, - request: TextGenerationRequest, - ) -> AsyncGenerator[GenerativeResponse, None]: - """ - Make a request to the aiohttp backend. - - Sends a prompt to the LLM server and streams the response tokens. - - :param request: The text generation request to submit. - :type request: TextGenerationRequest - :yield: A stream of GenerativeResponse objects. - :rtype: AsyncGenerator[GenerativeResponse, None] - """ - - async with aiohttp.ClientSession(timeout=self._timeout) as session: - logger.debug("Making request to aiohttp backend with prompt: {}", request.prompt) - - request_args = {} - if request.output_token_count is not None: - request_args.update( - { - "max_completion_tokens": request.output_token_count, - "stop": None, - "ignore_eos": True, - } - ) - elif settings.aiohttp.max_gen_tokens and settings.aiohttp.max_gen_tokens > 0: - request_args.update( - { - "max_tokens": settings.aiohttp.max_gen_tokens, - } - ) - - request_args.update(self._request_args) - - payload = { - "model": self._model, - "messages": [ - {"role": "user", "content": request.prompt}, - ], - "stream": True, - **request_args, - } - - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {self._api_key}", - } - - try: - async with session.post(url=self._api_url, json=payload, headers=headers) as response: - if response.status != 200: - error_message = await response.text() - logger.error("Request failed: {} - {}", response.status, error_message) - raise Exception(f"Failed to generate response: {error_message}") - - token_count = 0 - async for chunk_bytes in response.content: - chunk_bytes = chunk_bytes.strip() - if not chunk_bytes: - continue - - chunk = chunk_bytes.decode("utf-8").removeprefix("data: ") - if chunk == "[DONE]": - # Final response - yield GenerativeResponse( - type_="final", - prompt=request.prompt, - output_token_count=token_count, - prompt_token_count=request.prompt_token_count, - ) - else: - # Intermediate token response - token_count += 1 - data = json.loads(chunk) - delta = data["choices"][0]["delta"] - token = delta["content"] - yield GenerativeResponse( - type_="token_iter", - add_token=token, - prompt=request.prompt, - output_token_count=token_count, - prompt_token_count=request.prompt_token_count, - ) - except Exception as e: - logger.error("Error while making request: {}", e) - raise - - def available_models(self) -> List[str]: - """ - Retrieve a list of available models from the server. - """ - # This could include an API call to `self._api_url/models` if the server supports it. - logger.warning("Fetching available models is not implemented for aiohttp backend.") - return [] - - def validate_connection(self): - """ - Validate the connection to the backend server. - """ - logger.info("Connection validation is not explicitly implemented for aiohttp backend.") diff --git a/src/guidellm/config.py b/src/guidellm/config.py index 98dfb8f4..2d4e102a 100644 --- a/src/guidellm/config.py +++ b/src/guidellm/config.py @@ -106,9 +106,6 @@ class OpenAISettings(BaseModel): max_output_tokens: int = 16384 -class AiohttpSettings(OpenAISettings): - pass - class ReportGenerationSettings(BaseModel): """ Report generation settings for the application @@ -157,7 +154,6 @@ class Settings(BaseSettings): preferred_output_tokens_source: Optional[Literal["backend", "local"]] = None preferred_backend: Literal["openai"] = "openai" openai: OpenAISettings = OpenAISettings() - aiohttp: AiohttpSettings = AiohttpSettings() # Report settings report_generation: ReportGenerationSettings = ReportGenerationSettings() diff --git a/src/guidellm/core/result.py b/src/guidellm/core/result.py index 7bc91506..2670c105 100644 --- a/src/guidellm/core/result.py +++ b/src/guidellm/core/result.py @@ -183,7 +183,7 @@ def __iter__(self): """ return iter(self.results) - @computed_field # type: ignore + @computed_field # type: ignore[misc] @property def request_count(self) -> int: """ diff --git a/tests/unit/backend/test_openai_backend.py b/tests/unit/backend/test_openai_backend.py index db03c259..62ca369f 100644 --- a/tests/unit/backend/test_openai_backend.py +++ b/tests/unit/backend/test_openai_backend.py @@ -1,5 +1,6 @@ import time +import httpx import pytest from guidellm.backend import OpenAIHTTPBackend, ResponseSummary, StreamingTextResponse From 2b82bd5a3e53656836c553d7e9f75f71c91934c6 Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Sat, 8 Mar 2025 10:25:36 +0000 Subject: [PATCH 05/43] Finalize implementation, fix bugs, and ensure unit tests are passing / in place --- tests/unit/backend/test_openai_backend.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/backend/test_openai_backend.py b/tests/unit/backend/test_openai_backend.py index 62ca369f..db03c259 100644 --- a/tests/unit/backend/test_openai_backend.py +++ b/tests/unit/backend/test_openai_backend.py @@ -1,6 +1,5 @@ import time -import httpx import pytest from guidellm.backend import OpenAIHTTPBackend, ResponseSummary, StreamingTextResponse From a01db7d7650f039f80805f9e39bddeaa971f5bf4 Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Mon, 10 Mar 2025 15:27:15 +0000 Subject: [PATCH 06/43] Initial state for Scheduling system rework. Still needs testing and further integration / refactors. --- src/guidellm/backend/openai.py | 4 + src/guidellm/backend/response.py | 6 + src/guidellm/config.py | 1 + src/guidellm/executor/executor.py | 4 +- src/guidellm/scheduler/__init__.py | 36 +- src/guidellm/scheduler/backend_worker.py | 257 +++++++ src/guidellm/scheduler/load_generator.py | 196 ----- src/guidellm/scheduler/scheduler.py | 901 ++++++++++++++--------- src/guidellm/scheduler/strategy.py | 281 +++++++ src/guidellm/scheduler/test.py | 40 + 10 files changed, 1171 insertions(+), 555 deletions(-) create mode 100644 src/guidellm/scheduler/backend_worker.py delete mode 100644 src/guidellm/scheduler/load_generator.py create mode 100644 src/guidellm/scheduler/strategy.py create mode 100644 src/guidellm/scheduler/test.py diff --git a/src/guidellm/backend/openai.py b/src/guidellm/backend/openai.py index 7870a949..9380ffad 100644 --- a/src/guidellm/backend/openai.py +++ b/src/guidellm/backend/openai.py @@ -410,6 +410,8 @@ async def _iterative_completions_request( yield StreamingTextResponse( type_="start", + value="", + start_time=start_time, iter_count=iter_count, delta="", time=start_time, @@ -443,6 +445,8 @@ async def _iterative_completions_request( yield StreamingTextResponse( type_="iter", + value=response_value, + start_time=start_time, iter_count=iter_count, delta=delta, time=iter_time, diff --git a/src/guidellm/backend/response.py b/src/guidellm/backend/response.py index 699f41cc..8265a427 100644 --- a/src/guidellm/backend/response.py +++ b/src/guidellm/backend/response.py @@ -21,6 +21,8 @@ class StreamingTextResponse(BaseModel): A model representing the response content for a streaming text request. :param type_: The type of the response; either 'start' or 'iter'. + :param value: The value of the response up to this iteration. + :param start_time: The time.time() the request started. :param iter_count: The iteration count for the response. For 'start' this is 0 and for the first 'iter' it is 1. :param delta: The text delta added to the response for this stream iteration. @@ -30,6 +32,8 @@ class StreamingTextResponse(BaseModel): """ type_: StreamingResponseType + value: str + start_time: float iter_count: int delta: str time: float @@ -69,6 +73,7 @@ class ResponseSummary(BaseModel): :param prompt_tokens: The number of tokens in the prompt, if any usage was returned. :param output_tokens: The number of tokens in the output, if any usage was returned. :param request_id: The unique identifier for the request, if any. + :param error: The error message, if any, returned from making the request. """ value: str @@ -81,6 +86,7 @@ class ResponseSummary(BaseModel): response_prompt_tokens: Optional[int] = None response_output_tokens: Optional[int] = None request_id: Optional[str] = None + error: Optional[str] = None @computed_field # type: ignore[misc] @property diff --git a/src/guidellm/config.py b/src/guidellm/config.py index 2d4e102a..a16787dd 100644 --- a/src/guidellm/config.py +++ b/src/guidellm/config.py @@ -142,6 +142,7 @@ class Settings(BaseSettings): request_timeout: int = 60 * 5 # 5 minutes request_http2: bool = True max_concurrency: int = 512 + max_worker_processes: int = 10 num_sweep_profiles: int = 9 logging: LoggingSettings = LoggingSettings() diff --git a/src/guidellm/executor/executor.py b/src/guidellm/executor/executor.py index bfecf17f..030904b1 100644 --- a/src/guidellm/executor/executor.py +++ b/src/guidellm/executor/executor.py @@ -11,7 +11,7 @@ ProfileGenerator, ) from guidellm.request import RequestGenerator -from guidellm.scheduler import Scheduler, SchedulerResult +from guidellm.scheduler import Scheduler __all__ = ["Executor", "ExecutorResult"] @@ -38,7 +38,7 @@ class ExecutorResult: count_completed: int generation_modes: Sequence[ProfileGenerationMode] report: TextGenerationBenchmarkReport - scheduler_result: Optional[SchedulerResult] = None + scheduler_result = None current_index: Optional[int] = None current_profile: Optional[Profile] = None diff --git a/src/guidellm/scheduler/__init__.py b/src/guidellm/scheduler/__init__.py index 39485648..d6cabb33 100644 --- a/src/guidellm/scheduler/__init__.py +++ b/src/guidellm/scheduler/__init__.py @@ -1,4 +1,34 @@ -from .load_generator import LoadGenerationMode, LoadGenerator -from .scheduler import Scheduler, SchedulerResult +from .backend_worker import BackendRequestsWorker, GenerationRequest +from .scheduler import ( + RequestsWorker, + Scheduler, + SchedulerRequestInfo, + SchedulerResult, + SchedulerRunInfo, +) +from .strategy import ( + AsyncConstantStrategy, + AsyncPoissonStrategy, + ConcurrentStrategy, + SchedulingStrategy, + StrategyType, + SynchronousStrategy, + ThroughputStrategy, +) -__all__ = ["LoadGenerationMode", "LoadGenerator", "Scheduler", "SchedulerResult"] +__all__ = [ + "GenerationRequest", + "BackendRequestsWorker", + "Scheduler", + "SchedulerResult", + "SchedulerRunInfo", + "SchedulerRequestInfo", + "RequestsWorker", + "StrategyType", + "SchedulingStrategy", + "SynchronousStrategy", + "ThroughputStrategy", + "ConcurrentStrategy", + "AsyncConstantStrategy", + "AsyncPoissonStrategy", +] diff --git a/src/guidellm/scheduler/backend_worker.py b/src/guidellm/scheduler/backend_worker.py new file mode 100644 index 00000000..05609672 --- /dev/null +++ b/src/guidellm/scheduler/backend_worker.py @@ -0,0 +1,257 @@ +import asyncio +import math +import time +import uuid +from typing import ( + Any, + AsyncGenerator, + Dict, + Literal, + Optional, + Tuple, + Union, +) + +from pydantic import BaseModel, Field + +from guidellm.backend import ( + Backend, + RequestArgs, + ResponseSummary, + StreamingTextResponse, +) +from guidellm.scheduler.scheduler import RequestsWorker + +__all__ = ["GenerationRequest", "BackendRequestsWorker"] + + +class GenerationRequest(BaseModel): + """ + A class representing a request for generation. + This class is used to encapsulate the details of a generation request, + including the request ID, type, content, parameters, statistics, and constraints. + It is designed to be used with the BackendRequestsWorker class to handle + the generation process. + + :param request_id: The unique identifier for the request. + :param request_type: The type of request (e.g., text, chat). + :param content: The content for the request to send to the backend. + If request_type is 'text', this should be a string or list of strings + which will be resolved by backend.text_completions. + If request_type is 'chat', this should be a string, + a list of (str, Dict[str, Union[str, Dict[str, str]], Path, Image]), + or Any raw content which will be resolved by backend.chat_completions. + If raw content, raw_content=True must be passed in the params. + :param params: Additional parameters for the request passed in as kwargs. + For an http backend, these are passed into the body of the request. + :param stats: Statistics for the request, such as the number of prompt tokens. + Used for tracking and reporting purposes. + :param constraints: Constraints for the request, such as the maximum number + of output tokens. Used for controlling the behavior of the backend. + """ + + request_id: Optional[str] = Field( + default_factory=lambda: str(uuid.uuid4()), + description="The unique identifier for the request.", + ) + request_type: Literal["text", "chat"] = Field( + default="text", + description=( + "The type of request (e.g., text, chat). " + "If request_type is 'text', resolved by backend.text_completions. " + "If request_type is 'chat', resolved by backend.chat_completions." + ), + ) + content: Any = Field( + description=( + "The content for the request to send to the backend. " + "If request_type is 'text', this should be a string or list of strings " + "which will be resolved by backend.text_completions. " + "If request_type is 'chat', this should be a string, " + "a list of (str, Dict[str, Union[str, Dict[str, str]], Path, Image]), " + "or Any raw content which will be resolved by backend.chat_completions. " + "If raw content, raw_content=True must be passed in the params." + ) + ) + params: Dict[str, Any] = Field( + default_factory=dict, + description=( + "Additional parameters for the request that will be passed in as kwargs. " + "For an http backend, these are passed into the body of the request. " + ), + ) + stats: Dict[Literal["prompt_tokens"], int] = Field( + default_factory=dict, + description=( + "Statistics for the request, such as the number of prompt tokens. " + "Used for tracking and reporting purposes." + ), + ) + constraints: Dict[Literal["output_tokens"], int] = Field( + default_factory=dict, + description=( + "Constraints for the request, such as the maximum number of output tokens. " + "Used for controlling the behavior of the backend." + ), + ) + + +class BackendRequestsWorker(RequestsWorker): + """ + A class that handles the execution of requests using a backend. + This class is responsible for sending requests to the backend, + handling responses, and managing errors. + + :param backend: The backend to use for handling requests. + This should be an instance of Backend such as an OpenAIHTTPBackend. + """ + + def __init__(self, backend: Backend): + self.backend = backend + + async def resolve( + self, + request: GenerationRequest, + start_time: float, + timeout_time: float, + ) -> ResponseSummary: + """ + Resolve a request by sending it to the backend and handling the response. + This method sends the request to the backend, waits for a response, + and handles any errors that may occur during the process. + + :param request: The request to resolve. + :param start_time: The time to start the request. + :param timeout_time: The time to wait for a response before timing out. + If timeout_time is math.inf, the request will not timeout. + :return: A ResponseSummary object containing the response from the backend. + If an error occurs, the ResponseSummary will contain the error message. + """ + response = None + error: Optional[str] = None + + try: + request_func, request_kwargs = self._create_request_func_kwargs(request) + + async def _runner(): + # wrap function so we can enforce timeout and + # still return the latest state from the backend + async for resp in request_func(**request_kwargs): + nonlocal response + response = resp + + if (wait_time := start_time - time.time()) > 0: + await asyncio.sleep(wait_time) + + start_time = time.time() + await asyncio.wait_for( + _runner(), + timeout=timeout_time - time.time() if timeout_time < math.inf else None, + ) + + if not response: + raise ValueError( + f"No response received for request: {request} " + f"and backend: {self.backend}" + ) + if not isinstance(response, ResponseSummary): + raise ValueError( + f"Received no ResponseSummary for request: {request} " + f"and backend: {self.backend}, received: {response}" + ) + except asyncio.TimeoutError as texc: + error = str(texc) + except Exception as exc: # noqa: BLE001 + error = str(exc) + + return self._handle_response(request, response, error, start_time) + + def _create_request_func_kwargs( + self, + request: GenerationRequest, + ) -> Tuple[ + AsyncGenerator[Union[StreamingTextResponse, ResponseSummary], None], + Dict[str, Any], + ]: + request_func: AsyncGenerator[ + Union[StreamingTextResponse, ResponseSummary], None + ] + request_kwargs: Dict[str, Any] + + if request.request_type == "text": + request_func = self.backend.text_completions + request_kwargs = { + "prompt": request.content, + "request_id": request.request_id, + "prompt_token_count": request.stats.get("prompt_tokens", None), + "output_token_count": request.constraints.get("output_tokens", None), + **request.params, + } + elif request.request_type == "chat": + request_func = self.backend.chat_completions + request_kwargs = { + "content": request.content, + "request_id": request.request_id, + "prompt_token_count": request.stats.get("prompt_tokens", None), + "output_token_count": request.constraints.get("output_tokens", None), + **request.params, + } + else: + raise ValueError( + f"Invalid request type: {request.request_type} for {request}" + ) + + return request_func, request_kwargs + + def _handle_response( + self, + request: GenerationRequest, + response: Any, + error: Optional[str], + start_time: float, + ) -> ResponseSummary: + if response is None or not isinstance( + response, (ResponseSummary, StreamingTextResponse) + ): + # nothing received or invalid response, fill in defaults for error + if response: + error = str( + ValueError( + f"Invalid response: {type(response)} for request: {request}; " + ) + ) + (error or "") + + return ResponseSummary( + value="", + request_args=RequestArgs( + target=self.backend.target, + headers={}, + payload={}, + ), + start_time=start_time, + end_time=time.time(), + request_id=request.request_id, + error=error or "Unknown error", + ) + + if isinstance(response, StreamingTextResponse): + return ResponseSummary( + value=response.value, + request_args=RequestArgs( + target=self.backend.target, + headers={}, + payload={}, + ), + start_time=response.start_time, + end_time=time.time(), + request_prompt_tokens=request.stats.get("prompt_tokens", None), + request_output_tokens=None, + response_prompt_tokens=None, + response_output_tokens=response.iter_count, + request_id=request.request_id, + error=error or "Unknown error", + ) + + response.error = error + + return response diff --git a/src/guidellm/scheduler/load_generator.py b/src/guidellm/scheduler/load_generator.py deleted file mode 100644 index f629752a..00000000 --- a/src/guidellm/scheduler/load_generator.py +++ /dev/null @@ -1,196 +0,0 @@ -import time -from typing import Generator, Literal, Optional, get_args - -import numpy as np -from loguru import logger - -__all__ = ["LoadGenerationMode", "LoadGenerator"] - -LoadGenerationMode = Literal["synchronous", "constant", "poisson", "throughput"] - - -class LoadGenerator: - """ - Load Generator class that generates timestamps for load generation. - - This class supports multiple load generation modes: "constant", "poisson", - "throughput", and "synchronous". Each mode has its own method for generating - timestamps based on the rate provided during initialization. - - :param mode: The mode of load generation. Valid options are "constant", - "poisson", "throughput", and "synchronous". - :type mode: LoadGenerationMode - :param rate: The rate at which to generate timestamps. This value is - interpreted differently depending on the mode. - :type rate: float - - :raises ValueError: If an invalid mode is provided. - """ - - def __init__(self, mode: LoadGenerationMode, rate: Optional[float] = None): - """ - Initialize the Load Generator with the mode and rate. - - :param mode: The mode of load generation ("constant", "poisson", "throughput", - or "synchronous"). - :type mode: LoadGenerationMode - :param rate: The rate at which to generate timestamps. In the "constant" - mode, this represents the frequency of events. In the "poisson" mode, - it represents the average frequency. - :type rate: Optional[float] - """ - if mode not in get_args(LoadGenerationMode): - error = ValueError( - f"{mode} is not a valid Load Generation Mode. " - f"Valid options are {get_args(LoadGenerationMode)}" - ) - logger.error(error) - raise error - - if mode not in ["synchronous", "throughput"] and (rate is None or rate <= 0): - error = ValueError(f"Rate must be > 0 for mode: {mode}. Given: {rate}") - logger.error(error) - raise error - - self._mode = mode - self._rate = rate - logger.debug( - "Initialized LoadGenerator with mode: {mode}, rate: {rate}", - mode=mode, - rate=rate, - ) - - @property - def mode(self) -> LoadGenerationMode: - """ - Get the mode of load generation. - - :return: The mode of load generation. - :rtype: LoadGenerationMode - """ - return self._mode - - @property - def rate(self) -> Optional[float]: - """ - Get the rate of load generation. - - :return: The rate of load generation. - :rtype: Optional[float] - """ - return self._rate - - def times(self) -> Generator[float, None, None]: - """ - Generate timestamps for load generation based on the selected mode. - - :return: A generator that yields timestamps at which each load - should be initiated. - :rtype: Generator[float, None, None] - - :raises ValueError: If the mode is invalid. - """ - logger.debug(f"Generating timestamps using mode: {self._mode}") - - if self._mode == "throughput": - yield from self.throughput_times() - elif self._mode == "constant": - yield from self.constant_times() - elif self._mode == "poisson": - yield from self.poisson_times() - elif self._mode == "synchronous": - yield from self.synchronous_times() - else: - logger.error(f"Invalid mode encountered: {self._mode}") - raise ValueError(f"Invalid mode: {self._mode}") - - def synchronous_times(self) -> Generator[float, None, None]: - """ - Generate invalid timestamps for the "synchronous" mode. - - :return: A generator that yields a constant invalid timestamp (-1.0). - :rtype: Generator[float, None, None] - """ - logger.debug("Generating invalid timestamps for synchronous mode") - while True: - yield -1.0 - - def throughput_times(self) -> Generator[float, None, None]: - """ - Generate timestamps at the maximum rate possible, returning the current time. - - :return: A generator that yields the current time in seconds. - :rtype: Generator[float, None, None] - """ - logger.debug("Generating timestamps at throughput rate") - while True: - yield time.time() - - def constant_times(self) -> Generator[float, None, None]: - """ - Generate timestamps at a constant rate based on the specified rate. - - :return: A generator that yields timestamps incremented by 1/rate seconds. - :rtype: Generator[float, None, None] - """ - logger.debug("Generating constant rate timestamps with rate: {}", self._rate) - - if self._rate is None or self._rate == 0: - raise ValueError( - "Rate must be > 0 for constant mode, given: {}", self._rate - ) - - start_time = time.time() - time_increment = 1.0 / self._rate - counter = 0 - - while True: - yield_time = start_time + time_increment * counter - logger.debug(f"Yielding timestamp: {yield_time}") - yield yield_time - counter += 1 - - def poisson_times(self) -> Generator[float, None, None]: - """ - Generate timestamps based on a Poisson process, where the number - of requests to be sent per second is drawn from a Poisson distribution. - The inter arrival time between requests is exponentially distributed. - - :return: A generator that yields timestamps based on a Poisson distribution. - :rtype: Generator[float, None, None] - """ - logger.debug("Generating Poisson rate timestamps with rate: {}", self._rate) - - if self._rate is None or self._rate == 0: - raise ValueError("Rate must be > 0 for poisson mode, given: {}", self._rate) - - time_tracker = time.time() - rng = np.random.default_rng() - time_increment = 1.0 - - while True: - num_requests = rng.poisson(self._rate) - - if num_requests == 0: - yield time_tracker + time_increment - else: - inter_arrival_times = rng.exponential(1.0 / self._rate, num_requests) - logger.debug( - "Calculated new inter-arrival times for poisson process: {}", - inter_arrival_times, - ) - arrival_time_tracker = time_tracker - - for arrival_time in inter_arrival_times: - arrival_time_tracker += arrival_time - - if arrival_time_tracker > time_tracker + time_increment: - logger.debug( - "Arrival time tracker: {} is greater than current time", - arrival_time_tracker, - ) - break - - yield arrival_time_tracker - - time_tracker += time_increment # Move on to the next time period diff --git a/src/guidellm/scheduler/scheduler.py b/src/guidellm/scheduler/scheduler.py index 2f8c44fe..65d791f5 100644 --- a/src/guidellm/scheduler/scheduler.py +++ b/src/guidellm/scheduler/scheduler.py @@ -1,417 +1,610 @@ import asyncio +import concurrent.futures import math +import multiprocessing +import multiprocessing.queues +import os import time +from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import AsyncGenerator, Literal, Optional, Union, get_args +from typing import ( + Any, + AsyncGenerator, + Iterable, + Iterator, + List, + Literal, + Optional, + Tuple, + Union, +) -from loguru import logger +from pydantic import BaseModel -from guidellm.backend import Backend, ResponseSummary, StreamingTextResponse from guidellm.config import settings -from guidellm.core import ( - TextGenerationBenchmark, - TextGenerationError, - TextGenerationRequest, - TextGenerationResult, +from guidellm.scheduler.strategy import ( + AsyncConstantStrategy, + AsyncPoissonStrategy, + ConcurrentStrategy, + SchedulingStrategy, + SynchronousStrategy, + ThroughputStrategy, ) -from guidellm.request import RequestGenerator -from guidellm.scheduler.load_generator import LoadGenerationMode, LoadGenerator -__all__ = ["Scheduler", "SchedulerResult"] +__all__ = [ + "Scheduler", + "SchedulerResult", + "SchedulerRunInfo", + "SchedulerRequestInfo", + "RequestsWorker", +] -@dataclass -class SchedulerResult: +class RequestsWorker(ABC): """ - Represents the result of a single task execution within the Scheduler. - - :param completed: Indicates if the task is completed. - :type completed: bool - :param count_total: Total number of tasks to be executed. - :type count_total: int - :param count_completed: Number of tasks that have been completed so far. - :type count_completed: int - :param report: Benchmark data for the task execution. - :type benchmark: TextGenerationBenchmark - :param current_result: The result of the current request, if any. - :type current_result: Optional[Union[TextGenerationResult, Exception]] + An abstract base class for a worker that processes requests. + This class defines the interface for a worker that can resolve requests + asynchronously or synchronously within the Scheduler class. + Subclasses must implement the `resolve` method, + which takes a request directly given from the load generator, + along with the desired start_time for the request and a timeout_time. + The `resolve` method should return the response from the backend. """ - completed: bool - count_total: int - count_completed: int - benchmark: TextGenerationBenchmark - current_result: Optional[Union[TextGenerationResult, TextGenerationError]] = None + @abstractmethod + async def resolve( + self, + request: Any, + start_time: float, + timeout_time: float, + ) -> Any: + """ + An abstract method that must be implemented by subclasses. + This method should handle the resolution of a request through asyncio, + including any necessary backend processing and response handling. + + :param request: The request to be resolved generated by the load generator. + :param start_time: The desired start time for the request. + :param timeout_time: The timeout time for the request, if there is no timeout + given, then this will be math.inf. + :return: The response from the worker. + """ + ... -class Scheduler: +class SchedulerRunInfo(BaseModel): """ - Schedules and manages the execution of tasks for text generation requests. - - :param generator: The request generator that produces text generation requests. - :type generator: RequestGenerator - :param backend: The backend that processes the requests. - :type backend: Backend - :param mode: The mode of load generation (e.g., synchronous, asynchronous). - :type mode: LoadGenerationMode - :param rate: The rate at which requests are generated, if applicable. - :type rate: Optional[float] - :param max_number: Maximum number of requests to be processed. - :type max_number: Optional[int] - :param max_duration: Maximum duration in seconds for which requests - should be processed. - :type max_duration: Optional[float] - - :raises ValueError: If neither max_number nor max_duration is specified or - if they are not positive. + Information about the current run of the scheduler. + This class holds metadata about the scheduling run, + including the start and end times, the number of processes, + and the scheduling strategy used. + It also tracks the number of requests created, queued, pending, + and completed during the run. + + :param start_time: The start time of the scheduling run. + :param end_time: The end time of the scheduling run; + if None, then this will be math.inf. + :param end_number: The maximum number of requests to be processed; + if None, then this will be math.inf. + :param processes: The number of processes used in the scheduling run. + :param strategy: The scheduling strategy used in the run. + This should be an instance of SchedulingStrategy. + :param created_requests: The number of requests created during the run. + :param queued_requests: The number of requests queued during the run. + :param pending_requests: The number of requests pending during the run. + :param completed_requests: The number of requests completed during the run. """ - def __init__( - self, - generator: RequestGenerator, - backend: Backend, - mode: LoadGenerationMode = "synchronous", - rate: Optional[float] = None, - max_number: Optional[int] = None, - max_duration: Optional[float] = None, - ): - logger.info( - "Scheduler initialized with params: generator={}, backend={}, mode={}, " - "rate={}, max_number={}, max_duration={}", - generator, - backend, - mode, - rate, - max_number, - max_duration, - ) - - if mode not in get_args(LoadGenerationMode): - err = ValueError( - f"{mode} is not a valid Load Generation Mode. " - f"Valid options are {get_args(LoadGenerationMode)}" - ) - logger.error(err) - raise err - - if not max_number and not max_duration: - err = ValueError("Either max_number or max_duration must be specified") - logger.error(err) - raise err - - if max_number and max_number <= 0: - err = ValueError(f"max_number must be > 0, given: {max_number}") - logger.error(err) - raise err - - if max_duration and max_duration <= 0: - err = ValueError(f"max_duration must be > 0, given: {max_duration}") - logger.error(err) - raise err - - if mode in ["constant", "poisson"] and not rate: - err = ValueError(f"Rate must be > 0 for mode: {mode}. Given: {rate}") - logger.error(err) - raise err - - self._generator = generator - self._backend = backend - self._mode = mode - self._rate = rate - self._max_number = max_number - self._max_duration = max_duration - - self._load_generator = LoadGenerator(mode, rate) + start_time: float + end_time: float + end_number: float + processes: int + strategy: SchedulingStrategy - @property - def generator(self) -> RequestGenerator: - """ - The request generator that produces text generation requests. + created_requests: int = 0 + queued_requests: int = 0 + pending_requests: int = 0 + completed_requests: int = 0 - :return: The request generator instance. - :rtype: RequestGenerator - """ - return self._generator - @property - def backend(self) -> Backend: - """ - The backend that processes the requests. +class SchedulerRequestInfo(BaseModel): + """ + Information about a specific request run through the scheduler. + This class holds metadata about the request, including + the targeted start time, queued time, start time, end time, + and the process ID that handled the request. + + :param targeted_start_time: The targeted start time for the request (time.time()). + :param queued_time: The time the request was queued (time.time()). + :param start_time: The actual start time of the request (time.time()). + :param end_time: The end time of the request (time.time()). + :param process_id: The ID of the underlying process that handled the request. + """ - :return: The backend instance. - :rtype: Backend - """ - return self._backend + targeted_start_time: float = -1 + queued_time: float = -1 + start_time: float = -1 + end_time: float = -1 + process_id: int = -1 - @property - def mode(self) -> LoadGenerationMode: - """ - The mode of load generation (e.g., synchronous, asynchronous). - :return: The load generation mode. - :rtype: LoadGenerationMode - """ - return self._mode +class SchedulerResult(BaseModel): + """ + The yielded, iterative result for a scheduler run. + These are triggered on the start and end of the run, + as well as on the start and end of each request. + Depending on the type, it will hold the request and response + along with information and statistics about the request and general run. + + :param type_: The type of the result, which can be one of: + - "run_start": Indicates the start of the run. + - "run_complete": Indicates the completion of the run (teardown happens after). + - "request_start": Indicates the start of a request. + - "request_complete": Indicates the completion of a request. + :param request: The request that was processed. + :param response: The response from the worker for the request. + :param request_info: Information about the request, including + the targeted start time, queued time, start time, end time, + and the process ID that handled the request. + :param run_info: Information about the current run of the scheduler, + including the start and end times, the number of processes, + and the scheduling strategy used. + It also tracks the number of requests created, queued, pending, + and completed during the run. + """ - @property - def rate(self) -> Optional[float]: - """ - The rate at which requests are generated, if applicable. + type_: Literal["run_start", "run_complete", "request_start", "request_complete"] + request: Any + response: Any + request_info: Optional[SchedulerRequestInfo] + run_info: SchedulerRunInfo - :return: The rate of request generation. - :rtype: Optional[float] - """ - return self._rate - @property - def max_number(self) -> Optional[int]: - """ - Maximum number of requests to be processed. +@dataclass +class _WorkerProcessRequest: + request: Any + start_time: float + timeout_time: Optional[float] + queued_time: float - :return: The maximum number of requests. - :rtype: Optional[int] - """ - return self._max_number - @property - def max_duration(self) -> Optional[float]: - """ - Maximum duration in seconds for which requests should be processed. +@dataclass +class _WorkerProcessResponse: + type_: Literal["request_start", "request_complete"] + request: Any + response: Any + info: SchedulerRequestInfo - :return: The maximum duration in seconds. - :rtype: Optional[float] - """ - return self._max_duration - @property - def load_generator(self) -> LoadGenerator: - """ - The load generator responsible for generating load based on mode and rate. +class Scheduler: + """ + A class that handles the scheduling of requests to a worker. + This class is responsible for managing the lifecycle of the requests, + including their creation, queuing, and processing. + It uses a multiprocessing approach to handle requests concurrently + and efficiently, based on the specified scheduling strategy. + The Scheduler class is designed to work with a RequestsWorker, + which is an abstract base class that defines the interface for a worker + that can resolve requests asynchronously or synchronously. + The Scheduler class also supports different scheduling strategies, + including synchronous, throughput, and concurrent strategies. + + :param worker: The worker that will process the requests. + This should be an instance of RequestsWorker. + :param request_loader: An iterable that generates requests. + This can be a list, generator, or any other iterable. + The requests will be processed by the worker. + :param scheduling_strategy: The scheduling strategy to use. + Specifies the times at which requests will be sent as well how many + worker processes are used and if requests are scheduled sync or async. + This can be one of the following: + - "synchronous": Requests are sent synchronously. + - "throughput": Requests are sent at the maximum rate possible. + - An instance of SchedulingStrategy. + :param max_number: The maximum number of requests to process. + If None, then no limit is set and either the iterator must be exhaustible + or the max_duration must be set. + :param max_duration: The maximum duration for the scheduling run. + If None, then no limit is set and either the iterator must be exhaustible + or the max_number must be set. + :param num_processes: The number of processes to use for the worker. + If None, then the number of processes is set to the number of CPU cores + minus one, or the max_worker_processes setting if it is lower. + If the scheduling strategy is synchronous, then this is set to 1. + If the scheduling strategy is concurrent, then this is set to the number + of streams in the strategy. + """ - :return: The load generator instance. - :rtype: LoadGenerator - """ - return self._load_generator + def __init__( + self, + worker: RequestsWorker, + request_loader: Iterable[Any], + scheduling_strategy: Union[ + Literal["synchronous", "throughput"], SchedulingStrategy + ] = "throughput", + max_number: Optional[int] = None, + max_duration: Optional[float] = None, + num_processes: Optional[int] = None, + ): + if not isinstance(worker, RequestsWorker): + raise ValueError(f"Invalid worker: {worker}") - @property - def benchmark_mode(self) -> Literal["asynchronous", "synchronous", "throughput"]: - """ - The report mode for the scheduler. + if not isinstance(request_loader, Iterable): + raise ValueError(f"Invalid request_loader: {request_loader}") - :return: The report mode. - :rtype: Literal["asynchronous", "synchronous", "throughput"] - """ - if self._mode == "synchronous": - return "synchronous" + if scheduling_strategy == "synchronous": + scheduling_strategy = SynchronousStrategy() + elif scheduling_strategy == "throughput": + scheduling_strategy = ThroughputStrategy() - if self._mode == "throughput": - return "throughput" + if not isinstance(scheduling_strategy, SchedulingStrategy): + raise ValueError(f"Invalid scheduling strategy: {scheduling_strategy}") - return "asynchronous" + self._worker = worker + self._request_loader = request_loader + self._scheduling_strategy: SchedulingStrategy = scheduling_strategy + self._max_number = max_number + self._max_duration = max_duration + self._num_processes = num_processes async def run(self) -> AsyncGenerator[SchedulerResult, None]: """ - Run the scheduler to process requests based on the configured mode, rate, - maximum number, and maximum duration. - - :yield: The result of each task executed by the scheduler. - :rtype: Generator[SchedulerResult, None, None] + The main method that runs the scheduler. + This method is a generator that yields SchedulerResult objects + at the start and end of the run, as well as at the start and end + of each request. + It uses multiprocessing to handle requests concurrently + and efficiently, based on the specified scheduling strategy. + The method also handles the lifecycle of the requests, + including their creation, queuing, and processing. + The method is designed to be used as an asynchronous generator, + allowing it to be used with asyncio and other asynchronous frameworks. + + :return: An asynchronous generator that yields SchedulerResult objects. + Each SchedulerResult object contains information about the request, + the response, and the run information. """ - logger.info("Starting Scheduler run") - - benchmark = TextGenerationBenchmark(mode=self.benchmark_mode, rate=self.rate) - start_time = time.time() - end_time = start_time + self.max_duration if self.max_duration else math.inf - max_number = float(self.max_number) if self.max_number else math.inf - runner = self._run_sync if self._mode == "synchronous" else self._run_async - count_total = ( - self.max_number - if self.max_number - else round(self.max_duration) - if self.max_duration - else 0 - ) - # yield initial result for progress tracking - yield SchedulerResult( - completed=False, - count_total=count_total, - count_completed=0, - benchmark=benchmark, - ) - - run_count = 0 - async for res in runner(benchmark, end_time, max_number): - run_count += 1 - count_completed = ( - min(run_count, self.max_number) - if self.max_number - else round(time.time() - start_time) - if self.max_duration - else 0 + with ( + multiprocessing.Manager() as manager, + concurrent.futures.ProcessPoolExecutor() as executor, + ): + futures, requests_queue, responses_queue = await self._start_processes( + manager, executor ) + run_info, requests_iter, times_iter = self._run_setup(futures) + yield SchedulerResult( + type_="run_start", + request=None, + response=None, + request_info=None, + run_info=run_info, + ) + + while True: + if ( + requests_iter is None + and run_info.completed_requests >= run_info.created_requests + ): + # we've exhausted all requests we've wanted to run + # and yielded all responses + break + + if requests_iter is not None and not requests_queue.full(): + # we have space on the queue, try to add more requests + # if we've reached the limit number/time or we've exhausted requests + # then set requests_iter to None to stop adding more + try: + request_time = next(times_iter) + if (run_info.queued_requests >= run_info.end_number) or ( + request_time >= run_info.end_time + ): + raise StopIteration + + request = next(requests_iter) + requests_queue.put( + _WorkerProcessRequest( + request=request, + start_time=request_time, + timeout_time=run_info.end_time, + queued_time=time.time(), + ) + ) + run_info.created_requests += 1 + run_info.queued_requests += 1 + except StopIteration: + requests_iter = None + + try: + process_response: _WorkerProcessResponse = ( + responses_queue.get_nowait() + ) + + if process_response.type_ == "request_start": + run_info.pending_requests += 1 + run_info.queued_requests -= 1 + yield SchedulerResult( + type_="request_start", + request=process_response.request, + response=None, + request_info=process_response.info, + run_info=run_info, + ) + elif process_response.type_ == "request_complete": + run_info.pending_requests -= 1 + run_info.completed_requests += 1 + yield SchedulerResult( + type_="request_complete", + request=process_response.request, + response=process_response.response, + request_info=process_response.info, + run_info=run_info, + ) + else: + raise ValueError( + f"Invalid process response type: {process_response}" + ) + except multiprocessing.queues.Empty: + pass + + await asyncio.sleep(0.01) # yield control to the event loop yield SchedulerResult( - completed=False, - count_total=count_total, - count_completed=count_completed, - benchmark=benchmark, - current_result=res, + type_="run_complete", + request=None, + response=None, + request_info=None, + run_info=run_info, ) - logger.info("Scheduler run completed") - - yield SchedulerResult( - completed=True, - count_total=count_total, - count_completed=( - benchmark.request_count + benchmark.error_count - if self.max_number - else round(time.time() - start_time) - if self.max_duration - else 0 - ), - benchmark=benchmark, - ) + await self._stop_processes(futures, requests_queue) - async def _run_sync( - self, benchmark: TextGenerationBenchmark, end_time: float, max_number: float - ) -> AsyncGenerator[Union[TextGenerationResult, TextGenerationError], None]: - for index, (request, submit_at) in enumerate( - zip(self.generator, self.load_generator.times()) - ): - if index >= max_number or time.time() >= end_time: - break + def _run_setup( + self, processes: List[asyncio.Future] + ) -> Tuple[SchedulerRunInfo, Iterator[Any], Iterator[float]]: + requests_iter = iter(self._request_loader) + start_time = time.time() + times_iter = iter(self._scheduling_strategy.request_times()) + end_time = time.time() + (self._max_duration or math.inf) + end_number = self._max_number or math.inf - logger.debug( - "Running synchronous request={} at submit_at={}", - request, - submit_at, - ) - benchmark.request_started() - result = await self._scheduled_request(request, submit_at, end_time) - if result is not None: - benchmark.request_completed(result) - logger.debug("Request completed with output: {}", result) - yield result - - async def _run_async( - self, benchmark: TextGenerationBenchmark, end_time: float, max_number: float - ) -> AsyncGenerator[Union[TextGenerationResult, TextGenerationError], None]: - tasks = [] - pending = asyncio.Semaphore(settings.max_concurrency) - - for index, (request, submit_at) in enumerate( - zip(self.generator, self.load_generator.times()) - ): - # wait for number of pending tasks to be >= max_concurrency - await pending.acquire() + try: + # update end number if the request loader is finite and less than max + iter_length = len(self._request_loader) + if 0 < iter_length < end_number: + end_number = iter_length + except TypeError: + pass + + if end_number == math.inf and end_time is None: + pass # TODO: log warning + + info = SchedulerRunInfo( + start_time=start_time, + end_time=end_time, + end_number=end_number, + processes=len(processes), + strategy=self._scheduling_strategy, + ) - if index >= max_number or time.time() >= end_time or submit_at >= end_time: - break + return info, requests_iter, times_iter - logger.debug( - "Running asynchronous request={} at submit_at={}", - request, - submit_at, + async def _start_processes( + self, + manager, + executor: concurrent.futures.ProcessPoolExecutor, + ) -> Tuple[ + List[asyncio.Future], + multiprocessing.Queue, + multiprocessing.Queue, + ]: + cpu_cores = os.cpu_count() or 1 + + worker_type: Literal["sync", "async"] + requests_queue_limit: Optional[int] + max_concurrency: int + num_processes: int + + if isinstance(self._scheduling_strategy, SynchronousStrategy): + worker_type = "sync" + requests_queue_limit = 2 + num_processes = 1 + max_concurrency = -1 + elif isinstance(self._scheduling_strategy, ConcurrentStrategy): + worker_type = "sync" + num_processes = self._scheduling_strategy.streams + requests_queue_limit = ( + num_processes * 2 + ) # add 2 per process to ensure no idling + max_concurrency = -1 + elif isinstance( + self._scheduling_strategy, + (ThroughputStrategy, AsyncConstantStrategy, AsyncPoissonStrategy), + ): + worker_type = "async" + num_processes = self._num_processes or min( + max(1, cpu_cores - 1), settings.max_worker_processes + ) + max_concurrency = ( + self._scheduling_strategy.max_concurrency + if isinstance(self._scheduling_strategy, ThroughputStrategy) + else None + ) or settings.max_concurrency + requests_queue_limit = ( + max_concurrency + + num_processes # add 1 extra per process to ensure no idling + ) + max_concurrency = max_concurrency // num_processes # convert to per process + else: + raise ValueError( + f"Invalid scheduling strategy: {self._scheduling_strategy}" ) - def _completed(_task: asyncio.Task) -> None: - # NOTE: this is only ok because we don't use threads/processes - nonlocal pending - pending.release() - _res = _task.result() + requests_queue = manager.Queue(maxsize=requests_queue_limit) + responses_queue = manager.Queue() + + futures = [] + loop = asyncio.get_event_loop() + for process_id in range(num_processes): + if worker_type == "sync": + futures.append( + loop.run_in_executor( + executor, + self._worker_process_sync, + requests_queue, + responses_queue, + process_id, + ) + ) + elif worker_type == "async": + futures.append( + loop.run_in_executor( + executor, + self._worker_process_async, + requests_queue, + responses_queue, + max_concurrency, + process_id, + ) + ) + else: + raise ValueError(f"Invalid worker type: {worker_type}") - if _res: - benchmark.request_completed(_res) - logger.debug("Request completed: {}", _res) + await asyncio.sleep(0.1) # give time for processes to start - benchmark.request_started() - task = asyncio.create_task( - self._scheduled_request(request, submit_at, end_time) - ) - task.add_done_callback(_completed) - tasks.append(task) + return futures, requests_queue, responses_queue - # release control to the event loop for other tasks - await asyncio.sleep(0) + async def _stop_processes( + self, + futures: List[asyncio.Future], + requests_queue: multiprocessing.Queue, + ): + for _ in futures: + requests_queue.put(None) - for compl_task in asyncio.as_completed(tasks): - task_res = await compl_task - if task_res is not None: - yield task_res + await asyncio.gather(*futures) - async def _scheduled_request( - self, request: TextGenerationRequest, submit_at: float, end_time: float - ) -> Optional[Union[TextGenerationResult, TextGenerationError]]: - try: - if submit_at > end_time: - raise asyncio.TimeoutError( - f"Request submission time {submit_at} " - f"is greater than end time {end_time}" + def _worker_process_sync( + self, + requests_queue: multiprocessing.Queue, + results_queue: multiprocessing.Queue, + process_id: int, + ): + async def _process_runner(): + while True: + try: + process_request: Optional[_WorkerProcessRequest] = ( + requests_queue.get_nowait() + ) + except multiprocessing.queues.Empty: + await asyncio.sleep(0.01) + continue + + if process_request is None: # stop signal + break + + info = SchedulerRequestInfo( + targeted_start_time=process_request.start_time, + queued_time=process_request.queued_time, + start_time=time.time(), + end_time=-1, + process_id=process_id, + ) + results_queue.put( + _WorkerProcessResponse( + type_="request_start", + request=process_request.request, + response=None, + info=info, + ) + ) + response = await self._worker.resolve( + process_request.request, + process_request.start_time, + process_request.timeout_time, + ) + info.end_time = time.time() + results_queue.put( + _WorkerProcessResponse( + type_="request_complete", + request=process_request.request, + response=response, + info=info, + ) ) - if submit_at > time.time(): - await asyncio.sleep(submit_at - time.time()) + try: + asyncio.run(_process_runner()) + except Exception as exc: + print(exc) - timeout = ( - end_time - time.time() if end_time and end_time < math.inf else None - ) + def _worker_process_async( + self, + requests_queue: multiprocessing.Queue, + results_queue: multiprocessing.Queue, + max_concurrency: Optional[int], + process_id: int, + ): + async def _process_runner(): + pending = asyncio.Semaphore(max_concurrency) if max_concurrency else None + + while True: + try: + process_request: Optional[_WorkerProcessRequest] = ( + requests_queue.get_nowait() + ) + except multiprocessing.queues.Empty: + await asyncio.sleep(0.01) + continue + + if process_request is None: # stop signal + break + + if pending: + await pending.acquire() + + info = SchedulerRequestInfo( + targeted_start_time=process_request.start_time, + queued_time=process_request.queued_time, + start_time=time.time(), + end_time=-1, + process_id=process_id, + ) + results_queue.put( + _WorkerProcessResponse( + type_="request_start", + request=process_request.request, + response=None, + info=info, + ) + ) - return await asyncio.wait_for( - self._resolve_text_request(request), timeout=timeout - ) - except Exception as exc: # noqa: BLE001 - if not isinstance(exc, asyncio.TimeoutError): - logger.warning("Request {} failed: {}", request, exc) - - return TextGenerationError(request=request, message=str(exc)) - - async def _resolve_text_request( - self, request: TextGenerationRequest - ) -> TextGenerationResult: - final_resp = None - first_token_time = None - last_token_time = None - - if request.type_ == "text": - async for resp in self._backend.text_completions( # type: ignore[attr-defined] - prompt=request.prompt, - id_=request.id, - prompt_token_count=request.prompt_token_count, - output_token_count=request.output_token_count, - ): - if isinstance(resp, StreamingTextResponse) and resp.type_ == "iter": - first_token_time = first_token_time or resp.time - last_token_time = resp.time - - final_resp = resp - elif request.type_ == "chat": - async for resp in self._backend.chat_completions( # type: ignore[attr-defined] - content=request.prompt, - id_=request.id, - prompt_token_count=request.prompt_token_count, - output_token_count=request.output_token_count, - ): - if isinstance(resp, StreamingTextResponse) and resp.type_ == "iter": - first_token_time = first_token_time or resp.time - last_token_time = resp.time - - final_resp = resp - - if not final_resp or not isinstance(final_resp, ResponseSummary): - raise ValueError( - f"Invalid final response for request: {request} " - f"and backend: {self._backend}, recieved: {final_resp}" - ) + def _task_completed( + task: asyncio.Task, + request=process_request.request, + info=info, + ): + nonlocal pending + if pending: + pending.release() + response = task.result() + info.end_time = time.time() + results_queue.put( + _WorkerProcessResponse( + type_="request_complete", + request=request, + response=response, + info=info, + ) + ) + + task = asyncio.create_task( + self._worker.resolve( + process_request.request, + process_request.start_time, + process_request.timeout_time, + ) + ) + task.add_done_callback(_task_completed) - return TextGenerationResult( - request=request, - prompt_token_count=final_resp.prompt_tokens, - output=final_resp.value, - output_token_count=resp.output_tokens, - start_time=resp.start_time, - end_time=resp.end_time, - first_token_time=first_token_time, - last_token_time=last_token_time, - ) + asyncio.run(_process_runner()) diff --git a/src/guidellm/scheduler/strategy.py b/src/guidellm/scheduler/strategy.py new file mode 100644 index 00000000..7fdb1811 --- /dev/null +++ b/src/guidellm/scheduler/strategy.py @@ -0,0 +1,281 @@ +import math +import random +import time +from abc import ABC, abstractmethod +from typing import ( + Generator, + Literal, + Optional, +) + +from pydantic import BaseModel, Field + +__all__ = [ + "StrategyType", + "SchedulingStrategy", + "SynchronousStrategy", + "ThroughputStrategy", + "ConcurrentStrategy", + "AsyncConstantStrategy", + "AsyncPoissonStrategy", +] + + +StrategyType = Literal["synchronous", "throughput", "concurrent", "constant", "poisson"] + + +class SchedulingStrategy(ABC, BaseModel): + """ + An abstract base class for scheduling strategies. + This class defines the interface for scheduling requests and provides + a common structure for all scheduling strategies. + Subclasses should implement the `request_times` method to provide + specific scheduling behavior. + + :param type_: The type of scheduling strategy to use. + This should be one of the predefined strategy types. + """ + + type_: StrategyType = Field( + description="The type of scheduling strategy schedule requests with.", + ) + + @abstractmethod + def request_times(self) -> Generator[float, None, None]: + """ + A generator that yields timestamps for when requests should be sent. + This method should be implemented by subclasses to provide specific + scheduling behavior. + + :return: A generator that yields timestamps for request scheduling + or -1 for requests that should be sent immediately. + """ + ... + + +class SynchronousStrategy(SchedulingStrategy): + """ + A class representing a synchronous scheduling strategy. + This strategy schedules requests synchronously, one at a time, + with the maximum rate possible. + It inherits from the `SchedulingStrategy` base class and + implements the `request_times` method to provide the specific + behavior for synchronous scheduling. + + :param type_: The synchronous StrategyType to schedule requests synchronously. + """ + + type_: Literal["synchronous"] = "synchronous" + + def request_times(self) -> Generator[float, None, None]: + """ + A generator that yields time.time() so requests are sent immediately, + while scheduling them synchronously. + + :return: A generator that yields time.time() for immediate request scheduling. + """ + while True: + yield time.time() + + +class ThroughputStrategy(SchedulingStrategy): + """ + A class representing a throughput scheduling strategy. + This strategy schedules as many requests asynchronously as possible, + with the maximum rate possible. + It inherits from the `SchedulingStrategy` base class and + implements the `request_times` method to provide the specific + behavior for throughput scheduling. + + :param type_: The throughput StrategyType to schedule requests asynchronously. + """ + + type_: Literal["throughput"] = "throughput" + max_concurrency: Optional[int] = Field( + default=None, + description=( + "The maximum number of concurrent requests to schedule. " + "If set to None, the concurrency value from settings will be used. " + "This must be a positive integer greater than 0." + ), + gt=0, + ) + + def request_times(self) -> Generator[float, None, None]: + """ + A generator that yields time.time() so requests are sent + immediately, while scheduling as many asynchronously as possible. + + :return: A generator that yields time.time() for immediate request scheduling. + """ + while True: + yield time.time() + + +class ConcurrentStrategy(SchedulingStrategy): + """ + A class representing a concurrent scheduling strategy. + This strategy schedules requests concurrently with the specified + number of streams. + It inherits from the `SchedulingStrategy` base class and + implements the `request_times` method to provide the specific + behavior for concurrent scheduling. + + :param type_: The concurrent StrategyType to schedule requests concurrently. + :param streams: The number of concurrent streams to use for scheduling requests. + Each stream runs synchronously with the maximum rate possible. + This must be a positive integer. + """ + + type_: Literal["concurrent"] = "concurrent" + streams: int = Field( + description=( + "The number of concurrent streams to use for scheduling requests. " + "Each stream runs sychronously with the maximum rate possible. " + "This must be a positive integer." + ), + gt=0, + ) + + def request_times(self) -> Generator[float, None, None]: + """ + A generator that yields time.time() so requests are sent + immediately, while scheduling them concurrently with the specified + number of streams. + + :return: A generator that yields time.time() for immediate request scheduling. + """ + while True: + yield time.time() + + +class AsyncConstantStrategy(SchedulingStrategy): + """ + A class representing an asynchronous constant scheduling strategy. + This strategy schedules requests asynchronously at a constant request rate + in requests per second. + If initial_burst is set, it will send an initial burst of math.floor(rate) + requests to reach the target rate. + This is useful to ensure that the target rate is reached quickly + and then maintained. + It inherits from the `SchedulingStrategy` base class and + implements the `request_times` method to provide the specific + behavior for asynchronous constant scheduling. + + :param type_: The constant StrategyType to schedule requests asynchronously. + :param rate: The rate at which to schedule requests asynchronously in + requests per second. This must be a positive float. + :param initial_burst: True to send an initial burst of requests + (math.floor(self.rate)) to reach target rate. + False to not send an initial burst. + """ + + type_: Literal["constant"] = "constant" + rate: float = Field( + description=( + "The rate at which to schedule requests asynchronously in " + "requests per second. This must be a positive float." + ), + gt=0, + ) + initial_burst: bool = Field( + default=True, + description=( + "True to send an initial burst of requests (math.floor(self.rate)) " + "to reach target rate. False to not send an initial burst." + ), + ) + + def request_times(self) -> Generator[float, None, None]: + """ + A generator that yields timestamps for when requests should be sent. + This method schedules requests asynchronously at a constant rate + in requests per second. + If burst_time is set, it will send an initial burst of requests + to reach the target rate. + This is useful to ensure that the target rate is reached quickly + and then maintained. + + :return: A generator that yields timestamps for request scheduling. + """ + start_time = time.time() + constant_increment = 1.0 / self.rate + + # handle bursts first to get to the desired rate + if self.initial_burst is not None: + # calcualte total burst count based on sending initial at rate + # plus any within the time to ramp up + burst_count = math.floor(self.rate) + for _ in range(burst_count): + yield start_time + + start_time += constant_increment + + counter = 0 + + # continue with constant rate after bursting + while True: + yield start_time + constant_increment * counter + counter += 1 + + +class AsyncPoissonStrategy(SchedulingStrategy): + """ + A class representing an asynchronous Poisson scheduling strategy. + This strategy schedules requests asynchronously at a Poisson request rate + in requests per second. + If initial_burst is set, it will send an initial burst of math.floor(rate) + requests to reach the target rate. + It inherits from the `SchedulingStrategy` base class and + implements the `request_times` method to provide the specific + behavior for asynchronous Poisson scheduling. + + :param type_: The Poisson StrategyType to schedule requests asynchronously. + :param rate: The rate at which to schedule requests asynchronously in + requests per second. This must be a positive float. + :param initial_burst: True to send an initial burst of requests + (math.floor(self.rate)) to reach target rate. + False to not send an initial burst. + """ + + type_: Literal["poisson"] = "poisson" + rate: float = Field( + description=( + "The rate at which to schedule requests asynchronously in " + "requests per second. This must be a positive float." + ), + gt=0, + ) + initial_burst: bool = Field( + default=True, + description=( + "True to send an initial burst of requests (math.floor(self.rate)) " + "to reach target rate. False to not send an initial burst." + ), + ) + + def request_times(self) -> Generator[float, None, None]: + """ + A generator that yields timestamps for when requests should be sent. + This method schedules requests asynchronously at a Poisson rate + in requests per second. + The inter arrival time between requests is exponentially distributed + based on the rate. + + :return: A generator that yields timestamps for request scheduling. + """ + start_time = time.time() + + if self.initial_burst is not None: + # calcualte total burst count based on sending initial at rate + # plus any within the time to ramp up + burst_count = math.floor(self.rate) + for _ in range(burst_count): + yield start_time + else: + yield start_time + + while True: + inter_arrival_time = random.expovariate(self.rate) + start_time += inter_arrival_time + yield start_time diff --git a/src/guidellm/scheduler/test.py b/src/guidellm/scheduler/test.py new file mode 100644 index 00000000..671bacd3 --- /dev/null +++ b/src/guidellm/scheduler/test.py @@ -0,0 +1,40 @@ +import asyncio +import uuid + +from guidellm.backend.openai import OpenAIHTTPBackend +from guidellm.scheduler import Scheduler +from guidellm.scheduler.backend_worker import BackendRequestsWorker, GenerationRequest + + +def test_scheduler(): + backend = OpenAIHTTPBackend(target="http://192.168.4.13:8000") + backend.validate() + worker = BackendRequestsWorker( + backend=backend, + ) + request_loader = [ + GenerationRequest( + request_id=str(uuid.uuid4()), + request_type="text", + content="Create a test prompt for LLMs: ", + constraints={"output_tokens": 256}, + ) + for _ in range(1000) + ] + scheduler = Scheduler( + worker=worker, + request_loader=request_loader, + scheduling_strategy="throughput", + ) + + async def _run_scheduler(): + async for result in scheduler.run(): + print( + f": {result.run_info.processes} : {result.run_info.queued_requests} : {result.run_info.pending_requests} : {result.run_info.completed_requests}" + ) + + asyncio.run(_run_scheduler()) + + +if __name__ == "__main__": + test_scheduler() From 85dee09891461bb3fcb6cf56ec05e0e3c76e2d98 Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Mon, 10 Mar 2025 20:39:18 +0000 Subject: [PATCH 07/43] Enable temp testing script to work, refactor strategy to be more generic with how scheduler uses them --- src/guidellm/executor/profile_generator.py | 5 +- src/guidellm/scheduler/scheduler.py | 72 ++-- src/guidellm/scheduler/strategy.py | 379 +++++++++++++++++++-- src/guidellm/scheduler/test.py | 42 ++- 4 files changed, 410 insertions(+), 88 deletions(-) diff --git a/src/guidellm/executor/profile_generator.py b/src/guidellm/executor/profile_generator.py index 1f857f78..23cf8429 100644 --- a/src/guidellm/executor/profile_generator.py +++ b/src/guidellm/executor/profile_generator.py @@ -8,7 +8,6 @@ from guidellm.config import settings from guidellm.core import TextGenerationBenchmark, TextGenerationBenchmarkReport from guidellm.core.serializable import Serializable -from guidellm.scheduler import LoadGenerationMode __all__ = [ "Profile", @@ -33,7 +32,7 @@ class Profile(Serializable): :type args: Optional[Dict[str, Any]] """ - load_gen_mode: LoadGenerationMode + load_gen_mode: Any load_gen_rate: Optional[float] = None args: Dict[str, Any] = Field(default_factory=dict) @@ -229,7 +228,7 @@ def create_fixed_rate_profile( :return: The generated profile or None if index is out of range. :rtype: Optional[Profile] """ - modes_map: Dict[str, LoadGenerationMode] = { + modes_map: Dict[str, Any] = { "constant": "constant", "poisson": "poisson", } diff --git a/src/guidellm/scheduler/scheduler.py b/src/guidellm/scheduler/scheduler.py index 65d791f5..10061938 100644 --- a/src/guidellm/scheduler/scheduler.py +++ b/src/guidellm/scheduler/scheduler.py @@ -23,9 +23,6 @@ from guidellm.config import settings from guidellm.scheduler.strategy import ( - AsyncConstantStrategy, - AsyncPoissonStrategy, - ConcurrentStrategy, SchedulingStrategy, SynchronousStrategy, ThroughputStrategy, @@ -398,55 +395,29 @@ async def _start_processes( multiprocessing.Queue, multiprocessing.Queue, ]: - cpu_cores = os.cpu_count() or 1 - - worker_type: Literal["sync", "async"] - requests_queue_limit: Optional[int] - max_concurrency: int - num_processes: int - - if isinstance(self._scheduling_strategy, SynchronousStrategy): - worker_type = "sync" - requests_queue_limit = 2 - num_processes = 1 - max_concurrency = -1 - elif isinstance(self._scheduling_strategy, ConcurrentStrategy): - worker_type = "sync" - num_processes = self._scheduling_strategy.streams - requests_queue_limit = ( - num_processes * 2 - ) # add 2 per process to ensure no idling - max_concurrency = -1 - elif isinstance( - self._scheduling_strategy, - (ThroughputStrategy, AsyncConstantStrategy, AsyncPoissonStrategy), - ): - worker_type = "async" - num_processes = self._num_processes or min( - max(1, cpu_cores - 1), settings.max_worker_processes - ) - max_concurrency = ( - self._scheduling_strategy.max_concurrency - if isinstance(self._scheduling_strategy, ThroughputStrategy) - else None - ) or settings.max_concurrency - requests_queue_limit = ( - max_concurrency - + num_processes # add 1 extra per process to ensure no idling - ) - max_concurrency = max_concurrency // num_processes # convert to per process - else: - raise ValueError( - f"Invalid scheduling strategy: {self._scheduling_strategy}" - ) + processing_mode = self._scheduling_strategy.processing_mode - requests_queue = manager.Queue(maxsize=requests_queue_limit) + num_processes = self._scheduling_strategy.processes_limit + if num_processes is None: + cpu_cores = os.cpu_count() or 1 + num_processes = min(max(1, cpu_cores - 1), settings.max_worker_processes) + + num_processing_requests = self._scheduling_strategy.processing_requests_limit + if num_processing_requests is None: + num_processing_requests = settings.max_concurrency + num_processing_requests_per_process = num_processing_requests // num_processes + + num_queued_requests = self._scheduling_strategy.queued_requests_limit + if num_queued_requests is None: + num_queued_requests = num_processing_requests + num_processes + + requests_queue = manager.Queue(maxsize=num_queued_requests) responses_queue = manager.Queue() futures = [] loop = asyncio.get_event_loop() for process_id in range(num_processes): - if worker_type == "sync": + if processing_mode == "sync": futures.append( loop.run_in_executor( executor, @@ -456,19 +427,22 @@ async def _start_processes( process_id, ) ) - elif worker_type == "async": + elif processing_mode == "async": futures.append( loop.run_in_executor( executor, self._worker_process_async, requests_queue, responses_queue, - max_concurrency, + num_processing_requests_per_process, process_id, ) ) else: - raise ValueError(f"Invalid worker type: {worker_type}") + raise ValueError( + f"Invalid processing mode: {processing_mode} " + f"for strategy: {self._scheduling_strategy}" + ) await asyncio.sleep(0.1) # give time for processes to start diff --git a/src/guidellm/scheduler/strategy.py b/src/guidellm/scheduler/strategy.py index 7fdb1811..0edd68d2 100644 --- a/src/guidellm/scheduler/strategy.py +++ b/src/guidellm/scheduler/strategy.py @@ -14,14 +14,14 @@ "StrategyType", "SchedulingStrategy", "SynchronousStrategy", - "ThroughputStrategy", "ConcurrentStrategy", + "ThroughputStrategy", "AsyncConstantStrategy", "AsyncPoissonStrategy", ] -StrategyType = Literal["synchronous", "throughput", "concurrent", "constant", "poisson"] +StrategyType = Literal["synchronous", "concurrent", "throughput", "constant", "poisson"] class SchedulingStrategy(ABC, BaseModel): @@ -40,6 +40,66 @@ class SchedulingStrategy(ABC, BaseModel): description="The type of scheduling strategy schedule requests with.", ) + @property + @abstractmethod + def processing_mode(self) -> Literal["sync", "async"]: + """ + The processing mode for the scheduling strategy, either 'sync' or 'async'. + This property determines how the worker processes are setup: + either to run synchronously with one request at a time or asynchronously. + This property should be implemented by subclasses to return + the appropriate processing mode. + + :return: The processing mode for the scheduling strategy, + either 'sync' or 'async'. + """ + ... + + @property + @abstractmethod + def processes_limit(self) -> Optional[int]: + """ + The limit on the number of worker processes for the scheduling strategy + or None if the strategy does not restrict the number of processes. + This property determines how many worker processes are created + for the scheduling strategy. This property should be implemented + by subclasses to return the appropriate number of processes. + + :return: The number of processes for the scheduling strategy + or None if the strategy does not restrict the number of processes. + """ + ... + + @property + @abstractmethod + def queued_requests_limit(self) -> Optional[int]: + """ + The maximum number of queued requests for the scheduling strategy or None + if the strategy does not restrict the number of queued requests. + This property determines how many requests can be queued at one time + for the scheduling strategy. This property should be implemented + by subclasses to return the appropriate number of queued requests. + + :return: The maximum number of queued requests for the scheduling strategy + or None if the strategy does not restrict the number of queued requests. + """ + ... + + @property + @abstractmethod + def processing_requests_limit(self) -> Optional[int]: + """ + The maximum number of processing requests for the scheduling strategy + or None if the strategy does not restrict the number of processing requests. + This property determines how many requests can be processed at one time + for the scheduling strategy. This property should be implemented + by subclasses to return the appropriate number of processing requests. + + :return: The maximum number of processing requests for the scheduling strategy + or None if the strategy does not restrict the number of processing requests. + """ + ... + @abstractmethod def request_times(self) -> Generator[float, None, None]: """ @@ -67,44 +127,61 @@ class SynchronousStrategy(SchedulingStrategy): type_: Literal["synchronous"] = "synchronous" - def request_times(self) -> Generator[float, None, None]: + @property + def processing_mode(self) -> Literal["sync"]: """ - A generator that yields time.time() so requests are sent immediately, - while scheduling them synchronously. + The processing mode for the scheduling strategy, either 'sync' or 'async'. + This property determines how the worker processes are setup: + either to run synchronously with one request at a time or asynchronously. - :return: A generator that yields time.time() for immediate request scheduling. + :return: 'sync' for synchronous scheduling strategy + for the single worker process. """ - while True: - yield time.time() + return "sync" + @property + def processes_limit(self) -> Optional[int]: + """ + The limit on the number of worker processes for the scheduling strategy + or None if the strategy does not restrict the number of processes. + This property determines how many worker processes are created + for the scheduling strategy. -class ThroughputStrategy(SchedulingStrategy): - """ - A class representing a throughput scheduling strategy. - This strategy schedules as many requests asynchronously as possible, - with the maximum rate possible. - It inherits from the `SchedulingStrategy` base class and - implements the `request_times` method to provide the specific - behavior for throughput scheduling. + :return: 1 for the synchronous scheduling strategy to limit + the worker processes to one. + """ + return 1 - :param type_: The throughput StrategyType to schedule requests asynchronously. - """ + @property + def queued_requests_limit(self) -> Optional[int]: + """ + The maximum number of queued requests for the scheduling strategy or None + if the strategy does not restrict the number of queued requests. + This property determines how many requests can be queued at one time + for the scheduling strategy. - type_: Literal["throughput"] = "throughput" - max_concurrency: Optional[int] = Field( - default=None, - description=( - "The maximum number of concurrent requests to schedule. " - "If set to None, the concurrency value from settings will be used. " - "This must be a positive integer greater than 0." - ), - gt=0, - ) + :return: 1 for the synchronous scheduling strategy to limit + the queued requests to one that is ready to be processed. + """ + return 1 + + @property + def processing_requests_limit(self) -> Optional[int]: + """ + The maximum number of processing requests for the scheduling strategy + or None if the strategy does not restrict the number of processing requests. + This property determines how many requests can be processed at one time + for the scheduling strategy. + + :return: 1 for the synchronous scheduling strategy to limit + the processing requests to one that is ready to be processed. + """ + return 1 def request_times(self) -> Generator[float, None, None]: """ - A generator that yields time.time() so requests are sent - immediately, while scheduling as many asynchronously as possible. + A generator that yields time.time() so requests are sent immediately, + while scheduling them synchronously. :return: A generator that yields time.time() for immediate request scheduling. """ @@ -137,6 +214,57 @@ class ConcurrentStrategy(SchedulingStrategy): gt=0, ) + @property + def processing_mode(self) -> Literal["sync"]: + """ + The processing mode for the scheduling strategy, either 'sync' or 'async'. + This property determines how the worker processes are setup: + either to run synchronously with one request at a time or asynchronously. + + :return: 'sync' for synchronous scheduling strategy + for the multiple worker processes equal to streams. + """ + return "sync" + + @property + def processes_limit(self) -> Optional[int]: + """ + The limit on the number of worker processes for the scheduling strategy + or None if the strategy does not restrict the number of processes. + This property determines how many worker processes are created + for the scheduling strategy. + + :return: {self.streams} for the concurrent scheduling strategy to limit + the worker processes to the number of streams. + """ + return self.streams + + @property + def queued_requests_limit(self) -> Optional[int]: + """ + The maximum number of queued requests for the scheduling strategy or None + if the strategy does not restrict the number of queued requests. + This property determines how many requests can be queued at one time + for the scheduling strategy. + + :return: {self.streams} for the concurrent scheduling strategy to limit + the queued requests to the number of streams that are ready to be processed. + """ + return self.streams + + @property + def processing_requests_limit(self) -> Optional[int]: + """ + The maximum number of processing requests for the scheduling strategy + or None if the strategy does not restrict the number of processing requests. + This property determines how many requests can be processed at one time + for the scheduling strategy. + + :return: {self.streams} for the concurrent scheduling strategy to limit + the processing requests to the number of streams that ready to be processed. + """ + return self.streams + def request_times(self) -> Generator[float, None, None]: """ A generator that yields time.time() so requests are sent @@ -149,6 +277,95 @@ def request_times(self) -> Generator[float, None, None]: yield time.time() +class ThroughputStrategy(SchedulingStrategy): + """ + A class representing a throughput scheduling strategy. + This strategy schedules as many requests asynchronously as possible, + with the maximum rate possible. + It inherits from the `SchedulingStrategy` base class and + implements the `request_times` method to provide the specific + behavior for throughput scheduling. + + :param type_: The throughput StrategyType to schedule requests asynchronously. + """ + + type_: Literal["throughput"] = "throughput" + max_concurrency: Optional[int] = Field( + default=None, + description=( + "The maximum number of concurrent requests to schedule. " + "If set to None, the concurrency value from settings will be used. " + "This must be a positive integer greater than 0." + ), + gt=0, + ) + + @property + def processing_mode(self) -> Literal["async"]: + """ + The processing mode for the scheduling strategy, either 'sync' or 'async'. + This property determines how the worker processes are setup: + either to run synchronously with one request at a time or asynchronously. + + :return: 'async' for asynchronous scheduling strategy + for the multiple worker processes handling requests. + """ + return "async" + + @property + def processes_limit(self) -> Optional[int]: + """ + The limit on the number of worker processes for the scheduling strategy + or None if the strategy does not restrict the number of processes. + This property determines how many worker processes are created + for the scheduling strategy. + + :return: None for the throughput scheduling strategy to apply + no limit on the number of processes. + """ + return None + + @property + def queued_requests_limit(self) -> Optional[int]: + """ + The maximum number of queued requests for the scheduling strategy or None + if the strategy does not restrict the number of queued requests. + This property determines how many requests can be queued at one time + for the scheduling strategy. + + :return: None for the throughput scheduling strategy to apply + no limit on the number of queued requests. + """ + return None + + @property + def processing_requests_limit(self) -> Optional[int]: + """ + The maximum number of processing requests for the scheduling strategy + or None if the strategy does not restrict the number of processing requests. + This property determines how many requests can be processed at one time + for the scheduling strategy. + + :return: {self.max_concurrency} for the throughput scheduling strategy to limit + the processing requests to the maximum concurrency. + If max_concurrency is None, this will be set to None. + """ + return self.max_concurrency + + def request_times(self) -> Generator[float, None, None]: + """ + A generator that yields the start time.time() so requests are sent + immediately, while scheduling as many asynchronously as possible. + + :return: A generator that yields the start time.time() + for immediate request scheduling. + """ + start_time = time.time() + + while True: + yield start_time + + class AsyncConstantStrategy(SchedulingStrategy): """ A class representing an asynchronous constant scheduling strategy. @@ -186,6 +403,57 @@ class AsyncConstantStrategy(SchedulingStrategy): ), ) + @property + def processing_mode(self) -> Literal["async"]: + """ + The processing mode for the scheduling strategy, either 'sync' or 'async'. + This property determines how the worker processes are setup: + either to run synchronously with one request at a time or asynchronously. + + :return: 'async' for asynchronous scheduling strategy + for the multiple worker processes handling requests. + """ + return "async" + + @property + def processes_limit(self) -> Optional[int]: + """ + The limit on the number of worker processes for the scheduling strategy + or None if the strategy does not restrict the number of processes. + This property determines how many worker processes are created + for the scheduling strategy. + + :return: None for the async constant scheduling strategy to apply + no limit on the number of processes. + """ + return None + + @property + def queued_requests_limit(self) -> Optional[int]: + """ + The maximum number of queued requests for the scheduling strategy or None + if the strategy does not restrict the number of queued requests. + This property determines how many requests can be queued at one time + for the scheduling strategy. + + :return: None for the async constant scheduling strategy to apply + no limit on the number of queued requests. + """ + return None + + @property + def processing_requests_limit(self) -> Optional[int]: + """ + The maximum number of processing requests for the scheduling strategy + or None if the strategy does not restrict the number of processing requests. + This property determines how many requests can be processed at one time + for the scheduling strategy. + + :return: None for the async constant scheduling strategy to apply + no limit on the number of processing requests. + """ + return None + def request_times(self) -> Generator[float, None, None]: """ A generator that yields timestamps for when requests should be sent. @@ -254,6 +522,57 @@ class AsyncPoissonStrategy(SchedulingStrategy): ), ) + @property + def processing_mode(self) -> Literal["async"]: + """ + The processing mode for the scheduling strategy, either 'sync' or 'async'. + This property determines how the worker processes are setup: + either to run synchronously with one request at a time or asynchronously. + + :return: 'async' for asynchronous scheduling strategy + for the multiple worker processes handling requests. + """ + return "async" + + @property + def processes_limit(self) -> Optional[int]: + """ + The limit on the number of worker processes for the scheduling strategy + or None if the strategy does not restrict the number of processes. + This property determines how many worker processes are created + for the scheduling strategy. + + :return: None for the async poisson scheduling strategy to apply + no limit on the number of processes. + """ + return None + + @property + def queued_requests_limit(self) -> Optional[int]: + """ + The maximum number of queued requests for the scheduling strategy or None + if the strategy does not restrict the number of queued requests. + This property determines how many requests can be queued at one time + for the scheduling strategy. + + :return: None for the async poisson scheduling strategy to apply + no limit on the number of queued requests. + """ + return None + + @property + def processing_requests_limit(self) -> Optional[int]: + """ + The maximum number of processing requests for the scheduling strategy + or None if the strategy does not restrict the number of processing requests. + This property determines how many requests can be processed at one time + for the scheduling strategy. + + :return: None for the async poisson scheduling strategy to apply + no limit on the number of processing requests. + """ + return None + def request_times(self) -> Generator[float, None, None]: """ A generator that yields timestamps for when requests should be sent. diff --git a/src/guidellm/scheduler/test.py b/src/guidellm/scheduler/test.py index 671bacd3..f0aea393 100644 --- a/src/guidellm/scheduler/test.py +++ b/src/guidellm/scheduler/test.py @@ -2,11 +2,41 @@ import uuid from guidellm.backend.openai import OpenAIHTTPBackend -from guidellm.scheduler import Scheduler from guidellm.scheduler.backend_worker import BackendRequestsWorker, GenerationRequest +from guidellm.scheduler.scheduler import Scheduler +from guidellm.scheduler.strategy import ( + AsyncConstantStrategy, + AsyncPoissonStrategy, + ConcurrentStrategy, + StrategyType, +) -def test_scheduler(): +def test_scheduler(strategy_type: StrategyType, output_tokens: int = 256): + if strategy_type == "synchronous": + num_requests = 20 + strategy = "synchronous" + elif strategy_type == "concurrent": + num_requests = 100 + strategy = ConcurrentStrategy( + streams=6, + ) + elif strategy_type == "throughput": + num_requests = 1000 + strategy = "throughput" + elif strategy_type == "constant": + num_requests = 100 + strategy = AsyncConstantStrategy( + rate=5.5, + ) + elif strategy_type == "poisson": + num_requests = 100 + strategy = AsyncPoissonStrategy( + rate=5.5, + ) + else: + raise ValueError(f"Invalid strategy type: {strategy_type}") + backend = OpenAIHTTPBackend(target="http://192.168.4.13:8000") backend.validate() worker = BackendRequestsWorker( @@ -17,14 +47,14 @@ def test_scheduler(): request_id=str(uuid.uuid4()), request_type="text", content="Create a test prompt for LLMs: ", - constraints={"output_tokens": 256}, + constraints={"output_tokens": output_tokens}, ) - for _ in range(1000) + for _ in range(num_requests) ] scheduler = Scheduler( worker=worker, request_loader=request_loader, - scheduling_strategy="throughput", + scheduling_strategy=strategy, ) async def _run_scheduler(): @@ -37,4 +67,4 @@ async def _run_scheduler(): if __name__ == "__main__": - test_scheduler() + test_scheduler("poisson") From fb3fdd7375d35109f704361c77e239703f46cbad Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Sat, 15 Mar 2025 14:50:55 +0000 Subject: [PATCH 08/43] Polishing for scheduler --- src/guidellm/backend/response.py | 4 +- src/guidellm/config.py | 3 +- src/guidellm/scheduler/backend_worker.py | 15 +- src/guidellm/scheduler/scheduler.py | 218 ++++++++++++++--------- src/guidellm/scheduler/test.py | 8 +- 5 files changed, 143 insertions(+), 105 deletions(-) diff --git a/src/guidellm/backend/response.py b/src/guidellm/backend/response.py index 8265a427..f16fb891 100644 --- a/src/guidellm/backend/response.py +++ b/src/guidellm/backend/response.py @@ -79,8 +79,8 @@ class ResponseSummary(BaseModel): value: str request_args: RequestArgs iterations: int = 0 - start_time: float - end_time: float + start_time: Optional[float] + end_time: Optional[float] request_prompt_tokens: Optional[int] = None request_output_tokens: Optional[int] = None response_prompt_tokens: Optional[int] = None diff --git a/src/guidellm/config.py b/src/guidellm/config.py index a16787dd..24f3c704 100644 --- a/src/guidellm/config.py +++ b/src/guidellm/config.py @@ -143,8 +143,9 @@ class Settings(BaseSettings): request_http2: bool = True max_concurrency: int = 512 max_worker_processes: int = 10 - num_sweep_profiles: int = 9 + default_async_loop_sleep: float = 0.0001 logging: LoggingSettings = LoggingSettings() + num_sweep_profiles: int = 9 # Data settings dataset: DatasetSettings = DatasetSettings() diff --git a/src/guidellm/scheduler/backend_worker.py b/src/guidellm/scheduler/backend_worker.py index 05609672..01543f09 100644 --- a/src/guidellm/scheduler/backend_worker.py +++ b/src/guidellm/scheduler/backend_worker.py @@ -112,7 +112,6 @@ def __init__(self, backend: Backend): async def resolve( self, request: GenerationRequest, - start_time: float, timeout_time: float, ) -> ResponseSummary: """ @@ -121,7 +120,6 @@ async def resolve( and handles any errors that may occur during the process. :param request: The request to resolve. - :param start_time: The time to start the request. :param timeout_time: The time to wait for a response before timing out. If timeout_time is math.inf, the request will not timeout. :return: A ResponseSummary object containing the response from the backend. @@ -140,10 +138,6 @@ async def _runner(): nonlocal response response = resp - if (wait_time := start_time - time.time()) > 0: - await asyncio.sleep(wait_time) - - start_time = time.time() await asyncio.wait_for( _runner(), timeout=timeout_time - time.time() if timeout_time < math.inf else None, @@ -164,7 +158,7 @@ async def _runner(): except Exception as exc: # noqa: BLE001 error = str(exc) - return self._handle_response(request, response, error, start_time) + return self._handle_response(request, response, error) def _create_request_func_kwargs( self, @@ -208,7 +202,6 @@ def _handle_response( request: GenerationRequest, response: Any, error: Optional[str], - start_time: float, ) -> ResponseSummary: if response is None or not isinstance( response, (ResponseSummary, StreamingTextResponse) @@ -228,8 +221,8 @@ def _handle_response( headers={}, payload={}, ), - start_time=start_time, - end_time=time.time(), + start_time=None, + end_time=None, request_id=request.request_id, error=error or "Unknown error", ) @@ -243,7 +236,7 @@ def _handle_response( payload={}, ), start_time=response.start_time, - end_time=time.time(), + end_time=None, request_prompt_tokens=request.stats.get("prompt_tokens", None), request_output_tokens=None, response_prompt_tokens=None, diff --git a/src/guidellm/scheduler/scheduler.py b/src/guidellm/scheduler/scheduler.py index 10061938..546e5335 100644 --- a/src/guidellm/scheduler/scheduler.py +++ b/src/guidellm/scheduler/scheduler.py @@ -19,6 +19,7 @@ Union, ) +from loguru import logger from pydantic import BaseModel from guidellm.config import settings @@ -52,7 +53,6 @@ class RequestsWorker(ABC): async def resolve( self, request: Any, - start_time: float, timeout_time: float, ) -> Any: """ @@ -61,7 +61,6 @@ async def resolve( including any necessary backend processing and response handling. :param request: The request to be resolved generated by the load generator. - :param start_time: The desired start time for the request. :param timeout_time: The timeout time for the request, if there is no timeout given, then this will be math.inf. :return: The response from the worker. @@ -88,7 +87,9 @@ class SchedulerRunInfo(BaseModel): This should be an instance of SchedulingStrategy. :param created_requests: The number of requests created during the run. :param queued_requests: The number of requests queued during the run. - :param pending_requests: The number of requests pending during the run. + :param scheduled_requests: The number of requests scheduled during the run. + (requests pending being sent to the worker but recieved by a process) + :param processing_requests: The number of requests actively being run. :param completed_requests: The number of requests completed during the run. """ @@ -100,7 +101,8 @@ class SchedulerRunInfo(BaseModel): created_requests: int = 0 queued_requests: int = 0 - pending_requests: int = 0 + scheduled_requests: int = 0 + processing_requests: int = 0 completed_requests: int = 0 @@ -113,15 +115,18 @@ class SchedulerRequestInfo(BaseModel): :param targeted_start_time: The targeted start time for the request (time.time()). :param queued_time: The time the request was queued (time.time()). - :param start_time: The actual start time of the request (time.time()). - :param end_time: The end time of the request (time.time()). + :param scheduled_time: The time the request was scheduled (time.time()) + (any sleep time before the request was sent to the worker). + :param worker_start: The time the worker started processing request (time.time()). + :param worker_end: The time the worker finished processing request. (time.time()). :param process_id: The ID of the underlying process that handled the request. """ targeted_start_time: float = -1 queued_time: float = -1 - start_time: float = -1 - end_time: float = -1 + scheduled_time: float = -1 + worker_start: float = -1 + worker_end: float = -1 process_id: int = -1 @@ -150,7 +155,13 @@ class SchedulerResult(BaseModel): and completed during the run. """ - type_: Literal["run_start", "run_complete", "request_start", "request_complete"] + type_: Literal[ + "run_start", + "run_complete", + "request_scheduled", + "request_start", + "request_complete", + ] request: Any response: Any request_info: Optional[SchedulerRequestInfo] @@ -167,7 +178,7 @@ class _WorkerProcessRequest: @dataclass class _WorkerProcessResponse: - type_: Literal["request_start", "request_complete"] + type_: Literal["request_scheduled", "request_start", "request_complete"] request: Any response: Any info: SchedulerRequestInfo @@ -317,9 +328,19 @@ async def run(self) -> AsyncGenerator[SchedulerResult, None]: responses_queue.get_nowait() ) - if process_response.type_ == "request_start": - run_info.pending_requests += 1 + if process_response.type_ == "request_scheduled": run_info.queued_requests -= 1 + run_info.scheduled_requests += 1 + yield SchedulerResult( + type_="request_scheduled", + request=process_response.request, + response=None, + request_info=process_response.info, + run_info=run_info, + ) + elif process_response.type_ == "request_start": + run_info.scheduled_requests -= 1 + run_info.processing_requests += 1 yield SchedulerResult( type_="request_start", request=process_response.request, @@ -328,7 +349,7 @@ async def run(self) -> AsyncGenerator[SchedulerResult, None]: run_info=run_info, ) elif process_response.type_ == "request_complete": - run_info.pending_requests -= 1 + run_info.processing_requests -= 1 run_info.completed_requests += 1 yield SchedulerResult( type_="request_complete", @@ -344,7 +365,8 @@ async def run(self) -> AsyncGenerator[SchedulerResult, None]: except multiprocessing.queues.Empty: pass - await asyncio.sleep(0.01) # yield control to the event loop + # yield control to the event loop + await asyncio.sleep(settings.default_async_loop_sleep) yield SchedulerResult( type_="run_complete", @@ -374,7 +396,10 @@ def _run_setup( pass if end_number == math.inf and end_time is None: - pass # TODO: log warning + logger.warning( + "No end number or end time set, " + "scheduler will run indefinitely until the request loader is exhausted." + ) info = SchedulerRunInfo( start_time=start_time, @@ -467,50 +492,35 @@ def _worker_process_sync( async def _process_runner(): while True: try: - process_request: Optional[_WorkerProcessRequest] = ( - requests_queue.get_nowait() - ) + process_request: Optional[_WorkerProcessRequest] = requests_queue.get_nowait() except multiprocessing.queues.Empty: - await asyncio.sleep(0.01) + # yield control to the event loop + await asyncio.sleep(settings.default_async_loop_sleep) continue if process_request is None: # stop signal break - info = SchedulerRequestInfo( - targeted_start_time=process_request.start_time, + await self._worker_schedule_request( + worker=self._worker, + request=process_request.request, queued_time=process_request.queued_time, - start_time=time.time(), - end_time=-1, + start_time=process_request.start_time, + timeout_time=process_request.timeout_time, + results_queue=results_queue, process_id=process_id, ) - results_queue.put( - _WorkerProcessResponse( - type_="request_start", - request=process_request.request, - response=None, - info=info, - ) - ) - response = await self._worker.resolve( - process_request.request, - process_request.start_time, - process_request.timeout_time, - ) - info.end_time = time.time() - results_queue.put( - _WorkerProcessResponse( - type_="request_complete", - request=process_request.request, - response=response, - info=info, - ) - ) + # yield control to event loop + await asyncio.sleep(settings.default_async_loop_sleep) try: asyncio.run(_process_runner()) except Exception as exc: - print(exc) + logger.error( + f"Error in worker process {process_id}: {exc}", + exc_info=True, + stack_info=True, + ) def _worker_process_async( self, @@ -524,11 +534,10 @@ async def _process_runner(): while True: try: - process_request: Optional[_WorkerProcessRequest] = ( - requests_queue.get_nowait() - ) + process_request: Optional[_WorkerProcessRequest] = requests_queue.get_nowait() except multiprocessing.queues.Empty: - await asyncio.sleep(0.01) + # yield control to event loop + await asyncio.sleep(settings.default_async_loop_sleep) continue if process_request is None: # stop signal @@ -537,48 +546,83 @@ async def _process_runner(): if pending: await pending.acquire() - info = SchedulerRequestInfo( - targeted_start_time=process_request.start_time, - queued_time=process_request.queued_time, - start_time=time.time(), - end_time=-1, - process_id=process_id, - ) - results_queue.put( - _WorkerProcessResponse( - type_="request_start", - request=process_request.request, - response=None, - info=info, - ) - ) - - def _task_completed( - task: asyncio.Task, - request=process_request.request, - info=info, - ): + def _task_done(_: asyncio.Task): nonlocal pending if pending: pending.release() - response = task.result() - info.end_time = time.time() - results_queue.put( - _WorkerProcessResponse( - type_="request_complete", - request=request, - response=response, - info=info, - ) - ) task = asyncio.create_task( - self._worker.resolve( - process_request.request, - process_request.start_time, - process_request.timeout_time, + self._worker_schedule_request( + worker=self._worker, + request=process_request.request, + queued_time=process_request.queued_time, + start_time=process_request.start_time, + timeout_time=process_request.timeout_time, + results_queue=results_queue, + process_id=process_id, ) ) - task.add_done_callback(_task_completed) + task.add_done_callback(_task_done) + # yield control to event loop + await asyncio.sleep(settings.default_async_loop_sleep) + + try: + asyncio.run(_process_runner()) + except Exception as exc: + logger.error( + f"Error in worker process {process_id}: {exc}", + exc_info=True, + stack_info=True, + ) + + @staticmethod + async def _worker_schedule_request( + worker: RequestsWorker, + request: Any, + queued_time: float, + start_time: float, + timeout_time: float, + results_queue: multiprocessing.Queue, + process_id: int, + ): + info = SchedulerRequestInfo( + targeted_start_time=start_time, + queued_time=queued_time, + scheduled_time=time.time(), + worker_start=-1, + worker_end=-1, + process_id=process_id, + ) + results_queue.put( + _WorkerProcessResponse( + type_="request_scheduled", + request=request, + response=None, + info=info, + ) + ) + + if (wait_time := start_time - time.time()) > 0: + await asyncio.sleep(wait_time) - asyncio.run(_process_runner()) + info.worker_start = time.time() + results_queue.put( + _WorkerProcessResponse( + type_="request_start", + request=request, + response=None, + info=info, + ) + ) + + response = await worker.resolve(request, timeout_time) + + info.worker_end = time.time() + results_queue.put( + _WorkerProcessResponse( + type_="request_complete", + request=request, + response=response, + info=info, + ) + ) diff --git a/src/guidellm/scheduler/test.py b/src/guidellm/scheduler/test.py index f0aea393..179f59f7 100644 --- a/src/guidellm/scheduler/test.py +++ b/src/guidellm/scheduler/test.py @@ -25,9 +25,9 @@ def test_scheduler(strategy_type: StrategyType, output_tokens: int = 256): num_requests = 1000 strategy = "throughput" elif strategy_type == "constant": - num_requests = 100 + num_requests = 1000 strategy = AsyncConstantStrategy( - rate=5.5, + rate=0.1, ) elif strategy_type == "poisson": num_requests = 100 @@ -60,11 +60,11 @@ def test_scheduler(strategy_type: StrategyType, output_tokens: int = 256): async def _run_scheduler(): async for result in scheduler.run(): print( - f": {result.run_info.processes} : {result.run_info.queued_requests} : {result.run_info.pending_requests} : {result.run_info.completed_requests}" + f": {result.run_info.processes} : {result.run_info.queued_requests} : {result.run_info.scheduled_requests} {result.run_info.processing_requests} : {result.run_info.completed_requests}" ) asyncio.run(_run_scheduler()) if __name__ == "__main__": - test_scheduler("poisson") + test_scheduler("throughput") From 12835e0e179e0a959738555e2eec667b731d2ee0 Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Sat, 15 Mar 2025 15:06:19 +0000 Subject: [PATCH 09/43] styling fixes --- src/guidellm/scheduler/scheduler.py | 12 +++-- src/guidellm/scheduler/test.py | 70 ----------------------------- 2 files changed, 8 insertions(+), 74 deletions(-) delete mode 100644 src/guidellm/scheduler/test.py diff --git a/src/guidellm/scheduler/scheduler.py b/src/guidellm/scheduler/scheduler.py index 546e5335..4249ce11 100644 --- a/src/guidellm/scheduler/scheduler.py +++ b/src/guidellm/scheduler/scheduler.py @@ -492,7 +492,9 @@ def _worker_process_sync( async def _process_runner(): while True: try: - process_request: Optional[_WorkerProcessRequest] = requests_queue.get_nowait() + process_request: Optional[_WorkerProcessRequest] = ( + requests_queue.get_nowait() + ) except multiprocessing.queues.Empty: # yield control to the event loop await asyncio.sleep(settings.default_async_loop_sleep) @@ -515,7 +517,7 @@ async def _process_runner(): try: asyncio.run(_process_runner()) - except Exception as exc: + except Exception as exc: # noqa: BLE001 logger.error( f"Error in worker process {process_id}: {exc}", exc_info=True, @@ -534,7 +536,9 @@ async def _process_runner(): while True: try: - process_request: Optional[_WorkerProcessRequest] = requests_queue.get_nowait() + process_request: Optional[_WorkerProcessRequest] = ( + requests_queue.get_nowait() + ) except multiprocessing.queues.Empty: # yield control to event loop await asyncio.sleep(settings.default_async_loop_sleep) @@ -568,7 +572,7 @@ def _task_done(_: asyncio.Task): try: asyncio.run(_process_runner()) - except Exception as exc: + except Exception as exc: # noqa: BLE001 logger.error( f"Error in worker process {process_id}: {exc}", exc_info=True, diff --git a/src/guidellm/scheduler/test.py b/src/guidellm/scheduler/test.py deleted file mode 100644 index 179f59f7..00000000 --- a/src/guidellm/scheduler/test.py +++ /dev/null @@ -1,70 +0,0 @@ -import asyncio -import uuid - -from guidellm.backend.openai import OpenAIHTTPBackend -from guidellm.scheduler.backend_worker import BackendRequestsWorker, GenerationRequest -from guidellm.scheduler.scheduler import Scheduler -from guidellm.scheduler.strategy import ( - AsyncConstantStrategy, - AsyncPoissonStrategy, - ConcurrentStrategy, - StrategyType, -) - - -def test_scheduler(strategy_type: StrategyType, output_tokens: int = 256): - if strategy_type == "synchronous": - num_requests = 20 - strategy = "synchronous" - elif strategy_type == "concurrent": - num_requests = 100 - strategy = ConcurrentStrategy( - streams=6, - ) - elif strategy_type == "throughput": - num_requests = 1000 - strategy = "throughput" - elif strategy_type == "constant": - num_requests = 1000 - strategy = AsyncConstantStrategy( - rate=0.1, - ) - elif strategy_type == "poisson": - num_requests = 100 - strategy = AsyncPoissonStrategy( - rate=5.5, - ) - else: - raise ValueError(f"Invalid strategy type: {strategy_type}") - - backend = OpenAIHTTPBackend(target="http://192.168.4.13:8000") - backend.validate() - worker = BackendRequestsWorker( - backend=backend, - ) - request_loader = [ - GenerationRequest( - request_id=str(uuid.uuid4()), - request_type="text", - content="Create a test prompt for LLMs: ", - constraints={"output_tokens": output_tokens}, - ) - for _ in range(num_requests) - ] - scheduler = Scheduler( - worker=worker, - request_loader=request_loader, - scheduling_strategy=strategy, - ) - - async def _run_scheduler(): - async for result in scheduler.run(): - print( - f": {result.run_info.processes} : {result.run_info.queued_requests} : {result.run_info.scheduled_requests} {result.run_info.processing_requests} : {result.run_info.completed_requests}" - ) - - asyncio.run(_run_scheduler()) - - -if __name__ == "__main__": - test_scheduler("throughput") From 65869d7a857afd147e528b29fff748413a79bfb8 Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Mon, 17 Mar 2025 05:06:23 +0000 Subject: [PATCH 10/43] initial groundwork for new benchmark package and classes along with required changes, migrations, and fixes --- src/guidellm/backend/backend.py | 8 + src/guidellm/backend/openai.py | 48 +- src/guidellm/backend/response.py | 16 +- src/guidellm/benchmark/__init__.py | 33 + src/guidellm/benchmark/aggregator.py | 136 ++++ src/guidellm/benchmark/benchmark.py | 227 ++++++ src/guidellm/benchmark/benchmarker.py | 214 ++++++ src/guidellm/benchmark/entrypoints.py | 1 + src/guidellm/benchmark/profile.py | 366 ++++++++++ src/guidellm/config.py | 11 +- src/guidellm/core/__init__.py | 24 - src/guidellm/core/distribution.py | 190 ------ src/guidellm/core/report.py | 311 --------- src/guidellm/core/request.py | 49 -- src/guidellm/core/result.py | 585 ---------------- src/guidellm/dataset/__init__.py | 1 + src/guidellm/{request => dataset}/base.py | 0 src/guidellm/{request => dataset}/emulated.py | 0 src/guidellm/{request => dataset}/file.py | 0 .../{request => dataset}/transformers.py | 0 src/guidellm/executor/__init__.py | 10 - src/guidellm/executor/executor.py | 213 ------ src/guidellm/executor/profile_generator.py | 346 ---------- src/guidellm/objects/__init__.py | 10 + src/guidellm/objects/distribution.py | 315 +++++++++ .../{core => objects}/serializable.py | 0 src/guidellm/request/__init__.py | 12 +- src/guidellm/request/loader.py | 1 + src/guidellm/request/request.py | 79 +++ src/guidellm/scheduler/__init__.py | 38 +- src/guidellm/scheduler/backend_worker.py | 250 ------- src/guidellm/scheduler/result.py | 115 ++++ src/guidellm/scheduler/scheduler.py | 644 +++++------------- src/guidellm/scheduler/strategy.py | 280 +++----- src/guidellm/scheduler/types.py | 7 + src/guidellm/scheduler/worker.py | 412 +++++++++++ tests/unit/core/test_serializable.py | 2 +- 37 files changed, 2296 insertions(+), 2658 deletions(-) create mode 100644 src/guidellm/benchmark/__init__.py create mode 100644 src/guidellm/benchmark/aggregator.py create mode 100644 src/guidellm/benchmark/benchmark.py create mode 100644 src/guidellm/benchmark/benchmarker.py create mode 100644 src/guidellm/benchmark/entrypoints.py create mode 100644 src/guidellm/benchmark/profile.py delete mode 100644 src/guidellm/core/__init__.py delete mode 100644 src/guidellm/core/distribution.py delete mode 100644 src/guidellm/core/report.py delete mode 100644 src/guidellm/core/request.py delete mode 100644 src/guidellm/core/result.py create mode 100644 src/guidellm/dataset/__init__.py rename src/guidellm/{request => dataset}/base.py (100%) rename src/guidellm/{request => dataset}/emulated.py (100%) rename src/guidellm/{request => dataset}/file.py (100%) rename src/guidellm/{request => dataset}/transformers.py (100%) delete mode 100644 src/guidellm/executor/__init__.py delete mode 100644 src/guidellm/executor/executor.py delete mode 100644 src/guidellm/executor/profile_generator.py create mode 100644 src/guidellm/objects/__init__.py create mode 100644 src/guidellm/objects/distribution.py rename src/guidellm/{core => objects}/serializable.py (100%) create mode 100644 src/guidellm/request/loader.py create mode 100644 src/guidellm/request/request.py delete mode 100644 src/guidellm/scheduler/backend_worker.py create mode 100644 src/guidellm/scheduler/result.py create mode 100644 src/guidellm/scheduler/types.py create mode 100644 src/guidellm/scheduler/worker.py diff --git a/src/guidellm/backend/backend.py b/src/guidellm/backend/backend.py index e2b89f1e..6009ae32 100644 --- a/src/guidellm/backend/backend.py +++ b/src/guidellm/backend/backend.py @@ -102,6 +102,14 @@ def model(self) -> Optional[str]: """ ... + @property + @abstractmethod + def info(self) -> Dict[str, Any]: + """ + :return: The information about the backend. + """ + ... + def validate(self): """ Handle final setup and validate the backend is ready for use. diff --git a/src/guidellm/backend/openai.py b/src/guidellm/backend/openai.py index 9380ffad..cbf0aed0 100644 --- a/src/guidellm/backend/openai.py +++ b/src/guidellm/backend/openai.py @@ -19,6 +19,10 @@ __all__ = ["OpenAIHTTPBackend"] +TEXT_COMPLETIONS_PATH = "/v1/completions" +CHAT_COMPLETIONS_PATH = "/v1/chat/completions" + + @Backend.register("openai_http") class OpenAIHTTPBackend(Backend): """ @@ -61,6 +65,17 @@ def __init__( ): super().__init__(type_="openai_http") self._target = target or settings.openai.base_url + + if not self._target: + raise ValueError("Target URL must be provided for OpenAI HTTP backend.") + + if self._target.endswith("/v1") or self._target.endswith("/v1/"): + # backwards compatability, strip v1 off + self._target = self._target[:-3] + + if self._target.endswith("/"): + self._target = self._target[:-1] + self._model = model api_key = api_key or settings.openai.api_key @@ -94,6 +109,22 @@ def model(self) -> Optional[str]: """ return self._model + @property + def info(self) -> Dict[str, Any]: + """ + :return: The information about the backend. + """ + return { + "max_output_tokens": self.max_output_tokens, + "timeout": self.timeout, + "http2": self.http2, + "authorization": bool(self.authorization), + "organization": self.organization, + "project": self.project, + "text_completions_path": TEXT_COMPLETIONS_PATH, + "chat_completions_path": CHAT_COMPLETIONS_PATH, + } + def check_setup(self): """ Check if the backend is setup correctly and can be used for requests. @@ -379,12 +410,10 @@ async def _iterative_completions_request( headers: Dict, payload: Dict, ) -> AsyncGenerator[Union[StreamingTextResponse, ResponseSummary], None]: - target = f"{self.target}/v1/" - if type_ == "text": - target += "completions" + target = f"{self.target}{TEXT_COMPLETIONS_PATH}" elif type_ == "chat": - target += "chat/completions" + target = f"{self.target}{CHAT_COMPLETIONS_PATH}" else: raise ValueError(f"Unsupported type: {type_}") @@ -407,6 +436,8 @@ async def _iterative_completions_request( iter_count = 0 start_time = time.time() iter_time = start_time + first_iter_time: Optional[float] = None + last_iter_time: Optional[float] = None yield StreamingTextResponse( type_="start", @@ -440,14 +471,19 @@ async def _iterative_completions_request( data = json.loads(line.strip()[len("data: ") :]) if delta := self._extract_completions_delta_content(type_, data): + if first_iter_time is None: + first_iter_time = iter_time + last_iter_time = iter_time + iter_count += 1 response_value += delta yield StreamingTextResponse( type_="iter", value=response_value, - start_time=start_time, iter_count=iter_count, + start_time=start_time, + first_iter_time=first_iter_time, delta=delta, time=iter_time, request_id=request_id, @@ -477,6 +513,8 @@ async def _iterative_completions_request( ), start_time=start_time, end_time=iter_time, + first_iter_time=first_iter_time, + last_iter_time=last_iter_time, iterations=iter_count, request_prompt_tokens=request_prompt_tokens, request_output_tokens=request_output_tokens, diff --git a/src/guidellm/backend/response.py b/src/guidellm/backend/response.py index f16fb891..757b142f 100644 --- a/src/guidellm/backend/response.py +++ b/src/guidellm/backend/response.py @@ -67,11 +67,19 @@ class ResponseSummary(BaseModel): :param value: The final value returned from the request. :param request_args: The arguments used to make the request. + :param iterations: The number of iterations in the request. :param start_time: The time the request started. :param end_time: The time the request ended. - :param iterations: The number of iterations in the request. - :param prompt_tokens: The number of tokens in the prompt, if any usage was returned. - :param output_tokens: The number of tokens in the output, if any usage was returned. + :param first_iter_time: The time the first iteration was received. + :param last_iter_time: The time the last iteration was received. + :param request_prompt_tokens: The number of tokens measured in the prompt + for the request, if any. + :param request_output_tokens: The number of tokens enforced for the output + for the request, if any. + :param response_prompt_tokens: The number of tokens measured in the prompt + for the response, if any. + :param response_output_tokens: The number of tokens measured in the output + for the response, if any. :param request_id: The unique identifier for the request, if any. :param error: The error message, if any, returned from making the request. """ @@ -81,6 +89,8 @@ class ResponseSummary(BaseModel): iterations: int = 0 start_time: Optional[float] end_time: Optional[float] + first_iter_time: Optional[float] + last_iter_time: Optional[float] request_prompt_tokens: Optional[int] = None request_output_tokens: Optional[int] = None response_prompt_tokens: Optional[int] = None diff --git a/src/guidellm/benchmark/__init__.py b/src/guidellm/benchmark/__init__.py new file mode 100644 index 00000000..1efbffb7 --- /dev/null +++ b/src/guidellm/benchmark/__init__.py @@ -0,0 +1,33 @@ +from .aggregator import AGG, BenchmarkAggregator, GenerativeBenchmarkAggregator +from .benchmark import BENCH, Benchmark, GenerativeBenchmark +from .benchmarker import Benchmarker, BenchmarkerResult, GenerativeBenchmarker +from .profile import ( + AsyncProfile, + ConcurrentProfile, + Profile, + ProfileType, + SweepProfile, + SynchronousProfile, + ThroughputProfile, + create_profile, +) + +__all__ = [ + "AGG", + "BENCH", + "Benchmark", + "BenchmarkAggregator", + "GenerativeBenchmark", + "GenerativeBenchmarkAggregator", + "Benchmarker", + "BenchmarkerResult", + "GenerativeBenchmarker", + "AsyncProfile", + "ConcurrentProfile", + "Profile", + "ProfileType", + "SweepProfile", + "SynchronousProfile", + "ThroughputProfile", + "create_profile", +] diff --git a/src/guidellm/benchmark/aggregator.py b/src/guidellm/benchmark/aggregator.py new file mode 100644 index 00000000..ba4baadf --- /dev/null +++ b/src/guidellm/benchmark/aggregator.py @@ -0,0 +1,136 @@ +from abc import ABC, abstractmethod +from collections import defaultdict +from typing import DefaultDict, Generic, List, TypeVar + +from pydantic import Field + +from guidellm.backend import ResponseSummary +from guidellm.benchmark.benchmark import BENCH, Benchmark, GenerativeBenchmark +from guidellm.objects import Serializable +from guidellm.request import GenerationRequest +from guidellm.scheduler import ( + REQ, + RES, + SchedulerResult, +) + +__all__ = [ + "AGG", + "BenchmarkAggregator", + "GenerativeBenchmarkAggregator", +] + + +class BenchmarkAggregator(Generic[BENCH, REQ, RES], ABC, Serializable): + created_requests: int = 0 + queued_requests: int = 0 + scheduled_requests: int = 0 + processing_requests: int = 0 + completed_requests: int = 0 + successful_requests: int = 0 + errored_requests: int = 0 + + queued_time: float = 0.0 + scheduled_time: float = 0.0 + worker_time: float = 0.0 + targeted_worker_start_delay: float = 0.0 + process_idle_time: DefaultDict[int, float] = defaultdict(float) + process_idle_time_scratch: DefaultDict[int, float] = defaultdict(float) + + def add_base_result( + self, result: SchedulerResult[REQ, RES], is_error: bool = False + ): + self.created_requests = result.run_info.created_requests + self.queued_requests = result.run_info.queued_requests + self.scheduled_requests = result.run_info.scheduled_requests + self.processing_requests = result.run_info.processing_requests + self.completed_requests = result.run_info.completed_requests + + if result.type_ != "request_complete": + return + + if is_error: + self.errored_requests += 1 + else: + self.successful_requests += 1 + + self.queued_time += ( + result.request_info.scheduled_time - result.request_info.queued_time + ) + self.scheduled_time += ( + result.request_info.worker_start - result.request_info.scheduled_time + ) + + self.worker_time += ( + result.request_info.worker_end - result.request_info.worker_start + ) + self.worker_schedule_delay_total += ( + result.request_info.worker_start - result.request_info.targeted_start_time + ) + + first_process_request = ( + result.request_info.process_id not in self.process_idle_time_scratch + ) + if not first_process_request: + self.process_idle_time_scratch[result.request_info.process_id] -= ( + result.request_info.worker_start + ) + self.process_idle_time[result.request_info.process_id] = ( + self.process_idle_time_scratch[result.request_info.process_id] + ) + self.process_idle_time_scratch[result.request_info.process_id] += ( + result.request_info.worker_end + ) + + def add_result(self, result: SchedulerResult[REQ, RES]): + self.add_base_result(result) + + @abstractmethod + def compile(self) -> Benchmark[BENCH]: ... + + +AGG = TypeVar("AGG", bound=BenchmarkAggregator) + + +class GenerativeBenchmarkAggregator( + BenchmarkAggregator[GenerativeBenchmark, GenerationRequest, ResponseSummary] +): + results: List[SchedulerResult[GenerationRequest, ResponseSummary]] = Field( + default_factory=list, + description="The list of results for the benchmark.", + ) + + request_time_total: float = 0.0 + targeted_request_delay_total: float = 0.0 + time_to_first_token_total: float = 0.0 + inter_token_latency_total: float = 0.0 + prompt_tokens_total: int = 0 + output_tokens_total: int = 0 + + def add_result(self, result: SchedulerResult[GenerationRequest, ResponseSummary]): + is_error = bool(result.response.error) + self.add_base_result(result, is_error=is_error) + + if result.type_ != "request_complete": + return + + self.results.append(result) + + if not is_error: + self.request_time_total += (result.response.end_time or 0.0) - ( + result.response.start_time or 0.0 + ) + self.targeted_request_delay_total += (result.response.start_time or 0.0) - ( + result.request_info.targeted_start_time or 0.0 + ) + self.time_to_first_token_total += ( + result.response.first_iter_time or 0.0 + ) - (result.response.start_time or 0.0) + self.inter_token_latency_total += ( + result.response.last_iter_time or 0.0 + ) - (result.response.first_iter_time or 0.0) + self.prompt_tokens_total += result.response.prompt_tokens or 0 + self.output_tokens_total += result.response.output_tokens or 0 + + def compile(self) -> GenerativeBenchmark: + pass # TODO diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py new file mode 100644 index 00000000..28abd5aa --- /dev/null +++ b/src/guidellm/benchmark/benchmark.py @@ -0,0 +1,227 @@ +from typing import Any, Dict, List, Optional, TypeVar + +from pydantic import Field, computed_field + +from guidellm.benchmark.profile import Profile +from guidellm.objects import ( + DistributionSummary, + Serializable, + StatusDistributionSummary, +) +from guidellm.scheduler import SchedulingStrategy + +__all__ = [ + "BENCH", + "Benchmark", + "GenerativeBenchmark", +] + + +class BenchmarkSettings(Serializable): + profile: Profile + profile_index: int + strategy: SchedulingStrategy + max_number: Optional[int] + max_duration: Optional[float] + warmup_number: Optional[int] + warmup_duration: Optional[float] + cooldown_number: Optional[int] + cooldown_duration: Optional[float] + + +class BenchmarkRunStats(Serializable): + run_start_time: float + run_end_time: float + + completed: int + errored: int + total: int + + queued_time_avg: float + scheduled_time_avg: float + worker_time_avg: float + worker_delay_avg: float + request_delay_avg: float + process_idle_time_avg: float + + +class Benchmark(Serializable): + settings: BenchmarkSettings + run_stats: BenchmarkRunStats + worker_description: Serializable + requests_loader_description: Serializable + extras: Dict[str, Any] = Field( + default_factory=dict, + ) + + requests_per_second: StatusDistributionSummary + requests_concurrency: StatusDistributionSummary + + +BENCH = TypeVar("BENCH", bound=Benchmark) + + +class GenerativeTextResponseStats(Serializable): + request_id: str + prompt: str + output: str + prompt_tokens: int + output_tokens: int + start_time: float + end_time: float + first_token_time: float + last_token_time: float + + @computed_field + @property + def request_latency(self) -> float: + return self.end_time - self.start_time + + @computed_field + @property + def time_to_first_token_ms(self) -> float: + return 1000 * (self.first_token_time - self.start_time) + + @computed_field + @property + def inter_token_latency_ms(self) -> float: + if self.output_tokens <= 1: + return 0.0 + + return ( + 1000 + * (self.last_token_time - self.first_token_time) + / (self.output_tokens - 1) + ) + + @computed_field + @property + def output_tokens_per_second(self) -> float: + if (itl_ms := self.inter_token_latency_ms) == 0.0: + return 0.0 + + return 1000.0 / itl_ms + + +class GenerativeTextErrorStats(GenerativeTextResponseStats): + error: str + request_id: str + prompt: str + output: Optional[str] + prompt_tokens: int + output_tokens: Optional[int] + start_time: float + end_time: None = None # no end since it failed + first_token_time: Optional[float] + last_token_time: Optional[float] + + @computed_field + @property + def request_latency(self) -> None: + return None + + @computed_field + @property + def time_to_first_token_ms(self) -> Optional[float]: + if self.first_token_time is None: + return None + + return 1000 * (self.first_token_time - self.start_time) + + @computed_field + @property + def inter_token_latency_ms(self) -> Optional[float]: + if ( + self.output_tokens is None + or self.output_tokens <= 1 + or self.first_token_time is None + or self.last_token_time is None + ): + return None + + return ( + 1000 + * (self.last_token_time - self.first_token_time) + / (self.output_tokens - 1) + ) + + @computed_field + @property + def output_tokens_per_second(self) -> Optional[float]: + if (itl_ms := self.inter_token_latency_ms) is None: + return None + + return 1000.0 / itl_ms + + +class GenerativeBenchmark(Benchmark): + completed_requests: List[GenerativeTextResponseStats] = Field( + description="The list of completed requests.", + ) + completed_sampled_size: Optional[int] = None + errored_requests: List[GenerativeTextErrorStats] = Field( + description="The list of errored requests.", + ) + errored_sampled_size: Optional[int] = None + + start_time: float = Field( + description="The start time of the first request for the benchmark.", + ) + end_time: float = Field( + description="The end time of the last request for the benchmark.", + ) + + requests_latency: DistributionSummary = Field( + description="The distribution of latencies for the completed requests.", + ) + prompts_token_count: StatusDistributionSummary = Field( + description=( + "The distribution of token counts in the prompts for completed, " + "errored, and all requests." + ) + ) + outputs_token_count: StatusDistributionSummary = Field( + description=( + "The distribution of token counts in the outputs for completed, " + "errored, and all requests." + ) + ) + times_to_first_token_ms: StatusDistributionSummary = Field( + description=( + "The distribution of latencies to receiving the first token in " + "milliseconds for completed, errored, and all requests." + ), + ) + inter_token_latencies_ms: StatusDistributionSummary = Field( + description=( + "The distribution of latencies between tokens in milliseconds for " + "completed, errored, and all requests." + ), + ) + outputs_tokens_per_second: StatusDistributionSummary = Field( + description=( + "The distribution of output tokens per second for completed, " + "errored, and all requests." + ), + ) + + @computed_field + @property + def duration(self) -> float: + """ + :return: The duration of the benchmark in seconds from the start of the + first request to the end of the last request. + """ + return self.end_time - self.start_time + + @staticmethod + def from_stats( + completed: List[GenerativeTextResponseStats], + errored: List[GenerativeTextErrorStats], + settings: BenchmarkSettings, + run_stats: BenchmarkRunStats, + worker_description: Serializable, + requests_loader_description: Serializable, + extras: Dict[str, Any], + ) -> "GenerativeBenchmark": + pass # TODO diff --git a/src/guidellm/benchmark/benchmarker.py b/src/guidellm/benchmark/benchmarker.py new file mode 100644 index 00000000..cf84c6e5 --- /dev/null +++ b/src/guidellm/benchmark/benchmarker.py @@ -0,0 +1,214 @@ +import time +from abc import ABC, abstractmethod +from typing import Any, AsyncGenerator, Dict, Generic, Iterable, Literal, Optional + +from guidellm.backend import Backend, ResponseSummary +from guidellm.benchmark.aggregator import AGG, BENCH, GenerativeBenchmarkAggregator +from guidellm.benchmark.benchmark import GenerativeBenchmark +from guidellm.benchmark.profile import Profile +from guidellm.objects import Serializable +from guidellm.request import GenerationRequest +from guidellm.scheduler import ( + REQ, + RES, + GenerativeRequestsWorker, + RequestsWorker, + Scheduler, + SchedulerResult, + SchedulingStrategy, +) + +__all__ = ["Benchmarker", "BenchmarkerResult", "GenerativeBenchmarker"] + + +class BenchmarkerResult(Generic[AGG, BENCH, REQ, RES], Serializable): + type_: Literal[ + "run_start", + "run_complete", + "scheduler_start", + "scheduler_update", + "scheduler_complete", + "benchmark_compiled", + ] + start_time: float + end_number: int + profile: Profile + current_index: int + current_strategy: Optional[SchedulingStrategy] = None + current_aggregator: Optional[AGG[BENCH, REQ, RES]] = None + current_benchmark: Optional[BENCH] = None + current_result: Optional[SchedulerResult[REQ, RES]] = None + + +class Benchmarker(Generic[AGG, BENCH, REQ, RES], ABC): + def __init__( + self, + worker: RequestsWorker[REQ, RES], + request_loader: Iterable[REQ], + requests_loader_description: Optional[Serializable] = None, + benchmark_save_extras: Optional[Dict[str, Any]] = None, + ): + self.scheduler: Scheduler[REQ, RES] = Scheduler( + worker=worker, request_loader=request_loader + ) + self.requests_loader_description = requests_loader_description + self.benchmark_save_extras = benchmark_save_extras + + async def run( + self, + profile: Profile, + max_number_per_strategy: Optional[int], + max_duration_per_strategy: Optional[float], + warmup_number_per_strategy: Optional[float], + warmup_duration_per_strategy: Optional[float], + cooldown_number_per_strategy: Optional[int], + cooldown_duration_per_strategy: Optional[float], + ) -> AsyncGenerator[BenchmarkerResult[AGG, BENCH, REQ, RES], None]: + start_time = time.time() + end_number = len(profile) + current_index = -1 + + yield BenchmarkerResult( + type_="run_start", + start_time=start_time, + end_number=end_number, + profile=profile, + current_index=current_index, + current_strategy=None, + current_aggregator=None, + current_benchmark=None, + current_result=None, + ) + + while scheduling_strategy := profile.next_strategy(): + current_index += 1 + aggregator: AGG[BENCH, REQ, RES] = self.create_benchmark_aggregator( + profile=profile, + current_index=current_index, + strategy=scheduling_strategy, + max_number=max_number_per_strategy, + max_duration=max_duration_per_strategy, + warmup_number=warmup_number_per_strategy, + warmup_duration=warmup_duration_per_strategy, + cooldown_number=cooldown_number_per_strategy, + cooldown_duration=cooldown_duration_per_strategy, + ) + + yield BenchmarkerResult( + type_="scheduler_start", + start_time=start_time, + end_number=end_number, + profile=profile, + current_index=current_index, + current_strategy=scheduling_strategy, + current_aggregator=aggregator, + current_benchmark=None, + current_result=None, + ) + + async for result in self.scheduler.run( + scheduling_strategy=scheduling_strategy, + max_number=max_number_per_strategy, + max_duration=max_duration_per_strategy, + ): + aggregator.add_result(result) + + yield BenchmarkerResult( + type_="scheduler_update", + start_time=start_time, + end_number=end_number, + profile=profile, + current_index=current_index, + current_strategy=scheduling_strategy, + current_aggregator=aggregator, + current_benchmark=None, + current_result=result, + ) + + yield BenchmarkerResult( + type_="scheduler_complete", + start_time=start_time, + end_number=end_number, + profile=profile, + current_index=current_index, + current_strategy=scheduling_strategy, + current_aggregator=aggregator, + current_benchmark=None, + current_result=None, + ) + + benchmark: BENCH = aggregator.compile() + profile.completed_strategy( + average_rate=benchmark.requests_per_second.completed.mean, + average_concurrency=benchmark.requests_concurrency.completed.mean, + ) + + yield BenchmarkerResult( + type_="benchmark_compiled", + start_time=start_time, + end_number=end_number, + profile=profile, + current_index=current_index, + current_strategy=scheduling_strategy, + current_aggregator=None, + current_benchmark=benchmark, + current_result=None, + ) + + yield BenchmarkerResult( + type_="run_complete", + start_time=start_time, + end_number=end_number, + profile=profile, + current_index=current_index, + current_strategy=None, + current_aggregator=None, + current_benchmark=None, + current_result=None, + ) + + @abstractmethod + def create_benchmark_aggregator( + self, + profile: Profile, + current_index: int, + strategy: SchedulingStrategy, + max_number: Optional[int], + max_duration: Optional[float], + warmup_number: Optional[float], + warmup_duration: Optional[float], + cooldown_number: Optional[int], + cooldown_duration: Optional[float], + ) -> AGG[BENCH, REQ, RES]: ... + + +class GenerativeBenchmarker( + Benchmarker[ + GenerativeBenchmarkAggregator, + GenerativeBenchmark, + GenerationRequest, + ResponseSummary, + ], +): + def __init__( + self, + backend: Backend, + request_loader: Iterable[GenerationRequest], + ): + super().__init__( + worker=GenerativeRequestsWorker(backend), request_loader=request_loader + ) + + def create_benchmark_aggregator( + self, + profile: Profile, + current_index: int, + strategy: SchedulingStrategy, + max_number: Optional[int], + max_duration: Optional[float], + warmup_number: Optional[float], + warmup_duration: Optional[float], + cooldown_number: Optional[int], + cooldown_duration: Optional[float], + ) -> GenerativeBenchmarkAggregator: + return GenerativeBenchmarkAggregator() # TODO diff --git a/src/guidellm/benchmark/entrypoints.py b/src/guidellm/benchmark/entrypoints.py new file mode 100644 index 00000000..46409041 --- /dev/null +++ b/src/guidellm/benchmark/entrypoints.py @@ -0,0 +1 @@ +# TODO diff --git a/src/guidellm/benchmark/profile.py b/src/guidellm/benchmark/profile.py new file mode 100644 index 00000000..f3050eb0 --- /dev/null +++ b/src/guidellm/benchmark/profile.py @@ -0,0 +1,366 @@ +from abc import ABC, abstractmethod +from typing import List, Literal, Optional, Sequence, Union + +import numpy as np +from pydantic import Field + +from guidellm.objects import Serializable +from guidellm.scheduler import ( + AsyncConstantStrategy, + AsyncPoissonStrategy, + ConcurrentStrategy, + SchedulingStrategy, + StrategyType, + SynchronousStrategy, + ThroughputStrategy, +) + +__all__ = [ + "ProfileType", + "Profile", + "SynchronousProfile", + "ConcurrentProfile", + "ThroughputProfile", + "AsyncProfile", + "SweepProfile", + "create_profile", +] + +ProfileType = Literal["synchronous", "concurrent", "throughput", "async", "sweep"] + + +class Profile(ABC, Serializable): + type_: ProfileType = Field( + description="The type of benchmarking profile to use.", + ) + completed_strategies: int = Field( + default=0, + description="The number of scheduling strategies generated so far.", + ) + measured_rates: List[float] = Field( + default_factory=list, + description=("The average rates measured for the strategies that have run."), + ) + measured_concurrencies: List[float] = Field( + default_factory=list, + description=( + "The average concurrency measured for the strategies that have run." + ), + ) + + def completed_strategy(self, average_rate: float, average_concurrency: float): + self.measured_rates.append(average_rate) + self.measured_concurrencies.append(average_concurrency) + self.completed_strategies += 1 + + @abstractmethod + def strategy_types(self) -> List[StrategyType]: ... + + @abstractmethod + def next_strategy(self) -> Optional[SchedulingStrategy]: ... + + +class SynchronousProfile(Profile): + type_: Literal["synchronous"] = "synchronous" + + def strategy_types(self) -> List[StrategyType]: + return [self.type_] + + def next_strategy(self) -> Optional[SchedulingStrategy]: + if self.completed_strategies >= 1: + return None + + return SynchronousStrategy() + + @staticmethod + def from_standard_args( + rate_type: Union[StrategyType, ProfileType], + rate: Optional[Union[float, Sequence[float]]], + **kwargs, + ) -> "SynchronousProfile": + if rate_type != "synchronous": + raise ValueError("Rate type must be 'synchronous' for synchronous profile.") + + if rate is not None: + raise ValueError( + "Rate does not apply to synchronous profile, it must be set to None." + ) + + if kwargs: + raise ValueError( + "No additional arguments are allowed for synchronous profile." + ) + + return SynchronousProfile() + + +class ConcurrentProfile(Profile): + type_: Literal["concurrent"] = "concurrent" + streams: Union[int, Sequence[int]] = Field( + description="The number of concurrent streams to use.", + ) + + def strategy_types(self) -> List[StrategyType]: + num_strategies = len(self.streams) if isinstance(self.streams, Sequence) else 1 + + return [self.type_] * num_strategies + + def next_strategy(self) -> Optional[SchedulingStrategy]: + streams = self.streams if isinstance(self.streams, Sequence) else [self.streams] + + if self.completed_strategies >= len(streams): + return None + + return ConcurrentStrategy( + streams=streams[self.completed_strategies], + ) + + @staticmethod + def from_standard_args( + rate_type: Union[StrategyType, ProfileType], + rate: Optional[Union[float, Sequence[float]]], + **kwargs, + ) -> "ConcurrentProfile": + if rate_type != "concurrent": + raise ValueError("Rate type must be 'concurrent' for concurrent profile.") + + if not rate: + raise ValueError("Rate (streams) must be provided for concurrent profile.") + + if not isinstance(rate, Sequence): + rate = [rate] + + if not all(stream.is_integer() and stream > 0 for stream in rate): + raise ValueError( + f"All rate values (streams) must be positive integers, received {rate}" + ) + + if kwargs: + raise ValueError( + "No additional arguments are allowed for concurrent profile." + ) + + return ConcurrentProfile(streams=rate) + + +class ThroughputProfile(Profile): + type_: Literal["throughput"] = "throughput" + max_concurrency: Optional[int] = Field( + default=None, + description="The maximum number of concurrent requests that can be scheduled.", + ) + + def strategy_types(self) -> List[StrategyType]: + return [self.type_] + + def next_strategy(self) -> Optional[SchedulingStrategy]: + if self.completed_strategies >= 1: + return None + + return ThroughputStrategy( + max_concurrency=self.max_concurrency, + ) + + @staticmethod + def from_standard_args( + rate_type: Union[StrategyType, ProfileType], + rate: Optional[Union[float, Sequence[float]]], + **kwargs, + ) -> "ThroughputProfile": + if rate_type != "throughput": + raise ValueError("Rate type must be 'throughput' for throughput profile.") + + if rate is not None: + raise ValueError( + "Rate does not apply to throughput profile, it must be set to None." + ) + + return ThroughputProfile(**kwargs) + + +class AsyncProfile(ThroughputProfile): + type_: Literal["async"] = "async" + strategy_type: Literal["constant", "poisson"] = Field( + description="The type of asynchronous strategy to use.", + ) + rate: Union[float, Sequence[float]] = Field( + description="The rate of requests per second to use.", + ) + initial_burst: bool = Field( + default=True, + description=( + "True to send an initial burst of requests (math.floor(self.rate)) " + "to reach target rate. False to not send an initial burst." + ), + ) + + def strategy_types(self) -> List[StrategyType]: + num_strategies = len(self.rate) if isinstance(self.rate, Sequence) else 1 + + return [self.strategy_type] * num_strategies + + def next_strategy(self) -> Optional[SchedulingStrategy]: + rate = self.rate if isinstance(self.rate, Sequence) else [self.rate] + + if self.completed_strategies >= len(rate): + return None + + if self.strategy_type == "constant": + return AsyncConstantStrategy( + rate=rate[self.completed_strategies], + initial_burst=self.initial_burst, + max_concurrency=self.max_concurrency, + ) + elif self.strategy_type == "poisson": + return AsyncPoissonStrategy( + rate=rate[self.completed_strategies], + initial_burst=self.initial_burst, + max_concurrency=self.max_concurrency, + ) + else: + raise ValueError(f"Invalid strategy type: {self.strategy_type}") + + @staticmethod + def from_standard_args( + rate_type: Union[StrategyType, ProfileType], + rate: Optional[Union[float, Sequence[float]]], + **kwargs, + ) -> "AsyncProfile": + if rate_type not in ("async", "constant", "poisson"): + raise ValueError( + "Rate type must be in ('async', 'constant', 'poisson') " + f"for async profile. Received: {rate_type}" + ) + + if not rate: + raise ValueError("Rate must be provided for async profile.") + + if not isinstance(rate, Sequence): + rate = [rate] + + if not all(r.is_integer() and r > 0 for r in rate): + raise ValueError( + f"All rate values must be positive integers, received {rate}" + ) + + if rate_type == "async": + rate_type = "constant" # default to constant if not specified + + return AsyncProfile( + strategy_type=rate_type, + rate=rate, + **kwargs, + ) + + +class SweepProfile(AsyncProfile): + type_: Literal["sweep"] = "sweep" + sweep_size: int = Field( + description="The number of strategies to generate for the sweep.", + ) + rate: float = -1 + strategy_type: Literal["constant", "poisson"] = "constant" + + def strategy_types(self) -> List[StrategyType]: + return ( + ["synchronous"] + + [self.strategy_type] * (self.sweep_size - 2) + + ["throughput"] + ) + + def next_strategy(self) -> Optional[SchedulingStrategy]: + if self.completed_strategies >= self.sweep_size: + return None + + if self.completed_strategies == 0: + return SynchronousStrategy() + + if self.completed_strategies == 1: + return ThroughputStrategy( + max_concurrency=self.max_concurrency, + ) + + min_rate = self.measured_rates[0] + max_rate = self.measured_rates[1] + rates = np.linspace(min_rate, max_rate, self.sweep_size)[1:-1] + + if self.strategy_type == "constant": + return AsyncConstantStrategy( + rate=rates[self.completed_strategies - 2], + initial_burst=self.initial_burst, + max_concurrency=self.max_concurrency, + ) + elif self.strategy_type == "poisson": + return AsyncPoissonStrategy( + rate=rates[self.completed_strategies - 2], + initial_burst=self.initial_burst, + max_concurrency=self.max_concurrency, + ) + else: + raise ValueError(f"Invalid strategy type: {self.strategy_type}") + + @staticmethod + def from_standard_args( + rate_type: Union[StrategyType, ProfileType], + rate: Optional[Union[float, Sequence[float]]], + **kwargs, + ) -> "SweepProfile": + if rate_type != "sweep": + raise ValueError("Rate type must be 'sweep' for sweep profile.") + + if rate: + raise ValueError("Rate does not apply to sweep profile, it must be None.") + + if "sweep_size" not in kwargs: + raise ValueError("Sweep size must be provided for sweep profile.") + + if not isinstance(kwargs["sweep_size"], int) or kwargs["sweep_size"] <= 2: + raise ValueError( + "Sweep size must be a positive integer > 2, " + f"received {kwargs['sweep_size']}" + ) + + return SweepProfile(**kwargs) + + +def create_profile( + rate_type: Union[StrategyType, ProfileType], + rate: Optional[Union[float, Sequence[float]]], + **kwargs, +) -> "Profile": + if rate_type == "synchronous": + return SynchronousProfile.from_standard_args( + rate_type=rate_type, + rate=rate, + **kwargs, + ) + + if rate_type == "concurrent": + return ConcurrentProfile.from_standard_args( + rate_type=rate_type, + rate=rate, + **kwargs, + ) + + if rate_type == "throughput": + return ThroughputProfile.from_standard_args( + rate_type=rate_type, + rate=rate, + **kwargs, + ) + + if rate_type in ("async", "constant", "poisson"): + return AsyncProfile.from_standard_args( + rate_type=rate_type, + rate=rate, + **kwargs, + ) + + if rate_type == "sweep": + return SweepProfile.from_standard_args( + rate_type=rate_type, + rate=rate, + **kwargs, + ) + + raise ValueError(f"Invalid profile type: {rate_type}") diff --git a/src/guidellm/config.py b/src/guidellm/config.py index 24f3c704..c585bd92 100644 --- a/src/guidellm/config.py +++ b/src/guidellm/config.py @@ -139,13 +139,18 @@ class Settings(BaseSettings): # general settings env: Environment = Environment.PROD + default_async_loop_sleep: float = 10e-5 + logging: LoggingSettings = LoggingSettings() + num_sweep_profiles: int = 9 + + # HTTP settings request_timeout: int = 60 * 5 # 5 minutes request_http2: bool = True + + # Scheduler settings max_concurrency: int = 512 max_worker_processes: int = 10 - default_async_loop_sleep: float = 0.0001 - logging: LoggingSettings = LoggingSettings() - num_sweep_profiles: int = 9 + max_add_requests_per_loop: int = 20 # Data settings dataset: DatasetSettings = DatasetSettings() diff --git a/src/guidellm/core/__init__.py b/src/guidellm/core/__init__.py deleted file mode 100644 index e738aa76..00000000 --- a/src/guidellm/core/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -from .distribution import Distribution -from .report import GuidanceReport -from .request import TextGenerationRequest -from .result import ( - RequestConcurrencyMeasurement, - TextGenerationBenchmark, - TextGenerationBenchmarkReport, - TextGenerationError, - TextGenerationResult, -) -from .serializable import Serializable, SerializableFileType - -__all__ = [ - "Distribution", - "GuidanceReport", - "RequestConcurrencyMeasurement", - "Serializable", - "SerializableFileType", - "TextGenerationBenchmark", - "TextGenerationBenchmarkReport", - "TextGenerationError", - "TextGenerationRequest", - "TextGenerationResult", -] diff --git a/src/guidellm/core/distribution.py b/src/guidellm/core/distribution.py deleted file mode 100644 index 749d6818..00000000 --- a/src/guidellm/core/distribution.py +++ /dev/null @@ -1,190 +0,0 @@ -from typing import List, Sequence, Union - -import numpy as np -from loguru import logger -from pydantic import Field - -from guidellm.core.serializable import Serializable - -__all__ = ["Distribution"] - - -class Distribution(Serializable): - """ - A class to represent a statistical distribution and perform various - statistical analyses. - """ - - data: Sequence[float] = Field( - default_factory=list, - description="The data points of the distribution.", - ) - - def __str__(self): - return f"Distribution({self.describe()})" - - def __len__(self): - return len(self.data) - - @property - def mean(self) -> float: - """ - Calculate and return the mean of the distribution. - :return: The mean of the distribution. - """ - if not self.data: - logger.info("No data points available to calculate mean.") - return 0.0 - - mean_value = np.mean(self.data).item() - logger.debug(f"Calculated mean: {mean_value}") - return mean_value - - @property - def median(self) -> float: - """ - Calculate and return the median of the distribution. - :return: The median of the distribution. - """ - if not self.data: - logger.info("No data points available to calculate median.") - return 0.0 - - median_value = np.median(self.data).item() - logger.debug(f"Calculated median: {median_value}") - return median_value - - @property - def variance(self) -> float: - """ - Calculate and return the variance of the distribution. - :return: The variance of the distribution. - """ - if not self.data: - logger.info("No data points available to calculate variance.") - return 0.0 - - variance_value = np.var(self.data).item() - logger.debug(f"Calculated variance: {variance_value}") - return variance_value - - @property - def std_deviation(self) -> float: - """ - Calculate and return the standard deviation of the distribution. - :return: The standard deviation of the distribution. - """ - if not self.data: - logger.info("No data points available to calculate standard deviation.") - return 0.0 - - std_deviation_value = np.std(self.data).item() - logger.debug(f"Calculated standard deviation: {std_deviation_value}") - return std_deviation_value - - def percentile(self, percentile: float) -> float: - """ - Calculate and return the specified percentile of the distribution. - :param percentile: The desired percentile to calculate (0-100). - :return: The specified percentile of the distribution. - """ - if not self.data: - logger.info("No data points available to calculate percentile.") - return 0.0 - - percentile_value = np.percentile(self.data, percentile).item() - logger.debug(f"Calculated {percentile}th percentile: {percentile_value}") - return percentile_value - - def percentiles(self, percentiles: Union[List[int], List[float]]) -> List[float]: - """ - Calculate and return the specified percentiles of the distribution. - :param percentiles: A list of desired percentiles to calculate (0-100). - :return: A list of the specified percentiles of the distribution. - """ - if not self.data: - logger.info("No data points available to calculate percentiles.") - return [0.0] * len(percentiles) - - percentiles_values: List[float] = np.percentile(self.data, percentiles).tolist() # type: ignore # noqa: PGH003 - logger.debug(f"Calculated percentiles {percentiles}: {percentiles_values}") - return percentiles_values - - @property - def min(self) -> float: - """ - Return the minimum value of the distribution. - :return: The minimum value of the distribution. - """ - if not self.data: - logger.info("No data points available to calculate minimum.") - return 0.0 - - min_value: float = np.min(self.data).item() # type: ignore # noqa: PGH003 - logger.debug(f"Calculated min: {min_value}") - return min_value - - @property - def max(self) -> float: - """ - Return the maximum value of the distribution. - :return: The maximum value of the distribution. - """ - if not self.data: - logger.info("No data points available to calculate maximum.") - return 0.0 - - max_value: float = np.max(self.data).item() # type: ignore # noqa: PGH003 - logger.debug(f"Calculated max: {max_value}") - return max_value - - @property - def range(self) -> float: - """ - Calculate and return the range of the distribution (max - min). - :return: The range of the distribution. - """ - if not self.data: - logger.info("No data points available to calculate range.") - return 0.0 - - range_value = self.max - self.min - logger.debug(f"Calculated range: {range_value}") - return range_value - - def describe(self) -> dict: - """ - Return a dictionary describing various statistics of the distribution. - :return: A dictionary with statistical summaries of the distribution. - """ - description = { - "mean": self.mean, - "median": self.median, - "variance": self.variance, - "std_deviation": self.std_deviation, - "percentile_indices": [10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 99], - "percentile_values": self.percentiles( - [10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 99], - ), - "min": self.min, - "max": self.max, - "range": self.range, - } - logger.debug(f"Generated description: {description}") - return description - - def add_data(self, new_data: Sequence[float]): - """ - Add new data points to the distribution. - :param new_data: A list of new numerical data points to add. - """ - self.data = list(self.data) + list(new_data) - logger.debug(f"Added new data: {new_data}") - - def remove_data(self, remove_data: Sequence[float]): - """ - Remove specified data points from the distribution. - :param remove_data: A list of numerical data points to remove. - """ - self.data = [item for item in self.data if item not in remove_data] - logger.debug(f"Removed data: {remove_data}") diff --git a/src/guidellm/core/report.py b/src/guidellm/core/report.py deleted file mode 100644 index 584fe63c..00000000 --- a/src/guidellm/core/report.py +++ /dev/null @@ -1,311 +0,0 @@ -import time -from datetime import datetime -from typing import List, Optional - -from loguru import logger -from pydantic import Field -from rich.console import Console, Group -from rich.live import Live -from rich.panel import Panel -from rich.table import Table - -from guidellm.core.result import TextGenerationBenchmark, TextGenerationBenchmarkReport -from guidellm.core.serializable import Serializable - -__all__ = ["GuidanceReport"] - - -def _create_benchmark_report_details(report: TextGenerationBenchmarkReport) -> str: - """ - Create a detailed string representation of a benchmark report. - - :param report: The benchmark report to generate details for. - :type report: TextGenerationBenchmarkReport - :return: A string containing the backend, data, rate, and limits of - the benchmark report. - :rtype: str - """ - backend = ( - f"Backend(type={report.args.get('backend_type', 'N/A')}, " - f"target={report.args.get('target', 'N/A')}, " - f"model={report.args.get('model', 'N/A')})" - ) - data = ( - f"Data(type={report.args.get('data_type', 'N/A')}, " - f"source={report.args.get('data', 'N/A')}, " - f"tokenizer={report.args.get('tokenizer', 'N/A')})" - ) - rate = ( - f"Rate(type={report.args.get('mode', 'N/A')}, " - f"rate={report.args.get('rate', 'N/A')})" - ) - limits = ( - f"Limits(max_number={report.args.get('max_number', 'N/A')} requests, " - f"max_duration={report.args.get('max_duration', 'N/A')} sec)" - ) - - logger.debug( - "Created benchmark report details for backend={}, data={}, rate={}, limits={}", - backend, - data, - rate, - limits, - ) - - return backend + "\n" + data + "\n" + rate + "\n" + limits + "\n" - - -def _benchmark_rate_id(benchmark: TextGenerationBenchmark) -> str: - """ - Generate a string identifier for a benchmark rate. - - :param benchmark: The benchmark for which to generate the rate ID. - :type benchmark: TextGenerationBenchmark - :return: A string representing the benchmark rate ID. - :rtype: str - """ - rate_id = ( - f"{benchmark.mode}@{benchmark.rate:.2f} req/sec" - if benchmark.rate - else f"{benchmark.mode}" - ) - logger.debug("Generated benchmark rate ID: {}", rate_id) - return rate_id - - -def _create_benchmark_report_requests_summary( - report: TextGenerationBenchmarkReport, -) -> Table: - """ - Create a table summarizing the requests of a benchmark report. - - :param report: The benchmark report to summarize. - :type report: TextGenerationBenchmarkReport - :return: A rich Table object summarizing the requests. - :rtype: Table - """ - table = Table( - "Benchmark", - "Requests Completed", - "Request Failed", - "Duration", - "Start Time", - "End Time", - title="[magenta]Requests Data by Benchmark[/magenta]", - title_style="bold", - title_justify="left", - show_header=True, - ) - - for benchmark in report.benchmarks_sorted: - start_time_str = ( - datetime.fromtimestamp(benchmark.start_time).strftime("%H:%M:%S") - if benchmark.start_time - else "N/A" - ) - end_time_str = ( - datetime.fromtimestamp(benchmark.end_time).strftime("%H:%M:%S") - if benchmark.end_time - else "N/A" - ) - - table.add_row( - _benchmark_rate_id(benchmark), - f"{benchmark.request_count}/{benchmark.total_count}", - f"{benchmark.error_count}/{benchmark.total_count}", - f"{benchmark.duration:.2f} sec", - f"{start_time_str}", - f"{end_time_str}", - ) - logger.debug("Created requests summary table for the report.") - return table - - -def _create_benchmark_report_data_tokens_summary( - report: TextGenerationBenchmarkReport, -) -> Table: - """ - Create a table summarizing data tokens of a benchmark report. - - :param report: The benchmark report to summarize. - :type report: TextGenerationBenchmarkReport - :return: A rich Table object summarizing the data tokens. - :rtype: Table - """ - table = Table( - "Benchmark", - "Prompt", - "Prompt (1%, 5%, 10%, 50%, 90%, 95%, 99%)", - "Output", - "Output (1%, 5%, 10%, 50%, 90%, 95%, 99%)", - title="[magenta]Tokens Data by Benchmark[/magenta]", - title_style="bold", - title_justify="left", - show_header=True, - ) - - for benchmark in report.benchmarks_sorted: - table.add_row( - _benchmark_rate_id(benchmark), - f"{benchmark.prompt_token:.2f}", - ", ".join( - f"{percentile:.1f}" - for percentile in benchmark.prompt_token_percentiles.values() - ), - f"{benchmark.output_token:.2f}", - ", ".join( - f"{percentile:.1f}" - for percentile in benchmark.output_token_percentiles.values() - ), - ) - logger.debug("Created data tokens summary table for the report.") - return table - - -def _create_benchmark_report_dist_perf_summary( - report: TextGenerationBenchmarkReport, -) -> Table: - """ - Create a table summarizing distribution performance of a benchmark report. - - :param report: The benchmark report to summarize. - :type report: TextGenerationBenchmarkReport - :return: A rich Table object summarizing the performance statistics. - :rtype: Table - """ - table = Table( - "Benchmark", - "Request Latency [1%, 5%, 10%, 50%, 90%, 95%, 99%] (sec)", - "Time to First Token [1%, 5%, 10%, 50%, 90%, 95%, 99%] (ms)", - "Inter Token Latency [1%, 5%, 10%, 50%, 90%, 95%, 99%] (ms)", - title="[magenta]Performance Stats by Benchmark[/magenta]", - title_style="bold", - title_justify="left", - show_header=True, - ) - - for benchmark in report.benchmarks_sorted: - table.add_row( - _benchmark_rate_id(benchmark), - ", ".join( - f"{percentile:.2f}" - for percentile in benchmark.request_latency_percentiles.values() - ), - ", ".join( - f"{percentile:.1f}" - for percentile in benchmark.time_to_first_token_percentiles.values() - ), - ", ".join( - f"{percentile:.1f}" - for percentile in benchmark.inter_token_latency_percentiles.values() - ), - ) - logger.debug("Created distribution performance summary table for the report.") - return table - - -def _create_benchmark_report_summary(report: TextGenerationBenchmarkReport) -> Table: - """ - Create a summary table for a benchmark report. - - :param report: The benchmark report to summarize. - :type report: TextGenerationBenchmarkReport - :return: A rich Table object summarizing overall performance. - :rtype: Table - """ - table = Table( - "Benchmark", - "Requests per Second", - "Request Latency", - "Time to First Token", - "Inter Token Latency", - "Output Token Throughput", - title="[magenta]Performance Summary by Benchmark[/magenta]", - title_style="bold", - title_justify="left", - show_header=True, - ) - - for benchmark in report.benchmarks_sorted: - table.add_row( - _benchmark_rate_id(benchmark), - f"{benchmark.completed_request_rate:.2f} req/sec", - f"{benchmark.request_latency:.2f} sec", - f"{benchmark.time_to_first_token:.2f} ms", - f"{benchmark.inter_token_latency:.2f} ms", - f"{benchmark.output_token_throughput:.2f} tokens/sec", - ) - logger.debug("Created overall performance summary table for the report.") - return table - - -class GuidanceReport(Serializable): - """ - A class to manage the guidance reports that include the benchmarking details, - potentially across multiple runs, for saving and loading from disk. - - :param benchmarks: The list of benchmarking reports. - :type benchmarks: List[TextGenerationBenchmarkReport] - """ - - benchmarks: List[TextGenerationBenchmarkReport] = Field( - default_factory=list, description="The list of benchmark reports." - ) - - def print( - self, save_path: Optional[str] = None, continual_refresh: bool = False - ) -> None: - """ - Print the guidance report to the console. - - :param save_path: Optional path to save the report to disk. - :type save_path: Optional[str] - :param continual_refresh: Whether to continually refresh the report. - :type continual_refresh: bool - :return: None - """ - logger.info("Printing guidance report to console with save_path={}", save_path) - report_viz = Panel( - Group( - *[ - Panel( - Group( - _create_benchmark_report_details(benchmark), - "", - _create_benchmark_report_requests_summary(benchmark), - "", - _create_benchmark_report_data_tokens_summary(benchmark), - "", - _create_benchmark_report_dist_perf_summary(benchmark), - "", - _create_benchmark_report_summary(benchmark), - ), - title=( - f"[bold magenta]Benchmark Report " - f"{index + 1}[/bold magenta]" - ), - expand=True, - title_align="left", - ) - for index, benchmark in enumerate(self.benchmarks) - ], - ), - title=( - "[bold cyan]GuideLLM Benchmarks Report[/bold cyan] [italic]" - f"({save_path})[/italic]" - ), - expand=True, - title_align="left", - ) - console = Console() - - if continual_refresh: - logger.info("Starting live report with continual refresh.") - with Live(report_viz, refresh_per_second=1, console=console) as live: - while True: - live.update(report_viz) - time.sleep(1) - else: - console.print(report_viz) - - logger.info("Guidance report printing completed.") diff --git a/src/guidellm/core/request.py b/src/guidellm/core/request.py deleted file mode 100644 index 547ac60a..00000000 --- a/src/guidellm/core/request.py +++ /dev/null @@ -1,49 +0,0 @@ -import uuid -from typing import Any, Dict, Literal, Optional - -from pydantic import Field - -from guidellm.core.serializable import Serializable - - -class TextGenerationRequest(Serializable): - """ - A class to represent a text generation request for generative AI workloads. - """ - - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="The unique identifier for the request.", - ) - type_: Literal["text", "chat"] = Field( - default="text", - description="The type of text generation request (e.g., text, chat).", - ) - prompt: str = Field(description="The input prompt for the text generation.") - prompt_token_count: Optional[int] = Field( - default=None, - description="The number of tokens in the input prompt.", - ) - output_token_count: Optional[int] = Field( - default=None, - description="The number of tokens to generate.", - ) - params: Dict[str, Any] = Field( - default_factory=dict, - description="The parameters for the text generation request.", - ) - - def __str__(self) -> str: - prompt_short = ( - self.prompt[:32] + "..." - if self.prompt and len(self.prompt) > 32 # noqa: PLR2004 - else self.prompt - ) - - return ( - f"TextGenerationRequest(id={self.id}, " - f"type_={self.type_}" - f"prompt={prompt_short}, prompt_token_count={self.prompt_token_count}, " - f"output_token_count={self.output_token_count}, " - f"params={self.params})" - ) diff --git a/src/guidellm/core/result.py b/src/guidellm/core/result.py deleted file mode 100644 index 2670c105..00000000 --- a/src/guidellm/core/result.py +++ /dev/null @@ -1,585 +0,0 @@ -from time import time -from typing import Any, Dict, List, Literal, Optional, Union - -from loguru import logger -from pydantic import Field, computed_field - -from guidellm.core.distribution import Distribution -from guidellm.core.request import TextGenerationRequest -from guidellm.core.serializable import Serializable - -__all__ = [ - "RequestConcurrencyMeasurement", - "TextGenerationBenchmark", - "TextGenerationBenchmarkReport", - "TextGenerationError", - "TextGenerationResult", -] - - -DEFAULT_PERCENTILES = [1, 5, 10, 50, 90, 95, 99] - - -class TextGenerationResult(Serializable): - """ - A class to represent the result of a text generation request - for generative AI workloads. - """ - - request: TextGenerationRequest = Field( - description="The text generation request used to generate the result.", - ) - prompt_token_count: Optional[int] = Field( - default=None, - description="The number of tokens in the input prompt.", - ) - output: str = Field( - default_factory=str, - description="The generated output for the text generation.", - ) - output_token_count: Optional[int] = Field( - default=None, - description="The number of tokens in the output.", - ) - start_time: Optional[float] = Field( - default=None, - description="The absolute start time, in seconds, of the text generation.", - ) - end_time: Optional[float] = Field( - default=None, - description="The absolute end time, in seconds, of the text generation.", - ) - first_token_time: Optional[float] = Field( - default=None, - description="The absolute time, in seconds, the first token was received.", - ) - last_token_time: Optional[float] = Field( - default=None, - description="The absolute time, in seconds, the last token was received.", - ) - - @computed_field # type: ignore[misc] - @property - def request_latency(self) -> Optional[float]: - """ - Get the request latency in seconds. - - :return: The request latency in seconds. - """ - if not self.end_time or not self.start_time: - return None - - return self.end_time - self.start_time - - @computed_field # type: ignore[misc] - @property - def time_to_first_token(self) -> Optional[float]: - """ - Get the time taken to decode the first token in milliseconds. - - :return: The time taken to decode the first token in milliseconds. - """ - if not self.first_token_time or not self.start_time: - return None - - return 1000 * (self.first_token_time - self.start_time) - - @computed_field # type: ignore[misc] - @property - def inter_token_latency(self) -> Optional[float]: - """ - Get the average time between tokens in milliseconds. - - :return: The average time between tokens. - """ - if ( - not self.last_token_time - or not self.first_token_time - or not self.output_token_count - or self.output_token_count < 2 # noqa: PLR2004 - ): - return None - - return ( - 1000 - * (self.last_token_time - self.first_token_time) - / (self.output_token_count - 1) # ignore first token - ) - - @computed_field # type: ignore[misc] - @property - def output_tokens_per_second(self) -> Optional[float]: - """ - Get the average token throughput in tokens per second for the entire request. - Note, does not account for the time taken to decode the first token. - - :return: The average token throughput. - """ - itl = self.inter_token_latency - - if itl is None: - return None - - return 1000.0 / itl - - -class TextGenerationError(Serializable): - """ - A class to represent an error that occurred during a text generation request - for generative AI workloads. - """ - - request: TextGenerationRequest = Field( - description="The text generation request that resulted in an error.", - ) - message: str = Field( - description="The error message that occurred during text generation.", - ) - - -class RequestConcurrencyMeasurement(Serializable): - """ - A dataclass to represent the concurrency measurement of a request. - """ - - time: float = Field(description="The time of the measurement.") - completed: int = Field(description="The number of completed requests.") - errored: int = Field(description="The number of errored requests.") - processing: int = Field(description="The number of processing requests.") - - -class TextGenerationBenchmark(Serializable): - """ - A class to represent a report of text generation requests - (results and errors) for generative AI workloads. - This is a set of results and errors for a specific mode and rate. - """ - - mode: Literal["asynchronous", "synchronous", "throughput"] = Field( - description="The generation mode, one of 'async', 'sync', or 'throughput'." - ) - rate: Optional[float] = Field( - default=None, - description="The requested rate of requests per second.", - ) - results: List[TextGenerationResult] = Field( - default_factory=list, - description="The results of the text generation requests.", - ) - errors: List[TextGenerationError] = Field( - default_factory=list, - description="The errors of the text generation requests.", - ) - concurrencies: List[RequestConcurrencyMeasurement] = Field( - default_factory=list, - description="The concurrency measurements of the requests.", - ) - - def __iter__(self): - """ - Provide an iterator interface to iterate over the results. - - :return: An iterator over the results. - """ - return iter(self.results) - - @computed_field # type: ignore[misc] - @property - def request_count(self) -> int: - """ - Get the number of requests in the result. - - :return: The number of requests. - """ - return len(self.results) - - @computed_field # type: ignore[misc] - @property - def error_count(self) -> int: - """ - Get the number of errors in the result. - - :return: The number of errors. - """ - return len(self.errors) - - @computed_field # type: ignore[misc] - @property - def total_count(self) -> int: - """ - Get the total number of requests in the result. - - :return: The total number of requests. - """ - return self.request_count + self.error_count - - @computed_field # type: ignore[misc] - @property - def start_time(self) -> Optional[float]: - """ - Get the start time of the first request in the result. - - :return: The start time of the first request. - """ - return self.results[0].start_time if self.results else None - - @computed_field # type: ignore[misc] - @property - def end_time(self) -> Optional[float]: - """ - Get the end time of the last request in the result. - - :return: The end time of the last request. - """ - return self.results[-1].end_time if self.results else None - - @computed_field # type: ignore[misc] - @property - def duration(self) -> float: - """ - Get the duration of the result in seconds. - - :return: The duration of the result. - """ - return ( - self.end_time - self.start_time - if self.end_time and self.start_time - else 0.0 - ) - - @computed_field # type: ignore[misc] - @property - def completed_request_rate(self) -> float: - """ - Get the rate of requests per second in the result. - - :return: The rate of requests per second. - """ - return self.request_count / self.duration if self.duration else 0.0 - - @property - def request_latency_distribution(self) -> Distribution: - """ - Get the distribution of request latencies in seconds. - - :return: The distribution of request latencies. - """ - return Distribution( - data=[ - result.request_latency - for result in self.results - if result.request_latency - ] - ) - - @computed_field # type: ignore[misc] - @property - def request_latency(self) -> float: - """ - Get the average request latency in seconds. - - :return: The average request latency in seconds. - :rtype: float - """ - return self.request_latency_distribution.mean - - @computed_field # type: ignore[misc] - @property - def request_latency_percentiles(self) -> Dict[str, float]: - """ - Get standard percentiles of request latency in seconds. - - :return: A dictionary mapping percentile to request latency in seconds. - """ - if not self.results: - return {} - - values = self.request_latency_distribution.percentiles(DEFAULT_PERCENTILES) - - return dict(zip(map(str, DEFAULT_PERCENTILES), values)) - - @property - def ttft_distribution(self) -> Distribution: - """ - Get the distribution of time taken to decode the first token. - - :return: The distribution of time taken to decode the first token. - """ - return Distribution( - data=[ - result.time_to_first_token - for result in self.results - if result.time_to_first_token - ] - ) - - @computed_field # type: ignore[misc] - @property - def time_to_first_token(self) -> float: - """ - Get the time taken to decode the first token in milliseconds. - - :return: The time taken to decode the first token in milliseconds. - """ - return self.ttft_distribution.mean - - @computed_field # type: ignore[misc] - @property - def time_to_first_token_percentiles(self) -> Dict[str, float]: - """ - Get standard percentiles for time taken to decode the first token - in milliseconds. - - :return: A dictionary mapping percentile to time taken for the first token. - """ - if not self.results: - return {} - - values = self.ttft_distribution.percentiles(DEFAULT_PERCENTILES) - - return dict(zip(map(str, DEFAULT_PERCENTILES), values)) - - @property - def itl_distribution(self) -> Distribution: - """ - Get the distribution of time between tokens in milliseconds. - - :return: The distribution of time between tokens. - """ - return Distribution( - data=[ - result.inter_token_latency - for result in self.results - for _ in range( - result.output_token_count - 1 - if result.output_token_count and result.output_token_count > 1 - else 0 - ) - if (result.inter_token_latency) - ] - ) - - @computed_field # type: ignore[misc] - @property - def inter_token_latency(self) -> float: - """ - Get the average time between tokens in milliseconds. - - :return: The average time between tokens. - """ - return self.itl_distribution.mean - - @computed_field # type: ignore[misc] - @property - def inter_token_latency_percentiles(self) -> Dict[str, float]: - """ - Get standard percentiles for the time between tokens in milliseconds. - - :return: A dictionary mapping percentile to time between tokens. - """ - if not self.results: - return {} - - values = self.itl_distribution.percentiles(DEFAULT_PERCENTILES) - - return dict(zip(map(str, DEFAULT_PERCENTILES), values)) - - @computed_field # type: ignore[misc] - @property - def output_token_throughput(self) -> float: - """ - Get the average token throughput in tokens per second. - - :return: The average token throughput. - """ - output_tokens = sum( - result.output_token_count - for result in self.results - if result.output_token_count and result.output_token_count > 0 - ) - - return output_tokens / self.duration if self.duration else 0.0 - - @property - def prompt_token_distribution(self) -> Distribution: - """ - Get the distribution of prompt token counts. - - :return: The distribution of prompt token counts. - """ - return Distribution( - data=[ - result.prompt_token_count - for result in self.results - if result.prompt_token_count - ] - ) - - @computed_field # type: ignore[misc] - @property - def prompt_token(self) -> float: - """ - Get the average number of prompt tokens. - - :return: The average number of prompt tokens. - """ - return self.prompt_token_distribution.mean - - @computed_field # type: ignore[misc] - @property - def prompt_token_percentiles(self) -> Dict[str, float]: - """ - Get standard percentiles for number of prompt tokens. - - :return: A dictionary mapping percentile to number of prompt tokens. - """ - if not self.results: - return {} - - values = self.prompt_token_distribution.percentiles(DEFAULT_PERCENTILES) - - return dict(zip(map(str, DEFAULT_PERCENTILES), values)) - - @property - def output_token_distribution(self) -> Distribution: - """ - Get the distribution of output token counts. - - :return: The distribution of output token counts. - """ - return Distribution( - data=[ - result.output_token_count - for result in self.results - if result.output_token_count - ] - ) - - @computed_field # type: ignore[misc] - @property - def output_token(self) -> float: - """ - Get the average number of output tokens. - - :return: The average number of output tokens. - """ - return self.output_token_distribution.mean - - @computed_field # type: ignore[misc] - @property - def output_token_percentiles(self) -> Dict[str, float]: - """ - Get standard percentiles for number of output tokens. - - :return: List of percentiles of number of output tokens. - """ - if not self.results: - return {} - - values = self.output_token_distribution.percentiles(DEFAULT_PERCENTILES) - - return dict(zip(map(str, DEFAULT_PERCENTILES), values)) - - def request_started(self): - """ - Record the start of a generation request. - """ - if not self.concurrencies: - self.concurrencies = [ - RequestConcurrencyMeasurement( - time=time(), - completed=0, - errored=0, - processing=1, - ), - ] - else: - last = self.concurrencies[-1] - self.concurrencies.append( - RequestConcurrencyMeasurement( - time=time(), - completed=last.completed, - errored=last.errored, - processing=last.processing + 1, - ), - ) - - logger.info("Text generation request started") - - def request_completed( - self, - result: Union[TextGenerationResult, TextGenerationError], - ): - """ - Record the completion of a text generation request. - - :param result: The completed result or error. - :type result: Union[TextGenerationResult, TextGenerationError] - """ - if not self.concurrencies: - raise ValueError("Request completed without starting") - - if isinstance(result, TextGenerationError): - is_error = True - self.errors.append(result) - logger.info( - "Text generation request resulted in error: {}", - result.message, - ) - else: - if not result.start_time or not result.end_time: - raise ValueError("Start time and End time are not defined") - - is_error = False - self.results.append(result) - logger.info("Text generation request completed successfully: {}", result) - - last = self.concurrencies[-1] - self.concurrencies.append( - RequestConcurrencyMeasurement( - time=time(), - completed=last.completed + (not is_error), - errored=last.errored + is_error, - processing=last.processing - 1, - ) - ) - - -class TextGenerationBenchmarkReport(Serializable): - """ - A class to represent a report of text generation benchmarks - for generative AI workloads. - This is a collection of benchmarks for different modes and rates. - """ - - benchmarks: List[TextGenerationBenchmark] = Field( - default_factory=list, - description="The benchmarks of text generation requests.", - ) - args: Dict[str, Any] = Field( - default_factory=dict, - description="The arguments used for the benchmarks.", - ) - - def __iter__(self): - return iter(self.benchmarks) - - @property - def benchmarks_sorted(self) -> List[TextGenerationBenchmark]: - """ - Get the list of benchmarks sorted by request rate. - - :return: The sorted list of benchmarks. - :rtype: List[TextGenerationBenchmark] - """ - return sorted(self.benchmarks, key=lambda x: x.completed_request_rate) - - def add_benchmark(self, benchmark: TextGenerationBenchmark): - """ - Add a result to the report. - - :param benchmark: The result to add. - :type benchmark: TextGenerationBenchmark - """ - self.benchmarks.append(benchmark) - logger.debug("Added result: {}", benchmark) diff --git a/src/guidellm/dataset/__init__.py b/src/guidellm/dataset/__init__.py new file mode 100644 index 00000000..46409041 --- /dev/null +++ b/src/guidellm/dataset/__init__.py @@ -0,0 +1 @@ +# TODO diff --git a/src/guidellm/request/base.py b/src/guidellm/dataset/base.py similarity index 100% rename from src/guidellm/request/base.py rename to src/guidellm/dataset/base.py diff --git a/src/guidellm/request/emulated.py b/src/guidellm/dataset/emulated.py similarity index 100% rename from src/guidellm/request/emulated.py rename to src/guidellm/dataset/emulated.py diff --git a/src/guidellm/request/file.py b/src/guidellm/dataset/file.py similarity index 100% rename from src/guidellm/request/file.py rename to src/guidellm/dataset/file.py diff --git a/src/guidellm/request/transformers.py b/src/guidellm/dataset/transformers.py similarity index 100% rename from src/guidellm/request/transformers.py rename to src/guidellm/dataset/transformers.py diff --git a/src/guidellm/executor/__init__.py b/src/guidellm/executor/__init__.py deleted file mode 100644 index 7665e898..00000000 --- a/src/guidellm/executor/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from .executor import Executor, ExecutorResult -from .profile_generator import Profile, ProfileGenerationMode, ProfileGenerator - -__all__ = [ - "Executor", - "ExecutorResult", - "Profile", - "ProfileGenerationMode", - "ProfileGenerator", -] diff --git a/src/guidellm/executor/executor.py b/src/guidellm/executor/executor.py deleted file mode 100644 index 030904b1..00000000 --- a/src/guidellm/executor/executor.py +++ /dev/null @@ -1,213 +0,0 @@ -from dataclasses import dataclass -from typing import AsyncGenerator, Optional, Sequence, Union - -from loguru import logger - -from guidellm.backend import Backend -from guidellm.core import TextGenerationBenchmarkReport -from guidellm.executor.profile_generator import ( - Profile, - ProfileGenerationMode, - ProfileGenerator, -) -from guidellm.request import RequestGenerator -from guidellm.scheduler import Scheduler - -__all__ = ["Executor", "ExecutorResult"] - - -@dataclass -class ExecutorResult: - """ - Data class representing the result of executing tasks in the Executor. - - :param completed: Indicates whether all tasks have completed. - :type completed: bool - :param count_total: Total number of profiles. - :type count_total: int - :param count_completed: Number of completed profiles. - :type count_completed: int - :param report: A report report for text generation. - :type report: TextGenerationBenchmarkReport - :param scheduler_result: Optional scheduler result for the last task. - :type scheduler_result: Optional[SchedulerResult] - """ - - completed: bool - count_total: int - count_completed: int - generation_modes: Sequence[ProfileGenerationMode] - report: TextGenerationBenchmarkReport - scheduler_result = None - current_index: Optional[int] = None - current_profile: Optional[Profile] = None - - -class Executor: - """ - The Executor class manages the execution of tasks based on a given profile - generation mode and rate. It orchestrates the interaction between the backend, - request generator, and profile generator, and runs benchmarks accordingly. - - :param backend: The backend to run tasks against. - :type backend: Backend - :param request_generator: The generator that creates requests for execution. - :type request_generator: RequestGenerator - :param mode: The mode for profile generation (e.g., sweep, synchronous). - :type mode: ProfileGenerationMode - :param rate: The list of rates for load generation, or None. - :type rate: Optional[List[float]] - :param max_number: Maximum number of requests to generate for the scheduler - (a single report run), or None. - :type max_number: Optional[int] - :param max_duration: Maximum duration for generating requests for the scheduler, - (a single report run), or None. - :type max_duration: Optional[float] - """ - - def __init__( - self, - backend: Backend, - request_generator: RequestGenerator, - mode: ProfileGenerationMode = "sweep", - rate: Optional[Union[float, Sequence[float]]] = None, - max_number: Optional[int] = None, - max_duration: Optional[float] = None, - ): - self._backend = backend - self._generator = request_generator - self._max_number = max_number - self._max_duration = max_duration - self._profile_generator = ProfileGenerator(mode=mode, rate=rate) - logger.info("Executor initialized with mode: {}, rate: {}", mode, rate) - - @property - def backend(self) -> Backend: - """ - Returns the backend being used by the Executor. - - :return: Backend - :rtype: Backend - """ - return self._backend - - @property - def request_generator(self) -> RequestGenerator: - """ - Returns the request generator used by the Executor. - - :return: RequestGenerator - :rtype: RequestGenerator - """ - return self._generator - - @property - def profile_generator(self) -> ProfileGenerator: - """ - Returns the profile generator for generating profiles during execution. - - :return: ProfileGenerator - :rtype: ProfileGenerator - """ - return self._profile_generator - - @property - def max_number(self) -> Optional[int]: - """ - Returns the maximum number of requests to generate. - - :return: Maximum number of requests or None. - :rtype: Optional[int] - """ - return self._max_number - - @property - def max_duration(self) -> Optional[float]: - """ - Returns the maximum duration for generating requests. - - :return: Maximum duration in seconds or None. - :rtype: Optional[float] - """ - return self._max_duration - - async def run(self) -> AsyncGenerator[ExecutorResult, None]: - """ - Runs the Executor, generating and scheduling tasks based on the profile - generation mode. Yields results incrementally. - - :rtype: AsyncGenerator[ExecutorResult, None] - """ - report = TextGenerationBenchmarkReport() - report.args = { - # backend args - "backend_type": self.backend.type_, - "target": self.backend.target, - "model": self.backend.model, - # data args - "data_type": self.request_generator.type_, - "data": self.request_generator.source, - "tokenizer": self.request_generator.tokenizer.name_or_path, - # rate args - "mode": self.profile_generator.mode, - "rate": self.profile_generator.rates, - # limits args - "max_number": self.max_number, - "max_duration": self.max_duration, - } - profile_index = -1 - logger.info("Starting Executor run") - - yield ExecutorResult( - completed=False, - count_total=len(self.profile_generator), - count_completed=0, - generation_modes=self.profile_generator.profile_generation_modes, - report=report, - ) - - while profile := self.profile_generator.next(report): - logger.debug("Generated profile: {}", profile) - scheduler = Scheduler( - generator=self.request_generator, - backend=self.backend, - mode=profile.load_gen_mode, - rate=profile.load_gen_rate, - max_number=self.max_number or profile.args.get("max_number", None), - max_duration=self.max_duration, - ) - profile_index += 1 - - logger.info( - "Scheduling tasks with mode: {}, rate: {}", - profile.load_gen_mode, - profile.load_gen_rate, - ) - - async for scheduler_result in scheduler.run(): - if scheduler_result.completed: - report.add_benchmark(scheduler_result.benchmark) - logger.debug( - "Benchmark added for scheduler result: {}", - scheduler_result.benchmark, - ) - - yield ExecutorResult( - completed=False, - count_total=len(self.profile_generator), - count_completed=len(report.benchmarks), - generation_modes=self.profile_generator.profile_generation_modes, - report=report, - scheduler_result=scheduler_result, - current_index=profile_index, - current_profile=profile, - ) - - logger.info("Executor run completed") - yield ExecutorResult( - completed=True, - count_total=len(self.profile_generator), - count_completed=len(report.benchmarks), - generation_modes=self.profile_generator.profile_generation_modes, - report=report, - ) diff --git a/src/guidellm/executor/profile_generator.py b/src/guidellm/executor/profile_generator.py deleted file mode 100644 index 23cf8429..00000000 --- a/src/guidellm/executor/profile_generator.py +++ /dev/null @@ -1,346 +0,0 @@ -from typing import Any, Dict, List, Literal, Optional, Sequence, Union, get_args - -import numpy as np -from loguru import logger -from numpy._typing import NDArray -from pydantic import Field - -from guidellm.config import settings -from guidellm.core import TextGenerationBenchmark, TextGenerationBenchmarkReport -from guidellm.core.serializable import Serializable - -__all__ = [ - "Profile", - "ProfileGenerationMode", - "ProfileGenerator", -] - -ProfileGenerationMode = Literal[ - "sweep", "synchronous", "throughput", "constant", "poisson" -] - - -class Profile(Serializable): - """ - A data class representing a profile for load generation. - - :param load_gen_mode: The mode of load generation (e.g., constant, poisson). - :type load_gen_mode: LoadGenerationMode - :param load_gen_rate: The rate of load generation, if applicable. - :type load_gen_rate: Optional[float] - :param args: Additional arguments for the profile. - :type args: Optional[Dict[str, Any]] - """ - - load_gen_mode: Any - load_gen_rate: Optional[float] = None - args: Dict[str, Any] = Field(default_factory=dict) - - -class ProfileGenerator: - """ - Generates profiles based on different load generation modes. - - :param mode: The mode for profile generation (e.g., sweep, synchronous). - :type mode: ProfileGenerationMode - :param rate: The rate(s) for load generation; could be a float or list of floats. - :type rate: Optional[Union[float, Sequence[float]]] - """ - - def __init__( - self, - mode: ProfileGenerationMode, - rate: Optional[Union[float, Sequence[float]]] = None, - ): - if mode not in get_args(ProfileGenerationMode): - err = ValueError( - f"{mode} is not a valid Profile Generation Mode. " - f"Valid options are {get_args(ProfileGenerationMode)}" - ) - logger.error(err) - raise err - - self._mode = mode - - if self._mode in ("sweep", "throughput", "synchronous"): - if rate is not None: - err = ValueError(f"Rates are not applicable for {self._mode} mode") - logger.error(err) - raise err - self._rates = None - else: - if not rate: - err = ValueError(f"Rates are required for {self._mode} mode") - logger.error(err) - raise err - self._rates = rate if isinstance(rate, Sequence) else [rate] - - for rt in self._rates: - if rt <= 0: - err = ValueError( - f"Rate must be > 0 for mode: {self._mode}. Given: {rt}" - ) - logger.error(err) - raise err - - self._generated_count = 0 - - def __len__(self) -> int: - """ - Returns the number of profiles to generate based on the mode and rates. - - :return: The number of profiles. - :rtype: int - """ - if self._mode == "sweep": - return settings.num_sweep_profiles + 2 - - if self._mode in ("throughput", "synchronous"): - return 1 - - if not self._rates: - raise ValueError(f"Rates are required for {self._mode} mode") - - return len(self._rates) - - @property - def mode(self) -> ProfileGenerationMode: - """ - Returns the current mode of profile generation. - - :return: The profile generation mode. - :rtype: ProfileGenerationMode - """ - return self._mode - - @property - def rates(self) -> Optional[Sequence[float]]: - """ - Returns the list of rates for load generation, if any. - - :return: Sequence of rates or None if not applicable. - :rtype: Optional[Sequence[float]] - """ - return self._rates - - @property - def generated_count(self) -> int: - """ - Returns the current count of generated profiles. - - :return: The current count of generated profiles. - :rtype: int - """ - return self._generated_count - - @property - def profile_generation_modes(self) -> Sequence[ProfileGenerationMode]: - """ - Return the list of profile modes to be run in the report. - - :return: Sequence of profile modes to be run in the report. - :rtype: Sequence[ProfileGenerationMode] - """ - if self._mode == "sweep": - return ["synchronous", "throughput"] + ["constant"] * ( # type: ignore # noqa: PGH003 - settings.num_sweep_profiles - ) - - if self._mode in ["throughput", "synchronous"]: - return [self._mode] - - if self._rates is None: - raise ValueError(f"Rates are required for {self._mode} mode") - - if self._mode in ["constant", "poisson"]: - return [self._mode] * len(self._rates) - - raise ValueError(f"Invalid mode: {self._mode}") - - def next(self, current_report: TextGenerationBenchmarkReport) -> Optional[Profile]: - """ - Generates the next profile based on the current mode and report. - - :param current_report: The current report report. - :type current_report: TextGenerationBenchmarkReport - :return: The generated profile or None if no more profiles. - :rtype: Optional[Profile] - """ - logger.debug( - "Generating the next profile with mode: {}, current report: {}", - self.mode, - current_report, - ) - - if self.mode in ["constant", "poisson"]: - if not self.rates: - err = ValueError(f"Rates are required for {self.mode} mode") - logger.error(err) - raise err - - profile = self.create_fixed_rate_profile( - self.generated_count, - self.mode, - self.rates, - ) - elif self.mode == "synchronous": - profile = self.create_synchronous_profile(self.generated_count) - elif self.mode == "throughput": - profile = self.create_throughput_profile(self.generated_count) - elif self.mode == "sweep": - profile = self.create_sweep_profile( - self.generated_count, - sync_benchmark=( - current_report.benchmarks[0] if current_report.benchmarks else None - ), - throughput_benchmark=( - current_report.benchmarks[1] - if len(current_report.benchmarks) > 1 - else None - ), - ) - else: - err = ValueError(f"Invalid mode: {self.mode}") - logger.error(err) - raise err - - self._generated_count += 1 - logger.info( - "Generated profile: {}, total generated count: {}", - profile, - self._generated_count, - ) - return profile - - @staticmethod - def create_fixed_rate_profile( - index: int, mode: ProfileGenerationMode, rates: Sequence[float] - ) -> Optional[Profile]: - """ - Creates a profile with a fixed rate. - - :param index: The index of the rate in the list. - :type index: int - :param mode: The mode for profile generation (e.g., constant, poisson). - :type mode: ProfileGenerationMode - :param rates: The list of rates for load generation. - :type rates: Sequence[float] - :return: The generated profile or None if index is out of range. - :rtype: Optional[Profile] - """ - modes_map: Dict[str, Any] = { - "constant": "constant", - "poisson": "poisson", - } - - if mode not in modes_map: - err = ValueError(f"Invalid mode: {mode}") - logger.error(err) - raise err - - profile = ( - Profile( - load_gen_mode=modes_map[mode], - load_gen_rate=rates[index], - ) - if index < len(rates) - else None - ) - logger.debug("Created fixed rate profile: {}", profile) - return profile - - @staticmethod - def create_synchronous_profile(index: int) -> Optional[Profile]: - """ - Creates a profile with synchronous mode. - - :param index: The index of the profile to create. - :type index: int - :return: The generated profile or None if index is out of range. - :rtype: Optional[Profile] - """ - profile = ( - Profile( - load_gen_mode="synchronous", - load_gen_rate=None, - ) - if index < 1 - else None - ) - logger.debug("Created synchronous profile: {}", profile) - return profile - - @staticmethod - def create_throughput_profile(index: int) -> Optional[Profile]: - """ - Creates a profile with throughput mode. - - :param index: The index of the profile to create. - :type index: int - :return: The generated profile or None if index is out of range. - :rtype: Optional[Profile] - """ - profile = ( - Profile( - load_gen_mode="throughput", - load_gen_rate=None, - ) - if index < 1 - else None - ) - logger.debug("Created throughput profile: {}", profile) - return profile - - @staticmethod - def create_sweep_profile( - index: int, - sync_benchmark: Optional[TextGenerationBenchmark], - throughput_benchmark: Optional[TextGenerationBenchmark], - ) -> Optional[Profile]: - """ - Creates a profile with sweep mode, generating profiles between - synchronous and throughput benchmarks. - - :param index: The index of the profile to create. - :type index: int - :param sync_benchmark: The synchronous report data. - :type sync_benchmark: Optional[TextGenerationBenchmark] - :param throughput_benchmark: The throughput report data. - :type throughput_benchmark: Optional[TextGenerationBenchmark] - :return: The generated profile or None if index is out of range. - :rtype: Optional[Profile] - """ - if index < 0 or index >= settings.num_sweep_profiles + 2: - return None - - if index == 0: - return ProfileGenerator.create_synchronous_profile(0) - - if not sync_benchmark: - err = ValueError("Synchronous report is required for sweep mode") - logger.error(err) - raise err - - if index == 1: - throughput_profile: Profile = ProfileGenerator.create_throughput_profile(0) # type: ignore # noqa: PGH003 - return throughput_profile - - if not throughput_benchmark: - err = ValueError("Throughput report is required for sweep mode") - logger.error(err) - raise err - - min_rate = sync_benchmark.completed_request_rate - max_rate = throughput_benchmark.completed_request_rate - intermediate_rates: List[NDArray] = list( - np.linspace(min_rate, max_rate, settings.num_sweep_profiles + 1) - )[1:] - - return Profile( - load_gen_mode="constant", - load_gen_rate=( - float(load_gen_rate) - if (load_gen_rate := intermediate_rates[index - 2]) - else 1.0 # the fallback value - ), - ) diff --git a/src/guidellm/objects/__init__.py b/src/guidellm/objects/__init__.py new file mode 100644 index 00000000..d5c428b4 --- /dev/null +++ b/src/guidellm/objects/__init__.py @@ -0,0 +1,10 @@ +from .distribution import DistributionSummary, Percentiles, StatusDistributionSummary +from .serializable import Serializable, SerializableFileType + +__all__ = [ + "Percentiles", + "DistributionSummary", + "StatusDistributionSummary", + "Serializable", + "SerializableFileType", +] diff --git a/src/guidellm/objects/distribution.py b/src/guidellm/objects/distribution.py new file mode 100644 index 00000000..d25113d1 --- /dev/null +++ b/src/guidellm/objects/distribution.py @@ -0,0 +1,315 @@ +import math +from collections import defaultdict +from typing import List, Tuple + +import numpy as np + +from guidellm.objects import Serializable + +__all__ = [ + "Percentiles", + "DistributionSummary", + "StatusDistributionSummary", +] + + +class Percentiles(Serializable): + p001: float + p01: float + p05: float + p10: float + p25: float + p75: float + p90: float + p95: float + p99: float + p999: float + + @staticmethod + def from_values(values: List[float]) -> "Percentiles": + """ + Calculate percentiles from a list of values. + + :param values: A list of numerical values. + :return: An instance of Percentiles with calculated percentiles. + """ + if not values: + return Percentiles( + p001=0.0, + p01=0.0, + p05=0.0, + p10=0.0, + p25=0.0, + p75=0.0, + p90=0.0, + p95=0.0, + p99=0.0, + p999=0.0, + ) + + percentiles = np.percentile(values, [0.1, 1, 5, 10, 25, 75, 90, 95, 99, 99.9]) + return Percentiles( + p001=percentiles[0], + p01=percentiles[1], + p05=percentiles[2], + p10=percentiles[3], + p25=percentiles[4], + p75=percentiles[5], + p90=percentiles[6], + p95=percentiles[7], + p99=percentiles[8], + p999=percentiles[9], + ) + + +class DistributionSummary(Serializable): + mean: float + median: float + variance: float + std_dev: float + min: float + max: float + count: int + percentiles: Percentiles + + @staticmethod + def from_values(values: List[float]) -> "DistributionSummary": + """ + Create a DistributionSummary from a list of values. + + :param values: A list of numerical values. + :return: An instance of DistributionSummary. + """ + if not values: + return DistributionSummary( + mean=0.0, + median=0.0, + variance=0.0, + std_dev=0.0, + min=0.0, + max=0.0, + count=0, + percentiles=Percentiles.from_values([]), + ) + + return DistributionSummary( + mean=float(np.mean(values)), + median=float(np.median(values)), + variance=float(np.var(values)), + std_dev=float(np.std(values)), + min=float(np.min(values)), + max=float(np.max(values)), + count=len(values), + percentiles=Percentiles.from_values(values), + ) + + @staticmethod + def from_time_measurements( + measurements: List[Tuple[float, float]], + ) -> "DistributionSummary": + """ + Create a DistributionSummary from a list of time measurements of the form + (time, value), where time is the timestamp and value is the measurement. + + :param measurements: A list of tuples containing (time, value) pairs. + :return: An instance of DistributionSummary. + """ + if not measurements: + return DistributionSummary( + mean=0.0, + median=0.0, + variance=0.0, + std_dev=0.0, + min=0.0, + max=0.0, + count=0, + percentiles=Percentiles.from_values([]), + ) + + if len(measurements) == 1: + return DistributionSummary( + mean=measurements[0][1], + median=measurements[0][1], + variance=0.0, + std_dev=0.0, + min=measurements[0][1], + max=measurements[0][1], + count=1, + percentiles=Percentiles.from_values([measurements[0][1]]), + ) + + measurements.sort(key=lambda x: x[0]) + integral = sum( + (measurements[ind + 1][0] - measurements[ind][0]) * measurements[ind][1] + for ind in range(len(measurements) - 1) + ) + duration = measurements[-1][0] - measurements[0][0] + mean = integral / duration if duration > 0 else 0.0 + variance = ( + sum( + (measurements[ind + 1][0] - measurements[ind][0]) + * (measurements[ind][1] - mean) ** 2 + for ind in range(len(measurements) - 1) + ) + / duration + if duration > 0 + else 0.0 + ) + + value_durations_dict = defaultdict(float) + for ind in range(len(measurements) - 1): + value_durations_dict[measurements[ind][1]] += ( + measurements[ind + 1][0] - measurements[ind][0] + ) + value_durations = sorted( + [(duration, value) for value, duration in value_durations_dict.items()], + key=lambda x: x[0], + ) + + def _get_percentile(percentile: float) -> float: + target_duration = percentile / 100 * duration + cumulative_duration = 0.0 + for dur, val in value_durations: + cumulative_duration += dur + if cumulative_duration >= target_duration: + return val + return value_durations[-1][1] + + return DistributionSummary( + mean=mean, + median=_get_percentile(50.0), + variance=variance, + std_dev=math.sqrt(variance), + min=min([meas[1] for meas in measurements]), + max=max([meas[1] for meas in measurements]), + count=len(measurements), + percentiles=Percentiles( + p001=_get_percentile(0.1), + p01=_get_percentile(1.0), + p05=_get_percentile(5.0), + p10=_get_percentile(10.0), + p25=_get_percentile(25.0), + p75=_get_percentile(75.0), + p90=_get_percentile(90.0), + p95=_get_percentile(95.0), + p99=_get_percentile(99.0), + p999=_get_percentile(99.9), + ), + ) + + @staticmethod + def from_time_measurements_with_sampling( + measurements: List[Tuple[float, float]], + sample_time: float, + ) -> "DistributionSummary": + """ + Create a DistributionSummary from a list of time measurements of the form + (time, value), where time is the timestamp and value is the measurement. + This method samples the measurements at regular intervals defined by + sample_time. + + :param measurements: A list of tuples containing (time, value) pairs. + :param sample_time: The time interval for sampling. + :return: An instance of DistributionSummary. + """ + measurements.sort(key=lambda x: x[0]) + samples = [] + min_time = measurements[0][0] + max_time = measurements[-1][0] + sample_time + + for time_iter in np.arange( + min_time, + max_time, + sample_time, + ): + count = 0 + while measurements and measurements[0][0] <= time_iter: + count += measurements[0][1] + measurements.pop(0) + samples.append((time_iter, count)) + + return DistributionSummary.from_time_measurements(samples) + + +class StatusDistributionSummary(Serializable): + total: DistributionSummary + completed: DistributionSummary + errored: DistributionSummary + + @staticmethod + def from_values( + completed_values: List[float], + errored_values: List[float], + ) -> "StatusDistributionSummary": + """ + Create a StatusDistributionSummary from completed and errored values. + + :param completed_values: A list of numerical values for completed requests. + :param errored_values: A list of numerical values for errored requests. + :return: An instance of StatusDistributionSummary. + """ + return StatusDistributionSummary( + total=DistributionSummary.from_values( + completed_values + errored_values, + ), + completed=DistributionSummary.from_values(completed_values), + errored=DistributionSummary.from_values(errored_values), + ) + + @staticmethod + def from_time_measurements( + completed_measurements: List[Tuple[float, float]], + errored_measurements: List[Tuple[float, float]], + ) -> "StatusDistributionSummary": + """ + Create a StatusDistributionSummary from completed and errored time measurements. + + :param completed_measurements: A list of tuples containing (time, value) pairs + for completed requests. + :param errored_measurements: A list of tuples containing (time, value) pairs + for errored requests. + :return: An instance of StatusDistributionSummary. + """ + return StatusDistributionSummary( + total=DistributionSummary.from_time_measurements( + completed_measurements + errored_measurements, + ), + completed=DistributionSummary.from_time_measurements( + completed_measurements, + ), + errored=DistributionSummary.from_time_measurements( + errored_measurements, + ), + ) + + @staticmethod + def from_time_measurements_with_sampling( + completed_measurements: List[Tuple[float, float]], + errored_measurements: List[Tuple[float, float]], + sample_time: float, + ) -> "StatusDistributionSummary": + """ + Create a StatusDistributionSummary from completed and errored time measurements + with sampling. + + :param completed_measurements: A list of tuples containing (time, value) pairs + for completed requests. + :param errored_measurements: A list of tuples containing (time, value) pairs + for errored requests. + :param sample_time: The time interval for sampling. + :return: An instance of StatusDistributionSummary. + """ + return StatusDistributionSummary( + total=DistributionSummary.from_time_measurements_with_sampling( + completed_measurements + errored_measurements, + sample_time, + ), + completed=DistributionSummary.from_time_measurements_with_sampling( + completed_measurements, + sample_time, + ), + errored=DistributionSummary.from_time_measurements_with_sampling( + errored_measurements, + sample_time, + ), + ) diff --git a/src/guidellm/core/serializable.py b/src/guidellm/objects/serializable.py similarity index 100% rename from src/guidellm/core/serializable.py rename to src/guidellm/objects/serializable.py diff --git a/src/guidellm/request/__init__.py b/src/guidellm/request/__init__.py index 4feca91c..2a68a521 100644 --- a/src/guidellm/request/__init__.py +++ b/src/guidellm/request/__init__.py @@ -1,13 +1,5 @@ -from .base import GenerationMode, RequestGenerator -from .emulated import EmulatedConfig, EmulatedRequestGenerator -from .file import FileRequestGenerator -from .transformers import TransformersDatasetRequestGenerator +from .request import GenerationRequest __all__ = [ - "EmulatedConfig", - "EmulatedRequestGenerator", - "FileRequestGenerator", - "GenerationMode", - "RequestGenerator", - "TransformersDatasetRequestGenerator", + "GenerationRequest", ] diff --git a/src/guidellm/request/loader.py b/src/guidellm/request/loader.py new file mode 100644 index 00000000..46409041 --- /dev/null +++ b/src/guidellm/request/loader.py @@ -0,0 +1 @@ +# TODO diff --git a/src/guidellm/request/request.py b/src/guidellm/request/request.py new file mode 100644 index 00000000..a11cffad --- /dev/null +++ b/src/guidellm/request/request.py @@ -0,0 +1,79 @@ +import uuid +from typing import Any, Dict, Literal, Optional + +from pydantic import Field + +from guidellm.objects.serializable import Serializable + +__all__ = ["GenerationRequest"] + + +class GenerationRequest(Serializable): + """ + A class representing a request for generation. + This class is used to encapsulate the details of a generation request, + including the request ID, type, content, parameters, statistics, and constraints. + It is designed to be used with the BackendRequestsWorker class to handle + the generation process. + + :param request_id: The unique identifier for the request. + :param request_type: The type of request (e.g., text, chat). + :param content: The content for the request to send to the backend. + If request_type is 'text', this should be a string or list of strings + which will be resolved by backend.text_completions. + If request_type is 'chat', this should be a string, + a list of (str, Dict[str, Union[str, Dict[str, str]], Path, Image]), + or Any raw content which will be resolved by backend.chat_completions. + If raw content, raw_content=True must be passed in the params. + :param params: Additional parameters for the request passed in as kwargs. + For an http backend, these are passed into the body of the request. + :param stats: Statistics for the request, such as the number of prompt tokens. + Used for tracking and reporting purposes. + :param constraints: Constraints for the request, such as the maximum number + of output tokens. Used for controlling the behavior of the backend. + """ + + request_id: Optional[str] = Field( + default_factory=lambda: str(uuid.uuid4()), + description="The unique identifier for the request.", + ) + request_type: Literal["text", "chat"] = Field( + default="text", + description=( + "The type of request (e.g., text, chat). " + "If request_type is 'text', resolved by backend.text_completions. " + "If request_type is 'chat', resolved by backend.chat_completions." + ), + ) + content: Any = Field( + description=( + "The content for the request to send to the backend. " + "If request_type is 'text', this should be a string or list of strings " + "which will be resolved by backend.text_completions. " + "If request_type is 'chat', this should be a string, " + "a list of (str, Dict[str, Union[str, Dict[str, str]], Path, Image]), " + "or Any raw content which will be resolved by backend.chat_completions. " + "If raw content, raw_content=True must be passed in the params." + ) + ) + params: Dict[str, Any] = Field( + default_factory=dict, + description=( + "Additional parameters for the request that will be passed in as kwargs. " + "For an http backend, these are passed into the body of the request. " + ), + ) + stats: Dict[Literal["prompt_tokens"], int] = Field( + default_factory=dict, + description=( + "Statistics for the request, such as the number of prompt tokens. " + "Used for tracking and reporting purposes." + ), + ) + constraints: Dict[Literal["output_tokens"], int] = Field( + default_factory=dict, + description=( + "Constraints for the request, such as the maximum number of output tokens. " + "Used for controlling the behavior of the backend." + ), + ) diff --git a/src/guidellm/scheduler/__init__.py b/src/guidellm/scheduler/__init__.py index d6cabb33..e2addfc2 100644 --- a/src/guidellm/scheduler/__init__.py +++ b/src/guidellm/scheduler/__init__.py @@ -1,11 +1,5 @@ -from .backend_worker import BackendRequestsWorker, GenerationRequest -from .scheduler import ( - RequestsWorker, - Scheduler, - SchedulerRequestInfo, - SchedulerResult, - SchedulerRunInfo, -) +from .result import SchedulerRequestInfo, SchedulerResult, SchedulerRunInfo +from .scheduler import Scheduler from .strategy import ( AsyncConstantStrategy, AsyncPoissonStrategy, @@ -15,20 +9,30 @@ SynchronousStrategy, ThroughputStrategy, ) +from .types import REQ, RES +from .worker import ( + GenerativeRequestsWorker, + RequestsWorker, + WorkerProcessRequest, + WorkerProcessResult, +) __all__ = [ - "GenerationRequest", - "BackendRequestsWorker", - "Scheduler", + "SchedulerRequestInfo", "SchedulerResult", "SchedulerRunInfo", - "SchedulerRequestInfo", - "RequestsWorker", - "StrategyType", + "Scheduler", + "AsyncConstantStrategy", + "AsyncPoissonStrategy", + "ConcurrentStrategy", "SchedulingStrategy", + "StrategyType", "SynchronousStrategy", "ThroughputStrategy", - "ConcurrentStrategy", - "AsyncConstantStrategy", - "AsyncPoissonStrategy", + "REQ", + "RES", + "GenerativeRequestsWorker", + "RequestsWorker", + "WorkerProcessRequest", + "WorkerProcessResult", ] diff --git a/src/guidellm/scheduler/backend_worker.py b/src/guidellm/scheduler/backend_worker.py deleted file mode 100644 index 01543f09..00000000 --- a/src/guidellm/scheduler/backend_worker.py +++ /dev/null @@ -1,250 +0,0 @@ -import asyncio -import math -import time -import uuid -from typing import ( - Any, - AsyncGenerator, - Dict, - Literal, - Optional, - Tuple, - Union, -) - -from pydantic import BaseModel, Field - -from guidellm.backend import ( - Backend, - RequestArgs, - ResponseSummary, - StreamingTextResponse, -) -from guidellm.scheduler.scheduler import RequestsWorker - -__all__ = ["GenerationRequest", "BackendRequestsWorker"] - - -class GenerationRequest(BaseModel): - """ - A class representing a request for generation. - This class is used to encapsulate the details of a generation request, - including the request ID, type, content, parameters, statistics, and constraints. - It is designed to be used with the BackendRequestsWorker class to handle - the generation process. - - :param request_id: The unique identifier for the request. - :param request_type: The type of request (e.g., text, chat). - :param content: The content for the request to send to the backend. - If request_type is 'text', this should be a string or list of strings - which will be resolved by backend.text_completions. - If request_type is 'chat', this should be a string, - a list of (str, Dict[str, Union[str, Dict[str, str]], Path, Image]), - or Any raw content which will be resolved by backend.chat_completions. - If raw content, raw_content=True must be passed in the params. - :param params: Additional parameters for the request passed in as kwargs. - For an http backend, these are passed into the body of the request. - :param stats: Statistics for the request, such as the number of prompt tokens. - Used for tracking and reporting purposes. - :param constraints: Constraints for the request, such as the maximum number - of output tokens. Used for controlling the behavior of the backend. - """ - - request_id: Optional[str] = Field( - default_factory=lambda: str(uuid.uuid4()), - description="The unique identifier for the request.", - ) - request_type: Literal["text", "chat"] = Field( - default="text", - description=( - "The type of request (e.g., text, chat). " - "If request_type is 'text', resolved by backend.text_completions. " - "If request_type is 'chat', resolved by backend.chat_completions." - ), - ) - content: Any = Field( - description=( - "The content for the request to send to the backend. " - "If request_type is 'text', this should be a string or list of strings " - "which will be resolved by backend.text_completions. " - "If request_type is 'chat', this should be a string, " - "a list of (str, Dict[str, Union[str, Dict[str, str]], Path, Image]), " - "or Any raw content which will be resolved by backend.chat_completions. " - "If raw content, raw_content=True must be passed in the params." - ) - ) - params: Dict[str, Any] = Field( - default_factory=dict, - description=( - "Additional parameters for the request that will be passed in as kwargs. " - "For an http backend, these are passed into the body of the request. " - ), - ) - stats: Dict[Literal["prompt_tokens"], int] = Field( - default_factory=dict, - description=( - "Statistics for the request, such as the number of prompt tokens. " - "Used for tracking and reporting purposes." - ), - ) - constraints: Dict[Literal["output_tokens"], int] = Field( - default_factory=dict, - description=( - "Constraints for the request, such as the maximum number of output tokens. " - "Used for controlling the behavior of the backend." - ), - ) - - -class BackendRequestsWorker(RequestsWorker): - """ - A class that handles the execution of requests using a backend. - This class is responsible for sending requests to the backend, - handling responses, and managing errors. - - :param backend: The backend to use for handling requests. - This should be an instance of Backend such as an OpenAIHTTPBackend. - """ - - def __init__(self, backend: Backend): - self.backend = backend - - async def resolve( - self, - request: GenerationRequest, - timeout_time: float, - ) -> ResponseSummary: - """ - Resolve a request by sending it to the backend and handling the response. - This method sends the request to the backend, waits for a response, - and handles any errors that may occur during the process. - - :param request: The request to resolve. - :param timeout_time: The time to wait for a response before timing out. - If timeout_time is math.inf, the request will not timeout. - :return: A ResponseSummary object containing the response from the backend. - If an error occurs, the ResponseSummary will contain the error message. - """ - response = None - error: Optional[str] = None - - try: - request_func, request_kwargs = self._create_request_func_kwargs(request) - - async def _runner(): - # wrap function so we can enforce timeout and - # still return the latest state from the backend - async for resp in request_func(**request_kwargs): - nonlocal response - response = resp - - await asyncio.wait_for( - _runner(), - timeout=timeout_time - time.time() if timeout_time < math.inf else None, - ) - - if not response: - raise ValueError( - f"No response received for request: {request} " - f"and backend: {self.backend}" - ) - if not isinstance(response, ResponseSummary): - raise ValueError( - f"Received no ResponseSummary for request: {request} " - f"and backend: {self.backend}, received: {response}" - ) - except asyncio.TimeoutError as texc: - error = str(texc) - except Exception as exc: # noqa: BLE001 - error = str(exc) - - return self._handle_response(request, response, error) - - def _create_request_func_kwargs( - self, - request: GenerationRequest, - ) -> Tuple[ - AsyncGenerator[Union[StreamingTextResponse, ResponseSummary], None], - Dict[str, Any], - ]: - request_func: AsyncGenerator[ - Union[StreamingTextResponse, ResponseSummary], None - ] - request_kwargs: Dict[str, Any] - - if request.request_type == "text": - request_func = self.backend.text_completions - request_kwargs = { - "prompt": request.content, - "request_id": request.request_id, - "prompt_token_count": request.stats.get("prompt_tokens", None), - "output_token_count": request.constraints.get("output_tokens", None), - **request.params, - } - elif request.request_type == "chat": - request_func = self.backend.chat_completions - request_kwargs = { - "content": request.content, - "request_id": request.request_id, - "prompt_token_count": request.stats.get("prompt_tokens", None), - "output_token_count": request.constraints.get("output_tokens", None), - **request.params, - } - else: - raise ValueError( - f"Invalid request type: {request.request_type} for {request}" - ) - - return request_func, request_kwargs - - def _handle_response( - self, - request: GenerationRequest, - response: Any, - error: Optional[str], - ) -> ResponseSummary: - if response is None or not isinstance( - response, (ResponseSummary, StreamingTextResponse) - ): - # nothing received or invalid response, fill in defaults for error - if response: - error = str( - ValueError( - f"Invalid response: {type(response)} for request: {request}; " - ) - ) + (error or "") - - return ResponseSummary( - value="", - request_args=RequestArgs( - target=self.backend.target, - headers={}, - payload={}, - ), - start_time=None, - end_time=None, - request_id=request.request_id, - error=error or "Unknown error", - ) - - if isinstance(response, StreamingTextResponse): - return ResponseSummary( - value=response.value, - request_args=RequestArgs( - target=self.backend.target, - headers={}, - payload={}, - ), - start_time=response.start_time, - end_time=None, - request_prompt_tokens=request.stats.get("prompt_tokens", None), - request_output_tokens=None, - response_prompt_tokens=None, - response_output_tokens=response.iter_count, - request_id=request.request_id, - error=error or "Unknown error", - ) - - response.error = error - - return response diff --git a/src/guidellm/scheduler/result.py b/src/guidellm/scheduler/result.py new file mode 100644 index 00000000..41edc8c1 --- /dev/null +++ b/src/guidellm/scheduler/result.py @@ -0,0 +1,115 @@ +from typing import ( + Generic, + Literal, + Optional, +) + +from guidellm.objects import Serializable +from guidellm.scheduler.strategy import SchedulingStrategy +from guidellm.scheduler.types import REQ, RES + +__all__ = [ + "SchedulerResult", + "SchedulerRunInfo", + "SchedulerRequestInfo", +] + + +class SchedulerRunInfo(Serializable): + """ + Information about the current run of the scheduler. + This class holds metadata about the scheduling run, + including the start and end times, the number of processes, + and the scheduling strategy used. + It also tracks the number of requests created, queued, pending, + and completed during the run. + + :param start_time: The start time of the scheduling run. + :param end_time: The end time of the scheduling run; + if None, then this will be math.inf. + :param end_number: The maximum number of requests to be processed; + if None, then this will be math.inf. + :param processes: The number of processes used in the scheduling run. + :param strategy: The scheduling strategy used in the run. + This should be an instance of SchedulingStrategy. + :param created_requests: The number of requests created during the run. + :param queued_requests: The number of requests queued during the run. + :param scheduled_requests: The number of requests scheduled during the run. + (requests pending being sent to the worker but recieved by a process) + :param processing_requests: The number of requests actively being run. + :param completed_requests: The number of requests completed during the run. + """ + + start_time: float + end_time: float + end_number: float + processes: int + strategy: SchedulingStrategy + + created_requests: int = 0 + queued_requests: int = 0 + scheduled_requests: int = 0 + processing_requests: int = 0 + completed_requests: int = 0 + + +class SchedulerRequestInfo(Serializable): + """ + Information about a specific request run through the scheduler. + This class holds metadata about the request, including + the targeted start time, queued time, start time, end time, + and the process ID that handled the request. + + :param targeted_start_time: The targeted start time for the request (time.time()). + :param queued_time: The time the request was queued (time.time()). + :param scheduled_time: The time the request was scheduled (time.time()) + (any sleep time before the request was sent to the worker). + :param worker_start: The time the worker started processing request (time.time()). + :param worker_end: The time the worker finished processing request. (time.time()). + :param process_id: The ID of the underlying process that handled the request. + """ + + targeted_start_time: float = -1 + queued_time: float = -1 + scheduled_time: float = -1 + worker_start: float = -1 + worker_end: float = -1 + process_id: int = -1 + + +class SchedulerResult(Generic[REQ, RES], Serializable): + """ + The yielded, iterative result for a scheduler run. + These are triggered on the start and end of the run, + as well as on the start and end of each request. + Depending on the type, it will hold the request and response + along with information and statistics about the request and general run. + + :param type_: The type of the result, which can be one of: + - "run_start": Indicates the start of the run. + - "run_complete": Indicates the completion of the run (teardown happens after). + - "request_start": Indicates the start of a request. + - "request_complete": Indicates the completion of a request. + :param request: The request that was processed. + :param response: The response from the worker for the request. + :param request_info: Information about the request, including + the targeted start time, queued time, start time, end time, + and the process ID that handled the request. + :param run_info: Information about the current run of the scheduler, + including the start and end times, the number of processes, + and the scheduling strategy used. + It also tracks the number of requests created, queued, pending, + and completed during the run. + """ + + type_: Literal[ + "run_start", + "run_complete", + "request_scheduled", + "request_start", + "request_complete", + ] + request: REQ + response: RES + request_info: Optional[SchedulerRequestInfo] + run_info: SchedulerRunInfo diff --git a/src/guidellm/scheduler/scheduler.py b/src/guidellm/scheduler/scheduler.py index 4249ce11..5018f2ed 100644 --- a/src/guidellm/scheduler/scheduler.py +++ b/src/guidellm/scheduler/scheduler.py @@ -3,188 +3,37 @@ import math import multiprocessing import multiprocessing.queues -import os import time -from abc import ABC, abstractmethod -from dataclasses import dataclass from typing import ( Any, AsyncGenerator, + Generic, Iterable, Iterator, List, - Literal, Optional, Tuple, - Union, ) from loguru import logger -from pydantic import BaseModel from guidellm.config import settings -from guidellm.scheduler.strategy import ( - SchedulingStrategy, - SynchronousStrategy, - ThroughputStrategy, +from guidellm.scheduler.result import ( + SchedulerResult, + SchedulerRunInfo, +) +from guidellm.scheduler.strategy import SchedulingStrategy +from guidellm.scheduler.types import REQ, RES +from guidellm.scheduler.worker import ( + RequestsWorker, + WorkerProcessRequest, + WorkerProcessResult, ) -__all__ = [ - "Scheduler", - "SchedulerResult", - "SchedulerRunInfo", - "SchedulerRequestInfo", - "RequestsWorker", -] - - -class RequestsWorker(ABC): - """ - An abstract base class for a worker that processes requests. - This class defines the interface for a worker that can resolve requests - asynchronously or synchronously within the Scheduler class. - Subclasses must implement the `resolve` method, - which takes a request directly given from the load generator, - along with the desired start_time for the request and a timeout_time. - The `resolve` method should return the response from the backend. - """ - - @abstractmethod - async def resolve( - self, - request: Any, - timeout_time: float, - ) -> Any: - """ - An abstract method that must be implemented by subclasses. - This method should handle the resolution of a request through asyncio, - including any necessary backend processing and response handling. - - :param request: The request to be resolved generated by the load generator. - :param timeout_time: The timeout time for the request, if there is no timeout - given, then this will be math.inf. - :return: The response from the worker. - """ - ... - - -class SchedulerRunInfo(BaseModel): - """ - Information about the current run of the scheduler. - This class holds metadata about the scheduling run, - including the start and end times, the number of processes, - and the scheduling strategy used. - It also tracks the number of requests created, queued, pending, - and completed during the run. - - :param start_time: The start time of the scheduling run. - :param end_time: The end time of the scheduling run; - if None, then this will be math.inf. - :param end_number: The maximum number of requests to be processed; - if None, then this will be math.inf. - :param processes: The number of processes used in the scheduling run. - :param strategy: The scheduling strategy used in the run. - This should be an instance of SchedulingStrategy. - :param created_requests: The number of requests created during the run. - :param queued_requests: The number of requests queued during the run. - :param scheduled_requests: The number of requests scheduled during the run. - (requests pending being sent to the worker but recieved by a process) - :param processing_requests: The number of requests actively being run. - :param completed_requests: The number of requests completed during the run. - """ - - start_time: float - end_time: float - end_number: float - processes: int - strategy: SchedulingStrategy - - created_requests: int = 0 - queued_requests: int = 0 - scheduled_requests: int = 0 - processing_requests: int = 0 - completed_requests: int = 0 - - -class SchedulerRequestInfo(BaseModel): - """ - Information about a specific request run through the scheduler. - This class holds metadata about the request, including - the targeted start time, queued time, start time, end time, - and the process ID that handled the request. - - :param targeted_start_time: The targeted start time for the request (time.time()). - :param queued_time: The time the request was queued (time.time()). - :param scheduled_time: The time the request was scheduled (time.time()) - (any sleep time before the request was sent to the worker). - :param worker_start: The time the worker started processing request (time.time()). - :param worker_end: The time the worker finished processing request. (time.time()). - :param process_id: The ID of the underlying process that handled the request. - """ - - targeted_start_time: float = -1 - queued_time: float = -1 - scheduled_time: float = -1 - worker_start: float = -1 - worker_end: float = -1 - process_id: int = -1 - - -class SchedulerResult(BaseModel): - """ - The yielded, iterative result for a scheduler run. - These are triggered on the start and end of the run, - as well as on the start and end of each request. - Depending on the type, it will hold the request and response - along with information and statistics about the request and general run. - - :param type_: The type of the result, which can be one of: - - "run_start": Indicates the start of the run. - - "run_complete": Indicates the completion of the run (teardown happens after). - - "request_start": Indicates the start of a request. - - "request_complete": Indicates the completion of a request. - :param request: The request that was processed. - :param response: The response from the worker for the request. - :param request_info: Information about the request, including - the targeted start time, queued time, start time, end time, - and the process ID that handled the request. - :param run_info: Information about the current run of the scheduler, - including the start and end times, the number of processes, - and the scheduling strategy used. - It also tracks the number of requests created, queued, pending, - and completed during the run. - """ - - type_: Literal[ - "run_start", - "run_complete", - "request_scheduled", - "request_start", - "request_complete", - ] - request: Any - response: Any - request_info: Optional[SchedulerRequestInfo] - run_info: SchedulerRunInfo - - -@dataclass -class _WorkerProcessRequest: - request: Any - start_time: float - timeout_time: Optional[float] - queued_time: float - - -@dataclass -class _WorkerProcessResponse: - type_: Literal["request_scheduled", "request_start", "request_complete"] - request: Any - response: Any - info: SchedulerRequestInfo +__all__ = ["Scheduler"] -class Scheduler: +class Scheduler(Generic[REQ, RES]): """ A class that handles the scheduling of requests to a worker. This class is responsible for managing the lifecycle of the requests, @@ -202,37 +51,12 @@ class Scheduler: :param request_loader: An iterable that generates requests. This can be a list, generator, or any other iterable. The requests will be processed by the worker. - :param scheduling_strategy: The scheduling strategy to use. - Specifies the times at which requests will be sent as well how many - worker processes are used and if requests are scheduled sync or async. - This can be one of the following: - - "synchronous": Requests are sent synchronously. - - "throughput": Requests are sent at the maximum rate possible. - - An instance of SchedulingStrategy. - :param max_number: The maximum number of requests to process. - If None, then no limit is set and either the iterator must be exhaustible - or the max_duration must be set. - :param max_duration: The maximum duration for the scheduling run. - If None, then no limit is set and either the iterator must be exhaustible - or the max_number must be set. - :param num_processes: The number of processes to use for the worker. - If None, then the number of processes is set to the number of CPU cores - minus one, or the max_worker_processes setting if it is lower. - If the scheduling strategy is synchronous, then this is set to 1. - If the scheduling strategy is concurrent, then this is set to the number - of streams in the strategy. """ def __init__( self, - worker: RequestsWorker, - request_loader: Iterable[Any], - scheduling_strategy: Union[ - Literal["synchronous", "throughput"], SchedulingStrategy - ] = "throughput", - max_number: Optional[int] = None, - max_duration: Optional[float] = None, - num_processes: Optional[int] = None, + worker: RequestsWorker[REQ, RES], + request_loader: Iterable[REQ], ): if not isinstance(worker, RequestsWorker): raise ValueError(f"Invalid worker: {worker}") @@ -240,22 +64,15 @@ def __init__( if not isinstance(request_loader, Iterable): raise ValueError(f"Invalid request_loader: {request_loader}") - if scheduling_strategy == "synchronous": - scheduling_strategy = SynchronousStrategy() - elif scheduling_strategy == "throughput": - scheduling_strategy = ThroughputStrategy() + self.worker = worker + self.request_loader = request_loader - if not isinstance(scheduling_strategy, SchedulingStrategy): - raise ValueError(f"Invalid scheduling strategy: {scheduling_strategy}") - - self._worker = worker - self._request_loader = request_loader - self._scheduling_strategy: SchedulingStrategy = scheduling_strategy - self._max_number = max_number - self._max_duration = max_duration - self._num_processes = num_processes - - async def run(self) -> AsyncGenerator[SchedulerResult, None]: + async def run( + self, + scheduling_strategy: SchedulingStrategy, + max_number: Optional[int] = None, + max_duration: Optional[float] = None, + ) -> AsyncGenerator[SchedulerResult[REQ, RES], None]: """ The main method that runs the scheduler. This method is a generator that yields SchedulerResult objects @@ -268,19 +85,44 @@ async def run(self) -> AsyncGenerator[SchedulerResult, None]: The method is designed to be used as an asynchronous generator, allowing it to be used with asyncio and other asynchronous frameworks. + :param scheduling_strategy: The scheduling strategy to use. + Specifies the times at which requests will be sent as well how many + worker processes are used and if requests are scheduled sync or async. + This can be one of the following: + - "synchronous": Requests are sent synchronously. + - "throughput": Requests are sent at the maximum rate possible. + - An instance of SchedulingStrategy. + :param max_number: The maximum number of requests to process. + If None, then no limit is set and either the iterator must be exhaustible + or the max_duration must be set. + :param max_duration: The maximum duration for the scheduling run. + If None, then no limit is set and either the iterator must be exhaustible + or the max_number must be set. :return: An asynchronous generator that yields SchedulerResult objects. Each SchedulerResult object contains information about the request, the response, and the run information. """ + if scheduling_strategy is None or not isinstance( + scheduling_strategy, SchedulingStrategy + ): + raise ValueError(f"Invalid scheduling strategy: {scheduling_strategy}") + + if max_number is not None and max_number < 1: + raise ValueError(f"Invalid max_number: {max_number}") + + if max_duration is not None and max_duration < 0: + raise ValueError(f"Invalid max_duration: {max_duration}") with ( multiprocessing.Manager() as manager, concurrent.futures.ProcessPoolExecutor() as executor, ): futures, requests_queue, responses_queue = await self._start_processes( - manager, executor + manager, executor, scheduling_strategy + ) + run_info, requests_iter, times_iter = self._run_setup( + futures, scheduling_strategy, max_number, max_duration ) - run_info, requests_iter, times_iter = self._run_setup(futures) yield SchedulerResult( type_="run_start", request=None, @@ -298,72 +140,20 @@ async def run(self) -> AsyncGenerator[SchedulerResult, None]: # and yielded all responses break - if requests_iter is not None and not requests_queue.full(): - # we have space on the queue, try to add more requests - # if we've reached the limit number/time or we've exhausted requests - # then set requests_iter to None to stop adding more - try: - request_time = next(times_iter) - if (run_info.queued_requests >= run_info.end_number) or ( - request_time >= run_info.end_time - ): - raise StopIteration - - request = next(requests_iter) - requests_queue.put( - _WorkerProcessRequest( - request=request, - start_time=request_time, - timeout_time=run_info.end_time, - queued_time=time.time(), - ) - ) - run_info.created_requests += 1 - run_info.queued_requests += 1 - except StopIteration: - requests_iter = None - - try: - process_response: _WorkerProcessResponse = ( - responses_queue.get_nowait() - ) + requests_iter = self._add_requests( + requests_iter, + times_iter, + requests_queue, + run_info, + ) + await asyncio.sleep(0) # enable requests to start - if process_response.type_ == "request_scheduled": - run_info.queued_requests -= 1 - run_info.scheduled_requests += 1 - yield SchedulerResult( - type_="request_scheduled", - request=process_response.request, - response=None, - request_info=process_response.info, - run_info=run_info, - ) - elif process_response.type_ == "request_start": - run_info.scheduled_requests -= 1 - run_info.processing_requests += 1 - yield SchedulerResult( - type_="request_start", - request=process_response.request, - response=None, - request_info=process_response.info, - run_info=run_info, - ) - elif process_response.type_ == "request_complete": - run_info.processing_requests -= 1 - run_info.completed_requests += 1 - yield SchedulerResult( - type_="request_complete", - request=process_response.request, - response=process_response.response, - request_info=process_response.info, - run_info=run_info, - ) - else: - raise ValueError( - f"Invalid process response type: {process_response}" - ) - except multiprocessing.queues.Empty: - pass + iter_result = self._check_result_ready( + responses_queue, + run_info, + ) + if iter_result is not None: + yield iter_result # yield control to the event loop await asyncio.sleep(settings.default_async_loop_sleep) @@ -378,255 +168,189 @@ async def run(self) -> AsyncGenerator[SchedulerResult, None]: await self._stop_processes(futures, requests_queue) - def _run_setup( - self, processes: List[asyncio.Future] - ) -> Tuple[SchedulerRunInfo, Iterator[Any], Iterator[float]]: - requests_iter = iter(self._request_loader) - start_time = time.time() - times_iter = iter(self._scheduling_strategy.request_times()) - end_time = time.time() + (self._max_duration or math.inf) - end_number = self._max_number or math.inf - - try: - # update end number if the request loader is finite and less than max - iter_length = len(self._request_loader) - if 0 < iter_length < end_number: - end_number = iter_length - except TypeError: - pass - - if end_number == math.inf and end_time is None: - logger.warning( - "No end number or end time set, " - "scheduler will run indefinitely until the request loader is exhausted." - ) - - info = SchedulerRunInfo( - start_time=start_time, - end_time=end_time, - end_number=end_number, - processes=len(processes), - strategy=self._scheduling_strategy, - ) - - return info, requests_iter, times_iter - async def _start_processes( self, manager, executor: concurrent.futures.ProcessPoolExecutor, + scheduling_strategy: SchedulingStrategy, ) -> Tuple[ List[asyncio.Future], multiprocessing.Queue, multiprocessing.Queue, ]: - processing_mode = self._scheduling_strategy.processing_mode - - num_processes = self._scheduling_strategy.processes_limit - if num_processes is None: - cpu_cores = os.cpu_count() or 1 - num_processes = min(max(1, cpu_cores - 1), settings.max_worker_processes) - - num_processing_requests = self._scheduling_strategy.processing_requests_limit - if num_processing_requests is None: - num_processing_requests = settings.max_concurrency - num_processing_requests_per_process = num_processing_requests // num_processes - - num_queued_requests = self._scheduling_strategy.queued_requests_limit - if num_queued_requests is None: - num_queued_requests = num_processing_requests + num_processes - - requests_queue = manager.Queue(maxsize=num_queued_requests) + requests_queue = manager.Queue( + maxsize=scheduling_strategy.queued_requests_limit + ) responses_queue = manager.Queue() + per_process_requests_limit = scheduling_strategy.processing_requests_limit // ( + scheduling_strategy.num_processes + ) futures = [] loop = asyncio.get_event_loop() - for process_id in range(num_processes): - if processing_mode == "sync": + for process_id in range(scheduling_strategy.num_processes): + if scheduling_strategy.processing_mode == "sync": futures.append( loop.run_in_executor( executor, - self._worker_process_sync, + self.worker.process_loop_synchronous, requests_queue, responses_queue, process_id, ) ) - elif processing_mode == "async": + elif scheduling_strategy.processing_mode == "async": futures.append( loop.run_in_executor( executor, - self._worker_process_async, + self.worker.process_loop_asynchronous, requests_queue, responses_queue, - num_processing_requests_per_process, + per_process_requests_limit, process_id, ) ) else: raise ValueError( - f"Invalid processing mode: {processing_mode} " - f"for strategy: {self._scheduling_strategy}" + f"Invalid processing mode: {scheduling_strategy.processing_mode} " + f"for strategy: {scheduling_strategy}" ) await asyncio.sleep(0.1) # give time for processes to start return futures, requests_queue, responses_queue - async def _stop_processes( + def _run_setup( self, - futures: List[asyncio.Future], - requests_queue: multiprocessing.Queue, - ): - for _ in futures: - requests_queue.put(None) - - await asyncio.gather(*futures) + processes: List[asyncio.Future], + scheduling_strategy: SchedulingStrategy, + max_number: Optional[int], + max_duration: Optional[float], + ) -> Tuple[SchedulerRunInfo, Iterator[Any], Iterator[float]]: + requests_iter = iter(self.request_loader) + start_time = time.time() + times_iter = iter(scheduling_strategy.request_times()) + end_time = time.time() + (max_duration or math.inf) + end_number = max_number or math.inf - def _worker_process_sync( - self, - requests_queue: multiprocessing.Queue, - results_queue: multiprocessing.Queue, - process_id: int, - ): - async def _process_runner(): - while True: - try: - process_request: Optional[_WorkerProcessRequest] = ( - requests_queue.get_nowait() - ) - except multiprocessing.queues.Empty: - # yield control to the event loop - await asyncio.sleep(settings.default_async_loop_sleep) - continue + try: + # update end number if the request loader is finite and less than max + iter_length = len(self.request_loader) + if 0 < iter_length < end_number: + end_number = iter_length + except TypeError: + pass - if process_request is None: # stop signal - break + if end_number == math.inf and end_time is None: + logger.warning( + "No end number or end time set, " + "scheduler will run indefinitely until the request loader is exhausted." + ) - await self._worker_schedule_request( - worker=self._worker, - request=process_request.request, - queued_time=process_request.queued_time, - start_time=process_request.start_time, - timeout_time=process_request.timeout_time, - results_queue=results_queue, - process_id=process_id, - ) - # yield control to event loop - await asyncio.sleep(settings.default_async_loop_sleep) + info = SchedulerRunInfo( + start_time=start_time, + end_time=end_time, + end_number=end_number, + processes=len(processes), + strategy=scheduling_strategy, + ) - try: - asyncio.run(_process_runner()) - except Exception as exc: # noqa: BLE001 - logger.error( - f"Error in worker process {process_id}: {exc}", - exc_info=True, - stack_info=True, - ) + return info, requests_iter, times_iter - def _worker_process_async( + def _add_requests( self, + requests_iter: Optional[Iterator[Any]], + times_iter: Iterator[float], requests_queue: multiprocessing.Queue, - results_queue: multiprocessing.Queue, - max_concurrency: Optional[int], - process_id: int, - ): - async def _process_runner(): - pending = asyncio.Semaphore(max_concurrency) if max_concurrency else None + run_info: SchedulerRunInfo, + ) -> Optional[Iterator[Any]]: + if requests_iter is not None: + try: + added_count = 0 + + while ( + not requests_queue.full() + and added_count < settings.max_add_requests_per_loop + ): + if run_info.queued_requests >= run_info.end_number: + raise StopIteration - while True: - try: - process_request: Optional[_WorkerProcessRequest] = ( - requests_queue.get_nowait() + if request_time := next(times_iter) >= run_info.end_time: + raise StopIteration + + request = next(requests_iter) + work_req: WorkerProcessRequest[REQ] = WorkerProcessRequest( + request=request, + start_time=request_time, + timeout_time=run_info.end_time, ) - except multiprocessing.queues.Empty: - # yield control to event loop - await asyncio.sleep(settings.default_async_loop_sleep) - continue + requests_queue.put(work_req) - if process_request is None: # stop signal - break + run_info.created_requests += 1 + run_info.queued_requests += 1 + added_count += 1 + except StopIteration: + # we've reached the limit number, limit time, or exhausted the requests + # set to None to stop adding more and tell the loop no more requests + requests_iter = None - if pending: - await pending.acquire() - - def _task_done(_: asyncio.Task): - nonlocal pending - if pending: - pending.release() - - task = asyncio.create_task( - self._worker_schedule_request( - worker=self._worker, - request=process_request.request, - queued_time=process_request.queued_time, - start_time=process_request.start_time, - timeout_time=process_request.timeout_time, - results_queue=results_queue, - process_id=process_id, - ) - ) - task.add_done_callback(_task_done) - # yield control to event loop - await asyncio.sleep(settings.default_async_loop_sleep) + return requests_iter + def _check_result_ready( + self, + responses_queue: multiprocessing.Queue, + run_info: SchedulerRunInfo, + ) -> Optional[SchedulerResult]: try: - asyncio.run(_process_runner()) - except Exception as exc: # noqa: BLE001 - logger.error( - f"Error in worker process {process_id}: {exc}", - exc_info=True, - stack_info=True, + process_response: WorkerProcessResult[REQ, RES] = ( + responses_queue.get_nowait() ) + except multiprocessing.queues.Empty: + return None - @staticmethod - async def _worker_schedule_request( - worker: RequestsWorker, - request: Any, - queued_time: float, - start_time: float, - timeout_time: float, - results_queue: multiprocessing.Queue, - process_id: int, - ): - info = SchedulerRequestInfo( - targeted_start_time=start_time, - queued_time=queued_time, - scheduled_time=time.time(), - worker_start=-1, - worker_end=-1, - process_id=process_id, - ) - results_queue.put( - _WorkerProcessResponse( + if process_response.type_ == "request_scheduled": + run_info.queued_requests -= 1 + run_info.scheduled_requests += 1 + + return SchedulerResult( type_="request_scheduled", - request=request, + request=process_response.request, response=None, - info=info, + request_info=process_response.info, + run_info=run_info, ) - ) - if (wait_time := start_time - time.time()) > 0: - await asyncio.sleep(wait_time) + if process_response.type_ == "request_start": + run_info.scheduled_requests -= 1 + run_info.processing_requests += 1 - info.worker_start = time.time() - results_queue.put( - _WorkerProcessResponse( + return SchedulerResult( type_="request_start", - request=request, + request=process_response.request, response=None, - info=info, + request_info=process_response.info, + run_info=run_info, ) - ) - response = await worker.resolve(request, timeout_time) + if process_response.type_ == "request_complete": + run_info.processing_requests -= 1 + run_info.completed_requests += 1 - info.worker_end = time.time() - results_queue.put( - _WorkerProcessResponse( + return SchedulerResult( type_="request_complete", - request=request, - response=response, - info=info, + request=process_response.request, + response=process_response.response, + request_info=process_response.info, + run_info=run_info, ) - ) + raise ValueError(f"Invalid process response type: {process_response}") + + async def _stop_processes( + self, + futures: List[asyncio.Future], + requests_queue: multiprocessing.Queue, + ): + for _ in futures: + requests_queue.put(None) + + await asyncio.gather(*futures) diff --git a/src/guidellm/scheduler/strategy.py b/src/guidellm/scheduler/strategy.py index 0edd68d2..4c1b37e9 100644 --- a/src/guidellm/scheduler/strategy.py +++ b/src/guidellm/scheduler/strategy.py @@ -1,4 +1,5 @@ import math +import os import random import time from abc import ABC, abstractmethod @@ -8,7 +9,10 @@ Optional, ) -from pydantic import BaseModel, Field +from pydantic import Field + +from guidellm.config import settings +from guidellm.objects import Serializable __all__ = [ "StrategyType", @@ -24,7 +28,7 @@ StrategyType = Literal["synchronous", "concurrent", "throughput", "constant", "poisson"] -class SchedulingStrategy(ABC, BaseModel): +class SchedulingStrategy(ABC, Serializable): """ An abstract base class for scheduling strategies. This class defines the interface for scheduling requests and provides @@ -40,6 +44,38 @@ class SchedulingStrategy(ABC, BaseModel): description="The type of scheduling strategy schedule requests with.", ) + @property + def default_processes_limit(self) -> int: + """ + The default limit on the number of worker processes for the scheduling strategy. + + :return: The minimum between the number of CPU cores minus one + and the maximum number of worker processes allowed by settings. + """ + cpu_cores = os.cpu_count() or 1 + + return min(max(1, cpu_cores - 1), settings.max_worker_processes) + + @property + def default_queued_requests_limit(self) -> int: + """ + The default limit on the number of queued requests for the scheduling strategy. + + :return: The max concurrency value from settings, ensuring there are enough + requests even for the worst case scenario where the max concurrent requests + are pulled at once for processing. + """ + return settings.max_concurrency + + @property + def default_processing_requests_limit(self) -> int: + """ + The default limit on the number of active requests for the scheduling strategy. + + :return: The max concurrency value from settings. + """ + return settings.max_concurrency + @property @abstractmethod def processing_mode(self) -> Literal["sync", "async"]: @@ -57,16 +93,13 @@ def processing_mode(self) -> Literal["sync", "async"]: @property @abstractmethod - def processes_limit(self) -> Optional[int]: + def processes_limit(self) -> int: """ - The limit on the number of worker processes for the scheduling strategy - or None if the strategy does not restrict the number of processes. - This property determines how many worker processes are created - for the scheduling strategy. This property should be implemented - by subclasses to return the appropriate number of processes. + The limit on the number of worker processes for the scheduling strategy. + It determines how many worker processes are created + for the scheduling strategy and must be implemented by subclasses. - :return: The number of processes for the scheduling strategy - or None if the strategy does not restrict the number of processes. + :return: The number of processes for the scheduling strategy. """ ... @@ -74,14 +107,11 @@ def processes_limit(self) -> Optional[int]: @abstractmethod def queued_requests_limit(self) -> Optional[int]: """ - The maximum number of queued requests for the scheduling strategy or None - if the strategy does not restrict the number of queued requests. - This property determines how many requests can be queued at one time - for the scheduling strategy. This property should be implemented - by subclasses to return the appropriate number of queued requests. + The maximum number of queued requests for the scheduling strategy. + It determines how many requests can be queued at one time + for the scheduling strategy and must be implemented by subclasses. - :return: The maximum number of queued requests for the scheduling strategy - or None if the strategy does not restrict the number of queued requests. + :return: The maximum number of queued requests for the scheduling strategy. """ ... @@ -89,14 +119,11 @@ def queued_requests_limit(self) -> Optional[int]: @abstractmethod def processing_requests_limit(self) -> Optional[int]: """ - The maximum number of processing requests for the scheduling strategy - or None if the strategy does not restrict the number of processing requests. - This property determines how many requests can be processed at one time - for the scheduling strategy. This property should be implemented - by subclasses to return the appropriate number of processing requests. + The maximum number of processing requests for the scheduling strategy. + It determines how many requests can be processed at one time + for the scheduling strategy and must be implemented by subclasses. - :return: The maximum number of processing requests for the scheduling strategy - or None if the strategy does not restrict the number of processing requests. + :return: The maximum number of processing requests for the scheduling strategy. """ ... @@ -140,12 +167,11 @@ def processing_mode(self) -> Literal["sync"]: return "sync" @property - def processes_limit(self) -> Optional[int]: + def processes_limit(self) -> int: """ - The limit on the number of worker processes for the scheduling strategy - or None if the strategy does not restrict the number of processes. - This property determines how many worker processes are created - for the scheduling strategy. + The limit on the number of worker processes for the scheduling strategy. + It determines how many worker processes are created + for the scheduling strategy and must be implemented by subclasses. :return: 1 for the synchronous scheduling strategy to limit the worker processes to one. @@ -153,12 +179,11 @@ def processes_limit(self) -> Optional[int]: return 1 @property - def queued_requests_limit(self) -> Optional[int]: + def queued_requests_limit(self) -> int: """ - The maximum number of queued requests for the scheduling strategy or None - if the strategy does not restrict the number of queued requests. - This property determines how many requests can be queued at one time - for the scheduling strategy. + The maximum number of queued requests for the scheduling strategy. + It determines how many requests can be queued at one time + for the scheduling strategy and must be implemented by subclasses. :return: 1 for the synchronous scheduling strategy to limit the queued requests to one that is ready to be processed. @@ -166,12 +191,11 @@ def queued_requests_limit(self) -> Optional[int]: return 1 @property - def processing_requests_limit(self) -> Optional[int]: + def processing_requests_limit(self) -> int: """ - The maximum number of processing requests for the scheduling strategy - or None if the strategy does not restrict the number of processing requests. - This property determines how many requests can be processed at one time - for the scheduling strategy. + The maximum number of processing requests for the scheduling strategy. + It determines how many requests can be processed at one time + for the scheduling strategy and must be implemented by subclasses. :return: 1 for the synchronous scheduling strategy to limit the processing requests to one that is ready to be processed. @@ -227,12 +251,11 @@ def processing_mode(self) -> Literal["sync"]: return "sync" @property - def processes_limit(self) -> Optional[int]: + def processes_limit(self) -> int: """ - The limit on the number of worker processes for the scheduling strategy - or None if the strategy does not restrict the number of processes. - This property determines how many worker processes are created - for the scheduling strategy. + The limit on the number of worker processes for the scheduling strategy. + It determines how many worker processes are created + for the scheduling strategy and must be implemented by subclasses. :return: {self.streams} for the concurrent scheduling strategy to limit the worker processes to the number of streams. @@ -240,12 +263,11 @@ def processes_limit(self) -> Optional[int]: return self.streams @property - def queued_requests_limit(self) -> Optional[int]: + def queued_requests_limit(self) -> int: """ - The maximum number of queued requests for the scheduling strategy or None - if the strategy does not restrict the number of queued requests. - This property determines how many requests can be queued at one time - for the scheduling strategy. + The maximum number of queued requests for the scheduling strategy. + It determines how many requests can be queued at one time + for the scheduling strategy and must be implemented by subclasses. :return: {self.streams} for the concurrent scheduling strategy to limit the queued requests to the number of streams that are ready to be processed. @@ -253,12 +275,11 @@ def queued_requests_limit(self) -> Optional[int]: return self.streams @property - def processing_requests_limit(self) -> Optional[int]: + def processing_requests_limit(self) -> int: """ - The maximum number of processing requests for the scheduling strategy - or None if the strategy does not restrict the number of processing requests. - This property determines how many requests can be processed at one time - for the scheduling strategy. + The maximum number of processing requests for the scheduling strategy. + It determines how many requests can be processed at one time + for the scheduling strategy and must be implemented by subclasses. :return: {self.streams} for the concurrent scheduling strategy to limit the processing requests to the number of streams that ready to be processed. @@ -313,44 +334,43 @@ def processing_mode(self) -> Literal["async"]: return "async" @property - def processes_limit(self) -> Optional[int]: + def processes_limit(self) -> int: """ - The limit on the number of worker processes for the scheduling strategy - or None if the strategy does not restrict the number of processes. - This property determines how many worker processes are created - for the scheduling strategy. + The limit on the number of worker processes for the scheduling strategy. + It determines how many worker processes are created + for the scheduling strategy and must be implemented by subclasses. - :return: None for the throughput scheduling strategy to apply - no limit on the number of processes. + :return: The default processes limit since none is enforced for + asynchronous strategies. """ - return None + return self.default_processes_limit @property - def queued_requests_limit(self) -> Optional[int]: + def queued_requests_limit(self) -> int: """ - The maximum number of queued requests for the scheduling strategy or None - if the strategy does not restrict the number of queued requests. - This property determines how many requests can be queued at one time - for the scheduling strategy. + The maximum number of queued requests for the scheduling strategy. + It determines how many requests can be queued at one time + for the scheduling strategy and must be implemented by subclasses. - :return: None for the throughput scheduling strategy to apply - no limit on the number of queued requests. + :return: The processing requests limit to ensure that there are enough + requests even for the worst case scenario where the max concurrent + requests are pulled at once for processing. """ - return None + return self.processing_requests_limit @property - def processing_requests_limit(self) -> Optional[int]: + def processing_requests_limit(self) -> int: """ - The maximum number of processing requests for the scheduling strategy - or None if the strategy does not restrict the number of processing requests. - This property determines how many requests can be processed at one time - for the scheduling strategy. + The maximum number of processing requests for the scheduling strategy. + It determines how many requests can be processed at one time + for the scheduling strategy and must be implemented by subclasses. :return: {self.max_concurrency} for the throughput scheduling strategy to limit the processing requests to the maximum concurrency. - If max_concurrency is None, this will be set to None. + If max_concurrency is None, then the default processing requests limit + will be used. """ - return self.max_concurrency + return self.max_concurrency or self.default_processing_requests_limit def request_times(self) -> Generator[float, None, None]: """ @@ -366,7 +386,7 @@ def request_times(self) -> Generator[float, None, None]: yield start_time -class AsyncConstantStrategy(SchedulingStrategy): +class AsyncConstantStrategy(ThroughputStrategy): """ A class representing an asynchronous constant scheduling strategy. This strategy schedules requests asynchronously at a constant request rate @@ -403,57 +423,6 @@ class AsyncConstantStrategy(SchedulingStrategy): ), ) - @property - def processing_mode(self) -> Literal["async"]: - """ - The processing mode for the scheduling strategy, either 'sync' or 'async'. - This property determines how the worker processes are setup: - either to run synchronously with one request at a time or asynchronously. - - :return: 'async' for asynchronous scheduling strategy - for the multiple worker processes handling requests. - """ - return "async" - - @property - def processes_limit(self) -> Optional[int]: - """ - The limit on the number of worker processes for the scheduling strategy - or None if the strategy does not restrict the number of processes. - This property determines how many worker processes are created - for the scheduling strategy. - - :return: None for the async constant scheduling strategy to apply - no limit on the number of processes. - """ - return None - - @property - def queued_requests_limit(self) -> Optional[int]: - """ - The maximum number of queued requests for the scheduling strategy or None - if the strategy does not restrict the number of queued requests. - This property determines how many requests can be queued at one time - for the scheduling strategy. - - :return: None for the async constant scheduling strategy to apply - no limit on the number of queued requests. - """ - return None - - @property - def processing_requests_limit(self) -> Optional[int]: - """ - The maximum number of processing requests for the scheduling strategy - or None if the strategy does not restrict the number of processing requests. - This property determines how many requests can be processed at one time - for the scheduling strategy. - - :return: None for the async constant scheduling strategy to apply - no limit on the number of processing requests. - """ - return None - def request_times(self) -> Generator[float, None, None]: """ A generator that yields timestamps for when requests should be sent. @@ -487,7 +456,7 @@ def request_times(self) -> Generator[float, None, None]: counter += 1 -class AsyncPoissonStrategy(SchedulingStrategy): +class AsyncPoissonStrategy(ThroughputStrategy): """ A class representing an asynchronous Poisson scheduling strategy. This strategy schedules requests asynchronously at a Poisson request rate @@ -522,57 +491,6 @@ class AsyncPoissonStrategy(SchedulingStrategy): ), ) - @property - def processing_mode(self) -> Literal["async"]: - """ - The processing mode for the scheduling strategy, either 'sync' or 'async'. - This property determines how the worker processes are setup: - either to run synchronously with one request at a time or asynchronously. - - :return: 'async' for asynchronous scheduling strategy - for the multiple worker processes handling requests. - """ - return "async" - - @property - def processes_limit(self) -> Optional[int]: - """ - The limit on the number of worker processes for the scheduling strategy - or None if the strategy does not restrict the number of processes. - This property determines how many worker processes are created - for the scheduling strategy. - - :return: None for the async poisson scheduling strategy to apply - no limit on the number of processes. - """ - return None - - @property - def queued_requests_limit(self) -> Optional[int]: - """ - The maximum number of queued requests for the scheduling strategy or None - if the strategy does not restrict the number of queued requests. - This property determines how many requests can be queued at one time - for the scheduling strategy. - - :return: None for the async poisson scheduling strategy to apply - no limit on the number of queued requests. - """ - return None - - @property - def processing_requests_limit(self) -> Optional[int]: - """ - The maximum number of processing requests for the scheduling strategy - or None if the strategy does not restrict the number of processing requests. - This property determines how many requests can be processed at one time - for the scheduling strategy. - - :return: None for the async poisson scheduling strategy to apply - no limit on the number of processing requests. - """ - return None - def request_times(self) -> Generator[float, None, None]: """ A generator that yields timestamps for when requests should be sent. diff --git a/src/guidellm/scheduler/types.py b/src/guidellm/scheduler/types.py new file mode 100644 index 00000000..46ff4b9b --- /dev/null +++ b/src/guidellm/scheduler/types.py @@ -0,0 +1,7 @@ +from typing import TypeVar + +__all__ = ["REQ", "RES"] + + +REQ = TypeVar("REQ") +RES = TypeVar("RES") diff --git a/src/guidellm/scheduler/worker.py b/src/guidellm/scheduler/worker.py new file mode 100644 index 00000000..3416a02a --- /dev/null +++ b/src/guidellm/scheduler/worker.py @@ -0,0 +1,412 @@ +import asyncio +import math +import multiprocessing +import multiprocessing.queues +import time +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import ( + Any, + AsyncGenerator, + Dict, + Generic, + Literal, + Optional, + Tuple, + Union, +) + +from loguru import logger +from pydantic import Field + +from guidellm.backend import ( + Backend, + BackendType, + RequestArgs, + ResponseSummary, + StreamingTextResponse, +) +from guidellm.config import settings +from guidellm.objects import Serializable +from guidellm.request import GenerationRequest +from guidellm.scheduler.result import SchedulerRequestInfo +from guidellm.scheduler.types import REQ, RES + +__all__ = [ + "WorkerProcessRequest", + "WorkerProcessResult", + "RequestsWorker", + "GenerativeRequestsWorker", +] + + +@dataclass +class WorkerProcessRequest(Generic[REQ]): + request: REQ + start_time: float + timeout_time: Optional[float] + queued_time: float + + +@dataclass +class WorkerProcessResult(Generic[REQ, RES]): + type_: Literal["request_scheduled", "request_start", "request_complete"] + request: REQ + response: RES + info: SchedulerRequestInfo + + +class RequestsWorker(Generic[REQ, RES], ABC): + """ + An abstract base class for a worker that processes requests. + This class defines the interface for a worker that can resolve requests + asynchronously or synchronously within the Scheduler class. + Subclasses must implement the `resolve` method, + which takes a request directly given from the load generator, + along with the desired start_time for the request and a timeout_time. + The `resolve` method should return the response from the backend. + """ + + @property + @abstractmethod + def description(self) -> Serializable: + """ + An abstract property that must be implemented by subclasses. + This property should return a Serializable class representing the information + about the worker instance. + """ + ... + + @abstractmethod + async def resolve( + self, + request: REQ, + timeout_time: float, + ) -> RES: + """ + An abstract method that must be implemented by subclasses. + This method should handle the resolution of a request through asyncio, + including any necessary backend processing and response handling. + + :param request: The request to be resolved generated by the load generator. + :param timeout_time: The timeout time for the request, if there is no timeout + given, then this will be math.inf. + :return: The response from the worker. + """ + ... + + async def resolve_scheduler_request( + self, + request: Any, + queued_time: float, + start_time: float, + timeout_time: float, + results_queue: multiprocessing.Queue, + process_id: int, + ): + info = SchedulerRequestInfo( + targeted_start_time=start_time, + queued_time=queued_time, + scheduled_time=time.time(), + worker_start=-1, + worker_end=-1, + process_id=process_id, + ) + result: WorkerProcessResult[REQ, RES] = WorkerProcessResult( + type_="request_scheduled", + request=request, + response=None, + info=info, + ) + results_queue.put(result) + + if (wait_time := start_time - time.time()) > 0: + await asyncio.sleep(wait_time) + + info.worker_start = time.time() + result = WorkerProcessResult( + type_="request_start", + request=request, + response=None, + info=info, + ) + results_queue.put(result) + + response = await self.resolve(request, timeout_time) + + info.worker_end = time.time() + result = WorkerProcessResult( + type_="request_complete", + request=request, + response=response, + info=info, + ) + results_queue.put(result) + + def process_loop_synchronous( + self, + requests_queue: multiprocessing.Queue, + results_queue: multiprocessing.Queue, + process_id: int, + ): + async def _process_runner(): + while True: + try: + process_request: Optional[WorkerProcessRequest[REQ]] = ( + requests_queue.get_nowait() + ) + except multiprocessing.queues.Empty: + # yield control to the event loop + await asyncio.sleep(settings.default_async_loop_sleep) + continue + + if process_request is None: # stop signal + break + + await self.resolve_scheduler_request( + request=process_request.request, + queued_time=process_request.queued_time, + start_time=process_request.start_time, + timeout_time=process_request.timeout_time, + results_queue=results_queue, + process_id=process_id, + ) + + try: + asyncio.run(_process_runner()) + except Exception as exc: # noqa: BLE001 + logger.error( + f"Error in worker process {process_id}: {exc}", + exc_info=True, + stack_info=True, + ) + + def process_loop_asynchronous( + self, + requests_queue: multiprocessing.Queue, + results_queue: multiprocessing.Queue, + max_concurrency: Optional[int], + process_id: int, + ): + async def _process_runner(): + pending = asyncio.Semaphore(max_concurrency) if max_concurrency else None + + while True: + try: + process_request: Optional[WorkerProcessRequest[REQ]] = ( + requests_queue.get_nowait() + ) + except multiprocessing.queues.Empty: + # yield control to event loop + await asyncio.sleep(settings.default_async_loop_sleep) + continue + + if process_request is None: # stop signal + break + + if pending: + await pending.acquire() + + def _task_done(_: asyncio.Task): + nonlocal pending + if pending: + pending.release() + + task = asyncio.create_task( + self.resolve_scheduler_request( + request=process_request.request, + queued_time=process_request.queued_time, + start_time=process_request.start_time, + timeout_time=process_request.timeout_time, + results_queue=results_queue, + process_id=process_id, + ) + ) + task.add_done_callback(_task_done) + await asyncio.sleep(0) # enable start task immediately + + try: + asyncio.run(_process_runner()) + except Exception as exc: # noqa: BLE001 + logger.error( + f"Error in worker process {process_id}: {exc}", + exc_info=True, + stack_info=True, + ) + + +class GenerativeRequestsWorkerDescription(Serializable): + backend_type: BackendType + backend_target: str + backend_model: str + backend_info: Dict[str, Any] = Field( + default_factory=dict, + ) + + +class GenerativeRequestsWorker(RequestsWorker[GenerationRequest, ResponseSummary]): + """ + A class that handles the execution of requests using a backend. + This class is responsible for sending requests to the backend, + handling responses, and managing errors. + + :param backend: The backend to use for handling requests. + This should be an instance of Backend such as an OpenAIHTTPBackend. + """ + + def __init__(self, backend: Backend): + self.backend = backend + self.backend.validate() + + @property + def description(self) -> Serializable: + """ + Get the description of the worker. + :return: The description of the worker. + """ + return GenerativeRequestsWorkerDescription( + backend_type=self.backend.type_, + backend_target=self.backend.target, + backend_model=self.backend.model, + backend_info=self.backend.info, + ) + + async def resolve( + self, + request: GenerationRequest, + timeout_time: float, + ) -> ResponseSummary: + """ + Resolve a request by sending it to the backend and handling the response. + This method sends the request to the backend, waits for a response, + and handles any errors that may occur during the process. + + :param request: The request to resolve. + :param timeout_time: The time to wait for a response before timing out. + If timeout_time is math.inf, the request will not timeout. + :return: A ResponseSummary object containing the response from the backend. + If an error occurs, the ResponseSummary will contain the error message. + """ + response = None + error: Optional[str] = None + + try: + request_func, request_kwargs = self._create_request_func_kwargs(request) + + async def _runner(): + # wrap function so we can enforce timeout and + # still return the latest state from the backend + async for resp in request_func(**request_kwargs): + nonlocal response + response = resp + + await asyncio.wait_for( + _runner(), + timeout=timeout_time - time.time() if timeout_time < math.inf else None, + ) + + if not response: + raise ValueError( + f"No response received for request: {request} " + f"and backend: {self.backend}" + ) + if not isinstance(response, ResponseSummary): + raise ValueError( + f"Received no ResponseSummary for request: {request} " + f"and backend: {self.backend}, received: {response}" + ) + except asyncio.TimeoutError as texc: + error = str(texc) + except Exception as exc: # noqa: BLE001 + error = str(exc) + + return self._handle_response(request, response, error) + + def _create_request_func_kwargs( + self, + request: GenerationRequest, + ) -> Tuple[ + AsyncGenerator[Union[StreamingTextResponse, ResponseSummary], None], + Dict[str, Any], + ]: + request_func: AsyncGenerator[ + Union[StreamingTextResponse, ResponseSummary], None + ] + request_kwargs: Dict[str, Any] + + if request.request_type == "text": + request_func = self.backend.text_completions + request_kwargs = { + "prompt": request.content, + "request_id": request.request_id, + "prompt_token_count": request.stats.get("prompt_tokens", None), + "output_token_count": request.constraints.get("output_tokens", None), + **request.params, + } + elif request.request_type == "chat": + request_func = self.backend.chat_completions + request_kwargs = { + "content": request.content, + "request_id": request.request_id, + "prompt_token_count": request.stats.get("prompt_tokens", None), + "output_token_count": request.constraints.get("output_tokens", None), + **request.params, + } + else: + raise ValueError( + f"Invalid request type: {request.request_type} for {request}" + ) + + return request_func, request_kwargs + + def _handle_response( + self, + request: GenerationRequest, + response: Any, + error: Optional[str], + ) -> ResponseSummary: + if response is None or not isinstance( + response, (ResponseSummary, StreamingTextResponse) + ): + # nothing received or invalid response, fill in defaults for error + if response: + error = str( + ValueError( + f"Invalid response: {type(response)} for request: {request}; " + ) + ) + (error or "") + + return ResponseSummary( + value="", + request_args=RequestArgs( + target=self.backend.target, + headers={}, + payload={}, + ), + start_time=None, + end_time=None, + request_id=request.request_id, + error=error or "Unknown error", + ) + + if isinstance(response, StreamingTextResponse): + return ResponseSummary( + value=response.value, + request_args=RequestArgs( + target=self.backend.target, + headers={}, + payload={}, + ), + start_time=response.start_time, + end_time=None, + request_prompt_tokens=request.stats.get("prompt_tokens", None), + request_output_tokens=None, + response_prompt_tokens=None, + response_output_tokens=response.iter_count, + request_id=request.request_id, + error=error or "Unknown error", + ) + + response.error = error + + return response diff --git a/tests/unit/core/test_serializable.py b/tests/unit/core/test_serializable.py index ce0cec8a..b23ae062 100644 --- a/tests/unit/core/test_serializable.py +++ b/tests/unit/core/test_serializable.py @@ -3,7 +3,7 @@ import pytest -from guidellm.core.serializable import Serializable +from guidellm.objects.serializable import Serializable class ExampleModel(Serializable): From 53e094375fa9b8c33f69a01eb85236bcf1d6b1e2 Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Mon, 17 Mar 2025 20:51:55 +0000 Subject: [PATCH 11/43] finalize benchmark model objects --- src/guidellm/benchmark/benchmark.py | 542 ++++++++++++++++++++++---- src/guidellm/benchmark/benchmarker.py | 2 +- src/guidellm/benchmark/profile.py | 38 +- src/guidellm/objects/distribution.py | 423 ++++++++++++++------ 4 files changed, 807 insertions(+), 198 deletions(-) diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index 28abd5aa..487510e4 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -1,4 +1,6 @@ -from typing import Any, Dict, List, Optional, TypeVar +import random +import uuid +from typing import Any, Dict, List, Literal, Optional, TypeVar from pydantic import Field, computed_field @@ -17,74 +19,248 @@ ] -class BenchmarkSettings(Serializable): - profile: Profile - profile_index: int - strategy: SchedulingStrategy - max_number: Optional[int] - max_duration: Optional[float] - warmup_number: Optional[int] - warmup_duration: Optional[float] - cooldown_number: Optional[int] - cooldown_duration: Optional[float] +class BenchmarkArgs(Serializable): + """ + A serializable model representing the arguments used to specify a benchmark run + and how data was collected for it. + """ + + profile: Profile = Field( + description=( + "The profile used for the entire benchmark run that the strategy for " + "this benchmark was pulled from." + ) + ) + strategy_index: int = Field( + description=( + "The index of the strategy in the profile that was used for this benchmark." + ) + ) + strategy: SchedulingStrategy = Field( + description="The scheduling strategy used to run this benchmark. " + ) + max_number: Optional[int] = Field( + description="The maximum number of requests to run for this benchmark, if any." + ) + max_duration: Optional[float] = Field( + description="The maximum duration in seconds to run this benchmark, if any." + ) + warmup_number: Optional[int] = Field( + description=( + "The number of requests to run for the warmup phase of this benchmark, " + "if any. These are requests that were not included in the final results." + ) + ) + warmup_duration: Optional[float] = Field( + description=( + "The duration in seconds to run for the warmup phase of this benchmark, " + "if any. These are requests that were not included in the final results." + ) + ) + cooldown_number: Optional[int] = Field( + description=( + "The number of requests to run for the cooldown phase of this benchmark, " + "if any. These are requests that were not included in the final results." + ) + ) + cooldown_duration: Optional[float] = Field( + description=( + "The duration in seconds to run for the cooldown phase of this benchmark, " + "if any. These are requests that were not included in the final results." + ) + ) class BenchmarkRunStats(Serializable): - run_start_time: float - run_end_time: float + """ + A serializable model representing the run process statistics for the + entire benchmark run across all requests including warmup and cooldown. + """ - completed: int - errored: int - total: int + start_time: float = Field( + description="The start time of the benchmark run.", + ) + end_time: float = Field( + description="The end time of the benchmark run.", + ) - queued_time_avg: float - scheduled_time_avg: float - worker_time_avg: float - worker_delay_avg: float - request_delay_avg: float - process_idle_time_avg: float + total: int = Field( + description=( + "The total number of requests in the benchmark run, " + "including warmup and cooldown." + ), + ) + total_completed: int = Field( + description=( + "The total number of completed requests in the benchmark run, " + "including warmup and cooldown." + ), + ) + total_errored: int = Field( + description=( + "The total number of errored requests in the benchmark run, " + "including warmup and cooldown." + ) + ) + + queued_time_avg: float = Field( + description=( + "The average time spent in the queue for requests in the benchmark run." + ) + ) + scheduled_time_avg: float = Field( + description=( + "The average time spent in the scheduled state for requests in the " + "benchmark run." + ) + ) + worker_time_avg: float = Field( + description=( + "The average time spent running each request in the benchmark run." + ) + ) + worker_delay_avg: float = Field( + description=( + "The average delay between when a request was targeted to start at " + "and when it was started by the worker in the benchmark run." + ) + ) + resolve_delay_avg: float = Field( + description=( + "The average delay between when a request was targeted to start at " + "and when it was resolved/requested by the worker in the benchmark run." + ) + ) + process_idle_time_avg: float = Field( + description=( + "The average time spent in the idle state for each process in the " + "benchmark run where it wasn't actively running a request." + ) + ) class Benchmark(Serializable): - settings: BenchmarkSettings - run_stats: BenchmarkRunStats - worker_description: Serializable - requests_loader_description: Serializable + """ + The base serializable model representing a benchmark run and its results. + Specific benchmarker implementations should extend this model to include + additional information or metadata as needed. + + Note, requests_per_second and requests_concurrency are kept at this level + and are expected to be populated by the subclass implementation to ensure + the logic for Profiles can include more complicated logic for determining + what rates and concurrency values to use for subsequent strategies. + """ + + id_: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="The unique identifier for the benchmark.", + ) + run_id: str = Field( + description=( + "The unique identifier for the encompasing benchmark run that this " + "benchmark was a part of." + ) + ) + args: BenchmarkArgs = Field( + description=( + "The arguments used to specify how to run the benchmark and collect data." + ) + ) + run_stats: BenchmarkRunStats = Field( + description=( + "The process statistics for the entire benchmark run across all requests." + ) + ) + worker: Optional[Serializable] = Field( + description=( + "The description and specifics for the worker used to resolve requests " + "for this benchmark." + ) + ) + requests_loader: Optional[Serializable] = Field( + description=( + "The description and specifics for the request loader used to create " + "requests for this benchmark." + ) + ) extras: Dict[str, Any] = Field( - default_factory=dict, + description=( + "Any additional information or metadata that was passed for this benchmark." + ) ) - requests_per_second: StatusDistributionSummary - requests_concurrency: StatusDistributionSummary + requests_per_second: StatusDistributionSummary = Field( + description="The distribution of requests per second for the benchmark.", + ) + requests_concurrency: StatusDistributionSummary = Field( + description="The distribution of requests concurrency for the benchmark.", + ) BENCH = TypeVar("BENCH", bound=Benchmark) class GenerativeTextResponseStats(Serializable): - request_id: str - prompt: str - output: str - prompt_tokens: int - output_tokens: int - start_time: float - end_time: float - first_token_time: float - last_token_time: float + """ + A serializable model representing the request values, response values, and + statistics for a generative text response. + """ + + request_id: str = Field( + description="The unique identifier for the request.", + ) + request_type: Literal["text_completions", "chat_completions"] = Field( + description="The type of request made to the generative backend." + ) + prompt: str = Field( + description="The text prompt used for the generative request.", + ) + output: str = Field( + description="The generated text output from the generative request.", + ) + prompt_tokens: int = Field( + description="The number of tokens in the prompt text.", + ) + output_tokens: int = Field( + description="The number of tokens in the generated output text.", + ) + start_time: float = Field( + description="The time the request started.", + ) + end_time: float = Field( + description="The time the request ended.", + ) + first_token_time: float = Field( + description="The time the first token was received.", + ) + last_token_time: float = Field( + description="The time the last token was received.", + ) @computed_field @property def request_latency(self) -> float: + """ + :return: The duration of the request in seconds from the start to the end. + """ return self.end_time - self.start_time @computed_field @property def time_to_first_token_ms(self) -> float: + """ + :return: The time in milliseconds from the start of the request to the first + token received. + """ return 1000 * (self.first_token_time - self.start_time) @computed_field @property def inter_token_latency_ms(self) -> float: + """ + :return: The average time in milliseconds between generating tokens in the + output text. Note, does not include the time to generate the first token. + """ if self.output_tokens <= 1: return 0.0 @@ -97,6 +273,10 @@ def inter_token_latency_ms(self) -> float: @computed_field @property def output_tokens_per_second(self) -> float: + """ + :return: The average number of tokens generated per second in the output text. + Note, does not include the time to generate the first token. + """ if (itl_ms := self.inter_token_latency_ms) == 0.0: return 0.0 @@ -104,66 +284,127 @@ def output_tokens_per_second(self) -> float: class GenerativeTextErrorStats(GenerativeTextResponseStats): - error: str - request_id: str - prompt: str - output: Optional[str] - prompt_tokens: int - output_tokens: Optional[int] - start_time: float - end_time: None = None # no end since it failed - first_token_time: Optional[float] - last_token_time: Optional[float] - - @computed_field - @property - def request_latency(self) -> None: - return None + """ + A serializable model representing the request values, response values, and + statistics for a generative text response that errored. + Extends and overrides the GenerativeTextResponseStats model to include the + error message and optional properties given the error occurred. + """ + + error: str = Field( + description=( + "The error message for the error that occurred while making the request." + ) + ) + output: Optional[str] = Field( + default=None, + description=( + "The generated text output from the generative request, if any, " + "before the error occurred." + ), + ) + output_tokens: Optional[int] = Field( + default=None, + description=( + "The number of tokens in the generated output text, if any, " + "before the error occurred." + ), + ) + first_token_time: Optional[float] = Field( + default=None, + description=( + "The time the first token was received, if any, before the error occurred." + ), + ) + last_token_time: Optional[float] = Field( + default=None, + description=( + "The time the last token was received, if any, before the error occurred." + ), + ) @computed_field @property def time_to_first_token_ms(self) -> Optional[float]: + """ + :return: The time in milliseconds from the start of the request to the first + token received. None if the first token was not received. + """ if self.first_token_time is None: return None - return 1000 * (self.first_token_time - self.start_time) + return super().time_to_first_token_ms @computed_field @property def inter_token_latency_ms(self) -> Optional[float]: + """ + :return: The average time in milliseconds between generating tokens in the + output text. Note, does not include the time to generate the first token. + None if there were no output_tokens or the first token was not received. + """ if ( self.output_tokens is None - or self.output_tokens <= 1 or self.first_token_time is None or self.last_token_time is None ): return None - return ( - 1000 - * (self.last_token_time - self.first_token_time) - / (self.output_tokens - 1) - ) + return super().inter_token_latency_ms @computed_field @property def output_tokens_per_second(self) -> Optional[float]: - if (itl_ms := self.inter_token_latency_ms) is None: + """ + :return: The average number of tokens generated per second in the output text. + Note, does not include the time to generate the first token. None if there + were no output_tokens or the first token was not received. + """ + if self.inter_token_latency_ms is None: return None - return 1000.0 / itl_ms + return super().output_tokens_per_second class GenerativeBenchmark(Benchmark): + """ + A serializable model representing a benchmark run and its results for generative + requests and responses. Includes the completed and errored requests, the start + and end times for the benchmark, and the statistics for the requests and responses. + """ + + completed_total: int = Field( + description=( + "The total number of completed requests in the benchmark, " + "excluding warmup and cooldown." + ) + ) + completed_sampled_size: Optional[int] = Field( + default=None, + description=( + "The number of completed requests that were randomly sampled for " + "the benchmark. None if no sampling was applied." + ), + ) completed_requests: List[GenerativeTextResponseStats] = Field( description="The list of completed requests.", ) - completed_sampled_size: Optional[int] = None + errored_total: int = Field( + description=( + "The total number of errored requests in the benchmark, " + "excluding warmup and cooldown." + ) + ) + errored_sampled_size: Optional[int] = Field( + default=None, + description=( + "The number of errored requests that were randomly sampled for " + "the benchmark. None if no sampling was applied." + ), + ) errored_requests: List[GenerativeTextErrorStats] = Field( description="The list of errored requests.", ) - errored_sampled_size: Optional[int] = None - start_time: float = Field( description="The start time of the first request for the benchmark.", ) @@ -214,14 +455,175 @@ def duration(self) -> float: """ return self.end_time - self.start_time + def create_sampled( + self, sample_size: int, error_sample_size: Optional[int] = None + ) -> "GenerativeBenchmark": + """ + Create a new benchmark instance with a random sample of the completed and + errored requests based on the given sample sizes. If the sample sizes are + larger than the total number of requests, the sample sizes are capped at + the total number of requests. + + :param sample_size: The number of completed requests to sample. + :param error_sample_size: The number of errored requests to sample. + If None, defaults to the sample_size. + :return: A new benchmark instance with the sampled requests. + :raises ValueError: If the sample sizes are negative or if the + GenerativeBenchmark has already been sampled and the requested sample + sizes are larger than the previously sampled sizes. + """ + if error_sample_size is None: + error_sample_size = sample_size + + if sample_size < 0: + raise ValueError(f"Sample size must be non-negative, given {sample_size}") + if error_sample_size < 0: + raise ValueError( + f"Error sample size must be non-negative, given {error_sample_size}" + ) + + if ( + self.completed_sampled_size is not None + and sample_size > self.completed_sampled_size + ): + raise ValueError( + "The benchmark's completed response have already been sampled with " + f"size {self.completed_sampled_size} and cannot be resampled with " + f"a larger size, given: {sample_size}" + ) + if ( + self.errored_sampled_size is not None + and error_sample_size > self.errored_sampled_size + ): + raise ValueError( + "The benchmark's errored response have already been sampled with " + f"size {self.errored_sampled_size} and cannot be resampled with " + f"a larger size, given: {error_sample_size}" + ) + + sample_size = min(sample_size, len(self.completed_requests)) + error_sample_size = min(error_sample_size, len(self.errored_requests)) + + sampled_instance = self.model_copy() + sampled_instance.completed_sampled_size = sample_size + sampled_instance.completed_requests = random.sample( + self.completed_requests, sample_size + ) + sampled_instance.errored_sampled_size = error_sample_size + sampled_instance.errored_requests = random.sample( + self.errored_requests, error_sample_size + ) + + return sampled_instance + @staticmethod def from_stats( + run_id: str, completed: List[GenerativeTextResponseStats], errored: List[GenerativeTextErrorStats], - settings: BenchmarkSettings, + args: BenchmarkArgs, run_stats: BenchmarkRunStats, - worker_description: Serializable, - requests_loader_description: Serializable, - extras: Dict[str, Any], + worker: Optional[Serializable], + requests_loader: Optional[Serializable], + extras: Optional[Dict[str, Any]], ) -> "GenerativeBenchmark": - pass # TODO + """ + Create a GenerativeBenchmark instance from the given statistics and metadata. + Given the completed and errored requests, the benchmark will fill in the + remaining statistics for the various metrics required for a benchmark. + This is the preferred method for creating a GenerativeBenchmark instance + to ensure all statistics are properly calculated and populated. + + :param run_id: The unique identifier for the benchmark run. + :param completed: The list of completed requests. + :param errored: The list of errored requests. + :param args: The arguments used to specify how to run the benchmark + and collect data. + :param run_stats: The process statistics for the entire benchmark run across + all requests. + :param worker: The description and specifics for the worker used to resolve + requests. + :param requests_loader: The description and specifics for the request loader + used to create requests. + :param extras: Any additional information or metadata that was passed for + this benchmark. + :return: A GenerativeBenchmark instance with the given statistics and metadata + populated and calculated + """ + start_time = min(req.start_time for req in completed) if completed else 0.0 + + return GenerativeBenchmark( + run_id=run_id, + args=args, + run_stats=run_stats, + worker=worker, + requests_loader=requests_loader, + extras=extras or {}, + completed_total=len(completed), + completed_requests=completed, + errored_total=len(errored), + errored_requests=errored, + start_time=start_time, + end_time=max(req.end_time for req in completed) if completed else 0.0, + requests_per_second=StatusDistributionSummary.from_timestamped_values_per_frequency( + completed_values=( + [(start_time, 0.0)] # start time to cover full time range + + [(req.end_time, 1.0) for req in completed] + ), + errored_values=( + [(start_time, 0.0)] # start time to cover full time range + + [(req.end_time, 1.0) for req in errored if req.end_time] + ), + frequency=1.0, # 1 second + ), + requests_concurrency=StatusDistributionSummary.from_timestamped_interval_values( + completed_values=( + [(req.start_time, req.end_time, 1) for req in completed] + ), + errored_values=([(req.start_time, req.end_time, 1) for req in errored]), + ), + requests_latency=StatusDistributionSummary.from_values( + completed_values=[req.request_latency for req in completed], + errored_values=[req.request_latency for req in errored], + ), + prompts_token_count=StatusDistributionSummary.from_values( + completed_values=[req.prompt_tokens for req in completed], + errored_values=[req.prompt_tokens for req in errored], + ), + outputs_token_count=StatusDistributionSummary.from_values( + completed_values=[req.output_tokens for req in completed], + errored_values=[req.output_tokens for req in errored], + ), + times_to_first_token_ms=StatusDistributionSummary.from_values( + completed_values=[req.time_to_first_token_ms for req in completed], + errored_values=[req.time_to_first_token_ms for req in errored], + ), + inter_token_latencies_ms=StatusDistributionSummary.from_values( + completed_values=[ + req.inter_token_latency_ms + for req in completed + for _ in range(req.output_tokens - 1) + if req.output_tokens > 1 and req.inter_token_latency_ms + ], + errored_values=[ + req.inter_token_latency_ms + for req in errored + for _ in range(req.output_tokens - 1) + if req.output_tokens > 1 and req.inter_token_latency_ms + ], + ), + outputs_tokens_per_second=StatusDistributionSummary.from_values( + completed_values=[ + req.output_tokens_per_second + for req in completed + for _ in range(req.output_tokens - 1) + if req.output_tokens > 1 and req.output_tokens_per_second + ], + errored_values=[ + req.output_tokens_per_second + for req in errored + for _ in range(req.output_tokens - 1) + if req.output_tokens > 1 and req.output_tokens_per_second + ], + ), + ) diff --git a/src/guidellm/benchmark/benchmarker.py b/src/guidellm/benchmark/benchmarker.py index cf84c6e5..8d8a426c 100644 --- a/src/guidellm/benchmark/benchmarker.py +++ b/src/guidellm/benchmark/benchmarker.py @@ -65,7 +65,7 @@ async def run( cooldown_duration_per_strategy: Optional[float], ) -> AsyncGenerator[BenchmarkerResult[AGG, BENCH, REQ, RES], None]: start_time = time.time() - end_number = len(profile) + end_number = len(profile.strategy_types) current_index = -1 yield BenchmarkerResult( diff --git a/src/guidellm/benchmark/profile.py b/src/guidellm/benchmark/profile.py index f3050eb0..969b2a74 100644 --- a/src/guidellm/benchmark/profile.py +++ b/src/guidellm/benchmark/profile.py @@ -2,7 +2,7 @@ from typing import List, Literal, Optional, Sequence, Union import numpy as np -from pydantic import Field +from pydantic import Field, computed_field from guidellm.objects import Serializable from guidellm.scheduler import ( @@ -53,6 +53,8 @@ def completed_strategy(self, average_rate: float, average_concurrency: float): self.measured_concurrencies.append(average_concurrency) self.completed_strategies += 1 + @computed_field + @property @abstractmethod def strategy_types(self) -> List[StrategyType]: ... @@ -63,6 +65,7 @@ def next_strategy(self) -> Optional[SchedulingStrategy]: ... class SynchronousProfile(Profile): type_: Literal["synchronous"] = "synchronous" + @property def strategy_types(self) -> List[StrategyType]: return [self.type_] @@ -100,6 +103,7 @@ class ConcurrentProfile(Profile): description="The number of concurrent streams to use.", ) + @property def strategy_types(self) -> List[StrategyType]: num_strategies = len(self.streams) if isinstance(self.streams, Sequence) else 1 @@ -150,6 +154,7 @@ class ThroughputProfile(Profile): description="The maximum number of concurrent requests that can be scheduled.", ) + @property def strategy_types(self) -> List[StrategyType]: return [self.type_] @@ -194,6 +199,7 @@ class AsyncProfile(ThroughputProfile): ), ) + @property def strategy_types(self) -> List[StrategyType]: num_strategies = len(self.rate) if isinstance(self.rate, Sequence) else 1 @@ -259,13 +265,12 @@ class SweepProfile(AsyncProfile): description="The number of strategies to generate for the sweep.", ) rate: float = -1 - strategy_type: Literal["constant", "poisson"] = "constant" + rate_type: Literal["constant", "poisson"] = "constant" + @property def strategy_types(self) -> List[StrategyType]: return ( - ["synchronous"] - + [self.strategy_type] * (self.sweep_size - 2) - + ["throughput"] + ["synchronous"] + [self.rate_type] * (self.sweep_size - 2) + ["throughput"] ) def next_strategy(self) -> Optional[SchedulingStrategy]: @@ -284,20 +289,20 @@ def next_strategy(self) -> Optional[SchedulingStrategy]: max_rate = self.measured_rates[1] rates = np.linspace(min_rate, max_rate, self.sweep_size)[1:-1] - if self.strategy_type == "constant": + if self.rate_type == "constant": return AsyncConstantStrategy( rate=rates[self.completed_strategies - 2], initial_burst=self.initial_burst, max_concurrency=self.max_concurrency, ) - elif self.strategy_type == "poisson": + elif self.rate_type == "poisson": return AsyncPoissonStrategy( rate=rates[self.completed_strategies - 2], initial_burst=self.initial_burst, max_concurrency=self.max_concurrency, ) else: - raise ValueError(f"Invalid strategy type: {self.strategy_type}") + raise ValueError(f"Invalid strategy type: {self.rate_type}") @staticmethod def from_standard_args( @@ -308,19 +313,20 @@ def from_standard_args( if rate_type != "sweep": raise ValueError("Rate type must be 'sweep' for sweep profile.") - if rate: - raise ValueError("Rate does not apply to sweep profile, it must be None.") + if "sweep_size" in kwargs: + raise ValueError("Sweep size must not be provided, use rate instead.") - if "sweep_size" not in kwargs: - raise ValueError("Sweep size must be provided for sweep profile.") + if not rate: + raise ValueError( + "Rate (sweep_size) must be provided for concurrent profile." + ) - if not isinstance(kwargs["sweep_size"], int) or kwargs["sweep_size"] <= 2: + if not isinstance(rate, float) or not rate.is_integer() or rate <= 1: raise ValueError( - "Sweep size must be a positive integer > 2, " - f"received {kwargs['sweep_size']}" + f"Rate (sweep_size) must be a positive integer > 1, received {rate}" ) - return SweepProfile(**kwargs) + return SweepProfile(sweep_size=rate, **kwargs) def create_profile( diff --git a/src/guidellm/objects/distribution.py b/src/guidellm/objects/distribution.py index d25113d1..f26a8f30 100644 --- a/src/guidellm/objects/distribution.py +++ b/src/guidellm/objects/distribution.py @@ -3,6 +3,7 @@ from typing import List, Tuple import numpy as np +from pydantic import Field from guidellm.objects import Serializable @@ -14,21 +15,46 @@ class Percentiles(Serializable): - p001: float - p01: float - p05: float - p10: float - p25: float - p75: float - p90: float - p95: float - p99: float - p999: float + """ + A serializable model representing percentiles of a distribution. + """ + + p001: float = Field( + description="The 0.1th percentile of the distribution.", + ) + p01: float = Field( + description="The 1st percentile of the distribution.", + ) + p05: float = Field( + description="The 5th percentile of the distribution.", + ) + p10: float = Field( + description="The 10th percentile of the distribution.", + ) + p25: float = Field( + description="The 25th percentile of the distribution.", + ) + p75: float = Field( + description="The 75th percentile of the distribution.", + ) + p90: float = Field( + description="The 90th percentile of the distribution.", + ) + p95: float = Field( + description="The 95th percentile of the distribution.", + ) + p99: float = Field( + description="The 99th percentile of the distribution.", + ) + p999: float = Field( + description="The 99.9th percentile of the distribution.", + ) @staticmethod def from_values(values: List[float]) -> "Percentiles": """ Calculate percentiles from a list of values. + If the list is empty, all percentiles are set to 0. :param values: A list of numerical values. :return: An instance of Percentiles with calculated percentiles. @@ -63,23 +89,46 @@ def from_values(values: List[float]) -> "Percentiles": class DistributionSummary(Serializable): - mean: float - median: float - variance: float - std_dev: float - min: float - max: float - count: int - percentiles: Percentiles + """ + A serializable model representing a statistical summary for a given + distribution of numerical values. + """ + + mean: float = Field( + description="The mean/average of the distribution.", + ) + median: float = Field( + description="The median of the distribution.", + ) + variance: float = Field( + description="The variance of the distribution.", + ) + std_dev: float = Field( + description="The standard deviation of the distribution.", + ) + min: float = Field( + description="The minimum value of the distribution.", + ) + max: float = Field( + description="The maximum value of the distribution.", + ) + count: int = Field( + description="The number of values in the distribution.", + ) + percentiles: Percentiles = Field( + description="The percentiles of the distribution.", + ) @staticmethod def from_values(values: List[float]) -> "DistributionSummary": """ - Create a DistributionSummary from a list of values. + Calculate a distribution summary from a list of values. + If the list is empty, all values are set to 0. :param values: A list of numerical values. - :return: An instance of DistributionSummary. + :return: An instance of DistributionSummary with calculated values. """ + if not values: return DistributionSummary( mean=0.0, @@ -104,17 +153,28 @@ def from_values(values: List[float]) -> "DistributionSummary": ) @staticmethod - def from_time_measurements( - measurements: List[Tuple[float, float]], + def from_timestamped_values( + values: List[Tuple[float, float]], ) -> "DistributionSummary": """ - Create a DistributionSummary from a list of time measurements of the form - (time, value), where time is the timestamp and value is the measurement. - - :param measurements: A list of tuples containing (time, value) pairs. - :return: An instance of DistributionSummary. + Calculate a distribution summary from a list of timestamped values. + Specifically, this calculates the statistics assuming a piecewise + continuous distribution of values over time. + For example, rather than finding the average concurrency of requests + over a given time period, this will calculate that along with other + statistics such as the variance and percentiles. + If the list is empty, all values are set to 0. + If the list contains only one value, all values are set to that value. + Note, since this is calculating statistics over time, the values + should contain the entire time range. Generally, this means the first + value should be the start time with a measurement of 0. + + :param values: A list of timestamped numerical values of the form + (timestamp, value). + :return: An instance of DistributionSummary with calculated values. """ - if not measurements: + + if not values: return DistributionSummary( mean=0.0, median=0.0, @@ -126,41 +186,54 @@ def from_time_measurements( percentiles=Percentiles.from_values([]), ) - if len(measurements) == 1: + if len(values) == 1: return DistributionSummary( - mean=measurements[0][1], - median=measurements[0][1], + mean=values[0][1], + median=values[0][1], variance=0.0, std_dev=0.0, - min=measurements[0][1], - max=measurements[0][1], + min=values[0][1], + max=values[0][1], count=1, - percentiles=Percentiles.from_values([measurements[0][1]]), + percentiles=Percentiles.from_values([values[0][1]]), ) - measurements.sort(key=lambda x: x[0]) + # ensure values are sorted and piecewise continuous + # (combine any values at the same time) + tmp_values = sorted(values, key=lambda x: x[0]) + values = [] + epsilon = 1e-6 + + for val in tmp_values: + if values and abs(values[-1][0] - val[0]) < epsilon: + values[-1] = (val[0], val[1] + values[-1][1]) + else: + values.append(val) + + duration = values[-1][0] - values[0][0] + + # mean calculations integral = sum( - (measurements[ind + 1][0] - measurements[ind][0]) * measurements[ind][1] - for ind in range(len(measurements) - 1) + (values[ind + 1][0] - values[ind][0]) * values[ind][1] + for ind in range(len(values) - 1) ) - duration = measurements[-1][0] - measurements[0][0] mean = integral / duration if duration > 0 else 0.0 + + # variance calculations variance = ( sum( - (measurements[ind + 1][0] - measurements[ind][0]) - * (measurements[ind][1] - mean) ** 2 - for ind in range(len(measurements) - 1) + (values[ind + 1][0] - values[ind][0]) * (values[ind][1] - mean) ** 2 + for ind in range(len(values) - 1) ) / duration if duration > 0 else 0.0 ) + # percentile calculations value_durations_dict = defaultdict(float) - for ind in range(len(measurements) - 1): - value_durations_dict[measurements[ind][1]] += ( - measurements[ind + 1][0] - measurements[ind][0] - ) + for ind in range(len(values) - 1): + value_durations_dict[values[ind][1]] += values[ind + 1][0] - values[ind][0] value_durations = sorted( [(duration, value) for value, duration in value_durations_dict.items()], key=lambda x: x[0], @@ -180,9 +253,9 @@ def _get_percentile(percentile: float) -> float: median=_get_percentile(50.0), variance=variance, std_dev=math.sqrt(variance), - min=min([meas[1] for meas in measurements]), - max=max([meas[1] for meas in measurements]), - count=len(measurements), + min=min([meas[1] for meas in values]), + max=max([meas[1] for meas in values]), + count=len(values), percentiles=Percentiles( p001=_get_percentile(0.1), p01=_get_percentile(1.0), @@ -198,43 +271,110 @@ def _get_percentile(percentile: float) -> float: ) @staticmethod - def from_time_measurements_with_sampling( - measurements: List[Tuple[float, float]], - sample_time: float, + def from_timestamped_values_per_frequency( + values: List[Tuple[float, float]], + frequency: float, ) -> "DistributionSummary": """ - Create a DistributionSummary from a list of time measurements of the form - (time, value), where time is the timestamp and value is the measurement. - This method samples the measurements at regular intervals defined by - sample_time. - - :param measurements: A list of tuples containing (time, value) pairs. - :param sample_time: The time interval for sampling. - :return: An instance of DistributionSummary. + Calculate a distribution summary from a list of timestamped values + at a given frequency. + Specifically, this calculates the statistics assuming a piecewise + continuous distribution of values over time and then samples at + the given frequency from that distribution. + For example, rather than finding the average requests per second + over a given time period, this will calculate that along with other + statistics such as the variance and percentiles. + If the list is empty, all values are set to 0. + If the list contains only one value, all values are set to that value. + Note, since this is calculating statistics over time, the values + should contain the entire time range. Generally, this means the first + value should be the start time with a measurement of 0. + + :param values: A list of timestamped numerical values of the form + (timestamp, value). + :param frequency: The frequency to sample the distribution at + represented in the same units as the timestamps. + :return: An instance of DistributionSummary with calculated values. """ - measurements.sort(key=lambda x: x[0]) + values.sort(key=lambda x: x[0]) samples = [] - min_time = measurements[0][0] - max_time = measurements[-1][0] + sample_time + min_time = values[0][0] + max_time = values[-1][0] + frequency for time_iter in np.arange( min_time, max_time, - sample_time, + frequency, ): count = 0 - while measurements and measurements[0][0] <= time_iter: - count += measurements[0][1] - measurements.pop(0) + while values and values[0][0] <= time_iter: + count += values[0][1] + values.pop(0) samples.append((time_iter, count)) - return DistributionSummary.from_time_measurements(samples) + return DistributionSummary.from_timestamped_values(samples) + + @staticmethod + def from_timestamped_interval_values( + values: List[Tuple[float, float, float]], + ) -> "DistributionSummary": + """ + Calculate a distribution summary from a list of timestamped interval values, + that may or may note be overlapping in ranges. + Specifically, this calculates the statistics assuming a piecewise + continuous distribution of values over time. + For example, rather than finding the average concurrency of overlapping requests + over a given time period, this will calculate that along with other + statistics such as the variance and percentiles. + If the list is empty, all values are set to 0. + If the list contains only one value, all values are set to that value. + Note, since this is calculating statistics over time, the values + should contain the entire time range. + + :param values: A list of timestamped numerical values of the form + (start_time, end_time, value). + :return: An instance of DistributionSummary with calculated values. + """ + events_dict = defaultdict(int) + for start, end, count in values: + events_dict[start] += count + events_dict[end] -= count + + timestamped_values = [] + current_value = 0 + + for time, delta in sorted(events_dict.items()): + current_value += delta + timestamped_values.append((time, current_value)) + + return DistributionSummary.from_timestamped_values( + timestamped_values, + ) class StatusDistributionSummary(Serializable): - total: DistributionSummary - completed: DistributionSummary - errored: DistributionSummary + """ + A serializable model representing distribution summary statistics + based on groupings of status (e.g., completed, errored) for a given + distribution of numerical values. + Handles the total, completed, and errored distributions where the total + is the combination of the completed and errored distributions. + """ + + total: DistributionSummary = Field( + description="The distribution summary for all statuses (errored, completed).", + ) + completed: DistributionSummary = Field( + description=( + "The distribution summary for completed statuses " + "(e.g., successful requests)." + ) + ) + errored: DistributionSummary = Field( + description=( + "The distribution summary for errored statuses " "(e.g., failed requests)." + ) + ) @staticmethod def from_values( @@ -242,11 +382,13 @@ def from_values( errored_values: List[float], ) -> "StatusDistributionSummary": """ - Create a StatusDistributionSummary from completed and errored values. + Calculate distribution summaries from a list of values for + completed, errored, and the total combination of both. + If the lists are empty, all values are set to 0. - :param completed_values: A list of numerical values for completed requests. - :param errored_values: A list of numerical values for errored requests. - :return: An instance of StatusDistributionSummary. + :param completed_values: A list of numerical values for completed statuses. + :param errored_values: A list of numerical values for errored statuses. + :return: An instance of StatusDistributionSummary with calculated values. """ return StatusDistributionSummary( total=DistributionSummary.from_values( @@ -257,59 +399,118 @@ def from_values( ) @staticmethod - def from_time_measurements( - completed_measurements: List[Tuple[float, float]], - errored_measurements: List[Tuple[float, float]], + def from_timestamped_values( + completed_values: List[Tuple[float, float]], + errored_values: List[Tuple[float, float]], ) -> "StatusDistributionSummary": """ - Create a StatusDistributionSummary from completed and errored time measurements. + Calculate distribution summaries from a list of timestamped values for + completed, errored, and the total combination of both. + Specifically, this calculates the statistics assuming a piecewise + continuous distribution of values over time. + For example, rather than finding the average concurrency of requests + over a given time period, this will calculate that along with other + statistics such as the variance and percentiles. + If the lists are empty, all values are set to 0. + If the lists contain only one value, all values are set to that value. + Note, since this is calculating statistics over time, the values + should contain the entire time range. Generally, this means the first + value should be the start time with a measurement of 0. + + :param completed_values: A list of timestamped numerical values for + completed statuses. + :param errored_values: A list of timestamped numerical values for + errored statuses. + :return: An instance of StatusDistributionSummary with calculated values. + """ + return StatusDistributionSummary( + total=DistributionSummary.from_timestamped_values( + completed_values + errored_values, + ), + completed=DistributionSummary.from_timestamped_values( + completed_values, + ), + errored=DistributionSummary.from_timestamped_values( + errored_values, + ), + ) - :param completed_measurements: A list of tuples containing (time, value) pairs - for completed requests. - :param errored_measurements: A list of tuples containing (time, value) pairs - for errored requests. - :return: An instance of StatusDistributionSummary. + @staticmethod + def from_timestamped_values_per_frequency( + completed_values: List[Tuple[float, float]], + errored_values: List[Tuple[float, float]], + frequency: float, + ) -> "StatusDistributionSummary": + """ + Calculate distribution summaries from a list of timestamped values for + completed, errored, and the total combination of both at a given frequency. + Specifically, this calculates the statistics assuming a piecewise + continuous distribution of values over time and then samples at + the given frequency from that distribution. + For example, rather than finding the average requests per second + over a given time period, this will calculate that along with other + statistics such as the variance and percentiles. + If the lists are empty, all values are set to 0. + If the lists contain only one value, all values are set to that value. + Note, since this is calculating statistics over time, the values + should contain the entire time range. Generally, this means the first + value should be the start time with a measurement of 0. + + :param completed_values: A list of timestamped numerical values for + completed statuses. + :param errored_values: A list of timestamped numerical values for + errored statuses. + :param frequency: The frequency to sample the distribution at + represented in the same units as the timestamps. + :return: An instance of StatusDistributionSummary with calculated values. """ return StatusDistributionSummary( - total=DistributionSummary.from_time_measurements( - completed_measurements + errored_measurements, + total=DistributionSummary.from_timestamped_values_per_frequency( + completed_values + errored_values, + frequency, ), - completed=DistributionSummary.from_time_measurements( - completed_measurements, + completed=DistributionSummary.from_timestamped_values_per_frequency( + completed_values, + frequency, ), - errored=DistributionSummary.from_time_measurements( - errored_measurements, + errored=DistributionSummary.from_timestamped_values_per_frequency( + errored_values, + frequency, ), ) @staticmethod - def from_time_measurements_with_sampling( - completed_measurements: List[Tuple[float, float]], - errored_measurements: List[Tuple[float, float]], - sample_time: float, + def from_timestamped_interval_values( + completed_values: List[Tuple[float, float, float]], + errored_values: List[Tuple[float, float, float]], ) -> "StatusDistributionSummary": """ - Create a StatusDistributionSummary from completed and errored time measurements - with sampling. - - :param completed_measurements: A list of tuples containing (time, value) pairs - for completed requests. - :param errored_measurements: A list of tuples containing (time, value) pairs - for errored requests. - :param sample_time: The time interval for sampling. - :return: An instance of StatusDistributionSummary. + Calculate distribution summaries from a list of timestamped interval values for + completed, errored, and the total combination of both. + Specifically, this calculates the statistics assuming a piecewise + continuous distribution of values over time. + For example, rather than finding the average concurrency of overlapping requests + over a given time period, this will calculate that along with other + statistics such as the variance and percentiles. + If the lists are empty, all values are set to 0. + If the lists contain only one value, all values are set to that value. + Note, since this is calculating statistics over time, the values + should contain the entire time range. + + :param completed_values: A list of timestamped numerical values for + completed statuses. + :param errored_values: A list of timestamped numerical values for + errored statuses. + :return: An instance of StatusDistributionSummary with calculated values. """ return StatusDistributionSummary( - total=DistributionSummary.from_time_measurements_with_sampling( - completed_measurements + errored_measurements, - sample_time, + total=DistributionSummary.from_timestamped_interval_values( + completed_values + errored_values, ), - completed=DistributionSummary.from_time_measurements_with_sampling( - completed_measurements, - sample_time, + completed=DistributionSummary.from_timestamped_interval_values( + completed_values, ), - errored=DistributionSummary.from_time_measurements_with_sampling( - errored_measurements, - sample_time, + errored=DistributionSummary.from_timestamped_interval_values( + errored_values, ), ) From faacb633155f918a50e1a1b6eca27205d882f109 Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Sat, 22 Mar 2025 17:42:05 +0000 Subject: [PATCH 12/43] Latest state with datasets and benchmarker created --- src/guidellm/backend/response.py | 33 +- src/guidellm/benchmark/aggregator.py | 553 +++++++++++++++++++++++--- src/guidellm/benchmark/benchmark.py | 11 +- src/guidellm/benchmark/benchmarker.py | 35 +- src/guidellm/benchmark/entrypoints.py | 102 ++++- src/guidellm/benchmark/progress.py | 9 + src/guidellm/config.py | 4 +- src/guidellm/dataset/__init__.py | 17 +- src/guidellm/dataset/base.py | 200 ---------- src/guidellm/dataset/creator.py | 203 ++++++++++ src/guidellm/dataset/datasets.py | 60 +++ src/guidellm/dataset/emulated.py | 397 ------------------ src/guidellm/dataset/entrypoints.py | 31 ++ src/guidellm/dataset/file.py | 169 ++++---- src/guidellm/dataset/in_memory.py | 127 ++++++ src/guidellm/dataset/synthetic.py | 292 ++++++++++++++ src/guidellm/dataset/transformers.py | 103 ----- src/guidellm/request/__init__.py | 2 + src/guidellm/request/loader.py | 16 +- 19 files changed, 1488 insertions(+), 876 deletions(-) create mode 100644 src/guidellm/benchmark/progress.py delete mode 100644 src/guidellm/dataset/base.py create mode 100644 src/guidellm/dataset/creator.py create mode 100644 src/guidellm/dataset/datasets.py delete mode 100644 src/guidellm/dataset/emulated.py create mode 100644 src/guidellm/dataset/entrypoints.py create mode 100644 src/guidellm/dataset/in_memory.py create mode 100644 src/guidellm/dataset/synthetic.py delete mode 100644 src/guidellm/dataset/transformers.py diff --git a/src/guidellm/backend/response.py b/src/guidellm/backend/response.py index 757b142f..4c2d70b9 100644 --- a/src/guidellm/backend/response.py +++ b/src/guidellm/backend/response.py @@ -1,6 +1,5 @@ from typing import Any, Dict, Literal, Optional -from loguru import logger from pydantic import BaseModel, computed_field from guidellm.config import settings @@ -107,21 +106,7 @@ def prompt_tokens(self) -> Optional[int]: :return: The number of tokens in the prompt, if any. """ - if settings.preferred_prompt_tokens_source == "backend": - if self.response_prompt_tokens is None: - logger.warning( - "Preferred prompt tokens source is backend, but no prompt token " - f"values were returned with the response for {self}. " - "Defulating to request_prompt_tokens (if available)." - ) - return self.response_prompt_tokens or self.request_prompt_tokens - elif settings.preferred_prompt_tokens_source == "request": - if self.request_prompt_tokens is None: - logger.warning( - "Preferred prompt tokens source is request, but no prompt token " - f"values were returned with the request for {self}. " - "Defulating to response_prompt_tokens (if available)." - ) + if settings.preferred_prompt_tokens_source == "request": return self.request_prompt_tokens or self.response_prompt_tokens return self.response_prompt_tokens or self.request_prompt_tokens @@ -135,21 +120,7 @@ def output_tokens(self) -> Optional[int]: :return: The number of tokens in the output, if any. """ - if settings.preferred_output_tokens_source == "backend": - if self.response_output_tokens is None: - logger.warning( - "Preferred output tokens source is backend, but no output token " - f"values were returned with the response for {self}. " - "Defulating to request_output_tokens (if available)." - ) - return self.response_output_tokens or self.request_output_tokens - elif settings.preferred_output_tokens_source == "request": - if self.request_output_tokens is None: - logger.warning( - "Preferred output tokens source is request, but no output token " - f"values were returned with the request for {self}. " - "Defulating to response_output_tokens (if available)." - ) + if settings.preferred_output_tokens_source == "request": return self.request_output_tokens or self.response_output_tokens return self.response_output_tokens or self.request_output_tokens diff --git a/src/guidellm/benchmark/aggregator.py b/src/guidellm/benchmark/aggregator.py index ba4baadf..83128c1b 100644 --- a/src/guidellm/benchmark/aggregator.py +++ b/src/guidellm/benchmark/aggregator.py @@ -1,17 +1,40 @@ +import time from abc import ABC, abstractmethod from collections import defaultdict -from typing import DefaultDict, Generic, List, TypeVar +from typing import ( + Any, + DefaultDict, + Dict, + Generic, + List, + Literal, + Optional, + Tuple, + TypeVar, +) -from pydantic import Field +from pydantic import BaseModel, Field +from transformers import PreTrainedTokenizer # type: ignore # noqa: PGH003 from guidellm.backend import ResponseSummary -from guidellm.benchmark.benchmark import BENCH, Benchmark, GenerativeBenchmark +from guidellm.benchmark.benchmark import ( + BENCH, + Benchmark, + BenchmarkArgs, + BenchmarkRunStats, + GenerativeBenchmark, + GenerativeTextErrorStats, + GenerativeTextResponseStats, +) +from guidellm.benchmark.profile import Profile +from guidellm.config import settings from guidellm.objects import Serializable from guidellm.request import GenerationRequest from guidellm.scheduler import ( REQ, RES, SchedulerResult, + SchedulingStrategy, ) __all__ = [ @@ -21,34 +44,242 @@ ] -class BenchmarkAggregator(Generic[BENCH, REQ, RES], ABC, Serializable): - created_requests: int = 0 - queued_requests: int = 0 - scheduled_requests: int = 0 - processing_requests: int = 0 - completed_requests: int = 0 - successful_requests: int = 0 - errored_requests: int = 0 +class BenchmarkAggregator(Generic[BENCH, REQ, RES], ABC, BaseModel): + """ + A pydantic base class representing the base class for aggregating benchmark results. + The purpose is to receive and process results from a Benchmarker as it iterates + through a Scheduler for an individual benchmark run. + As results are added, lightweight statistics are updated and stored for immediate + progress and informational updates to the caller. + Once the benchmark run is complete, the `compile` method is called to finalize + the benchmark and return a Benchmark object with all the results and statistics + fully calculated. + """ + + run_id: str = Field( + description=( + "The unique identifier for the encompasing benchmark run that this " + "benchmark was a part of." + ) + ) + profile: Profile = Field( + description=( + "The profile used for the entire benchamrk run that the strategy for " + "the active benchmark was pulled from." + ) + ) + strategy_index: int = Field( + description=( + "The index of the strategy in the profile that was used for this benchmark." + ) + ) + strategy: SchedulingStrategy = Field( + description="The scheduling strategy used to run this benchmark. " + ) + max_number: Optional[int] = Field( + description="The maximum number of requests to run for this benchmark, if any." + ) + max_duration: Optional[float] = Field( + description="The maximum duration in seconds to run this benchmark, if any." + ) + warmup_number: Optional[int] = Field( + description=( + "The number of requests to run for the warmup phase of this benchmark, " + "if any. These are requests that were not included in the final results." + ) + ) + warmup_duration: Optional[float] = Field( + description=( + "The duration in seconds to run for the warmup phase of this benchmark, " + "if any. These are requests that were not included in the final results." + ) + ) + cooldown_number: Optional[int] = Field( + description=( + "The number of requests to run for the cooldown phase of this benchmark, " + "if any. These are requests that were not included in the final results." + ) + ) + cooldown_duration: Optional[float] = Field( + description=( + "The duration in seconds to run for the cooldown phase of this benchmark, " + "if any. These are requests that were not included in the final results." + ) + ) + worker_description: Optional[Serializable] = Field( + description=( + "The description and specifics for the worker used to resolve requests " + "for this benchmark." + ) + ) + request_loader_description: Optional[Serializable] = Field( + description=( + "The description and specifics for the request loader used to create " + "requests for this benchmark." + ) + ) + extras: Dict[str, Any] = Field( + description=( + "Any additional information or metadata that was passed for this benchmark." + ) + ) + + results: List[SchedulerResult[GenerationRequest, ResponseSummary]] = Field( + default_factory=list, + description=( + "The list of results from the benchmark, both completed and errored, " + "that were not within the warmup or cooldown periods." + ), + ) + + start_time: float = Field( + description=( + "The timestamp when the benchmark run started. Defaults to the current " + "time.time() on creation." + ), + default_factory=time.time, + ) + created_requests: int = Field( + description="The number of requests created for this benchmark run.", + default=0, + ) + queued_requests: int = Field( + description="The number of requests pending in queue for this benchmark run.", + default=0, + ) + scheduled_requests: int = Field( + description=( + "The number of requests scheduled (actively running but waiting for the " + "desired start time) for this benchmark run." + ), + default=0, + ) + processing_requests: int = Field( + description=( + "The number of requests actively being processed by the worker for this " + "benchmark run." + ), + default=0, + ) + completed_requests: int = Field( + description=( + "The number of requests completed for this benchmark run. This includes " + "requests within the warmup and cooldown period, if any, along with the " + "final results." + ), + default=0, + ) + successful_requests: int = Field( + description=( + "The number of requests that completed successfully without error. " + "This is a subset of the completed requests for any that did not error. " + "This includes requests within the warmup and cooldown period, if any, " + "along with the final results." + ), + default=0, + ) + errored_requests: int = Field( + description=( + "The number of requests that errored during processing. This is a subset " + "of the completed requests for any that errored. This includes requests " + "within the warmup and cooldown period, if any, " + "along with the final results." + ), + default=0, + ) + + queued_time: float = Field( + description=( + "The sum, in seconds, for time spent in queue for all requests that " + "completed within the benchmark run. This is the time from when the " + "request was created to when it was scheduled to be processed." + ), + default=0.0, + ) + scheduled_time: float = Field( + description=( + "The sum, in seconds, for time spent scheduled for all requests that " + "completed within the benchmark run. This is the time from when the " + "request was scheduled to be processed to when it was actually started." + ), + default=0.0, + ) + worker_time: float = Field( + description=( + "The sum, in seconds, for time spent processing for all requests that " + "completed within the benchmark run. This is the time from when the " + "request was started to when it was completed." + ), + default=0.0, + ) + targeted_worker_start_delay: float = Field( + description=( + "The sum, in seconds, for the delay between the targeted start time and " + "the actual start time for all requests that completed within the benchmark " + "run. This is the time from when the request was scheduled to be processed " + "to when it was actually started." + ), + default=0.0, + ) + process_idle_time: DefaultDict[int, float] = Field( + default_factory=lambda: defaultdict(float), + description=( + "The total idle time for each process that was used to process requests " + "for this benchmark run. This is the time that the process was not " + "actively processing a request." + ), + ) + process_idle_time_scratch: DefaultDict[int, float] = Field( + default_factory=lambda: defaultdict(float), + description=( + "A scratchpad for calculating the idle time for each process that was used " + "to process requests for this benchmark run. This is used to calculate the " + "total idle time for each process." + ), + ) + + def add_result(self, result: SchedulerResult[REQ, RES]): + """ + Add a result to the aggregator. This will update the internal statistics + and add the result to the list of results if it is not within the warmup or + cooldown period. + + :param result: The result to add to the aggregator. + """ + self.add_base_result(result) - queued_time: float = 0.0 - scheduled_time: float = 0.0 - worker_time: float = 0.0 - targeted_worker_start_delay: float = 0.0 - process_idle_time: DefaultDict[int, float] = defaultdict(float) - process_idle_time_scratch: DefaultDict[int, float] = defaultdict(float) + @abstractmethod + def compile(self) -> Benchmark[BENCH]: + """ + Compile the benchmark results and statistics into a Benchmark object. + This is required to be implemented by subclasses to finalize the benchmark + and return the compiled object. + """ + ... def add_base_result( self, result: SchedulerResult[REQ, RES], is_error: bool = False ): + """ + Helper function to update the base statistics for the aggregator and add the + result to the list of results if it is not within the warmup or cooldown period. + + :param result: The result to add to the aggregator. + :param is_error: A flag to indicate if the result was an error or not. + """ self.created_requests = result.run_info.created_requests self.queued_requests = result.run_info.queued_requests self.scheduled_requests = result.run_info.scheduled_requests self.processing_requests = result.run_info.processing_requests self.completed_requests = result.run_info.completed_requests - if result.type_ != "request_complete": - return + if result.type_ == "request_complete": + self._update_stats_from_result(result, is_error) + self._add_to_results_within_active_period(result) + def _update_stats_from_result( + self, result: SchedulerResult[REQ, RES], is_error: bool + ): if is_error: self.errored_requests += 1 else: @@ -82,11 +313,29 @@ def add_base_result( result.request_info.worker_end ) - def add_result(self, result: SchedulerResult[REQ, RES]): - self.add_base_result(result) + def _add_to_results_within_active_period(self, result: SchedulerResult[REQ, RES]): + start_time = result.request_info.worker_start + end_time = result.request_info.worker_end + completed_number = self.errored_requests + self.successful_requests - @abstractmethod - def compile(self) -> Benchmark[BENCH]: ... + if ( + (self.warmup_number and completed_number <= self.warmup_number) + or (self.warmup_duration and start_time <= self.warmup_duration) + or ( + self.cooldown_number + and self.max_number + and completed_number > self.max_number - self.cooldown_number + ) + or ( + self.cooldown_duration + and self.max_duration + and end_time >= self.max_duration - self.cooldown_duration + ) + ): + # within warmup or cooldown period + return + + self.results.append(result) AGG = TypeVar("AGG", bound=BenchmarkAggregator) @@ -95,42 +344,246 @@ def compile(self) -> Benchmark[BENCH]: ... class GenerativeBenchmarkAggregator( BenchmarkAggregator[GenerativeBenchmark, GenerationRequest, ResponseSummary] ): - results: List[SchedulerResult[GenerationRequest, ResponseSummary]] = Field( - default_factory=list, - description="The list of results for the benchmark.", + processor: Optional[PreTrainedTokenizer] = Field( + description=( + "The tokenizer to use for calculating token counts when none are " + "avaiable that match the preferred source." + ) ) - request_time_total: float = 0.0 - targeted_request_delay_total: float = 0.0 - time_to_first_token_total: float = 0.0 - inter_token_latency_total: float = 0.0 - prompt_tokens_total: int = 0 - output_tokens_total: int = 0 + request_time_total: float = Field( + default=0.0, + description=( + "The sum, in seconds, for the total time spent processing all requests " + "that completed within the benchmark run. This is the time from when the " + "request was created to when it was completed." + ), + ) + targeted_request_delay_total: float = Field( + default=0.0, + description=( + "The sum, in seconds, for the delay between the targeted start time and " + "the actual start time for all requests that completed within the " + "benchmark run. This is the time from when the request was scheduled to " + "be processed to when it was actually started." + ), + ) + time_to_first_token_total: float = Field( + default=0.0, + description=( + "The sum, in seconds, for the time from the start of the request to the " + "first token being generated for all requests that completed within the " + "benchmark run." + ), + ) + inter_token_latency_total: float = Field( + default=0.0, + description=( + "The sum, in seconds, for the time between each token being generated " + "for all requests that completed within the benchmark run." + ), + ) + prompt_tokens_total: int = Field( + default=0.0, + description=( + "The sum of the token count for the prompt for all requests that " + "completed, if available in the response." + ), + ) + output_tokens_total: int = Field( + default=0.0, + description=( + "The sum of the token count for the output for all requests that " + "completed, if available in the response." + ), + ) def add_result(self, result: SchedulerResult[GenerationRequest, ResponseSummary]): + """ + Add a result to the aggregator. This will update the internal statistics + and add the result to the list of results if it is not within the warmup or + cooldown period. + + :param result: The result to add to the aggregator. + """ is_error = bool(result.response.error) self.add_base_result(result, is_error=is_error) - if result.type_ != "request_complete": - return + if result.type_ == "request_complete": + self._update_generative_stats_from_result(result) - self.results.append(result) + def compile(self) -> GenerativeBenchmark: + """ + Compile the benchmark results and statistics into a GenerativeBenchmark object. + This is required to be implemented by subclasses to finalize the benchmark + and return the compiled object. + """ + completed, errored = self._compile_results() - if not is_error: - self.request_time_total += (result.response.end_time or 0.0) - ( - result.response.start_time or 0.0 + return GenerativeBenchmark.from_stats( + run_id=self.run_id, + completed=completed, + errored=errored, + args=BenchmarkArgs( + profile=self.profile, + strategy_index=self.strategy_index, + strategy=self.strategy, + max_number=self.max_number, + max_duration=self.max_duration, + warmup_number=self.warmup_number, + warmup_duration=self.warmup_duration, + cooldown_number=self.cooldown_number, + cooldown_duration=self.cooldown_duration, + ), + run_stats=BenchmarkRunStats( + start_time=self.start_time, + end_time=time.time(), + total=self.completed_requests, + total_completed=self.successful_requests, + total_errored=self.errored_requests, + queued_time_avg=( + self.queued_time / self.completed_requests + if self.completed_requests + else 0.0 + ), + scheduled_time_avg=( + self.scheduled_time / self.completed_requests + if self.completed_requests + else 0.0 + ), + worker_time_avg=( + self.worker_time / self.completed_requests + if self.completed_requests + else 0.0 + ), + worker_delay_avg=( + self.worker_schedule_delay_total / self.completed_requests + if self.completed_requests + else 0.0 + ), + resolve_delay_avg=( + self.targeted_request_delay_total / self.completed_requests + if self.completed_requests + else 0.0 + ), + process_idle_time_avg=( + sum(self.process_idle_time.values()) / self.completed_requests + if self.completed_requests + else 0.0 + ), + worker=self.worker_description, + request_loader=self.request_loader_description, + extras=self.extras, + ), + ) + + def _update_generative_stats_from_result( + self, result: SchedulerResult[GenerationRequest, ResponseSummary] + ): + duration = ( + result.response.end_time - result.response.start_time + if result.response.end_time and result.response.start_time + else 0.0 + ) + self.request_time_total += duration + + targeted_delay = ( + result.response.start_time - result.request_info.targeted_start_time + if result.response.start_time + else 0.0 + ) + self.targeted_request_delay_total += targeted_delay + + first_token_time = ( + result.response.first_iter_time - result.response.start_time + if result.response.first_iter_time and result.response.start_time + else 0.0 + ) + self.time_to_first_token_total += first_token_time + + tokens_latency = ( + result.response.last_iter_time - result.response.first_iter_time + if result.response.last_iter_time and result.response.first_iter_time + else 0.0 + ) + self.inter_token_latency_total += tokens_latency + + self.prompt_tokens_total += result.response.prompt_tokens or 0 + self.output_tokens_total += result.response.output_tokens or 0 + + def _compile_results( + self, + ) -> Tuple[List[GenerativeTextResponseStats, GenerativeTextErrorStats]]: + completed: List[GenerativeTextResponseStats] = [] + errored: List[GenerativeTextErrorStats] = [] + + for result in self.results: + prompt_tokens = self._compile_tokens_count( + value=str(result.request.content), + requests_tokens=result.response.request_prompt_tokens, + response_tokens=result.response.response_prompt_tokens, + preferred_tokens_source=settings.preferred_prompt_tokens_source, ) - self.targeted_request_delay_total += (result.response.start_time or 0.0) - ( - result.request_info.targeted_start_time or 0.0 + output_tokens = self._compile_tokens_count( + value=result.response.value, + requests_tokens=result.response.request_output_tokens, + response_tokens=result.response.response_output_tokens, + preferred_tokens_source=settings.preferred_output_tokens_source, ) - self.time_to_first_token_total += ( - result.response.first_iter_time or 0.0 - ) - (result.response.start_time or 0.0) - self.inter_token_latency_total += ( - result.response.last_iter_time or 0.0 - ) - (result.response.first_iter_time or 0.0) - self.prompt_tokens_total += result.response.prompt_tokens or 0 - self.output_tokens_total += result.response.output_tokens or 0 - def compile(self) -> GenerativeBenchmark: - pass # TODO + if result.response.error: + errored.append( + GenerativeTextErrorStats( + error=result.response.error, + request_id=result.request.request_id, + request_type=result.request.request_type, + prompt=str(result.request.content), + prompt_tokens=prompt_tokens, + output=result.response.value, + output_tokens=output_tokens, + start_time=result.response.start_time, + end_time=result.response.end_time, + first_token_time=result.response.first_iter_time, + last_token_time=result.response.last_iter_time, + ) + ) + else: + completed.append( + GenerativeTextResponseStats( + request_id=result.request.request_id, + request_type=result.request.request_type, + prompt=str(result.request.content), + prompt_tokens=prompt_tokens, + output=result.response.value, + output_tokens=output_tokens, + start_time=result.response.start_time, + end_time=result.response.end_time, + first_token_time=result.response.first_iter_time, + last_token_time=result.response.last_iter_time, + ) + ) + + return completed, errored + + def _compile_tokens_count( + self, + value: str, + requests_tokens: Optional[int], + response_tokens: Optional[int], + preferred_tokens_source: Optional[Literal["request", "response"]], + ) -> int: + if preferred_tokens_source is None and (requests_tokens or response_tokens): + return ( + response_tokens or requests_tokens + ) # trust response first if no preference + elif preferred_tokens_source == "response" and response_tokens: + return response_tokens + elif preferred_tokens_source == "request" and requests_tokens: + return requests_tokens + elif self.processor is None: + # no processor available, fall back on unpreferred source or 0 + return response_tokens or requests_tokens or 0 + + # no tokens that matched the preferred source, + # calculate locally based on the value + return len(self.processor.tokenize(value)) diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index 487510e4..aec8a13c 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -6,7 +6,6 @@ from guidellm.benchmark.profile import Profile from guidellm.objects import ( - DistributionSummary, Serializable, StatusDistributionSummary, ) @@ -14,7 +13,11 @@ __all__ = [ "BENCH", + "BenchmarkArgs", + "BenchmarkRunStats", "Benchmark", + "GenerativeTextResponseStats", + "GenerativeTextErrorStats", "GenerativeBenchmark", ] @@ -177,7 +180,7 @@ class Benchmark(Serializable): "for this benchmark." ) ) - requests_loader: Optional[Serializable] = Field( + request_loader: Optional[Serializable] = Field( description=( "The description and specifics for the request loader used to create " "requests for this benchmark." @@ -412,7 +415,7 @@ class GenerativeBenchmark(Benchmark): description="The end time of the last request for the benchmark.", ) - requests_latency: DistributionSummary = Field( + requests_latency: StatusDistributionSummary = Field( description="The distribution of latencies for the completed requests.", ) prompts_token_count: StatusDistributionSummary = Field( @@ -557,7 +560,7 @@ def from_stats( args=args, run_stats=run_stats, worker=worker, - requests_loader=requests_loader, + request_loader=requests_loader, extras=extras or {}, completed_total=len(completed), completed_requests=completed, diff --git a/src/guidellm/benchmark/benchmarker.py b/src/guidellm/benchmark/benchmarker.py index 8d8a426c..62e27bd6 100644 --- a/src/guidellm/benchmark/benchmarker.py +++ b/src/guidellm/benchmark/benchmarker.py @@ -2,6 +2,8 @@ from abc import ABC, abstractmethod from typing import Any, AsyncGenerator, Dict, Generic, Iterable, Literal, Optional +from transformers import PreTrainedTokenizer # type: ignore # noqa: PGH003 + from guidellm.backend import Backend, ResponseSummary from guidellm.benchmark.aggregator import AGG, BENCH, GenerativeBenchmarkAggregator from guidellm.benchmark.benchmark import GenerativeBenchmark @@ -48,6 +50,7 @@ def __init__( requests_loader_description: Optional[Serializable] = None, benchmark_save_extras: Optional[Dict[str, Any]] = None, ): + self.worker = worker self.scheduler: Scheduler[REQ, RES] = Scheduler( worker=worker, request_loader=request_loader ) @@ -170,8 +173,9 @@ async def run( @abstractmethod def create_benchmark_aggregator( self, + run_id: str, profile: Profile, - current_index: int, + strategy_index: int, strategy: SchedulingStrategy, max_number: Optional[int], max_duration: Optional[float], @@ -194,15 +198,23 @@ def __init__( self, backend: Backend, request_loader: Iterable[GenerationRequest], + request_loader_description: Optional[Serializable] = None, + benchmark_save_extras: Optional[Dict[str, Any]] = None, + processor: Optional[PreTrainedTokenizer] = None, ): super().__init__( - worker=GenerativeRequestsWorker(backend), request_loader=request_loader + worker=GenerativeRequestsWorker(backend), + request_loader=request_loader, + requests_loader_description=request_loader_description, + benchmark_save_extras=benchmark_save_extras, ) + self.processor = processor def create_benchmark_aggregator( self, + run_id: str, profile: Profile, - current_index: int, + strategy_index: int, strategy: SchedulingStrategy, max_number: Optional[int], max_duration: Optional[float], @@ -211,4 +223,19 @@ def create_benchmark_aggregator( cooldown_number: Optional[int], cooldown_duration: Optional[float], ) -> GenerativeBenchmarkAggregator: - return GenerativeBenchmarkAggregator() # TODO + return GenerativeBenchmarkAggregator( + processor=self.processor, + run_id=run_id, + profile=profile, + strategy_index=strategy_index, + strategy=strategy, + max_number=max_number, + max_duration=max_duration, + warmup_number=warmup_number, + warmup_duration=warmup_duration, + cooldown_number=cooldown_number, + cooldown_duration=cooldown_duration, + worker_description=self.worker.description, + request_loader_description=self.requests_loader_description, + extras=self.benchmark_save_extras, + ) diff --git a/src/guidellm/benchmark/entrypoints.py b/src/guidellm/benchmark/entrypoints.py index 46409041..85b0d3e1 100644 --- a/src/guidellm/benchmark/entrypoints.py +++ b/src/guidellm/benchmark/entrypoints.py @@ -1 +1,101 @@ -# TODO +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Optional, Union + +from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict +from transformers import PreTrainedTokenizer + +from guidellm.backend import Backend, BackendType +from guidellm.benchmark.benchmark import GenerativeBenchmark +from guidellm.benchmark.benchmarker import GenerativeBenchmarker +from guidellm.benchmark.profile import ProfileType, create_profile +from guidellm.benchmark.progress import BenchmarkerProgressDisplay +from guidellm.dataset import load_dataset +from guidellm.request import RequestLoader +from guidellm.scheduler import StrategyType + + +async def benchmark_generative_text( + target: str, + backend_type: BackendType, + backend_args: Optional[Dict[str, Any]], + model: Optional[str], + processor: Optional[Union[str, Path, PreTrainedTokenizer, Callable]], + processor_args: Optional[Dict[str, Any]], + data: Union[ + str, + Path, + Iterable[Union[str, Dict[str, Any]]], + Dataset, + DatasetDict, + IterableDataset, + IterableDatasetDict, + ], + data_args: Optional[Dict[str, Any]], + rate_type: Union[StrategyType, ProfileType], + rate: Optional[Union[int, float, List[Union[int, float]]]], + max_seconds: Optional[float], + max_requests: Optional[int], + warmup_percent: Optional[float], + cooldown_percent: Optional[float], + show_progress: bool, + output_path: Optional[Union[str, Path]], + output_type: Optional[str], + output_extras: Optional[Dict[str, Any]], +) -> List[GenerativeBenchmark]: + backend = Backend.create( + backend_type, target=target, model=model, **(backend_args or {}) + ) + backend.validate() + + if processor is None: + processor = backend.model + + if isinstance(processor, (str, Path)): + processor = PreTrainedTokenizer.from_pretrained( + processor, **(processor_args or {}) + ) + + dataset = load_dataset(data, data_args, processor) + request_loader, requests_loader_description, processor = RequestLoader( + dataset, processor, processor_args + ) + profile = create_profile(rate_type=rate_type, rate=rate) + + benchmarker = GenerativeBenchmarker( + backend=backend, + request_loader=request_loader, + request_loader_description=requests_loader_description, + benchmark_save_extras=output_extras, + processor=processor, + ) + progress = BenchmarkerProgressDisplay() if show_progress else None + benchmarks = [] + + async for result in benchmarker.run( + profile=profile, + max_number_per_strategy=max_requests, + max_duration_per_strategy=max_seconds, + warmup_number_per_strategy=( + round(max_requests * warmup_percent) + if max_requests and warmup_percent + else None + ), + warmup_duration_per_strategy=( + max_seconds * warmup_percent if max_seconds and warmup_percent else None + ), + cooldown_number_per_strategy=( + round(max_requests * cooldown_percent) + if max_requests and cooldown_percent + else None + ), + cooldown_duration_per_strategy=( + max_seconds * cooldown_percent if max_seconds and cooldown_percent else None + ), + ): + if progress: + progress.update(result) + + if result.type_ == "benchmark_compiled": + benchmarks.append(result.current_benchmark) + + return benchmarks diff --git a/src/guidellm/benchmark/progress.py b/src/guidellm/benchmark/progress.py new file mode 100644 index 00000000..853045df --- /dev/null +++ b/src/guidellm/benchmark/progress.py @@ -0,0 +1,9 @@ +from guidellm.benchmark.benchmarker import BenchmarkerResult + + +class BenchmarkerProgressDisplay: + def __init__(self): + pass + + def update(self, result: BenchmarkerResult): + pass diff --git a/src/guidellm/config.py b/src/guidellm/config.py index c585bd92..d9cbf5e3 100644 --- a/src/guidellm/config.py +++ b/src/guidellm/config.py @@ -157,8 +157,8 @@ class Settings(BaseSettings): emulated_data: EmulatedDataSettings = EmulatedDataSettings() # Request/stats settings - preferred_prompt_tokens_source: Optional[Literal["backend", "local"]] = None - preferred_output_tokens_source: Optional[Literal["backend", "local"]] = None + preferred_prompt_tokens_source: Optional[Literal["request", "response"]] = None + preferred_output_tokens_source: Optional[Literal["request", "response"]] = None preferred_backend: Literal["openai"] = "openai" openai: OpenAISettings = OpenAISettings() diff --git a/src/guidellm/dataset/__init__.py b/src/guidellm/dataset/__init__.py index 46409041..1980afd8 100644 --- a/src/guidellm/dataset/__init__.py +++ b/src/guidellm/dataset/__init__.py @@ -1 +1,16 @@ -# TODO +from .creator import ColumnInputTypes, DatasetCreator +from .datasets import HFDatasetsCreator +from .entrypoints import load_dataset +from .file import FileDatasetCreator +from .in_memory import InMemoryDatasetCreator +from .synthetic import SyntheticDatasetCreator + +__all__ = [ + "DatasetCreator", + "ColumnInputTypes", + "HFDatasetsCreator", + "load_dataset", + "FileDatasetCreator", + "InMemoryDatasetCreator", + "SyntheticDatasetCreator", +] diff --git a/src/guidellm/dataset/base.py b/src/guidellm/dataset/base.py deleted file mode 100644 index 9fd303e6..00000000 --- a/src/guidellm/dataset/base.py +++ /dev/null @@ -1,200 +0,0 @@ -import contextlib -import threading -import time -from abc import ABC, abstractmethod -from queue import Empty, Full, Queue -from typing import Iterator, Literal, Union - -from loguru import logger -from transformers import ( # type: ignore # noqa: PGH003 - AutoTokenizer, - PreTrainedTokenizer, -) - -from guidellm.core.request import TextGenerationRequest - -__all__ = ["GenerationMode", "RequestGenerator"] - - -GenerationMode = Literal["async", "sync"] - - -class RequestGenerator(ABC): - """ - A base class for request generators that generate result requests. - - :param type_: The type of the request generator. - :type type_: str - :param source: The data source for the request generator. - :type source: str - :param tokenizer: The tokenizer instance or the name/config to use - for tokenizing prompts. - :type tokenizer: Union[str, PreTrainedTokenizer] - :param mode: The generation mode, either 'async' or 'sync'. - :type mode: GenerationMode - :param async_queue_size: The size of the request queue. - :type async_queue_size: int - """ - - def __init__( - self, - type_: str, - source: str, - tokenizer: Union[str, PreTrainedTokenizer], - mode: GenerationMode = "async", - async_queue_size: int = 50, - ): - self._type = type_ - self._source = source - self._async_queue_size: int = async_queue_size - self._mode: str = mode - self._queue: Queue = Queue(maxsize=async_queue_size) - self._stop_event: threading.Event = threading.Event() - - if not tokenizer: - err = "Tokenizer must be provided for request generation" - logger.error(err) - raise ValueError(err) - - self._tokenizer = ( - AutoTokenizer.from_pretrained(tokenizer) - if isinstance(tokenizer, str) - else tokenizer - ) - logger.info("Tokenizer initialized for request generation: {}", self._tokenizer) - - if self._mode == "async": - self._thread = threading.Thread(target=self._populate_queue, daemon=True) - self._thread.start() - logger.info( - "RequestGenerator started in async mode with queue size: {}", - self._async_queue_size, - ) - - def __repr__(self) -> str: - """ - Return a string representation of the RequestGenerator. - - :return: String representation of the RequestGenerator. - :rtype: str - """ - return ( - f"RequestGenerator(" - f"mode={self._mode}, " - f"async_queue_size={self._async_queue_size}, " - f"tokenizer={self._tokenizer})" - ) - - def __iter__(self) -> Iterator[TextGenerationRequest]: - """ - Provide an iterator interface to generate new requests. - - :return: An iterator over result requests. - :rtype: Iterator[TextGenerationRequest] - """ - if self.mode == "async": - while not self._stop_event.is_set(): - try: - item = self._queue.get_nowait() - self._queue.task_done() - yield item - except Empty: - time.sleep(0.01) - continue - else: - while not self._stop_event.is_set(): - yield self.create_item() - - @abstractmethod - def __len__(self) -> int: - """ - Abstract method to get the length of the collection to be generated. - """ - - @abstractmethod - def create_item(self) -> TextGenerationRequest: - """ - Abstract method to create a new result request item. - - :return: A new result request. - :rtype: TextGenerationRequest - """ - - @property - def type_(self) -> str: - """ - Get the type of the request generator. - - :return: The type of the request generator. - :rtype: str - """ - return self._type - - @property - def source(self) -> str: - """ - Get the data source for the request generator. - - :return: The data source. - :rtype: str - """ - return self._source - - @property - def tokenizer(self) -> PreTrainedTokenizer: - """ - Get the tokenizer instance. - - :return: The tokenizer instance. - :rtype: PreTrainedTokenizer - """ - return self._tokenizer - - @property - def mode(self) -> str: - """ - Get the generation mode. - - :return: The generation mode. - :rtype: str - """ - return self._mode - - @property - def async_queue_size(self) -> int: - """ - Get the size of the request queue. - - :return: The size of the request queue. - :rtype: int - """ - return self._async_queue_size - - def stop(self): - """ - Stop the background task that populates the queue. - """ - logger.info("Stopping RequestGenerator...") - self._stop_event.set() - if self._mode == "async": - self._thread.join() - logger.info("RequestGenerator stopped") - - def _populate_queue(self): - """ - Populate the request queue in the background. - """ - - while not self._stop_event.is_set(): - with contextlib.suppress(Full): - if self._queue.qsize() < self._async_queue_size: - item = self.create_item() - self._queue.put(item, timeout=0.1) - logger.debug( - "Item added to queue. Current queue size: {}", - self._queue.qsize(), - ) - else: - time.sleep(0.1) - - logger.info("RequestGenerator stopped populating queue") diff --git a/src/guidellm/dataset/creator.py b/src/guidellm/dataset/creator.py new file mode 100644 index 00000000..7135d7e0 --- /dev/null +++ b/src/guidellm/dataset/creator.py @@ -0,0 +1,203 @@ +from abc import ABC, abstractmethod +from typing import Any, Dict, List, Literal, Optional, Tuple, Union + +from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict +from transformers import PreTrainedTokenizerBase + +__all__ = ["DatasetCreator", "ColumnInputTypes"] + +ColumnInputTypes = Literal[ + "text_column", "prompt_tokens_count_column", "output_tokens_count_column" +] + + +class DatasetCreator(ABC): + DEFAULT_SPLITS_TRAIN = [ + "train", + "training", + "train_set", + "training_set", + "train_dataset", + "training_dataset", + "train_data", + "training_data", + "pretrain", + "pretrain_set", + "pretrain_dataset", + "pretrain_data", + "pretraining", + ] + DEFAULT_SPLITS_CALIB = [ + "calibration", + "calib", + "cal", + "calibration_set", + "calib_set", + "cal_set", + "calibration_dataset", + "calib_dataset", + "cal_set", + "calibration_data", + "calib_data", + "cal_data", + ] + DEFAULT_SPLITS_VAL = [ + "validation", + "val", + "valid", + "validation_set", + "val_set", + "validation_dataset", + "val_dataset", + "validation_data", + "val_data", + "dev", + "dev_set", + "dev_dataset", + "dev_data", + ] + DEFAULT_SPLITS_TEST = [ + "test", + "testing", + "test_set", + "testing_set", + "test_dataset", + "testing_dataset", + "test_data", + "testing_data", + "eval", + "eval_set", + "eval_dataset", + "eval_data", + ] + DEFAULT_SPLITS_DATASET = {} + + @classmethod + def create( + cls, + data: Any, + data_args: Optional[Dict[str, Any]], + processor: PreTrainedTokenizerBase, + split_pref_order: Optional[List[str]] = None, + ) -> Tuple[Union[Dataset, IterableDataset], Dict[ColumnInputTypes, str]]: + if not cls.is_supported(data): + raise ValueError(f"Unsupported data type: {type(data)} given for {data}. ") + + split = cls.extract_args_split(data_args) + column_mappings = cls.extract_args_column_mappings(data_args, processor) + dataset = cls.handle_create(data, data_args, processor) + + if isinstance(dataset, (DatasetDict, IterableDatasetDict)): + dataset = cls.extract_dataset_split(dataset, split, split_pref_order) + + if not isinstance(dataset, (Dataset, IterableDataset)): + raise ValueError( + f"Unsupported data type: {type(dataset)} given for {dataset}." + ) + + return dataset, column_mappings + + @classmethod + def extract_args_split(cls, data_args: Dict[str, Any]) -> str: + split = "auto" + + if "split" in data_args: + split = data_args["split"] + del data_args["split"] + + return split + + @classmethod + def extract_args_column_mappings( + cls, + data_args: Dict[str, Any], + processor: PreTrainedTokenizerBase, + ) -> Dict[ColumnInputTypes, str]: + columns = {} + + if "prompt_column" in data_args: + columns["prompt_column"] = data_args["prompt_column"] + del data_args["prompt_column"] + + if "prompt_tokens_count_column" in data_args: + columns["prompt_tokens_count_column"] = data_args[ + "prompt_tokens_count_column" + ] + del data_args["prompt_tokens_count_column"] + + if "output_tokens_count_column" in data_args: + columns["output_tokens_count_column"] = data_args[ + "output_tokens_count_column" + ] + del data_args["output_tokens_count_column"] + + return columns + + @classmethod + def extract_dataset_name( + cls, dataset: Union[Dataset, IterableDataset, DatasetDict, IterableDatasetDict] + ) -> Optional[str]: + if isinstance(dataset, (DatasetDict, IterableDatasetDict)): + dataset = dataset[list(dataset.keys())[0]] + + if isinstance(dataset, (Dataset, IterableDataset)): + if not hasattr(dataset, "info") or not hasattr( + dataset.info, "dataset_name" + ): + return None + + return dataset.info.dataset_name + + raise ValueError(f"Unsupported data type: {type(dataset)} given for {dataset}.") + + @classmethod + def extract_dataset_split( + cls, + dataset: Union[DatasetDict, IterableDatasetDict], + specified_split: Union[Literal["auto"], str] = "auto", + split_pref_order: Optional[Union[Literal["auto"], List[str]]] = "auto", + ) -> Union[Dataset, IterableDataset]: + if not isinstance(dataset, (DatasetDict, IterableDatasetDict)): + raise ValueError( + f"Unsupported data type: {type(dataset)} given for {dataset}." + ) + + if specified_split != "auto": + if specified_split not in dataset: + raise ValueError( + f"Split {specified_split} not found in dataset {dataset}." + ) + + return dataset[specified_split] + + dataset_name = cls.extract_dataset_name(dataset) + + if dataset_name and dataset_name in cls.DEFAULT_SPLITS_DATASET: + return dataset[cls.DEFAULT_SPLITS_DATASET[dataset_name]] + + if split_pref_order == "auto": + split_pref_order = [ + *cls.DEFAULT_SPLITS_TEST, + *cls.DEFAULT_SPLITS_VAL, + *cls.DEFAULT_SPLITS_CALIB, + *cls.DEFAULT_SPLITS_TRAIN, + ] + + for test_split in split_pref_order or []: + if test_split in dataset: + return dataset[test_split] + + return dataset[list(dataset.keys())[0]] + + @classmethod + @abstractmethod + def is_supported(cls, data: Any, data_args: Optional[Dict[str, Any]]) -> bool: ... + + @classmethod + @abstractmethod + def handle_create( + cls, + data: Any, + data_args: Optional[Dict[str, Any]], + processor: PreTrainedTokenizerBase, + ) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]: ... diff --git a/src/guidellm/dataset/datasets.py b/src/guidellm/dataset/datasets.py new file mode 100644 index 00000000..9a3c60f8 --- /dev/null +++ b/src/guidellm/dataset/datasets.py @@ -0,0 +1,60 @@ +from pathlib import Path +from typing import Any, Dict, Optional, Union + +from datasets import ( + Dataset, + DatasetDict, + IterableDataset, + IterableDatasetDict, + get_dataset_config_info, + load_dataset, +) +from transformers import PreTrainedTokenizerBase + +from guidellm.dataset.creator import DatasetCreator + +__all__ = ["HFDatasetsCreator"] + + +class HFDatasetsCreator(DatasetCreator): + @classmethod + def is_supported(cls, data: Any, data_args: Optional[Dict[str, Any]]) -> bool: + if isinstance( + data, (Dataset, DatasetDict, IterableDataset, IterableDatasetDict) + ): + # base type is supported + return True + + if isinstance(data, (str, Path)) and (path := Path(data)).exists(): + # local folder or py file, assume supported + return path.is_dir() or path.suffix == ".py" + + if isinstance(data, (str, Path)): + try: + # try to load dataset + return get_dataset_config_info(data) is not None + except: + pass + + return False + + @classmethod + def handle_create( + cls, + data: Any, + data_args: Optional[Dict[str, Any]], + processor: PreTrainedTokenizerBase, + ) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]: + if isinstance(data, (str, Path)): + data = load_dataset(data, **(data_args or {})) + elif data_args: + raise ValueError( + f"data_args should not be provided when data is a {type(data)}" + ) + + if isinstance( + data, (Dataset, DatasetDict, IterableDataset, IterableDatasetDict) + ): + return data + + raise ValueError(f"Unsupported data type: {type(data)} given for {data}. ") diff --git a/src/guidellm/dataset/emulated.py b/src/guidellm/dataset/emulated.py deleted file mode 100644 index 7d481cb7..00000000 --- a/src/guidellm/dataset/emulated.py +++ /dev/null @@ -1,397 +0,0 @@ -import json -import math -from dataclasses import dataclass -from pathlib import Path -from typing import Dict, List, Optional, Tuple, Union - -import numpy as np -from loguru import logger -from transformers import PreTrainedTokenizer # type: ignore # noqa: PGH003 - -from guidellm.config import settings -from guidellm.core.request import TextGenerationRequest -from guidellm.request.base import GenerationMode, RequestGenerator -from guidellm.utils import clean_text, filter_text, load_text, split_text - -__all__ = ["EmulatedConfig", "EmulatedRequestGenerator", "EndlessTokens"] - - -@dataclass -class EmulatedConfig: - """ - Configuration for emulated text generation requests. - - Args: - prompt_tokens (int): Number of prompt tokens. - prompt_tokens_variance (Optional[int]): Variance for prompt tokens. - prompt_tokens_min (Optional[int]): Minimum number of prompt tokens. - prompt_tokens_max (Optional[int]): Maximum number of prompt tokens. - generated_tokens (Optional[int]): Number of generated tokens. - generated_tokens_variance (Optional[int]): Variance for generated tokens. - generated_tokens_min (Optional[int]): Minimum number of generated tokens. - generated_tokens_max (Optional[int]): Maximum number of generated tokens. - """ - - @staticmethod - def create_config(config: Optional[Union[str, Path, Dict]]) -> "EmulatedConfig": - """ - Create an EmulatedConfig instance from a configuration source. - - :param config: Configuration source, can be a dictionary, JSON string, - key=value string, or file path. - :type config: Union[str, Path, Dict] - :return: An instance of EmulatedConfig. - :rtype: EmulatedConfig - :raises FileNotFoundError: If the configuration file is not found. - :raises ValueError: If the configuration format is invalid. - """ - if not config: - logger.debug("Creating default configuration") - return EmulatedConfig(prompt_tokens=1024, generated_tokens=256) - - if isinstance(config, dict): - logger.debug("Loading configuration from dict: {}", config) - return EmulatedConfig(**config) - - if isinstance(config, Path) or ( - isinstance(config, str) and (config.endswith(".json") or "{" in config) - ): - logger.debug("Loading configuration from json: {}", config) - - if isinstance(config, str) and "{" in config: - json_text = config.strip() - else: - if isinstance(config, str): - config = Path(config) - - if not config.exists(): - raise FileNotFoundError(f"Configuration file not found: {config}") - - json_text = config.read_text(encoding="utf-8") - - json_dict = json.loads(json_text) - - return EmulatedConfig(**json_dict) - - if isinstance(config, str) and "=" in config: - logger.debug("Loading configuration from csv string: {}", config) - items = config.split(",") - config_dict = {} - for item in items: - key_value = item.strip().split("=") - if len(key_value) != 2: # noqa: PLR2004 - raise ValueError(f"Unexpected format for item: {item}") - key = key_value[0].strip() - value = ( - int(key_value[1].strip()) - if key_value[1].isnumeric() - else key_value[1] - ) - config_dict[key] = value - - return EmulatedConfig(**config_dict) # type: ignore # noqa: PGH003 - - raise ValueError( - f"Invalid configuration given for creation of EmulatedConfig: {config}" - ) - - prompt_tokens: int - prompt_tokens_variance: Optional[int] = None - prompt_tokens_min: Optional[int] = None - prompt_tokens_max: Optional[int] = None - - generated_tokens: Optional[int] = None - generated_tokens_variance: Optional[int] = None - generated_tokens_min: Optional[int] = None - generated_tokens_max: Optional[int] = None - - @property - def prompt_tokens_range(self) -> Tuple[int, int]: - """ - Get the range (min, max) of prompt tokens to generate. - - :return: The range of prompt tokens. - :rtype: Tuple[int, int] - """ - return self._token_range( - self.prompt_tokens, - self.prompt_tokens_variance, - self.prompt_tokens_min, - self.prompt_tokens_max, - ) - - @property - def output_tokens_range(self) -> Tuple[int, int]: - """ - Get the range (min, max) of output tokens to generate. - - :return: The range of generated tokens. - :rtype: Tuple[int, int] - """ - if not self.generated_tokens: - return 0, 0 - - return self._token_range( - self.generated_tokens, - self.generated_tokens_variance, - self.generated_tokens_min, - self.generated_tokens_max, - ) - - def sample_prompt_tokens(self, rng: np.random.Generator) -> int: - """ - Sample the number of prompt tokens to generate. - - :param rng: The random number generator to use. - :type rng: np.random.Generator - :return: The number of prompt tokens to create. - :rtype: int - """ - return self._sample_tokens( - self.prompt_tokens, - self.prompt_tokens_variance, - self.prompt_tokens_min, - self.prompt_tokens_max, - rng, - ) - - def sample_output_tokens(self, rng: np.random.Generator) -> Optional[int]: - """ - Sample the number of output tokens to generate. - - :param rng: The random number generator to use. - :type rng: np.random.Generator - :return: The number of output tokens to generate. - :rtype: Optional[int] - """ - if not self.generated_tokens: - return None - - return self._sample_tokens( - self.generated_tokens, - self.generated_tokens_variance, - self.generated_tokens_min, - self.generated_tokens_max, - rng, - ) - - @staticmethod - def _sample_tokens( - base: int, - variance: Optional[int], - min_tokens: Optional[int], - max_tokens: Optional[int], - rng: np.random.Generator, - ) -> int: - min_tokens, max_tokens = EmulatedConfig._token_range( - base, variance, min_tokens, max_tokens - ) - - if min_tokens == max_tokens: - return min_tokens - - if not variance: - return rng.integers(min_tokens, max_tokens + 1) - - rand = rng.normal(base, math.sqrt(variance)) - - return int(min(max(rand, min_tokens), max_tokens)) - - @staticmethod - def _token_range( - base: int, - variance: Optional[int], - min_tokens: Optional[int], - max_tokens: Optional[int], - ) -> Tuple[int, int]: - if not variance: - return ( - min_tokens or base, - max_tokens or base, - ) - - min_tokens = min_tokens if min_tokens and min_tokens > 0 else 1 - max_tokens = ( - max_tokens if max_tokens and max_tokens > base else base + 5 * variance - ) - - return min_tokens, max_tokens - - -class EndlessTokens(List[str]): - """ - A list subclass that allows for endless data generation. - """ - - def __init__( - self, - data: Union[str, Path], - filter_start: Optional[Union[str, int]] = None, - filter_end: Optional[Union[str, int]] = None, - clean_text_args: Optional[Dict[str, bool]] = None, - ): - """ - Initialize EndlessDataWords with data. - - :param data: Source text data. - :type data: str - """ - logger.debug("Loading data from: {}", data) - data = load_text(data) - data = filter_text(data, filter_start, filter_end) - data = ( - clean_text(data) - if not clean_text_args - else clean_text(data, **clean_text_args) - ) - self._tokens, self._token_separators, self._line_indices = split_text(data) - - super().__init__(self._tokens) - - @property - def line_indices(self) -> List[int]: - """ - Get the list of start indices for lines. - - :return: List of start indices. - :rtype: List[int] - """ - return self._line_indices - - def create_text(self, start: int, length: int) -> str: - """ - Create a text snippet from the specified range. - - :param start: Start index. - :type start: int - :param length: Length of the snippet. - :type length: int - :return: Text snippet. - :rtype: str - """ - start = start % len(self) - text = "" - buff_token_sep = "" - - for counter in range(length): - index = (start + counter) % len(self) - text += buff_token_sep + self[index] - buff_token_sep = self._token_separators[index] - - return text - - -class EmulatedRequestGenerator(RequestGenerator): - """ - A request generator that generates emulated requests based on a configuration. - - :param config: The configuration string, file path, or dictionary. - :type config: Union[str, Dict, Path] - :param random_seed: The random seed to use for generating requests. - :type random_seed: Optional[int] - :param tokenizer: The tokenizer instance or the name/config to use - for tokenizing prompts. - :type tokenizer: Optional[Union[str, PreTrainedTokenizer]] - :param mode: The generation mode, either 'async' or 'sync'. - :type mode: GenerationMode - :param async_queue_size: The size of the request queue. - :type async_queue_size: int - """ - - def __init__( - self, - config: Optional[Union[str, Path, Dict]], - random_seed: Optional[int] = None, - tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None, - mode: GenerationMode = "async", - async_queue_size: int = 50, - ): - """ - Initialize EmulatedRequestGenerator with configuration and tokenizer. - - :param config: Configuration source, can be a dictionary, - JSON string, or file path. - :type config: Optional[Union[str, Path, Dict]] - :param random_seed: Optional seed for random number generator. - :type random_seed: Optional[int] - :param tokenizer: Tokenizer instance or configuration for tokenizing prompts. - :type tokenizer: Optional[Union[str, PreTrainedTokenizer]] - :param mode: Mode of request generation, either 'async' or 'sync'. - :type mode: str - :param async_queue_size: Size of the asynchronous queue. - :type async_queue_size: int - """ - self._config = EmulatedConfig.create_config(config) - self._tokens = EndlessTokens( - settings.emulated_data.source, - settings.emulated_data.filter_start, - settings.emulated_data.filter_end, - ) - self._rng = np.random.default_rng(random_seed) - - # NOTE: Must be after all the parameters since the queue population - # function requires attributes above - super().__init__( - type_="emulated", - source=str(config), - tokenizer=tokenizer, - mode=mode, - async_queue_size=async_queue_size, - ) - - def __len__(self) -> int: - raise NotImplementedError( - "Can't get the length of the emulated dataset. " - "Check the `--data-type` CLI parameter." - ) - - def create_item(self) -> TextGenerationRequest: - """ - Create a new text generation request item from the data. - - :return: A new text generation request. - :rtype: TextGenerationRequest - """ - logger.debug("Creating new text generation request") - target_prompt_token_count = self._config.sample_prompt_tokens(self._rng) - prompt = self.sample_prompt(target_prompt_token_count) - prompt_token_count = len(self.tokenizer.tokenize(prompt)) - output_token_count = self._config.sample_output_tokens(self._rng) - logger.debug("Generated prompt: {}", prompt) - - return TextGenerationRequest( - prompt=prompt, - prompt_token_count=prompt_token_count, - output_token_count=output_token_count, - ) - - def sample_prompt(self, tokens: int) -> str: - """ - Sample a prompt with the specified number of tokens. - - :param tokens: Number of tokens for the prompt. - :type tokens: int - :return: Sampled prompt text. - :rtype: str - """ - start_line_index = self._rng.integers(0, len(self._tokens.line_indices)) - - # binary search to find the proper number of tokens for the prompt - # this is because tokenizers differ in tokenization behavior - left = 0 - right = left + 5 * tokens - - while left < right: - mid = (left + right) // 2 - prompt = self._tokens.create_text(start_line_index, mid) - token_count = len(self.tokenizer.tokenize(prompt)) - - if token_count == tokens: - return prompt - - if token_count < tokens: - left = mid + 1 - else: - right = mid - - return self._tokens.create_text(start_line_index, left) diff --git a/src/guidellm/dataset/entrypoints.py b/src/guidellm/dataset/entrypoints.py new file mode 100644 index 00000000..2ff06702 --- /dev/null +++ b/src/guidellm/dataset/entrypoints.py @@ -0,0 +1,31 @@ +from typing import Any, Dict, List, Optional, Union + +from datasets import Dataset, IterableDataset +from transformers import PreTrainedTokenizerBase + +from guidellm.dataset.datasets import HFDatasetsCreator +from guidellm.dataset.file import FileDatasetCreator +from guidellm.dataset.in_memory import InMemoryDatasetCreator +from guidellm.dataset.synthetic import SyntheticDatasetCreator + +__all__ = ["load_dataset"] + + +def load_dataset( + data: Any, + data_args: Optional[Dict[str, Any]], + processor: PreTrainedTokenizerBase, + split_pref_order: Optional[List[str]] = None, +) -> Union[Dataset, IterableDataset]: + creators = [ + InMemoryDatasetCreator, + SyntheticDatasetCreator, + FileDatasetCreator, + HFDatasetsCreator, + ] + + for creator in creators: + if creator.is_supported(data, data_args): + return creator.create(data, data_args, processor, split_pref_order) + + raise ValueError(f"Unsupported data type: {type(data)} given for {data}. ") diff --git a/src/guidellm/dataset/file.py b/src/guidellm/dataset/file.py index b187f7b4..47143aec 100644 --- a/src/guidellm/dataset/file.py +++ b/src/guidellm/dataset/file.py @@ -1,83 +1,88 @@ from pathlib import Path -from typing import Optional, Union - -from loguru import logger -from transformers import PreTrainedTokenizer # type: ignore # noqa: PGH003 - -from guidellm.config import settings -from guidellm.core.request import TextGenerationRequest -from guidellm.request.base import GenerationMode, RequestGenerator -from guidellm.utils import load_text_lines - -__all__ = ["FileRequestGenerator"] - - -class FileRequestGenerator(RequestGenerator): - """ - A request generator implementation for files. - - :param path: The path to the file containing the data. - :type path: Optional[Union[str, Path]] - :param tokenizer: The tokenizer instance or the name/config to use - for tokenizing prompts. - :type tokenizer: Union[str, PreTrainedTokenizer] - :param mode: The generation mode, either 'async' or 'sync'. - :type mode: str - :param async_queue_size: The size of the request queue. - :type async_queue_size: int - """ - - def __init__( - self, - path: Optional[Union[str, Path]], - tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None, - mode: GenerationMode = "async", - async_queue_size: int = 50, - ): - if not path: - raise ValueError("File path must be provided for FileRequestGenerator") - - self._path = path - self._data = load_text_lines( - path, - filters=settings.dataset.preferred_data_columns, - ) - self._iterator = iter(self._data) - - # NOTE: Must be after all the parameters since the queue population - # function requires attributes above - super().__init__( - type_="file", - source=str(path), - tokenizer=tokenizer, - mode=mode, - async_queue_size=async_queue_size, - ) - - def __len__(self) -> int: - """ - Return the number of text lines. - """ - - return len(self._data) - - def create_item(self) -> TextGenerationRequest: - """ - Create a new result request item from the data. - - :return: A new result request. - :rtype: TextGenerationRequest - """ - logger.debug("Creating new request item from file data") - - try: - data = next(self._iterator) - except StopIteration: - self._iterator = iter(self._data) - data = next(self._iterator) - - token_count = len(self.tokenizer.tokenize(data)) - request = TextGenerationRequest(prompt=data, prompt_token_count=token_count) - logger.debug("Created new TextGenerationRequest: {}", request) - - return request +from typing import Any, Dict, Optional, Union + +import pandas as pd +from datasets import ( + Dataset, + DatasetDict, + IterableDataset, + IterableDatasetDict, + load_dataset, +) +from transformers import PreTrainedTokenizerBase + +from guidellm.dataset.creator import DatasetCreator + +__all__ = ["FileDatasetCreator"] + + +class FileDatasetCreator(DatasetCreator): + SUPPORTED_TYPES = { + ".txt", + ".text", + ".csv", + ".json", + ".jsonl", + ".parquet", + ".arrow", + ".hdf5", + ".tar", + } + + @classmethod + def is_supported(cls, data: Any, data_args: Optional[Dict[str, Any]]) -> bool: + if isinstance(data, (str, Path)) and (path := Path(data)).exists(): + # local folder or py file, assume supported + return path.suffix.lower() in cls.SUPPORTED_TYPES + + return False + + @classmethod + def handle_create( + cls, + data: Any, + data_args: Optional[Dict[str, Any]], + processor: PreTrainedTokenizerBase, + ) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]: + if not isinstance(data, (str, Path)): + raise ValueError(f"Unsupported data type: {type(data)} given for {data}. ") + + path = Path(data) + if not path.exists(): + raise FileNotFoundError(f"File not found: {path}") + + if not path.is_file(): + raise ValueError(f"Unsupported data type: {path} given for {path}. ") + + if path.suffix.lower() not in cls.SUPPORTED_TYPES: + raise ValueError(f"Unsupported file type: {path.suffix} given for {path}. ") + + return cls.load_dataset(path, data_args) + + @classmethod + def load_dataset( + cls, path: Path, data_args: Optional[Dict[str, Any]] + ) -> Union[Dataset, IterableDataset]: + if path.suffix.lower() in {".txt", ".text"}: + with path.open("r") as file: + items = file.readlines() + + dataset = Dataset.from_dict({"text": items}, **(data_args or {})) + elif path.suffix.lower() == ".csv": + dataset = load_dataset("csv", data_files=path, **(data_args or {})) + elif path.suffix.lower() in {".json", ".jsonl"}: + dataset = load_dataset("json", data_files=path, **(data_args or {})) + elif path.suffix.lower() == ".parquet": + dataset = load_dataset("parquet", data_files=path, **(data_args or {})) + elif path.suffix.lower() == ".arrow": + dataset = load_dataset("arrow", data_files=path, **(data_args or {})) + elif path.suffix.lower() == ".hdf5": + dataset = Dataset.from_pandas(pd.read_hdf(path), **(data_args or {})) + elif path.suffix.lower() == ".db": + dataset = Dataset.from_sql(con=path, **(data_args or {})) + elif path.suffix.lower() == ".tar": + dataset = load_dataset("webdataset", data_files=path, **(data_args or {})) + else: + raise ValueError(f"Unsupported file type: {path.suffix} given for {path}. ") + + return dataset diff --git a/src/guidellm/dataset/in_memory.py b/src/guidellm/dataset/in_memory.py new file mode 100644 index 00000000..f41101e6 --- /dev/null +++ b/src/guidellm/dataset/in_memory.py @@ -0,0 +1,127 @@ +from typing import Any, Dict, Iterable, Optional, Union + +from datasets import ( + Dataset, + DatasetDict, + IterableDataset, + IterableDatasetDict, +) +from transformers import PreTrainedTokenizerBase + +from guidellm.dataset.creator import DatasetCreator + +__all__ = ["InMemoryDatasetCreator"] + + +class InMemoryDatasetCreator(DatasetCreator): + @classmethod + def is_supported(cls, data: Any, data_args: Optional[Dict[str, Any]]) -> bool: + return isinstance(data, Iterable) and not isinstance(data, str) + + @classmethod + def handle_create( + cls, + data: Any, + data_args: Optional[Dict[str, Any]], + processor: PreTrainedTokenizerBase, + ) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]: + if not isinstance(data, Iterable): + raise TypeError( + f"Unsupported data format. Expected Iterable[Any], got {type(data)}" + ) + + if not data: + raise ValueError("Data is empty") + + if isinstance(data, Dict): + # assume data is a dictionary of columns and values: {"c1": ["i1", "i2"]} + data_dict = cls.format_data_dict(data) + elif isinstance(data[0], Dict): + # assume data is a list of dictionaries: [{"c1": "i1"}, {"c1": "i2"}] + data_dict = cls.format_data_iterable_dicts(data) + else: + # assume data is a list of items with no columns: ["i1", "i2"] + data_dict = cls.format_data_iterable_values(data) + + return Dataset.from_dict(data_dict, **(data_args or {})) + + @classmethod + def format_data_dict(cls, data: Dict[Any, Any]) -> Dict[str, Any]: + if not isinstance(data, Dict): + raise TypeError( + f"Unsupported data format. Expected Dict[str, Iterable[Any]], " + f"got {type(data)}" + ) + + if not all( + isinstance(key, str) and isinstance(val, Iterable) + for key, val in data.items() + ): + raise TypeError( + "Unsupported data format. Expected Dict[str, Iterable[Any]], " + f"got {type(data)}" + ) + + samples = len(list(data.values())[0]) + if not all(len(val) == samples for val in data.values()): + raise ValueError( + "Unsupported data format. Not all columns have the same number samples " + f"for {data}" + ) + + return data + + @classmethod + def format_data_iterable_dicts( + cls, data: Iterable[Dict[Any, Any]] + ) -> Dict[str, Any]: + if not isinstance(data, Iterable): + raise TypeError( + f"Unsupported data format. Expected Iterable[Dict[str, Any]], " + f"got {type(data)}" + ) + + if not all(isinstance(item, Dict) for item in data): + raise TypeError( + f"Unsupported data format. Expected Iterable[Dict[str, Any]], " + f"got {type(data)}" + ) + + if not all(isinstance(key, str) for key in data[0]): + raise TypeError( + "Unsupported data format. Expected Dict[str, Any], " + f"but one of the items had a non string column for {data}" + ) + + columns = list(data[0].keys()) + if not all( + len(item) == len(columns) and all(key in item for key in columns) + for item in data + ): + raise ValueError( + "Unsupported data format. Not all items have the same columns " + f"for {data}" + ) + + data_dict = {key: [] for key in columns} + for item in data: + for key, value in item.items(): + data_dict[key].append(value) + + return data_dict + + @classmethod + def format_data_iterable_values(cls, data: Iterable[Any]) -> Dict[str, Any]: + if not isinstance(data, Iterable): + raise TypeError( + f"Unsupported data format. Expected Iterable[Iterable[Any]], " + f"got {type(data)}" + ) + + first_type = type(data[0]) + if not all(isinstance(item, first_type) for item in data): + raise TypeError( + f"Unsupported data format. Not all types are the same for {data}" + ) + + return {"data": list(data)} diff --git a/src/guidellm/dataset/synthetic.py b/src/guidellm/dataset/synthetic.py new file mode 100644 index 00000000..b69c950a --- /dev/null +++ b/src/guidellm/dataset/synthetic.py @@ -0,0 +1,292 @@ +import json +import random +from pathlib import Path +from typing import Any, Dict, Iterable, Iterator, Optional, Tuple, Union + +import yaml +from datasets import ( + Dataset, + DatasetDict, + IterableDataset, + IterableDatasetDict, +) +from pydantic import Field +from transformers import PreTrainedTokenizerBase + +from guidellm.config import settings +from guidellm.dataset.creator import ColumnInputTypes, DatasetCreator +from guidellm.objects import Serializable + +__all__ = ["SyntheticDatasetCreator"] + + +class SyntheticDatasetConfig(Serializable): + prompt_tokens: int = Field( + description="The average number of text tokens generated for prompts." + ) + prompt_tokens_variance: Optional[int] = Field( + description="The variance of the number of text tokens generated for prompts.", + default=None, + ) + prompt_tokens_min: Optional[int] = Field( + description="The minimum number of text tokens generated for prompts.", + default=None, + ) + prompt_tokens_max: Optional[int] = Field( + description="The maximum number of text tokens generated for prompts.", + default=None, + ) + output_tokens: int = Field( + description="The average number of text tokens generated for outputs.", + ) + output_tokens_variance: Optional[int] = Field( + description="The variance of the number of text tokens generated for outputs.", + default=None, + ) + output_tokens_min: Optional[int] = Field( + description="The minimum number of text tokens generated for outputs.", + default=None, + ) + output_tokens_max: Optional[int] = Field( + description="The maximum number of text tokens generated for outputs.", + default=None, + ) + samples: int = Field( + description="The number of samples to generate for the dataset.", + default=10000, + ) + seed: int = Field( + description="The seed to use for random number generation.", + default=42, + ) + + @staticmethod + def parse_str(data: Union[str, Path]) -> "SyntheticDatasetConfig": + if ( + isinstance(data, Path) + or data.strip().endswith(".config") + or data.strip().endswith(".yaml") + ): + return SyntheticDatasetConfig.parse_config_file(data) + + if data.strip().startswith("{"): + return SyntheticDatasetConfig.parse_json(data) + + if data.count("=") > 1: + return SyntheticDatasetConfig.parse_key_value_pairs(data) + + raise ValueError( + f"Unsupported data format. Expected JSON or key-value pairs, got {data}" + ) + + @staticmethod + def parse_json(data: str) -> "SyntheticDatasetConfig": + config_dict = json.loads(data.strip()) + + return SyntheticDatasetConfig(**config_dict) + + @staticmethod + def parse_key_value_pairs(data: str) -> "SyntheticDatasetConfig": + config_dict = {} + items = data.strip().split(",") + for item in items: + key, value = item.split("=") + config_dict[key.strip()] = ( + int(value.strip()) if value.strip().isnumeric() else value.strip() + ) + + return SyntheticDatasetConfig(**config_dict) + + @staticmethod + def parse_config_file(data: Union[str, Path]) -> "SyntheticDatasetConfig": + with Path(data).open("r") as file: + config_dict = yaml.safe_load(file) + + return SyntheticDatasetConfig(**config_dict) + + +class IntegerRangeSampler: + def __init__( + self, + average: int, + variance: Optional[int], + min_value: Optional[int], + max_value: Optional[int], + seed: int, + ): + self.average = average + self.variance = variance + self.min_value = min_value + self.max_value = max_value + self.seed = seed + self.rng = random.Random(seed) + + def __iter__(self) -> Iterator[int]: + calc_min = self.min_value + if not calc_min: + calc_min = max( + 0, self.average - 5 * self.variance if self.variance else self.average + ) + calc_max = self.max_value + if not calc_max: + calc_max = ( + self.average + 5 * self.variance if self.variance else self.average + ) + + while True: + if calc_min == calc_max: + yield calc_min + elif not self.variance: + yield self.rng.randint(calc_min, calc_max + 1) + else: + rand = self.rng.gauss(self.average, self.variance) + yield round(max(calc_min, min(calc_max, rand))) + + +class EndlessTextCreator: + """ + A list subclass that allows for endless data generation. + """ + + def __init__( + self, + data: Union[str, Path], + filter_start: Optional[Union[str, int]] = None, + filter_end: Optional[Union[str, int]] = None, + ): + self.data = data + text = load_text(data) + text = filter_text(data, filter_start, filter_end) + self.words = split_text(text) + + def create_text(self, start: int, length: int) -> str: + """ + Create a text snippet from the specified range. + + :param start: Start index. + :type start: int + :param length: Length of the snippet. + :type length: int + :return: Text snippet. + :rtype: str + """ + start = start % len(self) + text = "" + + for counter in range(length): + index = (start + counter) % len(self.words) + if counter > 0: + text += " " + text += self.words[index] + + return text + + +class SyntheticTextItemsGenerator(Iterable[Dict[str, Union[str, int]]]): + def __init__( + self, config: SyntheticDatasetConfig, processor: PreTrainedTokenizerBase + ): + self.config = config + self.processor = processor + self.tokens = [] + self.text_creator = EndlessTextCreator( + data=settings.emulated_data.source, + filter_start=settings.emulated_data.filter_start, + filter_end=settings.emulated_data.filter_end, + ) + + def __iter__(self) -> Iterator[Tuple[str, int, int]]: + prompt_tokens_sampler = IntegerRangeSampler( + average=self.config.prompt_tokens, + variance=self.config.prompt_tokens_variance, + min_value=self.config.prompt_tokens_min, + max_value=self.config.prompt_tokens_max, + seed=self.config.seed, + ) + output_tokens_sampler = IntegerRangeSampler( + average=self.config.output_tokens, + variance=self.config.output_tokens_variance, + min_value=self.config.output_tokens_min, + max_value=self.config.output_tokens_max, + seed=self.config.seed, + ) + start_index_sampler = random.Random(self.config.seed).randint( + 0, len(self.text_creator.words) + ) + + for _, prompt_tokens, output_tokens, start_index in zip( + range(self.config.samples), + prompt_tokens_sampler, + output_tokens_sampler, + start_index_sampler, + ): + yield { + "prompt": self._create_prompt(prompt_tokens, start_index), + "prompt_tokens_count": prompt_tokens, + "output_tokens_count": output_tokens, + } + + def _create_prompt(self, prompt_tokens: int, start_index: int) -> str: + left = start_index + right = start_index + 5 * prompt_tokens + + while left < right: + mid = (left + right) // 2 + test_prompt = self.text_creator.create_text(start_index, mid) + test_tokens = len(self.processor.tokenize(test_prompt)) + + if test_tokens == prompt_tokens: + return test_prompt + elif test_tokens < prompt_tokens: + left = mid + 1 + else: + right = mid + + return self.text_creator.create_text(start_index, left) + + +class SyntheticDatasetCreator(DatasetCreator): + @classmethod + def is_supported(cls, data: Any, data_args: Optional[Dict[str, Any]]) -> bool: + if ( + isinstance(data, Path) + and data.exists() + and data.suffix in {".config", ".yaml"} + ): + return True + + if isinstance(data, str): + data_str: str = data.strip() + if ( + data_str.startswith("{") + or data_str.count("=") > 1 + or data_str.endswith((".config", ".yaml")) + ): + return True + + return False + + @classmethod + def handle_create( + cls, + data: Any, + data_args: Optional[Dict[str, Any]], + processor: PreTrainedTokenizerBase, + ) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]: + config = SyntheticDatasetConfig.parse_str(data) + generator = SyntheticTextItemsGenerator(config, processor) + items = list(generator) + + return Dataset.from_list(items) + + @classmethod + def extract_args_column_mappings( + cls, data_args: Dict[str, Any], processor: PreTrainedTokenizerBase + ) -> Dict[ColumnInputTypes, str]: + super().extract_args_column_mappings(data_args) + + return { + "prompt_column": "prompt", + "prompt_tokens_count_column": "prompt_tokens_count", + "output_tokens_count_column": "output_tokens_count", + } diff --git a/src/guidellm/dataset/transformers.py b/src/guidellm/dataset/transformers.py deleted file mode 100644 index 3fd24040..00000000 --- a/src/guidellm/dataset/transformers.py +++ /dev/null @@ -1,103 +0,0 @@ -from pathlib import Path -from typing import Optional, Union - -from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict -from loguru import logger -from transformers import PreTrainedTokenizer # type: ignore # noqa: PGH003 - -from guidellm.core.request import TextGenerationRequest -from guidellm.request.base import GenerationMode, RequestGenerator -from guidellm.utils import ( - load_transformers_dataset, - resolve_transformers_dataset_column, -) - -__all__ = ["TransformersDatasetRequestGenerator"] - - -class TransformersDatasetRequestGenerator(RequestGenerator): - """ - A request generator implementation for Hugging Face datasets. - - :param dataset: The name of the Hugging Face dataset to use or the path - to a local dataset. - :type dataset_name: str - :param split: The split of the dataset to use (e.g., 'train', 'test'). - :type split: str - :param column: The column/field to use for generating requests. - :type column: str - :param tokenizer: The tokenizer instance or the name/config to use - for tokenizing prompts. - :type tokenizer: Union[str, PreTrainedTokenizer] - :param mode: The generation mode, either 'async' or 'sync'. - :type mode: str - :param async_queue_size: The size of the request queue. - :type async_queue_size: int - """ - - def __init__( - self, - dataset: Union[ - str, Path, DatasetDict, Dataset, IterableDatasetDict, IterableDataset - ], - split: Optional[str] = None, - column: Optional[str] = None, - tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None, - mode: GenerationMode = "async", - async_queue_size: int = 50, - **kwargs, - ): - self._dataset = dataset - self._split = split - self._column = column - self._kwargs = kwargs - - self._hf_dataset: Union[Dataset, IterableDataset] = load_transformers_dataset( - dataset, split=split, **kwargs - ) - self._hf_column = resolve_transformers_dataset_column( - self._hf_dataset, column=column - ) - self._hf_dataset_iterator = iter(self._hf_dataset) - - # NOTE: Must be after all the parameters since the queue population - # function requires attributes above - super().__init__( - type_="transformers_dataset", - source=str(dataset), - tokenizer=tokenizer, - mode=mode, - async_queue_size=async_queue_size, - ) - - def __len__(self) -> int: - if not isinstance(self._hf_dataset, Dataset): - raise ValueError("Can't get dataset size for IterableDataset object") - else: - return len(self._hf_dataset) - - def create_item(self) -> TextGenerationRequest: - """ - Create a new result request item from the dataset. - - :return: A new result request. - :rtype: TextGenerationRequest - """ - - logger.debug("Creating new request item from dataset") - - try: - data = next(self._hf_dataset_iterator) - except StopIteration: - self._hf_dataset_iterator = iter(self._hf_dataset) - data = next(self._hf_dataset_iterator) - - prompt = data[self._hf_column] - token_count = len(self.tokenizer.tokenize(prompt)) - request = TextGenerationRequest( - prompt=prompt, - prompt_token_count=token_count, - ) - logger.debug(f"Created new TextGenerationRequest: {request}") - - return request diff --git a/src/guidellm/request/__init__.py b/src/guidellm/request/__init__.py index 2a68a521..10ef0be7 100644 --- a/src/guidellm/request/__init__.py +++ b/src/guidellm/request/__init__.py @@ -1,5 +1,7 @@ +from .loader import RequestLoader from .request import GenerationRequest __all__ = [ "GenerationRequest", + "RequestLoader", ] diff --git a/src/guidellm/request/loader.py b/src/guidellm/request/loader.py index 46409041..e935d5e2 100644 --- a/src/guidellm/request/loader.py +++ b/src/guidellm/request/loader.py @@ -1 +1,15 @@ -# TODO +from pathlib import Path +from typing import Any, Callable, Dict, Optional, Union + +from datasets import Dataset, IterableDataset +from transformers import PreTrainedTokenizer + + +class RequestLoader: + def __init__( + self, + dataset: Union[Dataset, IterableDataset], + processor: Optional[Union[str, Path, PreTrainedTokenizer, Callable]], + processor_args: Optional[Dict[str, Any]], + ): + pass From 34f851c13077796d80b6fb212deb63ac47c3600b Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Fri, 28 Mar 2025 21:46:40 +0000 Subject: [PATCH 13/43] latest working state with data loaders and benchmarking API --- plot.png | Bin 0 -> 43285 bytes pyproject.toml | 1 + src/guidellm/__init__.py | 2 +- src/guidellm/backend/backend.py | 14 +- src/guidellm/backend/openai.py | 16 +- src/guidellm/benchmark/aggregator.py | 272 ++++---- src/guidellm/benchmark/benchmark.py | 240 +++++-- src/guidellm/benchmark/benchmarker.py | 124 +++- src/guidellm/benchmark/entrypoints.py | 49 +- src/guidellm/benchmark/profile.py | 32 +- src/guidellm/benchmark/progress.py | 429 +++++++++++- src/guidellm/benchmark/test.py | 41 ++ src/guidellm/dataset/creator.py | 51 +- src/guidellm/dataset/datasets.py | 4 +- src/guidellm/dataset/entrypoints.py | 18 +- src/guidellm/dataset/file.py | 4 +- src/guidellm/dataset/in_memory.py | 5 +- src/guidellm/dataset/synthetic.py | 133 +--- src/guidellm/main.py | 632 +++++++++--------- src/guidellm/objects/__init__.py | 8 +- src/guidellm/objects/distribution.py | 516 -------------- src/guidellm/objects/statistics.py | 516 ++++++++++++++ src/guidellm/objects/test.py | 364 ++++++++++ src/guidellm/request/__init__.py | 10 +- src/guidellm/request/loader.py | 262 +++++++- src/guidellm/request/request.py | 8 +- src/guidellm/scheduler/result.py | 2 +- src/guidellm/scheduler/scheduler.py | 77 ++- src/guidellm/scheduler/strategy.py | 17 +- src/guidellm/scheduler/worker.py | 7 +- src/guidellm/utils/__init__.py | 42 +- src/guidellm/utils/cli_params.py | 34 - src/guidellm/utils/hf_transformers.py | 35 + src/guidellm/utils/progress.py | 199 ------ src/guidellm/utils/random.py | 42 ++ src/guidellm/utils/text.py | 407 ++--------- src/guidellm/utils/transformers.py | 151 ----- tests/unit/core/test_distribution.py | 107 --- tests/unit/objects/test_distribution.py | 337 ++++++++++ .../{core => objects}/test_serializable.py | 0 tests/unit/utils/test_transformers.py | 2 +- 41 files changed, 3047 insertions(+), 2163 deletions(-) create mode 100644 plot.png create mode 100644 src/guidellm/benchmark/test.py delete mode 100644 src/guidellm/objects/distribution.py create mode 100644 src/guidellm/objects/statistics.py create mode 100644 src/guidellm/objects/test.py delete mode 100644 src/guidellm/utils/cli_params.py create mode 100644 src/guidellm/utils/hf_transformers.py delete mode 100644 src/guidellm/utils/progress.py create mode 100644 src/guidellm/utils/random.py delete mode 100644 src/guidellm/utils/transformers.py delete mode 100644 tests/unit/core/test_distribution.py create mode 100644 tests/unit/objects/test_distribution.py rename tests/unit/{core => objects}/test_serializable.py (100%) diff --git a/plot.png b/plot.png new file mode 100644 index 0000000000000000000000000000000000000000..17ce9190d0742f6cfca9a79e71905b8f939711fc GIT binary patch literal 43285 zcmdqJ2UL~mmMw|_EyYkq1QkjJ$zVbxn@|Cj%qAlSkeorXRVoHh6iF%wC^-s}H%So? z5K*FlpKI4CV2d}Ax9-pP@LIT3@6L-1l{Dxb>&mj&03jOl0G}~Qi=$3yCJ9nFQ`8S0Fxht1`yB)dZ z%8I4maw~ts(2$=i2CG$;e>=p#l6(0#KgJb1mw&tX_`h+vlN<>u+>A%AKd49%upQJ; z<=$!Bls)h1By80kv}v!$D8Lh zqUEx7(;l;;1(^V$(B7(O!w-Ma^48`YDD+^a*3ad$GdMNsEie{1$q@P#D6NAQS0nY=!uTcj>0Qk0s=KR_)jD{Oq$r; zx^=TAPBV^|ma=Qb&f=uZ4;LHr43eXC@?6R7KVcH>?H$oDY5nw2Eyd`CLh1{FL&Xla z?Cd!H_~YbWA**kn9&Yn~`LefiUa(@Yfy%G@WmouS=h^W-y-9zKDV1acZ-q%E4N5D2 z^M;DBOA4V<-j&f5K6UlR+u}oV!KZ2Oxw;y(e|{!p-g-AS@NLs9Bfmk!wZbR%qoo4Y zBke_wv(#Jl^kTYS{W&C~lw)76*|aA@?(I3vX*L=f-hGD;%Xx^+MD$k2az$4~6?=2_ zRjZp;CFxg;m5DE=Wn^*{7!9ej+LduUJaGQW%`bk3XNC)z0}@5`j$AXgW9}5P|7lUe z!y9W{;={v;DOgyTHTA7FPuRwGx!aqf_1&^elubZjcGte6M-@ynhh5c@4Op3(nW>8k z^Aq)^Db7y{46J)A<*cldI={Sl8xs@bILlLfLh#rz#Q-7e^6u|%P)cueiQh{qRH{RE$5GucPg7GikFv-%EZ&pXidz{a>cCS$7PH7Jm2Zmw&QH@J+(C zLe3K>R7zj+bB)+bFdWc|`)Ke=)FDN>th1+qnmU-?#Y8>s;^IQDWZCG$qkhb$UuC?n zMn)-0F+tsC_(#&Qti^>{A=^QJ`^iD$n%6!)VG$AR13wbZ1PI$!ui7l~hpDOQKm(QH zwv(qkM)kc{K!9=T<;Gi_TwK!kH#1jb=Z*i$+icid899TyRHE7#Qx_x{*vJ!RbiH=) z^7D(ZAMX_${;YFwwaV=f>QJ`2K!eqD7X9YjKgpUU#?U zv9ZkS%$qlFw!7uk7eN%^pt*z}YCT~~1oH_Z($jA&l;a*-$KXqH>Khw}Exmj!~C}i|h_0cksx7cANB_-028Trfm;&XWO z4(%~hS|0IRzk--t=FIsGDlc@D_)?M$sw#0mEBVyP<1smn z$tguEU-IUnC||tjzA)3rM-`(lGj0ByKgy%Ry)!_}NsZa@*PjE&8W4;1^ISJ3s&LQr zzb_jxzSi>b9{%HzhX{A5#V-*a2L)X(l7K%G)!W8=u=WX-RZJn!evucbPhn8bx$dKGr? z(#y{V$;C}g+H!JoDRx!s?KifIGC%3uYS$HDOK$u##u?-H4NK+O(Do^0Id*RDizFHu zr#6JG-@zH7Tl&&|yo%yB=E$v{a`D554-yMj1FcP6(&Kyi4ZJoni9}3~cG2t4(o;vj z3MV^GkE~o=LzC-0)*Bs<-KStblr_1>`FF(}XQz$3k6-j`n|vm+(@o-9fqQ~epupv+ zfz&4K59aBqsn58}x91*v>g<|n^ANYf=b|h+iq~)0P}yGed?1DLZT8%$A#-|%`SF^n z3~NI&w`A54R!2)K6#XY-zXk|Ln>1u>nk2i2{}B)4+I4T?OX2AZ{M{m;?WMSDD2zmC#gZxe!M?BLk%pCtyPvQ zthO61;l-Hg=;)*j2fq)~X75fp3vy;*1J(xzu$Cd7_-(gk4 zliE%f>2~y-4GRZ{q_FLv;`QqxnzKkE;XAmLZ|2@!G5wLN$FX?=K||YO)^n`+&SnF| zx|u=hP!gtif2?INMmrKYYi4e=%q@#IjtPNr)x(DmJ-?ReKkxdPZrS@I;cXNpK91ad zLx%Npon5`

B=0+BQl^Naz=PagYs45<`ALL1+CS-WD7F+Fr_OX6$>Kd7IVTlsVp4 zd+Ed!Hj5#qjZ8?R&WtfP8g+v_`@?mIJvoo|5b|LEbv?+7rJ$%Zv0 z3uUACnVs4B`1q2I>y??Er-H@iew`7W>3-Y$E%=T>P27OlNIxyz&Iqk6dmFvt08-9AQ zr7pudn#as5q`%+Dy8nlqda6m)a~ZM7o{I3kXXlx|FU-%<`I`n|K`;tgMX>bTqt|*T z%7!P-VzTevw^!U*BbpMQqRz(3DuLk5H^Sziudi?8L6?`*`njk`uJFlTrC2roECeNk zrtGYPGOzuIES{}cwWd2V{I&c*UGjJ_SA6$$SD-IKuv4$nH^ZPmuw%ZHfQ~hB6CjJk z;)VGkm%cdaU@`q5Hih=1M~~!z7x?W)uJfDxR4Egk^j{baT+|quN-?T^vy(?{`}DNg z*sO-%eJ1-0o^2(ZiM}QA#z@Og3qld1REM zSN6KNw6v;(FDC|pttZr1s~p2-jUW<5;zNfwWod&mi-|PdxAMJKm!xmdn3)dH8vbw_ zTXA0A83ea?vmW;5jj8q%{gELdA&x8ZKHOQGY~80!72L^o(Ju)2g8$~XlL}!MzbnXy zh9Z+jVt9Q;W#n{*1nr!sJLs(kf2!h#QNY9>BNoG}nQ?Qpv()oz;QTAwf}#T+B3Fec zYzFFTQ_Qi{8k{B@460*ZkjPzcLXG{FaS&gEz0jTB^}6C)@ENlCCC;7O>9COWrqU+3 z^+{~b{DitFA_Q5NT2A9iH*|HQF!4gpbGM%LBBfl+9`=2^nZ@lY(u4g#a?M!xTaR8c z6o0N-Yt`g5S!-;HENxH~KFkor(%PS~67GIdkQ%Sn{Ob#pTE#h%Opv!_jimtJN z`kJTLMHc5qZTk4sQ@%XCo;g-7Z37f#D}X{~n53yX76woEf%???It2EcDzVCK+KV&3 zu>)UT9G&iU_wW#4VZe}=P#my*A7gK*&!RXnxNt;lan4d*eRg8t@x(xVeYT90)Dad2 z+*U$zettd!{i0-qO2V7(sisYiL;+~O|Rd$Q9Fb8mOi~c zhrs2kv#VgU_?%f0@(pt_W6!Az@*RxL`rf;=E}dFwk$m{dVYbpqMdb z7MId)c6M}hbnh&E)m1-{?}44q+mwN|fVp?0=i?nyp~pJ9c_~mW#$*_Ort84jN6_PNVZq zS+^Vsq`rIa9s?>Ls-}ZnQ!2SMA78xV5E}}ND4hfq_d9DhGh)C|@E-cv(C`?r=g0Ug zqSnUEn>`!{Xa`bRuyaX%IOF;3DPxbe%_fmU<*Q}x_2DQ+&+`FqtdxOK_%V-+&4EN!u=jm4?)bRAg}dqqdI z+~}FlpW^Q_jz*z(&fY#Fb{>@&mT8GEUurUv#@7hNNY8X- zks4@U(G=3WkujUh>tm-+pLTC z8d8I5Mzme#Z(K;6&G{Z$P(-hmW)_S$Adx;&+#Dz7|0iwt9DfBec$6kJ1w{rcN)@Mx zTD>UM_hFhj2zvESIiLa3snq(fUy3mMWuFhZJL)|@d-iO^(n9O2jC|`bltm#aM6=Pl z#a|fVw+gUZ2_Q--QeI-+x^>eAF740G_jhjIC35SXh=@oMlJ`|HIth9JajZ0|!X!PP zR*YrWSHT-hD{uq#nsyuVzJ&)mM|F4KWN_7)ERseN@_zR0*^pgNQ%+8U|0d)56mP4A z?S_&g!^6*5VzsjBT2zhmJ32eV)-pTFYC8_!YiPK;lUpSSE0~LgrQNgsom#Bgd)>zk zZLKAIkJ=InncmN4<0>g_z9BN=^NN6Z=H}+p)jUP9RInA%h!>*cl?pb`-8L~Ey8vAN z)s~YA+|^*1Us0jZ($ZqXFXZ|9HSoS7p!~|&W5D$o#`WvgeShmA7KL)w5M@|?#=g9J z@gL}%W_rSH#1EX;+uvE`cm}vgsmiPFH);tIxAocznZmyR@{l zdYf`wp3N41`t&2~NrvrETu@LDmr%CXBk>dzMoF*an+tZEDuPOIo2zrcG>$yZk+NSE z@LJTN?y(FnlS>%b0wi+#sh0c3@$n+83=WcAf$_h7nE|X8RCf*x#GYreJBw}i4eZUp zjJAQh#?!V!kCME+15J;O{O1MOOlRpA#kO1ff7#VvSI9DWI|RTb8tl->uV1pLT7V(E zhBh%XF)>9cMf+&hV^JyrdONx##pf*8*N@O46aUeaz5BRP1U4!=KfltSfBres73gwI zzkI1WM2(%YTTxI*X`R&p9dK3EphfDZPiLzl|ieN*wHazSE z#EAlw%qFN}htR{>d&%Bbht|`uDy3W8kepr|?Yc;xQ#gal#UQzJw!XWqO`*Lc?GZLZ zPNg(oRtUnG9PmoYM9H*z{d+wb9@GNE!@~v*8Od9>ZA-pZxQW`c4*8M|1@4m;madDZ zwZ1ugy=R?0J4j_bVHWh-Wx=k&yl^i;4wYFBllz;V#UWPazkD~}+Lo-s9i^SKuw$>b zR6eu1_ME-fh=<+!3arZT$2q+G3t#9Se*E~+KT*7AZq&S`;pwQxl%r-^j&zlZ2K_>5 z`qYz$tmeIy9Fo(k65c#{&TvO_7R0I~HsSQazDz$~UrOZ9lnD@#mHvX3W0eXr9Q*g5 z784UgSxn%OwCPdD1k3{|>0B13!xIw|lYrP9x5fK@LS7rMH_fSSG#Y)9E)yt53fVks z-pui&kd;G6taQ2$1E3PJ2e>3Xd#0Cb<6a@f=Lav9Bd(0ndP{6!WQ+nOWUw$dO~5~j zxQ?D4)#iYaM{k1!THM%fl(iL8xpyeZIiGtnH8WFuHDSBZH%_U~ihkmYa|#R$46K{3 zjT%t>8EJx8I4N;4SnNMaV^#fHvvV#yS#+lFFtwMQ@a{sjqh}EkQrp3)_>ENbxV1hMr9Q7-y&4GPDW5ifkbm{N%fcx0{X)B8 z29A6suStL8QA6z7Isgsj7?ohur~1RK`5B-cQ9f($=y6(XrS=M)(GjmGXLR$j5|P`! z(=ETn=8^D%#W|j@^5u-tO3`76*gYtBZCd1kK3FC3MDGWT;!2)Prn(*8wf-aP-P{{# zAeyW|zd8n54c5LdOqn|k+*OkL7_{6a-0!=9!=sqD2-BwQmm+qFB>V^uWhxlgC2^Q} zesfE;!>qHiv2{`3N=t5^=#hxm%4#1Ri~}cofo>j-R?s<|}g#!7J)%g#eGa&O=izXPk<1*_;#O7DuF0V|t)MGt(aA(hxTuSSFhg5%G zg^aj&l|dDjC!zlOkdJwP8M(Lm+sP$dl_0f)v=6r&c#)k*jD)y5_n0w74B#7B3@OI^ z#F})(sz;gP=O4|yZbQ}+9!xt!A`xN8fz4yJa~f^dy?tMHBK?*l@U-*dye)P32AG_^ zPh|KOUhE7{K@AZf@`g!UuA^amM1F;(d0@O-p_b=pV0>m!Z|g7xq%({6)Ynpa%Yt2B zN=q+c_gG1Z9%5d}-F)OHh--{)k!|Pq-KLrQ-16UmldO*u6%%VTiE{;}KSQ<);TY@L z_74sYPK(jn^~I*HpG(rKj#0I-6?k#@nkxXFuW1u29yovHoulI7;-m_CDejU@(so2d zM9D%@8!$jQDX- zXM@FUpP%d%Zf|RQyW1q~Ls8X=+|c=^#rZ%^<(LTM_Fgdc(>`nm4;rAxLJ64A(bd&e zIqCaXyXIG?mTU_Pi}n6Y&8IO?h*&n)f>*8Gbh9?`#`Mh2aVIVrz<=xZ&s(w(Xn`UE zN1S$nD48s3XwZa=Q7RyIbVFMr?{n^6hbM10zXHX5P%_25B(#7%pWn1mg!+3Ar~o+& zi^PVhM66>%6(qiYZ)~jXJROSiaWmC;$KPuK=uaQtWpk}G(0QiFX0osqBpronGcdL= zJJg3F)TV@&$w5-WB^wJ;mtG|hoFw6e%*@`MK66G|u`hpQ@R0NH6c>Uo2t51A2HU+0 zV!C>IhEu;3tv@iUlSwgF_aw>5m9LbH5lR$MKVyht78YOn*jvJ$X-vh9!-37$YJk29=#ENTmDKqSO!Qb`Jz1Aj{xE874k z@UMtNypoLK&W;Hw^AJVYAC4Or(002?9|kQ-K4QGSXXNv1MBbL0`w9U<6| zjv8#%_~qo}ym|8`>3Y#tpy+)4;KwK=?Z>)@N z>A}5LqJZ1JV@D{8J}0On8F=Z~_0PXc`Z{lH9|r0AV9NobUx3_-MDd3asr5(0m;eRw=HFHu1mb`y`BfQX*h^#gZY_pf}cs5>NK2B55AbSZV=0D``*2! zb96KsZ?L*ZbwMZ-)E`=h4e z-*oWPn{3k#&?F9uk@Bn?nZ1d+Fp1z!O>u@NZ(E6cCZu4^yiWXW+mTLg5)P^lfI{(=esO(`aUo%cj|P?pHPrcGizYDpIm-|fNF z4OCCJKbL6-j>lfwE9R(zO0ODaJycKy9*`er`B4feqsUB#yfM6$nJsQ*sO9@Jx&DtV z46o&#aG9+J*g8m$?GmNio*i>`qYg6we;`<>Cl0Om#J2t>xx^1pn7f^won29YeNVV{{u<)cw8^Vilv%Kt#v&u151hlyY((kU z1`+4j6J~Lf)KWLoRsG#5we(GiwTCv4Pn<(6BLu>3&yfaBK`yS62!QIAwzg}aV#M!> z-M1StX?r!vjxlW+8Zu2nIqr8z=Pz?i&;7o=qV|=yS|75xL+;s)V(2KUs>evWd4Bwc z%Gt(zs)?UW8r(c=LC+NAZ5B6zxK)OV(=|Rm-n`y8mGE83>Sf;tf>9kCSH`N>nAzHv z+18upO*I*Z@nx1yg>IjiDSrO)r2&Ms(g+aua>)itZQsu`^8?R&4|S=0OW8B;SO!ug zbE3}h92V|I85x0}}h%Ta9#!b10w=-}=?;n5t|7 z^41(G$_Ym%-fZKH13>$(fq{XC!5H2zRQX|Cz!xJr)Y8+Vzk1CYM%zZa@WbGpsr9b( zw)Hs+Gtv7vnhe*`NMWze>uz8?_W8F>n~oz-K9=(4WI#-c9)>c`@Z!ab6lcA!{)g9W z*mY%UYHAnu&5^;JMdvo`SyvV?XJ4!1gL3ZBW(}Q~Q4jR){&64(R3j8l;VRdB|7uJHPsZf<>(m*~Ees5*&7;#;P>cYJhmmM+ zD#oaMhklnZ{AruaF5ojBvjDnDbk6am-W8TpIj%BF_f9wWvwnwXf_#H*h#^udT(4K+5T zWTC)rLmW-&Q>Z*_p#uf@>MaL{-5C0!)P9Ut0Yb3JPsh?zczuO#vC#70u97nq*0UE|c3KWT$Lp`r?_tW?bZ^eHW4dxamQcqj;l z!lBHR0s!Y&qyv4VhB)5*P7DwM8sp{Cd|R|u(A+kJ4(hQmw74J>pEdC^a?qVuk}Tfr zxsj5qGWvf-Z`QlAf;|N_>Sf4ImH0HEn4Z94Csjn-{R2PK6-aFdf5$O|P>Szr;?Jqs z0LM-~SaR{wKyhtw0Tqy!cI@2Q+3TtC^=iWB`g(OyhlxrIdPmC#(Iey|xvKv#{@{Vg zpJ-_Q$slxCQhE!xs@~7f4n!$NZncHUCdmEphBF}d6ahrqI}5s8Xmv7I)10ie4lR^k zts;RMYD-m|W_{HST#24&j;Q;2D84#s?!NdwZ$`c?S9ZuD$*}}1o`OhTn_opkbGzKC z_0u}?vExj)uaP&hQ{%>NJ*0_#Fmm}$yhi$wRk+<^c3wBGJ#DNsG-ja%(idBoZuZ~Y z?mzrYewf3HECK?Q+S=MU-c{_+v7+NY^8~{0WHyaw&Se3(M4}A={D5`HOA-qg^~y7B zx@c*3Nx^jh-DoP#V#At87UKOcupSlrND!&czNBQNZTXtD> zeqkU42?|CtYr&H*+J}bXVPgUAvm_ZXx`bSrcIp8$&BZ_1*v@;VS*j= zij&{4+MVi%`_c#{pJF8o0;Vhc&(7XH3Lhj=4>ky!WL=v1TjO+8-ypy%!N@O_0-nzy z3lsB_dJ`QpzJkYWk%NQdCFo-<$Km|_HALhq4G^xGmT`T~t(l>S+9K!8EKwh;a$H=H z-J}sEF_d!fU?gL6ab+~oX%+?X!ncC$kZkA%Jq_h>(v=T?aV5&&u{Bg-IlEo56<83X z3YkQvOk~uZ$g?$C4yAErsEkNi;RKS<9=?Von*lYTgB}FKls;Re33frj+QIK}n$(q( z^FRnuh+i3~=pJc2#}31)ZS}vCv(nLtaJ--}=kW#6AcP5e92;_3R41xD;lCiO^}W!0 zoqlze{|*L;KvXMh2HYR}3T&&~8^a?b@>i||5m>pANkj!4?+n0A|BPb1mL_;SgCFmL zQJ91RtCT=;2Z^83*VWPz3}*uMypFA|2DBp}AUCCoeBa+YJ39@^LnO&4O^Wu4#1gAw1Nkym~ zfgC)Ew2GSYyG>YazTUy5Dgy8Y{*IL^R}!}CYL)Z8yhD{(5=0hkavXh(R1tz3k5qlH zON>+`$Mg}>l~}1LxH}@LiUPRceO|_PjiXbz4=|G#Fw+%8IR&kul{i6043WGV5y?oe zgkV|JA&iHv+#a}vI2?ldsCI8&=tMo%>RypsvY-W%jO*<%AXp+c)WE$M=eSK_8{dV= z1ja!zfSChyq7{VBt?u!$G zz9Jj)(qihMQN;K6_e)RM^(Pc2!Kwww5p#A{Qbwi*q=R+EB>_$4OC)+`+KnEA{zwd0 z_9$4ZV3=?Mj`R$H6D1m{0-msP;`RciC_48mFGYGQJqmuHXv2?p*UdmC3x&gJls*8F;R)Xk#QPB5M=*788LQOvy$sL^Eo_Z65XsA`y=f z(V~=R@%KN5z9dzI%S5AWjso(7zT*G@dzL^mx>goqs*$>QvC?(p?lIy78vL1_QgW0o z+6guLBh>vtr|uZ()ghRwr>CbACfbKv^$p}m!?Et~#9Tpqa`bjND5@~E$;V87o~Tb5 zMPYG%#flX(AQSe8iDNbn@olEiH-B!;ciXXhcl3*+dbpDEOCjqhQq;WWyO) zYuL>%0T#YIVg9BX8pwkOze&pDBIDT9y-kY?sdcHQ@gV0*3JcG}%SPf2izIlVBx3-( z%Ya1HU~t&T{v$lpI0*xoJ*E4b_8v3(aR|0TBCeCb09hl+yzLzEQjobd$)46#ije)# zau{5s987b{NMu$G)-{oKqn#3Q^H(v)?;Qr|63tppLv@B)Zx;oLmZb;xr3{`*%dTSw z?dsJRJR!kGK>}0UO9`>sEBqp|*E_~Q zzT$ivrN6vDG_qLCH#&&3qbkFhKcJ{suD8g@&LKV8@cqk?UuFt*k=)Zhla;UEvRNOR zPuh_8)T2tF%UxqYMl{wX(Ngh4g{G~Bwr0*A!C;Lu#rzi52e~MEiDeu?iAp@ZirzV? zzQ((EN?tNl`mU{)=vCqv>+RY1tGs41A@{bqZFBdP8HTDjHLb_1xVQ!seVzn-sT@4OW;Hc*u_{VUt3j<& zK02dn%t=0@buyfz=4q7jFE4FYK3e4zPLWTPK@*>kuTR%o`ca<8+kUU%DJuK?-G(pK z^-X`9#mp-u{k?_BBaSP~`ae$k_c<<3YFQ;8x^bKTM)CHIP|r{$`?8P{7v8>aQ10Uq z4aKw<5xqD#Ii*mN$FS(nO$`%zq0xEP81JkALiOBAb9>Zry2E#5Vxk&6z4xnE(QBD* zorJd}z1`kU&tCJ^Lkb{$Kd(4<{~cGt{(4?dR+HAK0GN$~6x5A)o%?M#JPuYJ8%(F4+_7_>?Q(_c_mxQ}~VRDtBs0mP7rG0uq7%RI~F+gIV52 zxGGb}*R5ZlSZb)OGH~w&Sd2z^sbF)y0IP!sz*AL>>QSoLy@5&jVgy_~#rDXdDib2G z!Nie-m*vC#v~F36d$fWY*#?xv0^X+BcK0t=*XKvfu+Drq?`U-5WPJxcXe~guqVi_3Hsm z&{VOnWeozLX3EeKCwA;w!3F}(Q1*@dVy{y#fndnlGq<;Gq`iUv9XpZOhnL`=~~f*|H^;wRW&<(!atv=BB)g)5(x_R!zA2bG7% z_l?O4k{qMwQ41lKE;y!;abyt$$3T$ufLrGpapq$c6%}pj>rk`C`o#iMgmdKGz=jVW zSy)-2R$jIzX&F*xta|EYkZg)Dp$OSnl{OvdnRAI^eihi9IPd%EMgN|FqOTlDGI9F` z<|^#kO4{8oZhRdW#NedXPPbcmK&rZVdL1V_8<$bT8}^&C9P~z%@*7pMUQQZmTIRF= z*ym(wXC}i!90$@}jRq)Z9^A;F;o{-B1ay)jV>`Nf_39r7GqY5rq#orh0OqC)y?yhB zxQ;kjSkA(wNAjf=$~G&a^;q_=cDRf<*4F+tv>K1lR3JuF+CZR`xN;8n0eiGY&xqO2GaD z2X@fYhjS`L-2?%e0s5Fvbv7R*c^FvUORq&#QTlxm*jx=v5{ox=1I5)Ko{%}JS5 zU`kXtBoM_Arya_(Bt0Srb+rFjTZ!)DOg=y@}P!olCdP*`- z@+(f>8{PJGDDc+ieLHU(mcuUP_2NY&aDf3fGx6nm!x|qd?dx;vm7QLf^Xx#XZzgxt zA4s$1*f`^G%%E+A0zO9$R%61nqv0YN(jaLQX|(U-0p~4^uYHOFB*mm53KR&Y^#b5r z6#z^J)E7kL8#iv~11?}5a*T!!kSz+vfCq!VWJfseIAzuVR>a1)SF@W)-&hlXbi#ma zNzpxcH9Q+=xH!Rq+OhkD|)Rt^nT+4wVPIF{uzR_Z1+Oc5L1H2HRMcAX?D8 zkH#%GZrW5$2x)lZ?d&1vOFEMVZqAzlikx^2~Y@qrTm@E_>e&`yM?x1NLGuUXA#)`O>JQ zCv9qW*)%6?`p;1#j-i=;(7Qm5S~hgBoW|xRf899-AYR_ZHj{-43zXKDQRg1gics|V-Z!QJdxRhs-h`|P!Lnr zby8u9=THdSK5dt!IDmSF0tea-SO5t4fECUjgyA^o-v}fPVm@e~rjqSF4?2xw#FrafwkuN9$z#PXyy)g)JRPlo|K&7uxiT8$>QVj11RI zjC=2ug&j}A$oi_**Zx#4{0q*A^Ztp+~85wNDMa~IY|nt89kxNo*@GV4y=23`v@oGFgz?^zWE6b!97PeoI$};PKY^h3e=1}eky8e zFG+cdiKD-Dj-r!upJK2_R*idS2`ihqDW+#;a>!I*a}TyR*dDe3LCXT2+a28-Vvt;g z2kZ>Q2D#>yKA-J2Gn&%eSassp+-MG4$6c+(-qfwJ>f7@hoFxs|d&NeoxlCN%bH=u% zDZl2UdE%5TEWvELIO;in&N^cuu`D;Eyp6RY;7?Y`goFfQ>oCdaIe72>ebQzlH@TD)^MUHY;n?M zF&s54)kht``R0(>m9!`;6()Z@kGbrHZ|dT8#r(ATDK`^I(cay6k}+ z1Rlz0qwr*AKujn)DGR^{)sSLAXK@o05$HJbkWBu&cZVdqhE^wAc3p;zNdi?J(WZ#i zn#hQu7BBvwX)e}>N11rWDZVS7Q+vW*!F(YD4$@%DmMz2+uUU^_&=As784Nu*a8k_t zd$7&EJJeSa_=Qud-)uNT;#sPb0!OA`-$}~O&kmFzSSB-YSg>GP*;-x#3~71s0K}HR?t`oc1O3|cM$egXfz?} zmM4+?@MNOU1c>HEdc}x=2RM&?grV=N^J~V;cc=B@KMMZjvd@V2zyE51w|(TifZKNE z_lY)p#mtL}Fem>b@E!PdF-96o@xKJVPhNF6-r{e4EgRn7CgtJ#+q*{(Xy|gi&)%j@ zYfRpGg=Evz1{?M438=OPU>t~~xoOXdAks7wJBoy82%{|_8vtQLf$ewL1;%Z7clr)z zr1iXk&~xv%mCtKOCM+m5y8In2En(QHBRoTqqUU9LugvH(xb9<)4*$xKUKePuws^fO z;;(S(U|M#Dv!I2T?Ejf{=A@_I=k|6+8~$l~dZ&grpX(PAp1+9FD*B(57ynvowsWsm zIe6+eZL`$=+?7}#bni4i)Rrma(JQl4e_P=b4uSd8W#25&FR~UC`^@VLZhKY)ZD=`4 zn`=Eaf0Q|JM1tBRD>n=EAew$?g67If=zGueugbywhsxkaf?0-`>`PGA_-_;d2wGje z7QeJp@2|NzJbB-(ax=Kr`XySM5ZYknNlX<&^DTJkVitH~VD^Q?-%u6WoKtXCfJ6Q^ z|G^AN4#idY+(R_}Ui=6J0cG;H+V`|g$?J2y70~1Mk?T3fP*0rjnP|h(!E3Y==|__? ziXBEQSJ5=9okz>@!PddSuP#9=npeKRW>Otw*Cf0lVQ@oiT0oqa(Rb=EPuUaUOwSs|Om=3h}P^N=E{+?M*e!xXbTt0MpT z58IhqqyJ!z=BDLNy|8{H0mMq`u@Z8PzrVu5_B}hVP-RPErYsk)km8^Tt%L?k?sbT8AR)ycU@7US?rug3s$eWLUC5@JC$a;TRZh&!0-|YU(`pv@~ zU$~(3{JzwC?)Mr|NkyrqNqx(BRC*d3_GRnP?}&%M&;Jh%?N{v6UH3^MNJ$MpEcAYE zn3lF5xAam(LzCQ)te+Cqlb>(lHj92mHdGSdFpOa9n=?D` z`JFWE=X#~EhH?oWc%R$h8UiG3EG@;C^#*#8^xrQ(3E)1F$tj6AVm6LQgI73|;;&m9 z{PQ2~-Jjz8T*xWCqJ+m2D-k2lQ<{i((*1pnpAFntoz`jGe3kguo2q_l3l0@XQ(k^K z33GSstqdKw!}{d4Gb;S}9fV7I1Z28@8jy&0e+LCq2ooJw70<@YF*INCAu7!q^;Brf9Z?#F~qSRdLq|N1>o9 zXfptGMCKTqnYuhEM2n-jBDGxc@|>rq>rBQ3zbaX`f`)z7-!ImCmV5g@Q*E|5BX4mo z472R3C5QQD{rly@%RBdId517#9_;^KCQ+0`q2z!50|t0U`!(;-+Z{V4Pl+jGntgOt zaShtPuQ6z{GhgDqkcJPv0sbv+hz)u`)NZcjm(}ESPQCmdelIiY=Rn;mrx@}(rdmT0 zLfAhBR(JBZ@{oth(7>)V6*4(o1oH%- zYi$9cnEaR>S6HKh)*NN9ZzyTKo<0piAsU71nz*9!7C>(i?U5h73G=Y5l2)+{n*r4R zHXVJlQ1A8OvWSvFu0Y}3RU5F6y#BGqb08&#yQPGPu zVQPYggWoO_usaw8etldMc(8H!jtpSS2h*dWzlR(+f*;=J{~!+`7N#5*5D-9-&Iu-| zMF#CcNyj?c+u8s4BR5VI zG!eL}O*N=((yuR4wCp6{2!XbA^I5OoF?~;{xketSQ0PZ5EMK5`Lm1t=xVzB7ZbBn^ zDQp|Zy_iKi8j7Z|(u2k8Fsq5WM)x&MFMS=@De}0fB!z|R@QrMcMkKpDYXae7LZ51D84%GV18CKk(Qu|A0Yk0=p3ph z<=2-lU*w~(XM(Qay9k9%{Eh0VU#mevMMDUIE&gMR5WqSHNgq6C%ED&Rv8QM&h{T7Fe5D5U>xBaOkc5+6rhwi6XEvo00= z0K2-W9%vBaICMxBDj;XW9V~NpTKdK+RZpDK5lOlt0sU)mOM&r=C|#`JEbjetEz?sL zS{!WxTMuaymP$6kNS(iEy9S-n|3Oi-?SG{xqUq zDLVO6fs-Hix4pni?}}tOY1W0=@_xy2lb?R^v1objg|7K5)di%*R%L|YD<=0Es_dXB zZ>q6t{&TGt_%ImK;Q@l^b=gC7LM7Pu_a$1Jt7Kvx#%6z?RpKeO%~x`U>2l+#@iBOJzR7OrNh(L}@7N2HD1y8C_prGYJ?F$`)gS?VQHWca!gUyP>tlDhWQ-?Nj zr;DvyQTd6cgVedOHDxm2kP2vH)?sP0E}fH86NJ@N`dU<-NWer;fi>VnlV-qq_gy-D z%zO97lG9DFa-^iB%29Hn_lg|`#`?cv%6*<2-UM*7*dEyNPkCSo-83_ja+Mrz0y;}d zQc@1)<&lP}>5oU_pqy|3s6RV!z86F!u{}doL(;2V!=A&{&E)0cUczFP! zCr)eW5f6p_`dyoLTIB}Em2?Stq&|fDP*rSRGvbQ?obg;PoHzM)+*3nZw6Nu3Z^abW ztY3av8OcI-On;v#->mSyHUAze_zXrOTe7f~6ZK_x%6XQAN1c|LWx`O87TOmM>H)JdzQK^Vi7B z-$CGcdrL1#a<@Om)Op9)4XDea{ZSe2u@BU$*MM{-H%lat1dSO}|tUU5vD6jaFDLBA5Tb0ofE#L3O=~$+PY4 z*OYEOFxB@V*ZBJ@+6Q&zG^`@>ZrBAYmQ}z-jo0S3gFui$TFYnOB&)tzZFK| zcXm$h&&&17zYfURd$jspfeiL~*WT3e|Eg%-G`@qWwd4f_h;QHWIujKrnSK`)my^1` zSIN?xAK4+veBC}WA;4Duor9;AKl;D7IR5V`;&NHg;ylp1^&;zkI=#6pa0ZQ$)*fgz z*47+h&HveK=d!@~91qib14I$RUu!u}-`Fc?@u})Yy5hfXbX)4gN#yo8Ne5 zCerKI#UrhpSwg873t+GUS6Y{D8BR_V0;(YA1p(8!5;_r5bXvwIc~llIFRp01m`YSe z7u>jmb$0k~KtBPkAPR!WNwkGkp-HnU%fWWzu0tn5XNMDqhH*oNlD5;hcXSC55j+3V z>O7cW)gw<%@PQ*%`R2_C!j_NsRGcSAD~;j7F-I6es^_tZK{8tnAna+zG4E7{njpF?6e=~!sCzxuTLDO9RmZ#>ORk( zzePKLB;a)%CwQRTj-`>*=!YzfUZo8&F!Ay{K8?8{7@0H`{rK_Y^z7vqM?$@# z5Eu{9FXaUK8JzQjBY0HN?GH*iRB0<7s%Pn;(!SmE-Dt5+=fgjvf+btWU3*O7j^$91 z{9jete~_xzEEtCK_|H!0N3^HqIvh?ugi_nx?O%#+tk6>rNYS#yYBZBxnK<4LNz0|T zu=XuhNmmxPkiXSRUbfXpp@h<+X+DiZ7|gBLg@If7x4}09v5PC~BPsUL){GIPxqaZD zt2T2|Dc+_d9TTWR6yO0P?hRFLTJ80|PoK&@@=n(pmr@R&K=h~ZEss^kWTOKlV^&F0 z$QZgEezBa6y*;1xm@#p?g6C&C}CmA|=8yJiJ0) z>wf-gwRusOsE=^5u}TI92OHEThPrWC1YpCKc2}Iv^PLMp8Jj}(@!u52$6h>@{^0?Z zK^RRa=%pfuMkO1BLUi0w=ThX!N{meS8^H4$NC|a3%&-!`Y8pch@%l>L2RXSf9NzN5GsI@&+}QUZ^|1lV z)NM_r{jgK0Zl|C-*xSbVZSh2WR~e ztB_70EF=LZ%s&odsLWW2#}E%yTJsv=1v*+<(0;6V%R`#CiGGe#LEv@!k-T>GYIauE z{1ze9tP;J8Fgg~4M^=tjA_obg%kls^D(~0(|8r4LAcyN~z*3SGhJIQ3`c%`@WE>i) z3Fl)4^zy-vAQEXbqAzO3<6y5;A|5G1$u?zrBsCw@K+pSy(>h2bUF|I`!B{OSpb4zyym9$BI7Ldz%4A^C6S5zR z(40kqv}4zg+}!=+@sM*Ng2Y2y>+?}JkwZt-79O_Xb0?P9C z$4dc1RaLCGq~GhFmE1((H>#D#_J8kG?6+L+>J#y^<{`|RILwUnh1b?$VBS7>@Zj_$ zag$-x^}IJ>gB!u7XuTY3^)hIrJ+TFhr z8G>a`{HdgMnRpFxDjo42;ANkX(ePFozeA^RGMyace8hubc}j?dM+(xM6;BKb({AJu?VCtmWnswI5A=f zCFeLoDkq9MX%at4YrDJ>BmmNpLI7IvF7AOAv9AmB?Zy5%mD^Gnbu z_1e_#^5rLVB#oGff&ubG0sC|%9|NT7YuPDiSloW=IO$X_50ws3Wx}{uEZNzQ8~q4E z5!M}ipD8im@B*=V;-tXD=n{ljGlo=YTdcH``YPFu>7<_y+`}bsDF*c^vBac?!j|wl z`HhXJICt#X!;P~wvB~w}OTIU;h_yD379Mt(q2VqdXozrD<#}XpFfIk}=NDvTDDW1r z1_TaqB9eGkqFoLlE>flbKM7zk0#>8tHxlNkZuG={hgi)Ie-I3Jqa&NZF6FLUhCy#| zEE@_-PZn}66&a5moehr4Ttgf)W0i0=;x zADwhsfCpJI*U0FuS-$sfP+`t9J#3QYgj6AJT7%?X^f6V#)TiD)aQ#0BAJc$1+qY~9 zW_x|>G#tg4?h5Qax_}d6I7O1}vi~~=4UT_Hqa#{j$VN$O!Q#463WmsdRtP>j5_q?6 z-HMYgoEmPmJ(g&^j9rcHxWFsdtl6I?DjdXb&C`cvfEC9`Z zsB$+g(G3Mp_NupMNE!TxynyELJQhSv+Y&?nd_Bx9us~5_eno{K@+)Q8TZ4s?#!(ZB zSheGLLIcEY+PI_W+q`l8`VV~H@coKD$TZ7@fzSMlEB_#Xg`EO1X^zX2b{)&17k$7m z)r1q5_T^T^_S zBR2mb`9?bBNHadVf9mQ0_62BZX^Sc<-nXx{X84cfheKkJl6LKWBqmlTk;ci&o zVN_GTgg?rN44-|LO8n$dkx5gnJS;8bC^6hiw9DdxX9y0zz%nPjO`a^I%L09GDFhmj zEP#MTRubtM?P9uq_@AkZzp=E-;(RJB=Of8YF!PXJkw?j~==tQtNh@G7m0|Kj#0vv{ ztvzTXZq@zt=g$S(BhE;=$@b$0qBsa?$vpO6mmDDw^JCUJ?B7u#2fyR2P;%&yJ^nvX z2Xi_x-lqSk-woeaL#+lqJmQE&aHFGyNd7UKF2BsiD}g-9vwO$wxzcG#Xc;I+%;8u% zhS@9_F|T668YKleNsY+qol&Isow&+Csun1*X1FgCERYlSH61B&S$s_Nm{+V#7TBxC z3AO#>l`MO#EmHL+>mP&ibD|AMe95z*GavY|-`=G>2Ys@o#z>^7QG`{p0Ud zxR(wYm>e*o^xBbC`!9Geb2^)`v?}oCYTcM^P24pVogx{$Dw5$5y1y zefOAW`$SV$@V_bO(={^)r6E;<-iyi*$!&xbhSjHrxKrUT`&M-WksxjR5;@}c34RH} zm~CV@a+WMdmL6g!yyhVq1~QyX7jGQ2z+8U+omU3Hb0T(IS9jv-#s0)tgrj?9(IFoj z3+^$a#dY^Fv^0@TA*(Amu=OKJ+oXXX=b(7Mc+nG6iooZ^fMN_zN9*hfbZp?5japie zIy8QFDptk-WRei{=<#DEGy#!QP=R|(RbPS2N$VsX)r3ONp2&BjkH!&oyb()9oWIRF z9dLJ~V~SWJ$ze}8RyG<$Drv4KJ`Y0t7ojy)m5S(-*6F@iSedlwLC-Lj4M80*9XjUotr# z2R1nJmxb|om-Gw~43V$7{cX~DLk{M{87<{xq~SOuZgT`Wa)!tt+EDd|P?$AG*`iV- z4KT#mOv-S^}_Xr;EjTT$qYr{Ksb;e?{O@H7|xQj)NE%a9}2B2qO3k z?0k~C7-Gp1Kk@&x_a)$5=k2?{>TQ~#Wu{Fvqy^cs6j@rd7ozNxrHDui*+Qb4GKChq z5S5)Qkt|uJMfPOh6(M_3gmCWX$IRQzf8O{1Kj-?NbDeWt$928)3cv09{d}I!^W4vU z-_M(aimdSzlA%BY=?y>@4oEVo&dW2ejK!Ocl_ec1)MWbp6P?=^yIdxC%~K1Hn*GeI z`aIlUAWu5h1E}N8ZO97W0e2A?$}%V#iws{FbXFUfDO%=6#(UMCn?l> zIQCFS&qcuE0nlu@;#@(q)RlZvt5wh@m5<}arJnC&^&K75;Yi@g)|Nb-SbEYeJ4y>` z7SToQ!qZ!G{=?4BE+5GY(&ORY58B?}9vxO%)4V~`1z)De6d%R2kVx!g?KT83iUhQz zyJenY0&*69onF_h*L2-u*5`2rEb+(!T0YR@7V3a;D~2pBJsFBWvZqMyTf11^k2U_^ z9Rcaxvh@Qb>W8Qan$eqFq>T%581zF6lK7}nrV@DWssLnyDQW*unPgq4q9^h0khg@P zy|yLqf|MDc1xTKpKSbVkVC#Btk>e;81Gs!vShUBnr;2N_q!{@(r>gr#-CpY+F#D*| zN5fYrYBoMNPxwqtSm06W%f+-}L7bz`9+XW?Eo9{>JjH<=;JAD|t2EuqV=n}4S!U^y z>CkKaQ~*!@W4O-tv(Uei?X>$R$O$^&xa9#-`Ki>_#rRw$TItw_TbJTc_<7c!JO1H; z7s!NT$X!{fFR-YheBOGIkVwU3-T3f~-g!QHr`}IK-hm0CaUC1{Mz+nb!$_ z#%qAXfP|5da+QKS4-p~%p-~Q-`n#}Xf5v)b`&t@u|U~fLrwd>%9t8eY8>#JqaOim|F z5gT>inBj;F{+ZNmv_3Chym)lDFHj#J@qieK;uZ^O`^LfcqZ2e>i)~Ca?oAS-OJ-vD z2H9jCI-OCWFA+&yCab9Mq6=y;Z-4`bv?=xP3>D*aGd{sH zi4P9DlX{QZC$@=K=5fX%G1KtDVfSVoLSD`d&NxIa+FZr8T)3%z>!5_BDuf!mU?VRC z(-=a9NF2#zWi?f7VSz}(qzspzA02FTi%ApR(2t|_SPvq=o7(^~BcY!%g_|JS*FNaF zhT`VZff_4B*FKbPZci;f8BS|nk4!krlD7%CluFor+B&F^olGRxW<$qnQ5l90lplhO zK?e*{RS*J5--#IuX=z+#x!?3|0H>v z(>&J84DO_F-wb=n5paAf=9?XKqWl^ROcOA^L8V9@q+}n?24|Sho&O8;8Ji(ev4dW%0*5szUSowK z%Uj78i(e9tHzchpqs(@FJ|22tO<=qjz!J-c#v;gUw#ne?K zI|tON(S!-%ipC1PYz19LH% *$(r;%^(srS_ee#$F91Mh#`Qb58}aX7^);%(`GKB zEf6g);`Q%vS1e4z?{InOuc>Orf|P?ufTTHz)(9(RMN}A*mM9u*b&`&=^wdFOq8yda z{s&O|!?3p&Y#f&3f{RimbkfkFNnZyK$nhWaG~ZDsU2fXA=>J>oZByG*7ReQ;7FV z*Prk#q$5xdY$?6B%LPPX8S?d^f}I?9T@O-jW`alyL6dssAavX7WbT|9?OQr#HI}hR zDQ?+Yoa2UthO;zD388!T?M(9WANHpHlim1mJ3KKjP9;ons37xmb91ZC!@-A4<|niF zXLu|*x8v(V0rG=WWO(b7{xhzcsqQulXFN#|Dkt;FAZk`)ahPD=6)S?XTS*X&XfI^L zJca3?f!ZrqKjA$Lh;BN+oVa3)v<(Oh|EC!{=;<2pQuvAWr5G0gTwO>Wrd~(Av0)ZY zwmoEe-q_ffY`} z!#|Xg914JBB#5X=PHDtbpqh0m&QC%QdqG^lG07#14~*M#lZ!N};Io?L*|(@ z`QcP|?)+uds#V%LR|+2dXWe^dTa1r713VPuK=Y&~^H-)a_5zXZsnqHWP-eEQ;&*klx_WGQ4wfoW^ z)bKJYHRO6B$pAx49o-l>K;F&G-Fpa$wLfV-IXU~x0>6AYn?CMN?XT&h@-+Sg2O>G2 zc&1T!^+YQPC}m38Y@e$vtjha##&bo2w&@`DezC)7ea|mU4o29Nt7+^+dfyzTXAap0 zl24`(oo|Be!V6-$`X2mhS24xZCqiNptV22hM*AFDa$D43_k}S~yF&08y!8>Ef@ADk z+$6d)Z7eK2va+&92798BAf!Z~f@V~$*KtBw_BU|p;lqb-jkI6Zm^E*CIUpX^3?2(K zWKeBCa&Vk<7)noZ#(O<*6x_*J0tT`@_esq1hV2gxz(KxRQ;Wa{1Veh)&YhClYd}Pd z&Ek5Xm6#LjsC$+i=g5b#B5Kk3&Q!!|WUwcO@Q(1cwPSt5QcZXCoBnI)COm#2JEJO% zq-Eq|hc*{*j6|-$d2!?~izV)WBn=QmLeY)Zt}X$D?kaYaz1cI@tzZ8cE@tEeLKQOJ zagX;JF0K?}8Z)Za89LF4@n-^A5-Op(ojs_F-2Lk&6qlO^+p3KCM{`d~iiQt-@xq-583OpBNJ#cEWPr!kf5S~UedE-k63{ZnU=N?KT;8GU?<+7p&BwM zbTGJ~+rm*NJ;|gXXJZf;Eu=A-h-uCwvZzdVI}o-5mGx+RkrWwOpBhG}0TC}g1tEPJ zlRss)cJd^8k3x?=61q7kJe*|F180mgl@HlmWci5QL(NUTI)Atv=t8a;8WKW|o_2VD zkSU(e`-~|cA?U(%p}_~LYC(W90$FzJ;kkfC7#?Khh9rVB{Ddi-i3uS@A&9m`Ic$CC zzz0+&V3~mu-Wc|xR9>re;$gpcSYe~{5gipF5@{IqBhw=A`lyKCn>y9v3FqN3uhWRJkW2L$9{ zdwfGp&dmNy{xm?sOyN2~ZZJZr)bnNTbm3FE>Z+tp(fvA2wlvJagxHlJj)6zc&oB7? zTK58grL2=gpA(ol!B-~>jVhXUU>J4R&X_(u>Da5~&RyqF!w@(36THC|+sYtED8(lc z4lkGgy?@d?@IZM|7V&3>#^L$cF*bSU_uttRNz+ykKfHs2^lnxCEUNztTA?SKf+Wqi zqXxA;s58m44-|gl&b7mRS1Ha}t7FE&^$jTot7(f&u8eBjqPqokUI}le^VTn zk4$CI@P%*Uz~}w&*+Gy-0&&u`ps2`;i)I8O>mg6e%+HWJ!v|F@}k^ZCJHW z?FCL%4k{*CN7gUsM#nNKBZuc>2jx~7C4B8Gymzno0?fA~K?x?bnRp!`0~gAXDpO1I z+!(e_49dxx*poC4(6&DWU|U;xV$`*kZ)>2*Pe2nCx-(^nuz=H&%!n|fj<+)Tzt$K= zeTe>C;-M_`hul5mGQ|EO6A}4=lQZbtN%B}kqlgMiESZJ2WFw~vkr}bA(2zoXy$_1s zhYH>2k!2!dk$NyFuyT@h7JBrkbik~FReT@@QVKwIyyw)_XT=$VUFDL|2s$#m?mAvs zd~!j{27q~s?H!BTCW&ZZ-R3?P?0cFIPY)imS&)G-$vq-hQApQ1oL+JThGm~&ED{X` zY%E{?3&<5H@tUL8@1bZoc6x67U+bNf*$87IS5dgbG%Hu6TiBOQcIu6$NAOpi&5zo(;JBgtna2~2ZV7*Cb1=mjzRv;S zwRZwSXY9+Xe|!f~hj#`^#bL+*4#ldbhd7tXYlXI%_5mm?rifiB)c`7y;;sPyL#k;V zxEC~xiCVg>L<<#}#k}=wqX3=I+f#efpWd4$58r;?7uWDwg-mEV`vsBQ0EsS2sfU*l z^!}lmL>)_=WLeP&LF|**={|k15=8y}r{7j1+TMpao^Sp7^-T{+GLB4*nwC?q4`(R6 z(1A)fb}3~PrD`~a#K9&fj{|qw;{*ev4k9lC9c^yBYS+}O!aI`!E;x5I4GN&AJBnGP zEqmw@sJ4V+aka9oCe!{mXuWPv&DlHggD6mk zYM73LzAPRT=8p4K{x_|5-_rlUzYoM|LsXKxD*yrE0K#* zp$gWE0E{a^RwL&K$kbSRvS4&hP&a<2a9iv(0#uK;FiKG}s75i(19=7-kRMspL9=BX ztNs^S+#_j>f0->>R6By3L+uHe69;1;-!?+`Kx0J27CIC}WC?(y1R%?GiQq-d{Jt&z z*Xev~XQlyS0f~_?bCSeJHr0;g5_s(z6BtGAz#I(#i@Z;8d#oGT1ewjU&2~vBEaj)3 z`kw+rKmP1|Fq6>C=)TDF7ctxL#Yp^oaW#~P3=QW=r1WnDBPugBF|iL7jS@}*9eex- zQz@7=rgF^qE-x>yQ0RB6fjct}*!?R3sxN=`IuGCxc`skS9K`U*Y5g7T%FZZJBev6By_>>MOJ{B6@Ts|LRtZk&0ymo#&9*iIR4916lHfv1a|WR=k%;*v!+ z0H3gZ9qdacpI`2|7GdPhgB+&m$OI;7zDLtS1H=$=HKl18Gz1dazg%kn*KbV%%Iyc$AK4ma zkT(3}5kMm4%rr{d0F(eSC54)_wzj^$e&N!kk+3jdb+QEwIpQwE>^4r-Z`Z@liMsTm zl&Gk$6D@tuIGTCRNW_`&UtrtmTKZ7`yXcQDWkvhSN}a$74Z{Vew*Ir~(t*jUw*6~@l4A@O_3KMpV>`Xw4B3)LzO zuYr}8ia{0yX)4RP`A7UzL#g{ZTy>m(=7dA2q0|ZcP&kia9ym+KHEmJIs&|GWWuq({Pd{QXc-6hG#>C(Fv>XG*C3TW#JL0hR95GT-j2+@qV_@(9jNe&8wiV z(rs!;)d!eE-0e53H|$i}wuN7i|A94!pnpj23Qq21tDfxxz>l9Akl-Mj+Ze2N8Z<#< zTd+n+0u9(CNw_-`h}NQoUx((N2N6+COUth}rDZNYcMYd-#wYh^#cjQ(HiQn{h_v?a znV9fvl-b>TD*5;*7PMXXQl2!t~oI;iU&wuCCslQs<9ym#@OCobHW{(bGAk#QO?EM%@@L&LU( z#)VIdq^+as?`FhDs2VIjrEa8n!DSUr3%9GZS{Z_oPo5fgzTBwR@ZzG^NfnvKyjDOu zV$+!G@33r*?H<(wss~%FS4Rd%sN!GCgqf_Esb9~MJ1(RvRff|{`^n_xc7HnK+_EIH z3pmTXa8BIZ$#axcZ{9c|e&Vt0HI{xQ{koeuvk9hW%p`D26XH&VjxmF%26e4+xjxR- zHji)1k<*9rwrP}Kdshf=kh5@pUeHw6aCh^%+&%a{TJMDCrx`@Fl~51=}Z|sNHDep_t=rZE+z2x6>uTn=h&U!|KC(H}h}k z_5RO&=dE$Xm8)a)(#xdteMh`Bmay$XvuqjCXl8kN+`~^L*ruvo-o3JId1-C^4!U@D z6H7k3xScs^hFGm6LQ@17-d^}SM8dA>$TQA^Pwy$aX{BiSNzD|x2F{Hb4-285@&7j7 zaT~UCC~Z?RajSC+xp+?*|1xl=>qn`2SS&zxKZ>RHvi zMI7^m9Z%ah4d+Vo(6&D?0 z{DSCi>N~*OJOpe6v5|d(ESf@XT^+H%R2C(Dbk<|9l^FlvqIgHhCx|U;6}`AIxj@o$ zA>XG_Tb*W3E^-(g1y4iLaj%yd-5;&21 z-nkPC(NQ!YBnZ@B10s#;nE+5@3JZ;80AmT|f&Pjnq*MQ5$QDo#X&5pv#**?Wq4e%` zgt8|kB_$}*K2gjq&ACh3^3B zG(ha7W)lt@8jcp3Jb}Imnc6d_mIJ6FdLm%`42%MIFD}lc42+bb$bX6F02V+j&G0q` zear2v5nevxMq$8o+L|vFr5f8ZC!=xLuQRg^!HJ}ca3guVKE&usZ)3(Gq>!LOn_KZ( zSwm@Lf{*B3b3=`ov@}>D`dP%S*2-*t0Rf}|{$gld*i)+xASFEAMvm(x!r4v1SLC<5 zYoUqK4M0Mqa^%Qe8oE$->EcDQ)1-lIB$Rs=w0DN2lKS~^Mck1Kap4&6q-XL#@KcOAV-a>~e(HAZ4XM$Rj|zr7jQ}wN|g!dh1DTAWs&wD!O}+ z^}36)B_FUwH;x!S@NprUYK)T;1LhmXNQWRQ$_RdW2hQ$jmxs$t48Ot%`i7aCz1ItB zWMi29pK?gYiP5HtOn3su*N&rgHQ3{IQv9=26)0$ypyZAYz#nELaPN%6qBaah=jrzK zeDyDe<|9LV{+83Zece=oxKma`?DUgthM0-h`W>&<8mJb}4&2B!esN&}?L)X(^8*4P zIyw3}BuYY;sT2Jd%P&Ly>tAiUj}BT?Ow2x9C7LRRJBenJ*k;V4nmbB zZMQSpdi!m-lTKCgzC_{_|8NGk)uHI5*fB=1bk34M-i<1l1>0c~DqN@3Qg}et&aOt$ z%kxfb*?8xA|MvZkBOWI<_!A8ojhn7~fw*!t4V_T9kiG}%YX1Ib2yxJmrbap#tAQZN zgwre?)cLg;_K$Zo;Rgci)|HM&JlLBEHx_E>quFU@dDBlXf;~VuV2%yL=uEV0(Y5s>mMR{Jgg6>#H2L(8iPM0I zXM%jPgHN>-&@HVeF;QIJ_jo^g%`V`f3^V~yf{U5y#(_;Bbh3*>#A{}jLGqxqzZ z7R?=1gt!eJ8$GXNM&7S}@$x0j5&(0l3{{F1I5x`+u%`X)oNfN0;Y`6=8r4SyFKD7t z-DO6JYFcyouxK#)^QL8TwMk+P8PSkl9HVpT!-5*SajZ*kLK$=hSoc}MxssR!b4O`v zGzJeHcYWn0IMn2U>Jd)tRj5-qYKQuo;y`aYuD1Xsi|ItZ4E+E`!p>XTGhYh;7N~jq z@HrOzZa)2*6C2hp4Gi%JPdx{P#IY}SXZAG7?>g(W|I*+Lr_O*)I%Zw=!_>tMu`>nm8F+o5v0leh2w%ev8FDUXA-9ha>APy8M z!hL2-qT@-r7a^TA8kUE{sf=^t%N|nTV;~RUCUuxhJ6PYwG=Vi?8Tih6Kquoq=ZYdTHv%Ji2rqnn%csez7R0Fl-~;6lM}_*DEr z)I9&9oKU3Z!Flp0e!hoby=bwQ`o_B7c=(V-Q+~EljA5*5KiGXsbT{ChGj^PiBcR`| zasGVfj#2R{?bQ%j>IM)~0et6-S_cR;=Bcr0qWuQu^!+mPK$4}to-uFJ3faMCIdHqI zpaqk2CyoWa4z5WJc!jAHy&t~f098WHLGDVReh~p+q^~=vQw&`_CNsvqZ?pZhE(j;n z!AkE*EGEGgu|CoD@!$d^sfdHF45}P;gK4H1c(Dwl(;*8X8X}=5)OfZ;46Pj91o2-4 z+J&;mSbf1ZP=KywZM#WRBU}s>SGZ}Gl$eh=vffrmR z?@R7t)Z%5tHldDg;bxHYs%S(b*v@J|1Zd|Y+N|j~vCSBc%@iprFmHxb=Cm3yY)076 z6lpC(NPIHieeHYM-_V2onx+>lgc@b8%&edGl?T~j?eIRiz!9T%j@J$Y6IPAqS$q4m z!kg`Mj?IxMfBH7Zmp{>a%a=DZKxnHOf08V416TncC+AD8xH%aSws0HhmeGhc;9z6Z z2tqV+aQUtP!O9j(vFZ8&RYi&$i65wYN6~0hR_M_RRbZ z@KpykQz<6c9&=Q%wN1t5uxfsO^*Gz(EJ)Tr0nP0~0Dg_`*aLLvfdk6JC9MKOXw~8d zVhS{uOPXcB756d?Mz8Un?#tt|y#)`?VV69B6qUr%==+jSF!p=xtaG3W&t24P8k@SC zR4$Y$k$Mw|VhPBf9F1pT_~Z)-ZY<580tug>X#~~ib6=1;9p*q~D`vCzoL9QTt_@Ec zG8tlXKBrC(+wUANT{OF=Y-P+gmo7Zj%_5`dk2ywG=HOC84iKLoI13Rp5=UZ{2`Azl zO~7HVjeXLeFZ}|dL-o*Vq`)2gV`x10$Y?CCi!zNCl&-MK%BqC121$%a8|c6hjWrA4 zIB;lFal;MK-S9luz*?tmL4(@{dk~zMAsaezHQSRXAAJuN4FzI{kx#FoTKNRPrxRnJ z=zH|?x&4bO4;$F_F4JW~siT5o0trW_$-9$&w1xW;X8uK%T4Nv8hA&iqX_VMOk3sW)!?59_2|=I1<+~dEIGMF%gqKKZUdHKd?NPt(q-W6V)c`U2+Yh5cv;1xLu=P) z6;Jwl35mx|mW!0(p-aBT&dC&+kOMy6f}qRab$F?PsO z5qykREfG{;MmN}l|KPl~31t069CHs|1Bt>a;fIKw-@wn`zwYq*FbHo!(C;uiCKvri zh?phun`|WfQOL!XpIYl_xy$03217gSjFIutn=P|%PgaSJzZqRA#wUh>1&u74fFzp_ za5?5%mLg#HbjfHSl~zImd>-srVpbt`Gg}j&a`#FsQbBtl2??|F(_&0L4Mmv&P_Cpsz4&o4PXkbjR+vv|&wcf&dv7kfj!F(_4 zYH#dm+$9a#cW8fL5T_7$cEgmh?%lnO-wvUV&8Wj3&JUOVRnD+6aZe=}2 z0gyb9GNt9V?|y=cq|+vAG!RD)d+(|>folVHVs0+daB>e(Q&kN?gh02dL*j)QQjG#I z0%&S(&&dt`3bfEp3bD^TQ3Lor1>%%)AFa_*Q6 z9E0j$2aHgZ(3$cAn+OGTWpxtFdMVV{9$ky8M8I>jEd2Z6+p>n6Xg~9&F`K>!(;8#D#;JytLwO@C64DNy%=G+AX$c zRkNB^PX-0C2F}v7ozQB=5Tm7F zj=R$wnH0(SZ<;}x=aEH%c>0q;;;$R&tpw*`UMsn?z?^%9EX_tJGO{pNb^44&K(nFn z6N|h+7QV^;%WY3&?y?Wr5*X!fC{khP(&~Oscyj%1_0UhOA?-MJ0?aaME8a5L<^btH zf?*`i&jhX*PCVon&22i&h>%%q&P0L!1ty#DojS)-11?!05kl7iq=_Zx_f6!VV^plx zLyMpztz>xq%ou3|gIin>8VQx_gHgsWA`-uC$brwGKHyo3l(V_T+?F=>oFZGcJ6l`Fa zS8$5MK{Vy&u>134&$L|qEtW{Mi<~Mx!hA%#evyyw1+_g$KM4jBelH3l*l>x|h2Nh^ zYVrX(za|*8RSb*o&gA-7W44(2B2&M@bEQ`IbC0uva6?~z0xb)sW1gAUojXEk3}`r? zQJQ?!f4)!GhETVrxIeum{Id9k^eI7NgwvNi6V#{g1l^c0@n665o{2`#9{5b$g0c}B zu~5t~(r_+6Orz4s5dtL+Ha==4tLjs(N=ODEnt#{T6%FDvPG0Fxpm+cJ-Mr;7keHxo zBB~CtOm_r1sp6qJGs7&OZ=VdJu2`Gah8*jN8L)d%Q=Su3N~pui9al)-*#G%6aW>(FY1~viH2wJY%;)XRRk=O^J2J~fa(z0l zoXfL)Coe8y7KwP$(%NcBRw0R~VAi8mx`3Vi#9&`jUuVg(=dd~|#P&8B|9Hc3iEJAX z70c>zycJ&@-H9*|OFUV(VL)bLKqd^?T^p#sgIgf_SdaF;IE`Y|LOU6ah(G^x@Djq!O+eHA1*r5oUQ7q2uTZ67w)2F^l94AifYC zXn!Sv2O6Q*(}q$&6by!S$c7S!mg2xMf!|YJe*QWz?!+k0rlLgau6q?AWq`1)07V9R zX)am``j>y0Ge;bQagv9IfxEe346`4UQ8jR0vqe%*Zy^tTq@l=y(SdUbV%`87{2(dz z2f|j|-I!6g3B`R*Sy>q45`hy zYaQR z$FOq4Enz9=bCE_1 z)}P)hwZJ2dum}TJQAmZv+zYcXBZrYK!w~9~NXG)=;+t>ZqgG z6~>O|Q8xOK-g4y~XjuF(_$|0NAtfKViYwQ^3pTy#mi|wl#N@pN6NYLaYP@w66u{H2 zQvTnkr>6QLakCEs-|;q9@Teq>8ElJ+jy)q;iMl}nr2Dbq;l?1Vt1rAzqZ*BmRAsF* zvcjZvec)h+seMnbbN1VFs?~&X2_cJrRD04@tBBqk9I$j+tTWjTEa>On!q&bc^WCQY z(6rThQJno~846dp%73qefnfW9{E+}BqX-8#><>to#k$&*9WJQ+f`XQk6Ii)Oi3-k- zcU8sFxq?n*w*x>f1PZEWszKAPI05fOXc zp;%ibOIZP#68>l!t{8}M9|F$iE-o%aoXFUZ4-45Q!2ay z$vZ~rYH-liu-cZX>iACn^PWHcZ#|HDOORL5HQ0D=1vmVMwGc0Ena z2pogXJ@5viX)^Cs8n&EwaJ@b9)oyfH-~2Mr05?tx|suzk1#-6~+uLUzawuQqcY zdl{0FqKS#gF=eBPLJ4AFC|uOX$H!?}Ad0Ea@WSKg;khEZ@dly`C3PV9#65INh}H$! zf(E70BuUte5LAr(Mpaq)sc%IyM2$4RiwazFy#c)NddIxYAz5QRrw7jpv_$hYhK*;G zEW9QBF;$YYKXhcJ-kG_>%41XeWBHuSJ&ZYRKrvOp&n8DQI*S0o+^2a}_wKD4L;73= z#>pqF;1p>=`B6|7J`)fGW(BF`tR&goh;%t_fe=inHC1E)bGfABrUEqRGGXkiUons}pcMgiZP}H8$;CmBogdD&qCYzvr04O^b zsWSO?z*}tXym|8=3K6lk`jMSI2Hgl{pfhjB)LnqV0lSt=B0S){)U+nnh`Gb_gIsE{ z%Q%^rTzodR-b%fwr85QDuvL_AEZ^+u{k*zR<^i4 zi#5=dDnX!Il6a6b79%)U9$9s%kFJK9GtD-hD69^I;oQq4n?7C81wNjSegPHqq1FvU zO4RK0!BzU+ThP7k`s{8YJ@m63`uDrZlJ>PBdhv`N>tpr&bJe27k0 zp7ra?ez|;E&Ay-piAy;>EAp@K+C#;6`o?TCHJ242FFlp{rI{7-&SMzXbc+0`3=(S! zX=qruq<^z-#k(q-3ehVG(Z#E8Y|@AKn*r8IAr59kgjxk~_@rPy7}!jwVqwjp8!&P9 zMTK09fq=K+<+cPgE??XT(HGoK9BGVPCY3JMCiczAA~ z&K%CESSjv{RH%M$RR0nXQAFFiPf7&Yb@I`bL#A4!AklEty05V({s!Db{2|3(>gebQ z+MIE3*2H<#=glR@(6a~w(w9?LSC?e``sow!Xw>wzV9=O~8|MaE zOG8~)iDMG~9f~Tp8K~rhOdP*F<0Sb$$Z3*EoExq8AzyioteseFWL66KJPijYC4JWx zpa$MTI-Mnp+1b0zTtRkKBJL}QcQhpsjTm&YB?@Ix@rteZ=POK3D2P}rsJX(xHNyc_ z1Hh#!2tUf95nHR5NpKb{Bt2=jlEaxpA&x;37$aN1|CSTL==!QG!X+y+CkZCD$_92xxuxr<4 z%Bny+Q-qwP5^hjTi61O7rWJpWYWm16;c4;Gbct+=RP@|_1XOYLS% z9e~G1F06^ka32O|0b3eD7$%SnkO2K0XFmBL0c(KOP2GWTa5YK}=lU=-1Btxglp_Kb zDq@TxFqPMMCsr>OX)woh68DSXU2^e5nL@pDEQ!p@$S#yLN9-~zt(Tn(#UT)h0FSv8 zN?ui0E?*X}|Kkp*gEX_%!wh+92Xc){G-SdFwL;2p3+w^}nNZv`VsE;uDkaK_` zqj;}?px{23z74jc;wN$Im@URS67G%ev4Dukho^y{w{M-XN^kmML+$O~2g`ndivsdi zarC?wEnN7X>|eN`JKqLvP${9*D04b6{(#1tnl>;kzwlEXfWWVDW5n$~8G}_@`*;g^z4xu-fg5SDb zz%j`(AGsiyeUacGl&Ae2Is_af@$}HZKp9R_B8!r%04ZGPDN9S0$DL8TYc3m}&xn5L zXCb_gNfkl8{R?xv0p3Nt*dzk-f9-ck6SMNOQm=jV+S-G+%aGlrxHES9;fwzR(jukg literal 0 HcmV?d00001 diff --git a/pyproject.toml b/pyproject.toml index e0b47007..c8ae6196 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ dependencies = [ "loguru", "numpy", "pillow", + "protobuf", "pydantic>=2.0.0", "pydantic-settings>=2.0.0", "pyyaml>=6.0.0", diff --git a/src/guidellm/__init__.py b/src/guidellm/__init__.py index e5620188..28c66117 100644 --- a/src/guidellm/__init__.py +++ b/src/guidellm/__init__.py @@ -14,6 +14,6 @@ from .config import settings from .logger import configure_logger, logger -from .main import generate_benchmark_report +# from .main import generate_benchmark_report __all__ = ["configure_logger", "logger", "settings", "generate_benchmark_report"] diff --git a/src/guidellm/backend/backend.py b/src/guidellm/backend/backend.py index 6009ae32..115c0e11 100644 --- a/src/guidellm/backend/backend.py +++ b/src/guidellm/backend/backend.py @@ -1,4 +1,3 @@ -import asyncio from abc import ABC, abstractmethod from pathlib import Path from typing import Any, AsyncGenerator, Dict, List, Literal, Optional, Type, Union @@ -110,7 +109,7 @@ def info(self) -> Dict[str, Any]: """ ... - def validate(self): + async def validate(self): """ Handle final setup and validate the backend is ready for use. If not successful, raises the appropriate exception. @@ -121,13 +120,10 @@ def validate(self): if not models: raise ValueError("No models available for the backend") - async def _test_request(): - async for _ in self.text_completions( - prompt="Test connection", output_token_count=1 - ): # type: ignore[attr-defined] - pass - - asyncio.run(_test_request()) + async for _ in self.text_completions( + prompt="Test connection", output_token_count=1 + ): # type: ignore[attr-defined] + pass @abstractmethod def check_setup(self): diff --git a/src/guidellm/backend/openai.py b/src/guidellm/backend/openai.py index cbf0aed0..46bf034f 100644 --- a/src/guidellm/backend/openai.py +++ b/src/guidellm/backend/openai.py @@ -202,7 +202,7 @@ async def text_completions( # type: ignore[override] try: async for resp in self._iterative_completions_request( - type_="text", + type_="text_completions", request_id=request_id, request_prompt_tokens=prompt_token_count, request_output_tokens=output_token_count, @@ -277,7 +277,7 @@ async def chat_completions( # type: ignore[override] try: async for resp in self._iterative_completions_request( - type_="chat", + type_="chat_completions", request_id=request_id, request_prompt_tokens=prompt_token_count, request_output_tokens=output_token_count, @@ -403,16 +403,16 @@ def _create_chat_messages( async def _iterative_completions_request( self, - type_: Literal["text", "chat"], + type_: Literal["text_completions", "chat_completions"], request_id: Optional[str], request_prompt_tokens: Optional[int], request_output_tokens: Optional[int], headers: Dict, payload: Dict, ) -> AsyncGenerator[Union[StreamingTextResponse, ResponseSummary], None]: - if type_ == "text": + if type_ == "text_completions": target = f"{self.target}{TEXT_COMPLETIONS_PATH}" - elif type_ == "chat": + elif type_ == "chat_completions": target = f"{self.target}{CHAT_COMPLETIONS_PATH}" else: raise ValueError(f"Unsupported type: {type_}") @@ -525,15 +525,15 @@ async def _iterative_completions_request( @staticmethod def _extract_completions_delta_content( - type_: Literal["text", "chat"], data: Dict + type_: Literal["text_completions", "chat_completions"], data: Dict ) -> Optional[str]: if "choices" not in data or not data["choices"]: return None - if type_ == "text": + if type_ == "text_completions": return data["choices"][0]["text"] - if type_ == "chat": + if type_ == "chat_completions": return data["choices"][0]["delta"]["content"] raise ValueError(f"Unsupported type: {type_}") diff --git a/src/guidellm/benchmark/aggregator.py b/src/guidellm/benchmark/aggregator.py index 83128c1b..19780f68 100644 --- a/src/guidellm/benchmark/aggregator.py +++ b/src/guidellm/benchmark/aggregator.py @@ -1,9 +1,8 @@ import time from abc import ABC, abstractmethod -from collections import defaultdict +from pathlib import Path from typing import ( Any, - DefaultDict, Dict, Generic, List, @@ -11,15 +10,14 @@ Optional, Tuple, TypeVar, + Union, ) from pydantic import BaseModel, Field -from transformers import PreTrainedTokenizer # type: ignore # noqa: PGH003 from guidellm.backend import ResponseSummary from guidellm.benchmark.benchmark import ( BENCH, - Benchmark, BenchmarkArgs, BenchmarkRunStats, GenerativeBenchmark, @@ -28,7 +26,7 @@ ) from guidellm.benchmark.profile import Profile from guidellm.config import settings -from guidellm.objects import Serializable +from guidellm.objects import RunningStats, Serializable from guidellm.request import GenerationRequest from guidellm.scheduler import ( REQ, @@ -36,6 +34,7 @@ SchedulerResult, SchedulingStrategy, ) +from guidellm.utils import check_load_processor __all__ = [ "AGG", @@ -44,7 +43,7 @@ ] -class BenchmarkAggregator(Generic[BENCH, REQ, RES], ABC, BaseModel): +class BenchmarkAggregator(ABC, BaseModel, Generic[BENCH, REQ, RES]): """ A pydantic base class representing the base class for aggregating benchmark results. The purpose is to receive and process results from a Benchmarker as it iterates @@ -187,55 +186,53 @@ class BenchmarkAggregator(Generic[BENCH, REQ, RES], ABC, BaseModel): ), default=0, ) + in_warmup: bool = Field( + description=( + "A flag to indicate if the benchmark is currently in the warmup phase." + ), + default=False, + exclude=True, + ) + in_cooldown: bool = Field( + description=( + "A flag to indicate if the benchmark is currently in the cooldown phase." + ), + default=False, + exclude=True, + ) - queued_time: float = Field( + queued_time: RunningStats = Field( description=( - "The sum, in seconds, for time spent in queue for all requests that " + "The running statistics for the time spent in queue for all requests that " "completed within the benchmark run. This is the time from when the " "request was created to when it was scheduled to be processed." ), - default=0.0, + default_factory=RunningStats, ) - scheduled_time: float = Field( + scheduled_time: RunningStats = Field( description=( - "The sum, in seconds, for time spent scheduled for all requests that " + "The running statistics for the time spent scheduled for all requests that " "completed within the benchmark run. This is the time from when the " "request was scheduled to be processed to when it was actually started." ), - default=0.0, + default_factory=RunningStats, ) - worker_time: float = Field( + worker_time: RunningStats = Field( description=( - "The sum, in seconds, for time spent processing for all requests that " + "The running statistics for the time spent processing for all requests that " "completed within the benchmark run. This is the time from when the " "request was started to when it was completed." ), - default=0.0, + default_factory=RunningStats, ) - targeted_worker_start_delay: float = Field( + targeted_worker_start_delay: RunningStats = Field( description=( - "The sum, in seconds, for the delay between the targeted start time and " + "The running statistics for the delay between the targeted start time and " "the actual start time for all requests that completed within the benchmark " "run. This is the time from when the request was scheduled to be processed " "to when it was actually started." ), - default=0.0, - ) - process_idle_time: DefaultDict[int, float] = Field( - default_factory=lambda: defaultdict(float), - description=( - "The total idle time for each process that was used to process requests " - "for this benchmark run. This is the time that the process was not " - "actively processing a request." - ), - ) - process_idle_time_scratch: DefaultDict[int, float] = Field( - default_factory=lambda: defaultdict(float), - description=( - "A scratchpad for calculating the idle time for each process that was used " - "to process requests for this benchmark run. This is used to calculate the " - "total idle time for each process." - ), + default_factory=RunningStats, ) def add_result(self, result: SchedulerResult[REQ, RES]): @@ -249,7 +246,7 @@ def add_result(self, result: SchedulerResult[REQ, RES]): self.add_base_result(result) @abstractmethod - def compile(self) -> Benchmark[BENCH]: + def compile(self) -> BENCH: """ Compile the benchmark results and statistics into a Benchmark object. This is required to be implemented by subclasses to finalize the benchmark @@ -285,117 +282,112 @@ def _update_stats_from_result( else: self.successful_requests += 1 - self.queued_time += ( + self.queued_time.update( result.request_info.scheduled_time - result.request_info.queued_time ) - self.scheduled_time += ( + self.scheduled_time.update( result.request_info.worker_start - result.request_info.scheduled_time ) - - self.worker_time += ( + self.worker_time.update( result.request_info.worker_end - result.request_info.worker_start ) - self.worker_schedule_delay_total += ( + self.targeted_worker_start_delay.update( result.request_info.worker_start - result.request_info.targeted_start_time ) - first_process_request = ( - result.request_info.process_id not in self.process_idle_time_scratch - ) - if not first_process_request: - self.process_idle_time_scratch[result.request_info.process_id] -= ( - result.request_info.worker_start - ) - self.process_idle_time[result.request_info.process_id] = ( - self.process_idle_time_scratch[result.request_info.process_id] - ) - self.process_idle_time_scratch[result.request_info.process_id] += ( - result.request_info.worker_end - ) - def _add_to_results_within_active_period(self, result: SchedulerResult[REQ, RES]): start_time = result.request_info.worker_start end_time = result.request_info.worker_end completed_number = self.errored_requests + self.successful_requests + if (self.warmup_number and completed_number <= self.warmup_number) or ( + self.warmup_duration and start_time <= self.warmup_duration + ): + # within warmup period + self.in_warmup = True + return + if ( - (self.warmup_number and completed_number <= self.warmup_number) - or (self.warmup_duration and start_time <= self.warmup_duration) - or ( - self.cooldown_number - and self.max_number - and completed_number > self.max_number - self.cooldown_number - ) - or ( - self.cooldown_duration - and self.max_duration - and end_time >= self.max_duration - self.cooldown_duration - ) + self.cooldown_number + and completed_number > self.max_number - self.cooldown_number + ) or ( + self.cooldown_duration + and end_time >= self.max_duration - self.cooldown_duration ): - # within warmup or cooldown period + # within cooldown period + self.in_cooldown = True return + self.in_warmup = False + self.in_cooldown = False self.results.append(result) -AGG = TypeVar("AGG", bound=BenchmarkAggregator) +AGG = TypeVar("AGG", bound=BenchmarkAggregator[BENCH, REQ, RES]) class GenerativeBenchmarkAggregator( BenchmarkAggregator[GenerativeBenchmark, GenerationRequest, ResponseSummary] ): - processor: Optional[PreTrainedTokenizer] = Field( + processor: Optional[Union[str, Path, Any]] = Field( description=( "The tokenizer to use for calculating token counts when none are " "avaiable that match the preferred source." ) ) - request_time_total: float = Field( - default=0.0, + targeted_request_delay: RunningStats = Field( description=( - "The sum, in seconds, for the total time spent processing all requests " - "that completed within the benchmark run. This is the time from when the " - "request was created to when it was completed." - ), - ) - targeted_request_delay_total: float = Field( - default=0.0, - description=( - "The sum, in seconds, for the delay between the targeted start time and " + "The running statistics for the delay between the targeted start time and " "the actual start time for all requests that completed within the " "benchmark run. This is the time from when the request was scheduled to " "be processed to when it was actually started." ), + default_factory=RunningStats, ) - time_to_first_token_total: float = Field( - default=0.0, + request_latency: RunningStats = Field( description=( - "The sum, in seconds, for the time from the start of the request to the " + "The running statistics for the time spent processing all requests that " + "completed within the benchmark run. This is the time from when the " + "request was created to when it was completed." + ), + default_factory=RunningStats, + ) + time_to_first_token: RunningStats = Field( + description=( + "The running statistics for the time from the start of the request to the " "first token being generated for all requests that completed within the " "benchmark run." ), + default_factory=RunningStats, ) - inter_token_latency_total: float = Field( - default=0.0, + inter_token_latency: RunningStats = Field( description=( - "The sum, in seconds, for the time between each token being generated " + "The running statistics for the time between each token being generated " "for all requests that completed within the benchmark run." ), + default_factory=RunningStats, ) - prompt_tokens_total: int = Field( - default=0.0, + prompt_tokens: RunningStats = Field( description=( - "The sum of the token count for the prompt for all requests that " - "completed, if available in the response." + "The running statistics for the token count for the prompt for all " + "requests that completed, if available in the response." + ), + default_factory=RunningStats, + ) + output_tokens: RunningStats = Field( + description=( + "The running statistics for the token count for the output for all " + "requests that completed, if available in the response." ), + default_factory=RunningStats, ) - output_tokens_total: int = Field( - default=0.0, + total_tokens: RunningStats = Field( description=( - "The sum of the token count for the output for all requests that " + "The running statistics for the total token count for all requests that " "completed, if available in the response." ), + default_factory=RunningStats, ) def add_result(self, result: SchedulerResult[GenerationRequest, ResponseSummary]): @@ -406,7 +398,7 @@ def add_result(self, result: SchedulerResult[GenerationRequest, ResponseSummary] :param result: The result to add to the aggregator. """ - is_error = bool(result.response.error) + is_error = result.type_ == "request_complete" and result.response.error self.add_base_result(result, is_error=is_error) if result.type_ == "request_complete": @@ -441,79 +433,63 @@ def compile(self) -> GenerativeBenchmark: total=self.completed_requests, total_completed=self.successful_requests, total_errored=self.errored_requests, - queued_time_avg=( - self.queued_time / self.completed_requests - if self.completed_requests - else 0.0 - ), - scheduled_time_avg=( - self.scheduled_time / self.completed_requests - if self.completed_requests - else 0.0 - ), - worker_time_avg=( - self.worker_time / self.completed_requests - if self.completed_requests - else 0.0 - ), - worker_delay_avg=( - self.worker_schedule_delay_total / self.completed_requests - if self.completed_requests - else 0.0 - ), - resolve_delay_avg=( - self.targeted_request_delay_total / self.completed_requests - if self.completed_requests - else 0.0 - ), - process_idle_time_avg=( - sum(self.process_idle_time.values()) / self.completed_requests - if self.completed_requests - else 0.0 - ), - worker=self.worker_description, - request_loader=self.request_loader_description, - extras=self.extras, + queued_time_avg=self.queued_time.mean, + scheduled_time_avg=self.scheduled_time.mean, + worker_time_avg=self.worker_time.mean, + worker_delay_avg=self.targeted_worker_start_delay.mean, + resolve_delay_avg=self.targeted_request_delay.mean, ), + worker=self.worker_description, + requests_loader=self.request_loader_description, + extras=self.extras, ) def _update_generative_stats_from_result( self, result: SchedulerResult[GenerationRequest, ResponseSummary] ): - duration = ( + if self.request_latency.count == 0: + self.request_latency.start_time = self.start_time + self.targeted_request_delay.start_time = self.start_time + self.time_to_first_token.start_time = self.start_time + self.inter_token_latency.start_time = self.start_time + self.prompt_tokens.start_time = self.start_time + self.output_tokens.start_time = self.start_time + self.total_tokens.start_time = self.start_time + + self.request_latency.update( result.response.end_time - result.response.start_time if result.response.end_time and result.response.start_time else 0.0 ) - self.request_time_total += duration - - targeted_delay = ( + self.targeted_request_delay.update( result.response.start_time - result.request_info.targeted_start_time if result.response.start_time else 0.0 ) - self.targeted_request_delay_total += targeted_delay - - first_token_time = ( - result.response.first_iter_time - result.response.start_time + self.time_to_first_token.update( + (result.response.first_iter_time - result.response.start_time) * 1000.0 if result.response.first_iter_time and result.response.start_time else 0.0 ) - self.time_to_first_token_total += first_token_time - - tokens_latency = ( - result.response.last_iter_time - result.response.first_iter_time - if result.response.last_iter_time and result.response.first_iter_time - else 0.0 + if result.response.output_tokens > 1: + self.inter_token_latency.update( + (result.response.last_iter_time - result.response.first_iter_time) + * 1000.0, + count=result.response.output_tokens - 1, + ) + self.prompt_tokens.update( + result.response.prompt_tokens or 0, + ) + self.output_tokens.update( + result.response.output_tokens or 0, + ) + self.total_tokens.update( + (result.response.prompt_tokens or 0) + (result.response.output_tokens or 0), ) - self.inter_token_latency_total += tokens_latency - - self.prompt_tokens_total += result.response.prompt_tokens or 0 - self.output_tokens_total += result.response.output_tokens or 0 def _compile_results( self, - ) -> Tuple[List[GenerativeTextResponseStats, GenerativeTextErrorStats]]: + ) -> Tuple[List[Union[GenerativeTextResponseStats, GenerativeTextErrorStats]]]: completed: List[GenerativeTextResponseStats] = [] errored: List[GenerativeTextErrorStats] = [] @@ -584,6 +560,10 @@ def _compile_tokens_count( # no processor available, fall back on unpreferred source or 0 return response_tokens or requests_tokens or 0 + self.processor = check_load_processor( + self.processor, + error_msg="Processor/Tokenizer is required for calculating token counts.", + ) # no tokens that matched the preferred source, # calculate locally based on the value return len(self.processor.tokenize(value)) diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index aec8a13c..a7ba095b 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -134,12 +134,6 @@ class BenchmarkRunStats(Serializable): "and when it was resolved/requested by the worker in the benchmark run." ) ) - process_idle_time_avg: float = Field( - description=( - "The average time spent in the idle state for each process in the " - "benchmark run where it wasn't actively running a request." - ) - ) class Benchmark(Serializable): @@ -257,6 +251,20 @@ def time_to_first_token_ms(self) -> float: """ return 1000 * (self.first_token_time - self.start_time) + @computed_field + @property + def time_per_output_token_ms(self) -> float: + """ + :return: The average time in milliseconds per output token generated. + This includes the time to generate the first token and all other tokens. + """ + if self.output_tokens == 0: + return 0.0 + + return ( + 1000 * (self.last_token_time - self.first_token_time) / self.output_tokens + ) + @computed_field @property def inter_token_latency_ms(self) -> float: @@ -273,17 +281,28 @@ def inter_token_latency_ms(self) -> float: / (self.output_tokens - 1) ) + @computed_field + @property + def tokens_per_second(self) -> float: + """ + :return: The average number of tokens generated per second in the prompt and + output text. + """ + if (latency := self.request_latency) == 0.0: + return 0.0 + + return (self.prompt_tokens + self.output_tokens) / latency + @computed_field @property def output_tokens_per_second(self) -> float: """ - :return: The average number of tokens generated per second in the output text. - Note, does not include the time to generate the first token. + :return: The average number of output tokens generated per second. """ - if (itl_ms := self.inter_token_latency_ms) == 0.0: + if (latency := self.request_latency) == 0.0: return 0.0 - return 1000.0 / itl_ms + return self.output_tokens / latency class GenerativeTextErrorStats(GenerativeTextResponseStats): @@ -338,6 +357,19 @@ def time_to_first_token_ms(self) -> Optional[float]: return super().time_to_first_token_ms + @computed_field + @property + def time_per_output_token_ms(self) -> Optional[float]: + """ + :return: The average time in milliseconds per output token generated. + This includes the time to generate the first token and all other tokens. + None if the output_tokens is None or 0. + """ + if self.output_tokens is None or self.output_tokens == 0: + return None + + return super().time_per_output_token + @computed_field @property def inter_token_latency_ms(self) -> Optional[float]: @@ -436,6 +468,13 @@ class GenerativeBenchmark(Benchmark): "milliseconds for completed, errored, and all requests." ), ) + times_per_output_tokens_ms: StatusDistributionSummary = Field( + description=( + "The distribution of latencies per output token in milliseconds for " + "completed, errored, and all requests. " + "This includes the time to generate the first token and all other tokens." + ), + ) inter_token_latencies_ms: StatusDistributionSummary = Field( description=( "The distribution of latencies between tokens in milliseconds for " @@ -448,6 +487,12 @@ class GenerativeBenchmark(Benchmark): "errored, and all requests." ), ) + tokens_per_second: StatusDistributionSummary = Field( + description=( + "The distribution of tokens per second, including prompt and output tokens " + "for completed, errored, and all requests." + ), + ) @computed_field @property @@ -568,22 +613,19 @@ def from_stats( errored_requests=errored, start_time=start_time, end_time=max(req.end_time for req in completed) if completed else 0.0, - requests_per_second=StatusDistributionSummary.from_timestamped_values_per_frequency( - completed_values=( - [(start_time, 0.0)] # start time to cover full time range - + [(req.end_time, 1.0) for req in completed] - ), - errored_values=( - [(start_time, 0.0)] # start time to cover full time range - + [(req.end_time, 1.0) for req in errored if req.end_time] + requests_per_second=StatusDistributionSummary.from_request_times( + completed_requests=( + [(req.start_time, req.end_time) for req in completed] ), - frequency=1.0, # 1 second + errored_requests=[(req.start_time, req.end_time) for req in errored], + distribution_type="rate", ), - requests_concurrency=StatusDistributionSummary.from_timestamped_interval_values( - completed_values=( - [(req.start_time, req.end_time, 1) for req in completed] + requests_concurrency=StatusDistributionSummary.from_request_times( + completed_requests=( + [(req.start_time, req.end_time) for req in completed] ), - errored_values=([(req.start_time, req.end_time, 1) for req in errored]), + errored_requests=[(req.start_time, req.end_time) for req in errored], + distribution_type="concurrency", ), requests_latency=StatusDistributionSummary.from_values( completed_values=[req.request_latency for req in completed], @@ -601,32 +643,134 @@ def from_stats( completed_values=[req.time_to_first_token_ms for req in completed], errored_values=[req.time_to_first_token_ms for req in errored], ), + times_per_output_tokens_ms=StatusDistributionSummary.from_values( + completed_values=( + [ + req.time_per_output_token_ms + for req in completed + if req.output_tokens > 0 + ] + ), + errored_values=( + [ + req.time_per_output_token_ms + for req in errored + if req.output_tokens > 0 + ] + ), + completed_weights=( + [req.output_tokens for req in completed if req.output_tokens > 0] + ), + errored_weights=( + [req.output_tokens for req in errored if req.output_tokens > 0] + ), + ), inter_token_latencies_ms=StatusDistributionSummary.from_values( - completed_values=[ - req.inter_token_latency_ms - for req in completed - for _ in range(req.output_tokens - 1) - if req.output_tokens > 1 and req.inter_token_latency_ms - ], - errored_values=[ - req.inter_token_latency_ms - for req in errored - for _ in range(req.output_tokens - 1) - if req.output_tokens > 1 and req.inter_token_latency_ms - ], + completed_values=( + [ + req.inter_token_latency_ms + for req in completed + if req.output_tokens > 1 + ] + ), + errored_values=( + [ + req.inter_token_latency_ms + for req in errored + if req.output_tokens > 1 + ] + ), + completed_weights=( + [ + req.output_tokens - 1 + for req in completed + if req.output_tokens > 1 + ] + ), + errored_weights=( + [req.output_tokens - 1 for req in errored if req.output_tokens > 1] + ), + ), + outputs_tokens_per_second=StatusDistributionSummary.from_iterable_request_times( + completed_requests=( + [ + (req.start_time, req.end_time) + for req in completed + if req.output_tokens > 0 + ] + ), + errored_requests=( + [ + (req.start_time, req.end_time) + for req in errored + if req.output_tokens > 0 + ] + ), + completed_first_iter_times=( + [req.first_token_time for req in completed if req.output_tokens > 0] + ), + errored_first_iter_times=( + [req.first_token_time for req in errored if req.output_tokens > 0] + ), + completed_iter_counts=( + [req.output_tokens for req in completed if req.output_tokens > 0] + ), + errored_iter_counts=( + [req.output_tokens for req in errored if req.output_tokens > 0] + ), ), - outputs_tokens_per_second=StatusDistributionSummary.from_values( - completed_values=[ - req.output_tokens_per_second - for req in completed - for _ in range(req.output_tokens - 1) - if req.output_tokens > 1 and req.output_tokens_per_second - ], - errored_values=[ - req.output_tokens_per_second - for req in errored - for _ in range(req.output_tokens - 1) - if req.output_tokens > 1 and req.output_tokens_per_second - ], + tokens_per_second=StatusDistributionSummary.from_iterable_request_times( + completed_requests=( + [ + (req.start_time, req.end_time) + for req in completed + if req.prompt_tokens + req.output_tokens > 0 + ] + ), + errored_requests=( + [ + (req.start_time, req.end_time) + for req in errored + if req.prompt_tokens + req.output_tokens > 0 + ] + ), + completed_first_iter_times=( + [ + req.first_token_time + for req in completed + if req.prompt_tokens + req.output_tokens > 0 + ] + ), + errored_first_iter_times=( + [ + req.first_token_time + for req in errored + if req.prompt_tokens + req.output_tokens > 0 + ] + ), + completed_iter_counts=( + [ + req.prompt_tokens + req.output_tokens + for req in completed + if req.prompt_tokens + req.output_tokens > 0 + ] + ), + errored_iter_counts=( + [ + req.prompt_tokens + req.output_tokens + for req in errored + if req.prompt_tokens + req.output_tokens > 0 + ] + ), + completed_first_iter_counts=( + [ + req.prompt_tokens or 1 + for req in completed + if req.output_tokens > 0 + ] + ), + errored_first_iter_counts=( + [req.prompt_tokens or 1 for req in errored if req.output_tokens > 0] + ), ), ) diff --git a/src/guidellm/benchmark/benchmarker.py b/src/guidellm/benchmark/benchmarker.py index 62e27bd6..51636e82 100644 --- a/src/guidellm/benchmark/benchmarker.py +++ b/src/guidellm/benchmark/benchmarker.py @@ -1,7 +1,19 @@ import time +import uuid from abc import ABC, abstractmethod -from typing import Any, AsyncGenerator, Dict, Generic, Iterable, Literal, Optional +from pathlib import Path +from typing import ( + Any, + AsyncGenerator, + Dict, + Generic, + Iterable, + Literal, + Optional, + Union, +) +from pydantic import Field from transformers import PreTrainedTokenizer # type: ignore # noqa: PGH003 from guidellm.backend import Backend, ResponseSummary @@ -23,7 +35,7 @@ __all__ = ["Benchmarker", "BenchmarkerResult", "GenerativeBenchmarker"] -class BenchmarkerResult(Generic[AGG, BENCH, REQ, RES], Serializable): +class BenchmarkerResult(Serializable, Generic[AGG, BENCH, REQ, RES]): type_: Literal[ "run_start", "run_complete", @@ -37,11 +49,77 @@ class BenchmarkerResult(Generic[AGG, BENCH, REQ, RES], Serializable): profile: Profile current_index: int current_strategy: Optional[SchedulingStrategy] = None - current_aggregator: Optional[AGG[BENCH, REQ, RES]] = None + current_aggregator: Optional[AGG] = None current_benchmark: Optional[BENCH] = None current_result: Optional[SchedulerResult[REQ, RES]] = None +class BenchmarkerStrategyLimits(Serializable): + requests_loader_size: Optional[int] = Field( + description="Size of the request loader.", + ) + max_number_per_strategy: Optional[int] = Field( + description="Maximum number of requests to process per strategy.", + ge=0, + ) + max_duration_per_strategy: Optional[float] = Field( + description="Maximum duration (in seconds) to process requests per strategy.", + ge=0, + ) + warmup_percent_per_strategy: Optional[float] = Field( + description="Percentage of requests to use for warmup.", + ge=0, + le=1, + ) + cooldown_percent_per_strategy: Optional[float] = Field( + description="Percentage of requests to use for cooldown.", + ge=0, + le=1, + ) + + @property + def max_number(self) -> Optional[int]: + if self.max_number_per_strategy is not None: + return self.max_number_per_strategy + + if self.requests_loader_size is not None: + return self.requests_loader_size + + return None + + @property + def max_duration(self) -> Optional[float]: + return self.max_duration_per_strategy + + @property + def warmup_number(self) -> Optional[int]: + if self.warmup_percent_per_strategy is None or self.max_number is None: + return None + + return int(self.warmup_percent_per_strategy * self.max_number) + + @property + def warmup_duration(self) -> Optional[float]: + if self.warmup_percent_per_strategy is None or self.max_duration is None: + return None + + return self.warmup_percent_per_strategy * self.max_duration + + @property + def cooldown_number(self) -> Optional[int]: + if self.cooldown_percent_per_strategy is None or self.max_number is None: + return None + + return int(self.cooldown_percent_per_strategy * self.max_number) + + @property + def cooldown_duration(self) -> Optional[float]: + if self.cooldown_percent_per_strategy is None or self.max_duration is None: + return None + + return self.cooldown_percent_per_strategy * self.max_duration + + class Benchmarker(Generic[AGG, BENCH, REQ, RES], ABC): def __init__( self, @@ -62,14 +140,25 @@ async def run( profile: Profile, max_number_per_strategy: Optional[int], max_duration_per_strategy: Optional[float], - warmup_number_per_strategy: Optional[float], - warmup_duration_per_strategy: Optional[float], - cooldown_number_per_strategy: Optional[int], - cooldown_duration_per_strategy: Optional[float], + warmup_percent_per_strategy: Optional[float], + cooldown_percent_per_strategy: Optional[float], ) -> AsyncGenerator[BenchmarkerResult[AGG, BENCH, REQ, RES], None]: + try: + requests_loader_size = len(self.scheduler.request_loader) + except Exception: + requests_loader_size = None + + strategy_limits = BenchmarkerStrategyLimits( + requests_loader_size=requests_loader_size, + max_number_per_strategy=max_number_per_strategy, + max_duration_per_strategy=max_duration_per_strategy, + warmup_percent_per_strategy=warmup_percent_per_strategy, + cooldown_percent_per_strategy=cooldown_percent_per_strategy, + ) start_time = time.time() end_number = len(profile.strategy_types) current_index = -1 + run_id = str(uuid.uuid4()) yield BenchmarkerResult( type_="run_start", @@ -86,15 +175,16 @@ async def run( while scheduling_strategy := profile.next_strategy(): current_index += 1 aggregator: AGG[BENCH, REQ, RES] = self.create_benchmark_aggregator( + run_id=run_id, profile=profile, - current_index=current_index, + strategy_index=current_index, strategy=scheduling_strategy, - max_number=max_number_per_strategy, - max_duration=max_duration_per_strategy, - warmup_number=warmup_number_per_strategy, - warmup_duration=warmup_duration_per_strategy, - cooldown_number=cooldown_number_per_strategy, - cooldown_duration=cooldown_duration_per_strategy, + max_number=strategy_limits.max_number, + max_duration=strategy_limits.max_duration, + warmup_number=strategy_limits.warmup_number, + warmup_duration=strategy_limits.warmup_duration, + cooldown_number=strategy_limits.cooldown_number, + cooldown_duration=strategy_limits.cooldown_duration, ) yield BenchmarkerResult( @@ -183,7 +273,7 @@ def create_benchmark_aggregator( warmup_duration: Optional[float], cooldown_number: Optional[int], cooldown_duration: Optional[float], - ) -> AGG[BENCH, REQ, RES]: ... + ) -> AGG: ... class GenerativeBenchmarker( @@ -200,7 +290,7 @@ def __init__( request_loader: Iterable[GenerationRequest], request_loader_description: Optional[Serializable] = None, benchmark_save_extras: Optional[Dict[str, Any]] = None, - processor: Optional[PreTrainedTokenizer] = None, + processor: Optional[Union[str, Path, PreTrainedTokenizer]] = None, ): super().__init__( worker=GenerativeRequestsWorker(backend), @@ -237,5 +327,5 @@ def create_benchmark_aggregator( cooldown_duration=cooldown_duration, worker_description=self.worker.description, request_loader_description=self.requests_loader_description, - extras=self.benchmark_save_extras, + extras=self.benchmark_save_extras or {}, ) diff --git a/src/guidellm/benchmark/entrypoints.py b/src/guidellm/benchmark/entrypoints.py index 85b0d3e1..fa5d0aa3 100644 --- a/src/guidellm/benchmark/entrypoints.py +++ b/src/guidellm/benchmark/entrypoints.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Any, Callable, Dict, Iterable, List, Optional, Union +from typing import Any, Callable, Dict, Iterable, List, Literal, Optional, Union from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict from transformers import PreTrainedTokenizer @@ -9,8 +9,7 @@ from guidellm.benchmark.benchmarker import GenerativeBenchmarker from guidellm.benchmark.profile import ProfileType, create_profile from guidellm.benchmark.progress import BenchmarkerProgressDisplay -from guidellm.dataset import load_dataset -from guidellm.request import RequestLoader +from guidellm.request import GenerativeRequestLoader from guidellm.scheduler import StrategyType @@ -31,6 +30,7 @@ async def benchmark_generative_text( IterableDatasetDict, ], data_args: Optional[Dict[str, Any]], + data_sampler: Optional[Literal["random"]], rate_type: Union[StrategyType, ProfileType], rate: Optional[Union[int, float, List[Union[int, float]]]], max_seconds: Optional[float], @@ -41,30 +41,35 @@ async def benchmark_generative_text( output_path: Optional[Union[str, Path]], output_type: Optional[str], output_extras: Optional[Dict[str, Any]], + random_seed: int, ) -> List[GenerativeBenchmark]: backend = Backend.create( backend_type, target=target, model=model, **(backend_args or {}) ) - backend.validate() + await backend.validate() if processor is None: processor = backend.model - if isinstance(processor, (str, Path)): - processor = PreTrainedTokenizer.from_pretrained( - processor, **(processor_args or {}) - ) - - dataset = load_dataset(data, data_args, processor) - request_loader, requests_loader_description, processor = RequestLoader( - dataset, processor, processor_args + request_loader = GenerativeRequestLoader( + data=data, + data_args=data_args, + processor=processor, + processor_args=processor_args, + shuffle=data_sampler == "random", + iter_type=( + "finite" # assume a finite dataset is our limit + if max_requests is None and max_seconds is None + else "infinite" # default to infinite so we don't run out of data + ), + random_seed=random_seed, ) profile = create_profile(rate_type=rate_type, rate=rate) benchmarker = GenerativeBenchmarker( backend=backend, request_loader=request_loader, - request_loader_description=requests_loader_description, + request_loader_description=request_loader.description, benchmark_save_extras=output_extras, processor=processor, ) @@ -75,22 +80,8 @@ async def benchmark_generative_text( profile=profile, max_number_per_strategy=max_requests, max_duration_per_strategy=max_seconds, - warmup_number_per_strategy=( - round(max_requests * warmup_percent) - if max_requests and warmup_percent - else None - ), - warmup_duration_per_strategy=( - max_seconds * warmup_percent if max_seconds and warmup_percent else None - ), - cooldown_number_per_strategy=( - round(max_requests * cooldown_percent) - if max_requests and cooldown_percent - else None - ), - cooldown_duration_per_strategy=( - max_seconds * cooldown_percent if max_seconds and cooldown_percent else None - ), + warmup_percent_per_strategy=warmup_percent, + cooldown_percent_per_strategy=cooldown_percent, ): if progress: progress.update(result) diff --git a/src/guidellm/benchmark/profile.py b/src/guidellm/benchmark/profile.py index 969b2a74..4f7cd70c 100644 --- a/src/guidellm/benchmark/profile.py +++ b/src/guidellm/benchmark/profile.py @@ -198,6 +198,13 @@ class AsyncProfile(ThroughputProfile): "to reach target rate. False to not send an initial burst." ), ) + random_seed: int = Field( + default=42, + description=( + "The random seed to use for the asynchronous strategy. " + "This is used to generate random numbers for the Poisson strategy." + ), + ) @property def strategy_types(self) -> List[StrategyType]: @@ -222,6 +229,7 @@ def next_strategy(self) -> Optional[SchedulingStrategy]: rate=rate[self.completed_strategies], initial_burst=self.initial_burst, max_concurrency=self.max_concurrency, + random_seed=self.random_seed, ) else: raise ValueError(f"Invalid strategy type: {self.strategy_type}") @@ -230,6 +238,7 @@ def next_strategy(self) -> Optional[SchedulingStrategy]: def from_standard_args( rate_type: Union[StrategyType, ProfileType], rate: Optional[Union[float, Sequence[float]]], + random_seed: int, **kwargs, ) -> "AsyncProfile": if rate_type not in ("async", "constant", "poisson"): @@ -255,6 +264,7 @@ def from_standard_args( return AsyncProfile( strategy_type=rate_type, rate=rate, + random_seed=random_seed, **kwargs, ) @@ -270,7 +280,7 @@ class SweepProfile(AsyncProfile): @property def strategy_types(self) -> List[StrategyType]: return ( - ["synchronous"] + [self.rate_type] * (self.sweep_size - 2) + ["throughput"] + ["synchronous"] + ["throughput"] + [self.rate_type] * (self.sweep_size - 2) ) def next_strategy(self) -> Optional[SchedulingStrategy]: @@ -287,7 +297,7 @@ def next_strategy(self) -> Optional[SchedulingStrategy]: min_rate = self.measured_rates[0] max_rate = self.measured_rates[1] - rates = np.linspace(min_rate, max_rate, self.sweep_size)[1:-1] + rates = np.linspace(min_rate, max_rate, self.sweep_size - 1)[1:] if self.rate_type == "constant": return AsyncConstantStrategy( @@ -308,6 +318,7 @@ def next_strategy(self) -> Optional[SchedulingStrategy]: def from_standard_args( rate_type: Union[StrategyType, ProfileType], rate: Optional[Union[float, Sequence[float]]], + random_seed: int, **kwargs, ) -> "SweepProfile": if rate_type != "sweep": @@ -321,17 +332,28 @@ def from_standard_args( "Rate (sweep_size) must be provided for concurrent profile." ) - if not isinstance(rate, float) or not rate.is_integer() or rate <= 1: + if ( + not isinstance(rate, (int, float)) + or (isinstance(rate, float) and not rate.is_integer()) + or rate <= 1 + ): raise ValueError( f"Rate (sweep_size) must be a positive integer > 1, received {rate}" ) - return SweepProfile(sweep_size=rate, **kwargs) + if not kwargs: + kwargs = {} + + if "strategy_type" not in kwargs: + kwargs["strategy_type"] = "constant" + + return SweepProfile(sweep_size=rate, random_seed=random_seed, **kwargs) def create_profile( rate_type: Union[StrategyType, ProfileType], rate: Optional[Union[float, Sequence[float]]], + random_seed: int = 42, **kwargs, ) -> "Profile": if rate_type == "synchronous": @@ -359,6 +381,7 @@ def create_profile( return AsyncProfile.from_standard_args( rate_type=rate_type, rate=rate, + random_seed=random_seed, **kwargs, ) @@ -366,6 +389,7 @@ def create_profile( return SweepProfile.from_standard_args( rate_type=rate_type, rate=rate, + random_seed=random_seed, **kwargs, ) diff --git a/src/guidellm/benchmark/progress.py b/src/guidellm/benchmark/progress.py index 853045df..1e2e3dd4 100644 --- a/src/guidellm/benchmark/progress.py +++ b/src/guidellm/benchmark/progress.py @@ -1,9 +1,432 @@ +import math +import time +from abc import ABC +from dataclasses import dataclass +from datetime import datetime +from typing import Dict, List, Optional, Union + +from rich.console import Group +from rich.live import Live +from rich.panel import Panel +from rich.progress import ( + BarColumn, + Progress, + ProgressColumn, + SpinnerColumn, + TaskID, + TaskProgressColumn, + TextColumn, + TimeElapsedColumn, + TimeRemainingColumn, +) + from guidellm.benchmark.benchmarker import BenchmarkerResult +from guidellm.scheduler import ( + SchedulingStrategy, + StrategyType, +) + +SCHEDULING_STRATEGY_DESCRIPTIONS: Dict[StrategyType, str] = { + "synchronous": "synchronous", + "concurrent": "concurrent@{RATE}", + "throughput": "throughput", + "constant": "constant@{RATE}", + "poisson": "poisson@{RATE}", +} + + +@dataclass +class BenchmarkerTaskProgressState: + task_id: TaskID + strategy: Union[StrategyType, SchedulingStrategy] + started: bool = False + compiling: bool = False + ended: bool = False + + start_time: Optional[float] = None + max_number: Optional[int] = None + max_duration: Optional[float] = None + in_warmup: bool = False + in_cooldown: bool = False + + requests_rate: float = 0 + requests_latency: float = 0 + requests_processing: int = 0 + requests_completed: int = 0 + requests_errored: int = 0 + + output_tokens: float = 0 + prompt_tokens: float = 0 + output_tokens_rate: float = 0 + total_tokens_rate: float = 0 + tokens_ttft: float = 0 + tokens_itl: float = 0 + + @property + def description(self) -> str: + if self.strategy in StrategyType.__args__: + return SCHEDULING_STRATEGY_DESCRIPTIONS.get( + self.strategy, self.strategy + ).format(RATE="##" if self.strategy == "concurrent" else "#.##") + + rate = "" + + if hasattr(self.strategy, "streams"): + rate = f"{self.strategy.streams:>2}" + elif hasattr(self.strategy, "rate"): + rate = f"{self.strategy.rate:.2f}" + + return SCHEDULING_STRATEGY_DESCRIPTIONS.get( + self.strategy.type_, self.strategy.type_ + ).format(RATE=rate) + + @property + def total(self) -> Optional[float]: + if self.max_number is None and self.max_duration is None: + return None + + return 1000 + + @property + def completed(self) -> int: + if self.ended: + return 1000 + + if self.max_number is None and self.max_duration is None: + return 0 + + number = self.requests_completed + self.requests_errored + number_percent = ( + number / float(self.max_number) * 1000 if self.max_number else -math.inf + ) + duration_percent = ( + (time.time() - self.start_time) / self.max_duration * 1000 + if self.max_duration + else -math.inf + ) + + return min(int(max(number_percent, duration_percent)), 1000) + + @property + def fields(self) -> Dict[str, str]: + return { + "start_time": self.formatted_start_time, + "progress_status": self.formatted_progress_status, + "requests_summary": self.formatted_requests_summary, + "tokens_summary": self.formatted_tokens_summary, + } + + @property + def formatted_start_time(self) -> str: + if self.start_time is None: + return "--:--:--" + + return datetime.fromtimestamp(self.start_time).strftime("%H:%M:%S") + + @property + def formatted_progress_status(self) -> str: + if self.ended: + status = "complete" + elif self.compiling: + status = "compiling" + elif self.started and self.in_warmup: + status = "warmup" + elif self.started and self.in_cooldown: + status = "cooldown" + elif self.started: + status = "running" + else: + status = "pending" + + return status.ljust(9) + + @property + def formatted_requests_summary(self) -> str: + if not self.started: + return " " + + return ( + "Req: " + f"({self.requests_rate:.2f} / sec, " + f"{self.requests_latency:.2f}s Lat, " + f"{self.requests_processing:>3.1f} Conc, " + f"{self.requests_completed:>3} Comp, " + f"{self.requests_errored:>3} Err)" + ) + + @property + def formatted_tokens_summary(self) -> str: + if not self.started: + return " " + return ( + "Tok: " + f"({self.output_tokens_rate:.1f} gen/sec, " + f"{self.total_tokens_rate:.1f} tot/sec, " + f"{self.tokens_ttft:.1f}ms TTFT, " + f"{self.tokens_itl:.1f}ms ITL, " + f"{self.prompt_tokens:.0f} Prompt, " + f"{self.output_tokens:.0f} Gen)" + ) -class BenchmarkerProgressDisplay: + +class BenchmarkerProgressDisplay(ABC): def __init__(self): - pass + """ + Progress display view: + | Benchmarks -----------------------------------------------------------------| + | [T] N (S) Req: (#/sec, #sec, #proc, #com, #err) Tok: (#/sec, #TTFT, #ITL) | + | [T] % N (S) Req: (#/sec, #sec, #proc, #com, #err) Tok: (#/sec, #TTFT, #ITL) | + | ... | + | ----------------------------------------------------------------------------| + SP Running... [BAR] (#/#) [ ELAPSED < ETA ] + """ + self.started = False + self.benchmarker_tasks_progress = Progress(*self.create_task_progress_columns()) + self.benchmarker_tasks_panel = Panel( + self.benchmarker_tasks_progress, + title="Benchmarks", + title_align="left", + expand=True, + ) + self.benchmarker_progress = Progress( + TextColumn("Generating..."), + BarColumn(bar_width=None), + TextColumn( + "({task.fields[completed_benchmarks]}/{task.fields[total_benchmarks]})" + ), + TextColumn("["), + TimeElapsedColumn(), + TextColumn("<"), + TimeRemainingColumn(), + TextColumn("]"), + ) + self.benchmarker_live = Live( + Group( + self.benchmarker_tasks_panel, + self.benchmarker_progress, + ), + redirect_stdout=True, + redirect_stderr=True, + ) + self.active_task: Optional[TaskID] = None + self.benchmarker_tasks: List[BenchmarkerTaskProgressState] = [] + self.progress_task: Optional[TaskID] = None def update(self, result: BenchmarkerResult): - pass + if result.type_ == "run_start": + if self.started: + raise RuntimeError("Progress display already started.") + + self.handle_start(result) + self.started = True + elif result.type_ == "run_complete": + if not self.started: + raise RuntimeError("Progress display not started.") + + self.handle_end(result) + self.started = False + else: + if not self.started: + raise RuntimeError("Progress display not started.") + + self.handle_update(result) + + def handle_start(self, result: BenchmarkerResult): + self.benchmarker_live.start() + + for index, strategy_type in enumerate(result.profile.strategy_types): + task_id = self.benchmarker_tasks_progress.add_task( + description=strategy_type, + start=False, + total=None, + completed=0, + visible=False, + ) + task_progress_state = self.create_task_progress_state( + task_id=task_id, + index=index, + strategy_type=strategy_type, + result=result, + ) + self.benchmarker_tasks.append(task_progress_state) + self.benchmarker_tasks_progress.update( + task_id, + description=task_progress_state.description, + visible=True, + **task_progress_state.fields, + ) + + self.progress_task = self.benchmarker_progress.add_task( + "", + total=len(self.benchmarker_tasks) * 1000, + completed_benchmarks=0, + total_benchmarks=len(self.benchmarker_tasks), + ) + + def handle_update(self, result: BenchmarkerResult): + current_state = self.benchmarker_tasks[result.current_index] + + if result.type_ == "scheduler_start": + self.handle_update_scheduler_start(current_state, result) + self.active_task = current_state.task_id + elif result.type_ == "scheduler_update": + self.handle_update_scheduler_update(current_state, result) + elif result.type_ == "scheduler_complete": + self.handle_update_scheduler_complete(current_state, result) + elif result.type_ == "benchmark_compiled": + self.handle_update_benchmark_compiled(current_state, result) + else: + raise ValueError(f"Unknown result type: {result.type_}") + + self.benchmarker_tasks_progress.update( + current_state.task_id, + description=current_state.description, + completed=current_state.completed, + total=current_state.total, + **current_state.fields, + ) + self.benchmarker_progress.update( + self.progress_task, + completed=(result.current_index * 1000) + current_state.completed, + total=1000 * len(self.benchmarker_tasks), + completed_benchmarks=( + result.current_index + (1 if current_state.ended else 0) + ), + total_benchmarks=len(self.benchmarker_tasks), + ) + + if current_state.ended: + self.benchmarker_tasks_progress.stop_task(current_state.task_id) + self.active_task = None + + def handle_update_scheduler_start( + self, progress_state: BenchmarkerTaskProgressState, result: BenchmarkerResult + ): + if self.active_task is not None: + raise RuntimeError("Active task already set.") + + progress_state.strategy = result.current_strategy + progress_state.started = True + progress_state.start_time = result.current_aggregator.start_time + progress_state.max_number = result.current_aggregator.max_number + progress_state.max_duration = result.current_aggregator.max_duration + + def handle_update_scheduler_update( + self, progress_state: BenchmarkerTaskProgressState, result: BenchmarkerResult + ): + if self.active_task is None: + raise RuntimeError("Active task not set.") + + if self.active_task != progress_state.task_id: + raise RuntimeError("Active task does not match current task.") + + progress_state.in_warmup = result.current_aggregator.in_warmup + progress_state.in_cooldown = result.current_aggregator.in_cooldown + progress_state.requests_rate = result.current_aggregator.successful_requests / ( + time.time() - progress_state.start_time + ) + progress_state.requests_latency = result.current_aggregator.request_latency.mean + progress_state.requests_processing = ( + result.current_aggregator.processing_requests + ) + progress_state.requests_completed = ( + result.current_aggregator.successful_requests + ) + progress_state.requests_errored = result.current_aggregator.errored_requests + progress_state.output_tokens = result.current_aggregator.output_tokens.mean + progress_state.prompt_tokens = result.current_aggregator.prompt_tokens.mean + progress_state.output_tokens_rate = result.current_aggregator.output_tokens.rate + progress_state.total_tokens_rate = result.current_aggregator.total_tokens.rate + progress_state.tokens_ttft = result.current_aggregator.time_to_first_token.mean + progress_state.tokens_itl = result.current_aggregator.inter_token_latency.mean + + def handle_update_scheduler_complete( + self, progress_state: BenchmarkerTaskProgressState, result: BenchmarkerResult + ): + if self.active_task is None: + raise RuntimeError("Active task not set.") + + if self.active_task != progress_state.task_id: + raise RuntimeError("Active task does not match current task.") + + progress_state.in_warmup = False + progress_state.in_cooldown = False + progress_state.compiling = True + + def handle_update_benchmark_compiled( + self, progress_state: BenchmarkerTaskProgressState, result: BenchmarkerResult + ): + if self.active_task is None: + raise RuntimeError("Active task not set.") + + if self.active_task != progress_state.task_id: + raise RuntimeError("Active task does not match current task.") + + progress_state.compiling = False + progress_state.ended = True + progress_state.requests_rate = ( + result.current_benchmark.requests_per_second.completed.mean + ) + progress_state.requests_latency = ( + result.current_benchmark.requests_latency.completed.mean + ) + progress_state.requests_processing = ( + result.current_benchmark.requests_concurrency.completed.mean + ) + progress_state.requests_completed = result.current_benchmark.completed_total + progress_state.requests_errored = result.current_benchmark.errored_total + progress_state.output_tokens = ( + result.current_benchmark.outputs_token_count.completed.mean + ) + progress_state.prompt_tokens = ( + result.current_benchmark.prompts_token_count.completed.mean + ) + progress_state.output_tokens_rate = ( + result.current_benchmark.outputs_tokens_per_second.completed.mean + ) + progress_state.total_tokens_rate = ( + result.current_benchmark.tokens_per_second.completed.mean + ) + progress_state.tokens_ttft = ( + result.current_benchmark.times_to_first_token_ms.completed.mean + ) + progress_state.tokens_itl = ( + result.current_benchmark.inter_token_latencies_ms.completed.mean + ) + + def handle_end(self, result: BenchmarkerResult): + self.benchmarker_progress.update( + self.progress_task, + completed=len(self.benchmarker_tasks) * 1000, + total=len(self.benchmarker_tasks) * 1000, + completed_benchmarks=len(self.benchmarker_tasks), + total_benchmarks=len(self.benchmarker_tasks), + ) + self.benchmarker_progress.stop_task(self.progress_task) + self.benchmarker_live.stop() + self.active_task = None + self.benchmarker_tasks = [] + self.progress_task = None + + def create_task_progress_columns(self) -> List[ProgressColumn]: + return [ + TextColumn("[{task.fields[start_time]}]"), + SpinnerColumn(), + TaskProgressColumn(), + TextColumn("{task.description}"), + TextColumn("({task.fields[progress_status]})"), + TextColumn(" "), + TextColumn("{task.fields[requests_summary]}"), + TextColumn(" "), + TextColumn("{task.fields[tokens_summary]}"), + ] + + def create_task_progress_state( + self, + task_id: TaskID, + index: int, + strategy_type: StrategyType, + result: BenchmarkerResult, + ) -> BenchmarkerTaskProgressState: + return BenchmarkerTaskProgressState(task_id=task_id, strategy=strategy_type) diff --git a/src/guidellm/benchmark/test.py b/src/guidellm/benchmark/test.py new file mode 100644 index 00000000..df88daa6 --- /dev/null +++ b/src/guidellm/benchmark/test.py @@ -0,0 +1,41 @@ +import asyncio +import json + +from guidellm.benchmark.entrypoints import benchmark_generative_text + + +def run_benchmark_synthetic(): + results = asyncio.run( + benchmark_generative_text( + target="http://192.168.4.13:8000", + backend_type="openai_http", + backend_args=None, + model="neuralmagic/Qwen2.5-7B-quantized.w8a8", + processor=None, + processor_args=None, + data='{"prompt_tokens": 128, "output_tokens": 64}', + data_args=None, + data_sampler=None, + rate_type="sweep", + rate=5, + max_seconds=None, + max_requests=50, + warmup_percent=None, + cooldown_percent=None, + show_progress=True, + output_path=None, + output_type=None, + output_extras=None, + random_seed=42, + ) + ) + + dict_output = { + "benchmarks": [res.model_dump() for res in results], + } + with open("benchmarks.json", "w") as f: + json.dump(dict_output, f, indent=4) + + +if __name__ == "__main__": + run_benchmark_synthetic() diff --git a/src/guidellm/dataset/creator.py b/src/guidellm/dataset/creator.py index 7135d7e0..41eccd8a 100644 --- a/src/guidellm/dataset/creator.py +++ b/src/guidellm/dataset/creator.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +from pathlib import Path from typing import Any, Dict, List, Literal, Optional, Tuple, Union from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict @@ -77,15 +78,19 @@ def create( cls, data: Any, data_args: Optional[Dict[str, Any]], - processor: PreTrainedTokenizerBase, + processor: Optional[Union[str, Path, PreTrainedTokenizerBase]], + processor_args: Optional[Dict[str, Any]], + random_seed: int = 42, split_pref_order: Optional[List[str]] = None, ) -> Tuple[Union[Dataset, IterableDataset], Dict[ColumnInputTypes, str]]: - if not cls.is_supported(data): + if not cls.is_supported(data, data_args): raise ValueError(f"Unsupported data type: {type(data)} given for {data}. ") split = cls.extract_args_split(data_args) - column_mappings = cls.extract_args_column_mappings(data_args, processor) - dataset = cls.handle_create(data, data_args, processor) + column_mappings = cls.extract_args_column_mappings(data_args) + dataset = cls.handle_create( + data, data_args, processor, processor_args, random_seed + ) if isinstance(dataset, (DatasetDict, IterableDatasetDict)): dataset = cls.extract_dataset_split(dataset, split, split_pref_order) @@ -98,10 +103,10 @@ def create( return dataset, column_mappings @classmethod - def extract_args_split(cls, data_args: Dict[str, Any]) -> str: + def extract_args_split(cls, data_args: Optional[Dict[str, Any]]) -> str: split = "auto" - if "split" in data_args: + if data_args and "split" in data_args: split = data_args["split"] del data_args["split"] @@ -110,26 +115,26 @@ def extract_args_split(cls, data_args: Dict[str, Any]) -> str: @classmethod def extract_args_column_mappings( cls, - data_args: Dict[str, Any], - processor: PreTrainedTokenizerBase, + data_args: Optional[Dict[str, Any]], ) -> Dict[ColumnInputTypes, str]: columns = {} - if "prompt_column" in data_args: - columns["prompt_column"] = data_args["prompt_column"] - del data_args["prompt_column"] + if data_args: + if "prompt_column" in data_args: + columns["prompt_column"] = data_args["prompt_column"] + del data_args["prompt_column"] - if "prompt_tokens_count_column" in data_args: - columns["prompt_tokens_count_column"] = data_args[ - "prompt_tokens_count_column" - ] - del data_args["prompt_tokens_count_column"] + if "prompt_tokens_count_column" in data_args: + columns["prompt_tokens_count_column"] = data_args[ + "prompt_tokens_count_column" + ] + del data_args["prompt_tokens_count_column"] - if "output_tokens_count_column" in data_args: - columns["output_tokens_count_column"] = data_args[ - "output_tokens_count_column" - ] - del data_args["output_tokens_count_column"] + if "output_tokens_count_column" in data_args: + columns["output_tokens_count_column"] = data_args[ + "output_tokens_count_column" + ] + del data_args["output_tokens_count_column"] return columns @@ -199,5 +204,7 @@ def handle_create( cls, data: Any, data_args: Optional[Dict[str, Any]], - processor: PreTrainedTokenizerBase, + processor: Optional[Union[str, Path, PreTrainedTokenizerBase]], + processor_args: Optional[Dict[str, Any]], + random_seed: int, ) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]: ... diff --git a/src/guidellm/dataset/datasets.py b/src/guidellm/dataset/datasets.py index 9a3c60f8..4575d920 100644 --- a/src/guidellm/dataset/datasets.py +++ b/src/guidellm/dataset/datasets.py @@ -43,7 +43,9 @@ def handle_create( cls, data: Any, data_args: Optional[Dict[str, Any]], - processor: PreTrainedTokenizerBase, + processor: Optional[Union[str, Path, PreTrainedTokenizerBase]], + processor_args: Optional[Dict[str, Any]], + random_seed: int, ) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]: if isinstance(data, (str, Path)): data = load_dataset(data, **(data_args or {})) diff --git a/src/guidellm/dataset/entrypoints.py b/src/guidellm/dataset/entrypoints.py index 2ff06702..b7389f1e 100644 --- a/src/guidellm/dataset/entrypoints.py +++ b/src/guidellm/dataset/entrypoints.py @@ -1,8 +1,9 @@ -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Tuple, Union from datasets import Dataset, IterableDataset from transformers import PreTrainedTokenizerBase +from guidellm.dataset.creator import ColumnInputTypes, DatasetCreator from guidellm.dataset.datasets import HFDatasetsCreator from guidellm.dataset.file import FileDatasetCreator from guidellm.dataset.in_memory import InMemoryDatasetCreator @@ -15,9 +16,11 @@ def load_dataset( data: Any, data_args: Optional[Dict[str, Any]], processor: PreTrainedTokenizerBase, + processor_args: Optional[Dict[str, Any]], + random_seed: int = 42, split_pref_order: Optional[List[str]] = None, -) -> Union[Dataset, IterableDataset]: - creators = [ +) -> Tuple[Union[Dataset, IterableDataset], Dict[ColumnInputTypes, str]]: + creators: List[DatasetCreator] = [ InMemoryDatasetCreator, SyntheticDatasetCreator, FileDatasetCreator, @@ -26,6 +29,13 @@ def load_dataset( for creator in creators: if creator.is_supported(data, data_args): - return creator.create(data, data_args, processor, split_pref_order) + return creator.create( + data, + data_args, + processor, + processor_args, + random_seed, + split_pref_order, + ) raise ValueError(f"Unsupported data type: {type(data)} given for {data}. ") diff --git a/src/guidellm/dataset/file.py b/src/guidellm/dataset/file.py index 47143aec..1118db87 100644 --- a/src/guidellm/dataset/file.py +++ b/src/guidellm/dataset/file.py @@ -42,7 +42,9 @@ def handle_create( cls, data: Any, data_args: Optional[Dict[str, Any]], - processor: PreTrainedTokenizerBase, + processor: Optional[Union[str, Path, PreTrainedTokenizerBase]], + processor_args: Optional[Dict[str, Any]], + random_seed: int, ) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]: if not isinstance(data, (str, Path)): raise ValueError(f"Unsupported data type: {type(data)} given for {data}. ") diff --git a/src/guidellm/dataset/in_memory.py b/src/guidellm/dataset/in_memory.py index f41101e6..bd531b1f 100644 --- a/src/guidellm/dataset/in_memory.py +++ b/src/guidellm/dataset/in_memory.py @@ -1,3 +1,4 @@ +from pathlib import Path from typing import Any, Dict, Iterable, Optional, Union from datasets import ( @@ -23,7 +24,9 @@ def handle_create( cls, data: Any, data_args: Optional[Dict[str, Any]], - processor: PreTrainedTokenizerBase, + processor: Optional[Union[str, Path, PreTrainedTokenizerBase]], + processor_args: Optional[Dict[str, Any]], + random_seed: int, ) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]: if not isinstance(data, Iterable): raise TypeError( diff --git a/src/guidellm/dataset/synthetic.py b/src/guidellm/dataset/synthetic.py index b69c950a..6b7a57d2 100644 --- a/src/guidellm/dataset/synthetic.py +++ b/src/guidellm/dataset/synthetic.py @@ -16,6 +16,7 @@ from guidellm.config import settings from guidellm.dataset.creator import ColumnInputTypes, DatasetCreator from guidellm.objects import Serializable +from guidellm.utils import EndlessTextCreator, IntegerRangeSampler, check_load_processor __all__ = ["SyntheticDatasetCreator"] @@ -53,11 +54,7 @@ class SyntheticDatasetConfig(Serializable): ) samples: int = Field( description="The number of samples to generate for the dataset.", - default=10000, - ) - seed: int = Field( - description="The seed to use for random number generation.", - default=42, + default=1000, ) @staticmethod @@ -105,89 +102,16 @@ def parse_config_file(data: Union[str, Path]) -> "SyntheticDatasetConfig": return SyntheticDatasetConfig(**config_dict) -class IntegerRangeSampler: - def __init__( - self, - average: int, - variance: Optional[int], - min_value: Optional[int], - max_value: Optional[int], - seed: int, - ): - self.average = average - self.variance = variance - self.min_value = min_value - self.max_value = max_value - self.seed = seed - self.rng = random.Random(seed) - - def __iter__(self) -> Iterator[int]: - calc_min = self.min_value - if not calc_min: - calc_min = max( - 0, self.average - 5 * self.variance if self.variance else self.average - ) - calc_max = self.max_value - if not calc_max: - calc_max = ( - self.average + 5 * self.variance if self.variance else self.average - ) - - while True: - if calc_min == calc_max: - yield calc_min - elif not self.variance: - yield self.rng.randint(calc_min, calc_max + 1) - else: - rand = self.rng.gauss(self.average, self.variance) - yield round(max(calc_min, min(calc_max, rand))) - - -class EndlessTextCreator: - """ - A list subclass that allows for endless data generation. - """ - - def __init__( - self, - data: Union[str, Path], - filter_start: Optional[Union[str, int]] = None, - filter_end: Optional[Union[str, int]] = None, - ): - self.data = data - text = load_text(data) - text = filter_text(data, filter_start, filter_end) - self.words = split_text(text) - - def create_text(self, start: int, length: int) -> str: - """ - Create a text snippet from the specified range. - - :param start: Start index. - :type start: int - :param length: Length of the snippet. - :type length: int - :return: Text snippet. - :rtype: str - """ - start = start % len(self) - text = "" - - for counter in range(length): - index = (start + counter) % len(self.words) - if counter > 0: - text += " " - text += self.words[index] - - return text - - class SyntheticTextItemsGenerator(Iterable[Dict[str, Union[str, int]]]): def __init__( - self, config: SyntheticDatasetConfig, processor: PreTrainedTokenizerBase + self, + config: SyntheticDatasetConfig, + processor: PreTrainedTokenizerBase, + random_seed: int, ): self.config = config self.processor = processor + self.random_seed = random_seed self.tokens = [] self.text_creator = EndlessTextCreator( data=settings.emulated_data.source, @@ -201,25 +125,23 @@ def __iter__(self) -> Iterator[Tuple[str, int, int]]: variance=self.config.prompt_tokens_variance, min_value=self.config.prompt_tokens_min, max_value=self.config.prompt_tokens_max, - seed=self.config.seed, + random_seed=self.random_seed, ) output_tokens_sampler = IntegerRangeSampler( average=self.config.output_tokens, variance=self.config.output_tokens_variance, min_value=self.config.output_tokens_min, max_value=self.config.output_tokens_max, - seed=self.config.seed, - ) - start_index_sampler = random.Random(self.config.seed).randint( - 0, len(self.text_creator.words) + random_seed=self.random_seed, ) + rand = random.Random(self.random_seed) - for _, prompt_tokens, output_tokens, start_index in zip( + for _, prompt_tokens, output_tokens in zip( range(self.config.samples), prompt_tokens_sampler, output_tokens_sampler, - start_index_sampler, ): + start_index = rand.randint(0, len(self.text_creator.words)) yield { "prompt": self._create_prompt(prompt_tokens, start_index), "prompt_tokens_count": prompt_tokens, @@ -227,8 +149,8 @@ def __iter__(self) -> Iterator[Tuple[str, int, int]]: } def _create_prompt(self, prompt_tokens: int, start_index: int) -> str: - left = start_index - right = start_index + 5 * prompt_tokens + left = max(1, start_index - 2 * prompt_tokens) + right = start_index + 2 * prompt_tokens while left < right: mid = (left + right) // 2 @@ -271,19 +193,36 @@ def handle_create( cls, data: Any, data_args: Optional[Dict[str, Any]], - processor: PreTrainedTokenizerBase, + processor: Optional[Union[str, Path, PreTrainedTokenizerBase]], + processor_args: Optional[Dict[str, Any]], + random_seed: int, ) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]: + processor = check_load_processor( + processor, + processor_args, + error_msg=( + "Processor/tokenizer required for synthetic dataset generation." + ), + ) + config = SyntheticDatasetConfig.parse_str(data) - generator = SyntheticTextItemsGenerator(config, processor) + generator = SyntheticTextItemsGenerator(config, processor, random_seed) items = list(generator) - return Dataset.from_list(items) + return Dataset.from_list(items, **(data_args or {})) @classmethod def extract_args_column_mappings( - cls, data_args: Dict[str, Any], processor: PreTrainedTokenizerBase + cls, + data_args: Optional[Dict[str, Any]], ) -> Dict[ColumnInputTypes, str]: - super().extract_args_column_mappings(data_args) + data_args_columns = super().extract_args_column_mappings(data_args) + + if data_args_columns: + raise ValueError( + f"Column mappings are not supported for synthetic datasets. " + f"Got {data_args_columns}" + ) return { "prompt_column": "prompt", diff --git a/src/guidellm/main.py b/src/guidellm/main.py index e7363c6e..56b2456c 100644 --- a/src/guidellm/main.py +++ b/src/guidellm/main.py @@ -1,346 +1,346 @@ -import asyncio -from typing import Any, Literal, Mapping, Optional, Union, get_args +# import asyncio +# from typing import Any, Literal, Mapping, Optional, Union, get_args -import click -from loguru import logger -from transformers import AutoTokenizer # type: ignore[import-untyped] +# import click +# from loguru import logger +# from transformers import AutoTokenizer # type: ignore[import-untyped] -from guidellm.backend import Backend, BackendType -from guidellm.core import GuidanceReport, TextGenerationBenchmarkReport -from guidellm.executor import Executor, ProfileGenerationMode -from guidellm.request import ( - EmulatedRequestGenerator, - FileRequestGenerator, - TransformersDatasetRequestGenerator, -) -from guidellm.request.base import RequestGenerator -from guidellm.utils import BenchmarkReportProgress, cli_params +# from guidellm.backend import Backend, BackendType +# from guidellm.core import GuidanceReport, TextGenerationBenchmarkReport +# from guidellm.executor import Executor, ProfileGenerationMode +# from guidellm.request import ( +# EmulatedRequestGenerator, +# FileRequestGenerator, +# TransformersDatasetRequestGenerator, +# ) +# from guidellm.request.base import RequestGenerator +# from guidellm.utils import BenchmarkReportProgress, cli_params -__all__ = ["generate_benchmark_report"] +# __all__ = ["generate_benchmark_report"] -@click.command() -@click.option( - "--target", - type=str, - required=True, - help=( - "The target path or url for the backend to evaluate. " - "Ex: 'http://localhost:8000'" - ), -) -@click.option( - "--backend", - type=click.Choice(get_args(BackendType)), - default="openai_http", - help=( - "The backend to use for benchmarking. " - "The default is OpenAI Server enabling compatability with any server that " - "follows the OpenAI spec including vLLM." - ), -) -@click.option( - "--model", - type=str, - default=None, - help=( - "The Model to use for benchmarking. If not provided, it will use " - "the first available model provided the backend supports listing models." - ), -) -@click.option( - "--data", - type=str, - required=True, - help=( - "The data source to use for benchmarking. " - "Depending on the data-type, it should be a " - "path to a data file containing prompts to run (ex: data.txt), " - "a HuggingFace dataset name (ex: 'neuralmagic/LLM_compression_calibration'), " - "or a configuration for emulated data " - "(ex: 'prompt_tokens=128,generated_tokens=128')." - ), -) -@click.option( - "--data-type", - type=click.Choice(["emulated", "file", "transformers"]), - required=True, - help=( - "The type of data to use for benchmarking. " - "Use 'emulated' for synthetic data, 'file' for a file, or 'transformers' " - "for a HuggingFace dataset. Specify the data source with the --data flag." - ), -) -@click.option( - "--tokenizer", - type=str, - default=None, - help=( - "The tokenizer to use for calculating the number of prompt tokens. " - "This should match the tokenizer used by the model." - "By default, it will use the --model flag to determine the tokenizer. " - "If not provided and the model is not available, will raise an error. " - "Ex: 'neuralmagic/Meta-Llama-3.1-8B-quantized.w8a8'" - ), -) -@click.option( - "--rate-type", - type=click.Choice(get_args(ProfileGenerationMode)), - default="sweep", - help=( - "The type of request rate to use for benchmarking. " - "Use sweep to run a full range from synchronous to throughput (default), " - "synchronous for sending requests one after the other, " - "throughput to send requests as fast as possible, " - "constant for a fixed request rate, " - "or poisson for a real-world variable request rate." - ), -) -@click.option( - "--rate", - type=float, - default=None, - help=( - "The request rate to use for constant and poisson rate types. " - "To run multiple, provide the flag multiple times. " - ), - multiple=True, -) -@click.option( - "--max-seconds", - type=int, - default=120, - help=( - "The maximum number of seconds for each benchmark run. " - "Either max-seconds, max-requests, or both must be set. " - "The default is 120 seconds. " - "Note, this is the maximum time for each rate supplied, not the total time. " - "This value should be large enough to allow for " - "the server's performance to stabilize." - ), -) -@click.option( - "--max-requests", - type=cli_params.MAX_REQUESTS, - default=None, - help=( - "The maximum number of requests for each benchmark run. " - "Either max-seconds, max-requests, or both must be set. " - "Note, this is the maximum number of requests for each rate supplied, " - "not the total number of requests. " - "This value should be large enough to allow for " - "the server's performance to stabilize." - ), -) -@click.option( - "--output-path", - type=str, - default=None, - help=( - "The output path to save the output report to for loading later. " - "Ex: guidance_report.json. " - "The default is None, meaning no output is saved and results are only " - "printed to the console." - ), -) -@click.option( - "--enable-continuous-refresh", - is_flag=True, - default=False, - help=( - "Enable continual refreshing of the output table in the CLI " - "until the user exits. " - ), -) -def generate_benchmark_report_cli( - target: str, - backend: BackendType, - model: Optional[str], - data: Optional[str], - data_type: Literal["emulated", "file", "transformers"], - tokenizer: Optional[str], - rate_type: ProfileGenerationMode, - rate: Optional[float], - max_seconds: Optional[int], - max_requests: Union[Literal["dataset"], int, None], - output_path: str, - enable_continuous_refresh: bool, -): - """ - Generate a benchmark report for a specified backend and dataset. - """ - generate_benchmark_report( - target=target, - backend=backend, - model=model, - data=data, - data_type=data_type, - tokenizer=tokenizer, - rate_type=rate_type, - rate=rate, - max_seconds=max_seconds, - max_requests=max_requests, - output_path=output_path, - cont_refresh_table=enable_continuous_refresh, - ) +# @click.command() +# @click.option( +# "--target", +# type=str, +# required=True, +# help=( +# "The target path or url for the backend to evaluate. " +# "Ex: 'http://localhost:8000'" +# ), +# ) +# @click.option( +# "--backend", +# type=click.Choice(get_args(BackendType)), +# default="openai_http", +# help=( +# "The backend to use for benchmarking. " +# "The default is OpenAI Server enabling compatability with any server that " +# "follows the OpenAI spec including vLLM." +# ), +# ) +# @click.option( +# "--model", +# type=str, +# default=None, +# help=( +# "The Model to use for benchmarking. If not provided, it will use " +# "the first available model provided the backend supports listing models." +# ), +# ) +# @click.option( +# "--data", +# type=str, +# required=True, +# help=( +# "The data source to use for benchmarking. " +# "Depending on the data-type, it should be a " +# "path to a data file containing prompts to run (ex: data.txt), " +# "a HuggingFace dataset name (ex: 'neuralmagic/LLM_compression_calibration'), " +# "or a configuration for emulated data " +# "(ex: 'prompt_tokens=128,generated_tokens=128')." +# ), +# ) +# @click.option( +# "--data-type", +# type=click.Choice(["emulated", "file", "transformers"]), +# required=True, +# help=( +# "The type of data to use for benchmarking. " +# "Use 'emulated' for synthetic data, 'file' for a file, or 'transformers' " +# "for a HuggingFace dataset. Specify the data source with the --data flag." +# ), +# ) +# @click.option( +# "--tokenizer", +# type=str, +# default=None, +# help=( +# "The tokenizer to use for calculating the number of prompt tokens. " +# "This should match the tokenizer used by the model." +# "By default, it will use the --model flag to determine the tokenizer. " +# "If not provided and the model is not available, will raise an error. " +# "Ex: 'neuralmagic/Meta-Llama-3.1-8B-quantized.w8a8'" +# ), +# ) +# @click.option( +# "--rate-type", +# type=click.Choice(get_args(ProfileGenerationMode)), +# default="sweep", +# help=( +# "The type of request rate to use for benchmarking. " +# "Use sweep to run a full range from synchronous to throughput (default), " +# "synchronous for sending requests one after the other, " +# "throughput to send requests as fast as possible, " +# "constant for a fixed request rate, " +# "or poisson for a real-world variable request rate." +# ), +# ) +# @click.option( +# "--rate", +# type=float, +# default=None, +# help=( +# "The request rate to use for constant and poisson rate types. " +# "To run multiple, provide the flag multiple times. " +# ), +# multiple=True, +# ) +# @click.option( +# "--max-seconds", +# type=int, +# default=120, +# help=( +# "The maximum number of seconds for each benchmark run. " +# "Either max-seconds, max-requests, or both must be set. " +# "The default is 120 seconds. " +# "Note, this is the maximum time for each rate supplied, not the total time. " +# "This value should be large enough to allow for " +# "the server's performance to stabilize." +# ), +# ) +# @click.option( +# "--max-requests", +# type=cli_params.MAX_REQUESTS, +# default=None, +# help=( +# "The maximum number of requests for each benchmark run. " +# "Either max-seconds, max-requests, or both must be set. " +# "Note, this is the maximum number of requests for each rate supplied, " +# "not the total number of requests. " +# "This value should be large enough to allow for " +# "the server's performance to stabilize." +# ), +# ) +# @click.option( +# "--output-path", +# type=str, +# default=None, +# help=( +# "The output path to save the output report to for loading later. " +# "Ex: guidance_report.json. " +# "The default is None, meaning no output is saved and results are only " +# "printed to the console." +# ), +# ) +# @click.option( +# "--enable-continuous-refresh", +# is_flag=True, +# default=False, +# help=( +# "Enable continual refreshing of the output table in the CLI " +# "until the user exits. " +# ), +# ) +# def generate_benchmark_report_cli( +# target: str, +# backend: BackendType, +# model: Optional[str], +# data: Optional[str], +# data_type: Literal["emulated", "file", "transformers"], +# tokenizer: Optional[str], +# rate_type: ProfileGenerationMode, +# rate: Optional[float], +# max_seconds: Optional[int], +# max_requests: Union[Literal["dataset"], int, None], +# output_path: str, +# enable_continuous_refresh: bool, +# ): +# """ +# Generate a benchmark report for a specified backend and dataset. +# """ +# generate_benchmark_report( +# target=target, +# backend=backend, +# model=model, +# data=data, +# data_type=data_type, +# tokenizer=tokenizer, +# rate_type=rate_type, +# rate=rate, +# max_seconds=max_seconds, +# max_requests=max_requests, +# output_path=output_path, +# cont_refresh_table=enable_continuous_refresh, +# ) -def generate_benchmark_report( - target: str, - data: Optional[str], - data_type: Literal["emulated", "file", "transformers"], - backend: BackendType = "openai_http", - backend_kwargs: Optional[Mapping[str, Any]] = None, - model: Optional[str] = None, - tokenizer: Optional[str] = None, - rate_type: ProfileGenerationMode = "sweep", - rate: Optional[float] = None, - max_seconds: Optional[int] = 120, - max_requests: Union[Literal["dataset"], int, None] = None, - output_path: Optional[str] = None, - cont_refresh_table: bool = False, -) -> GuidanceReport: - """ - Generate a benchmark report for a specified backend and dataset. +# def generate_benchmark_report( +# target: str, +# data: Optional[str], +# data_type: Literal["emulated", "file", "transformers"], +# backend: BackendType = "openai_http", +# backend_kwargs: Optional[Mapping[str, Any]] = None, +# model: Optional[str] = None, +# tokenizer: Optional[str] = None, +# rate_type: ProfileGenerationMode = "sweep", +# rate: Optional[float] = None, +# max_seconds: Optional[int] = 120, +# max_requests: Union[Literal["dataset"], int, None] = None, +# output_path: Optional[str] = None, +# cont_refresh_table: bool = False, +# ) -> GuidanceReport: +# """ +# Generate a benchmark report for a specified backend and dataset. - :param target: The target URL or path for the backend to evaluate. - :param backend: The backend type to use for benchmarking. - :param model: The model to benchmark; - defaults to the first available if not specified. - :param data: The data source for benchmarking, - which may be a path, dataset name, or config. - :param data_type: The type of data to use, - such as 'emulated', 'file', or 'transformers'. - :param tokenizer: The tokenizer to use for token counting, - defaulting to Llama 3.1 if not provided. - :param rate_type: The rate type for requests during benchmarking. - :param rate: The specific request rate for constant and poisson rate types. - :param max_seconds: Maximum duration for each benchmark run in seconds. - :param max_requests: Maximum number of requests per benchmark run. - :param output_path: Path to save the output report file. - :param cont_refresh_table: Continually refresh the table in the CLI - until the user exits. - :param backend_kwargs: Additional keyword arguments for the backend. - """ - logger.info( - "Generating benchmark report with target: {}, backend: {}", target, backend - ) +# :param target: The target URL or path for the backend to evaluate. +# :param backend: The backend type to use for benchmarking. +# :param model: The model to benchmark; +# defaults to the first available if not specified. +# :param data: The data source for benchmarking, +# which may be a path, dataset name, or config. +# :param data_type: The type of data to use, +# such as 'emulated', 'file', or 'transformers'. +# :param tokenizer: The tokenizer to use for token counting, +# defaulting to Llama 3.1 if not provided. +# :param rate_type: The rate type for requests during benchmarking. +# :param rate: The specific request rate for constant and poisson rate types. +# :param max_seconds: Maximum duration for each benchmark run in seconds. +# :param max_requests: Maximum number of requests per benchmark run. +# :param output_path: Path to save the output report file. +# :param cont_refresh_table: Continually refresh the table in the CLI +# until the user exits. +# :param backend_kwargs: Additional keyword arguments for the backend. +# """ +# logger.info( +# "Generating benchmark report with target: {}, backend: {}", target, backend +# ) - # Create backend - backend_inst = Backend.create( - type_=backend, - target=target, - model=model, - **(backend_kwargs or {}), - ) - backend_inst.validate() +# # Create backend +# backend_inst = Backend.create( +# type_=backend, +# target=target, +# model=model, +# **(backend_kwargs or {}), +# ) +# backend_inst.validate() - request_generator: RequestGenerator +# request_generator: RequestGenerator - # Create tokenizer and request generator - tokenizer_inst = tokenizer - if not tokenizer_inst: - try: - tokenizer_inst = AutoTokenizer.from_pretrained(backend_inst.model) - except Exception as err: - raise ValueError( - "Could not load model's tokenizer, " - "--tokenizer must be provided for request generation" - ) from err +# # Create tokenizer and request generator +# tokenizer_inst = tokenizer +# if not tokenizer_inst: +# try: +# tokenizer_inst = AutoTokenizer.from_pretrained(backend_inst.model) +# except Exception as err: +# raise ValueError( +# "Could not load model's tokenizer, " +# "--tokenizer must be provided for request generation" +# ) from err - if data_type == "emulated": - request_generator = EmulatedRequestGenerator( - config=data, tokenizer=tokenizer_inst - ) - elif data_type == "file": - request_generator = FileRequestGenerator(path=data, tokenizer=tokenizer_inst) - elif data_type == "transformers": - request_generator = TransformersDatasetRequestGenerator( - dataset=data, tokenizer=tokenizer_inst - ) - else: - raise ValueError(f"Unknown data type: {data_type}") +# if data_type == "emulated": +# request_generator = EmulatedRequestGenerator( +# config=data, tokenizer=tokenizer_inst +# ) +# elif data_type == "file": +# request_generator = FileRequestGenerator(path=data, tokenizer=tokenizer_inst) +# elif data_type == "transformers": +# request_generator = TransformersDatasetRequestGenerator( +# dataset=data, tokenizer=tokenizer_inst +# ) +# else: +# raise ValueError(f"Unknown data type: {data_type}") - if data_type == "emulated" and max_requests == "dataset": - raise ValueError("Cannot use 'dataset' for emulated data") +# if data_type == "emulated" and max_requests == "dataset": +# raise ValueError("Cannot use 'dataset' for emulated data") - # Create executor - executor = Executor( - backend=backend_inst, - request_generator=request_generator, - mode=rate_type, - rate=rate if rate_type in ("constant", "poisson") else None, - max_number=( - len(request_generator) if max_requests == "dataset" else max_requests - ), - max_duration=max_seconds, - ) +# # Create executor +# executor = Executor( +# backend=backend_inst, +# request_generator=request_generator, +# mode=rate_type, +# rate=rate if rate_type in ("constant", "poisson") else None, +# max_number=( +# len(request_generator) if max_requests == "dataset" else max_requests +# ), +# max_duration=max_seconds, +# ) - # Run executor - logger.debug( - "Running executor with args: {}", - { - "backend": backend, - "request_generator": request_generator, - "mode": rate_type, - "rate": rate, - "max_number": max_requests, - "max_duration": max_seconds, - }, - ) - report = asyncio.run(_run_executor_for_result(executor)) +# # Run executor +# logger.debug( +# "Running executor with args: {}", +# { +# "backend": backend, +# "request_generator": request_generator, +# "mode": rate_type, +# "rate": rate, +# "max_number": max_requests, +# "max_duration": max_seconds, +# }, +# ) +# report = asyncio.run(_run_executor_for_result(executor)) - # Save and print report - guidance_report = GuidanceReport() - guidance_report.benchmarks.append(report) +# # Save and print report +# guidance_report = GuidanceReport() +# guidance_report.benchmarks.append(report) - if output_path: - guidance_report.save_file(output_path) +# if output_path: +# guidance_report.save_file(output_path) - guidance_report.print( - save_path=output_path if output_path is not None else "stdout", - continual_refresh=cont_refresh_table, - ) +# guidance_report.print( +# save_path=output_path if output_path is not None else "stdout", +# continual_refresh=cont_refresh_table, +# ) - return guidance_report +# return guidance_report -async def _run_executor_for_result(executor: Executor) -> TextGenerationBenchmarkReport: - report = None - progress = BenchmarkReportProgress() - started = False +# async def _run_executor_for_result(executor: Executor) -> TextGenerationBenchmarkReport: +# report = None +# progress = BenchmarkReportProgress() +# started = False - async for result in executor.run(): - if not started: - progress.start(result.generation_modes) # type: ignore # noqa: PGH003 - started = True +# async for result in executor.run(): +# if not started: +# progress.start(result.generation_modes) # type: ignore # noqa: PGH003 +# started = True - if result.current_index is not None: - description = f"{result.current_profile.load_gen_mode}" # type: ignore # noqa: PGH003 - if result.current_profile.load_gen_mode in ("constant", "poisson"): # type: ignore # noqa: PGH003 - description += f"@{result.current_profile.load_gen_rate:.2f} req/s" # type: ignore # noqa: PGH003 +# if result.current_index is not None: +# description = f"{result.current_profile.load_gen_mode}" # type: ignore # noqa: PGH003 +# if result.current_profile.load_gen_mode in ("constant", "poisson"): # type: ignore # noqa: PGH003 +# description += f"@{result.current_profile.load_gen_rate:.2f} req/s" # type: ignore # noqa: PGH003 - progress.update_benchmark( - index=result.current_index, - description=description, - completed=result.scheduler_result.completed, # type: ignore # noqa: PGH003 - completed_count=result.scheduler_result.count_completed, # type: ignore # noqa: PGH003 - completed_total=result.scheduler_result.count_total, # type: ignore # noqa: PGH003 - start_time=result.scheduler_result.benchmark.start_time, # type: ignore # noqa: PGH003 - req_per_sec=result.scheduler_result.benchmark.completed_request_rate, # type: ignore # noqa: PGH003 - ) +# progress.update_benchmark( +# index=result.current_index, +# description=description, +# completed=result.scheduler_result.completed, # type: ignore # noqa: PGH003 +# completed_count=result.scheduler_result.count_completed, # type: ignore # noqa: PGH003 +# completed_total=result.scheduler_result.count_total, # type: ignore # noqa: PGH003 +# start_time=result.scheduler_result.benchmark.start_time, # type: ignore # noqa: PGH003 +# req_per_sec=result.scheduler_result.benchmark.completed_request_rate, # type: ignore # noqa: PGH003 +# ) - if result.completed: - report = result.report - break +# if result.completed: +# report = result.report +# break - progress.finish() +# progress.finish() - if not report: - raise ValueError("No report generated by executor") +# if not report: +# raise ValueError("No report generated by executor") - return report +# return report -if __name__ == "__main__": - generate_benchmark_report_cli() +# if __name__ == "__main__": +# generate_benchmark_report_cli() diff --git a/src/guidellm/objects/__init__.py b/src/guidellm/objects/__init__.py index d5c428b4..724e5930 100644 --- a/src/guidellm/objects/__init__.py +++ b/src/guidellm/objects/__init__.py @@ -1,5 +1,10 @@ -from .distribution import DistributionSummary, Percentiles, StatusDistributionSummary from .serializable import Serializable, SerializableFileType +from .statistics import ( + DistributionSummary, + Percentiles, + RunningStats, + StatusDistributionSummary, +) __all__ = [ "Percentiles", @@ -7,4 +12,5 @@ "StatusDistributionSummary", "Serializable", "SerializableFileType", + "RunningStats", ] diff --git a/src/guidellm/objects/distribution.py b/src/guidellm/objects/distribution.py deleted file mode 100644 index f26a8f30..00000000 --- a/src/guidellm/objects/distribution.py +++ /dev/null @@ -1,516 +0,0 @@ -import math -from collections import defaultdict -from typing import List, Tuple - -import numpy as np -from pydantic import Field - -from guidellm.objects import Serializable - -__all__ = [ - "Percentiles", - "DistributionSummary", - "StatusDistributionSummary", -] - - -class Percentiles(Serializable): - """ - A serializable model representing percentiles of a distribution. - """ - - p001: float = Field( - description="The 0.1th percentile of the distribution.", - ) - p01: float = Field( - description="The 1st percentile of the distribution.", - ) - p05: float = Field( - description="The 5th percentile of the distribution.", - ) - p10: float = Field( - description="The 10th percentile of the distribution.", - ) - p25: float = Field( - description="The 25th percentile of the distribution.", - ) - p75: float = Field( - description="The 75th percentile of the distribution.", - ) - p90: float = Field( - description="The 90th percentile of the distribution.", - ) - p95: float = Field( - description="The 95th percentile of the distribution.", - ) - p99: float = Field( - description="The 99th percentile of the distribution.", - ) - p999: float = Field( - description="The 99.9th percentile of the distribution.", - ) - - @staticmethod - def from_values(values: List[float]) -> "Percentiles": - """ - Calculate percentiles from a list of values. - If the list is empty, all percentiles are set to 0. - - :param values: A list of numerical values. - :return: An instance of Percentiles with calculated percentiles. - """ - if not values: - return Percentiles( - p001=0.0, - p01=0.0, - p05=0.0, - p10=0.0, - p25=0.0, - p75=0.0, - p90=0.0, - p95=0.0, - p99=0.0, - p999=0.0, - ) - - percentiles = np.percentile(values, [0.1, 1, 5, 10, 25, 75, 90, 95, 99, 99.9]) - return Percentiles( - p001=percentiles[0], - p01=percentiles[1], - p05=percentiles[2], - p10=percentiles[3], - p25=percentiles[4], - p75=percentiles[5], - p90=percentiles[6], - p95=percentiles[7], - p99=percentiles[8], - p999=percentiles[9], - ) - - -class DistributionSummary(Serializable): - """ - A serializable model representing a statistical summary for a given - distribution of numerical values. - """ - - mean: float = Field( - description="The mean/average of the distribution.", - ) - median: float = Field( - description="The median of the distribution.", - ) - variance: float = Field( - description="The variance of the distribution.", - ) - std_dev: float = Field( - description="The standard deviation of the distribution.", - ) - min: float = Field( - description="The minimum value of the distribution.", - ) - max: float = Field( - description="The maximum value of the distribution.", - ) - count: int = Field( - description="The number of values in the distribution.", - ) - percentiles: Percentiles = Field( - description="The percentiles of the distribution.", - ) - - @staticmethod - def from_values(values: List[float]) -> "DistributionSummary": - """ - Calculate a distribution summary from a list of values. - If the list is empty, all values are set to 0. - - :param values: A list of numerical values. - :return: An instance of DistributionSummary with calculated values. - """ - - if not values: - return DistributionSummary( - mean=0.0, - median=0.0, - variance=0.0, - std_dev=0.0, - min=0.0, - max=0.0, - count=0, - percentiles=Percentiles.from_values([]), - ) - - return DistributionSummary( - mean=float(np.mean(values)), - median=float(np.median(values)), - variance=float(np.var(values)), - std_dev=float(np.std(values)), - min=float(np.min(values)), - max=float(np.max(values)), - count=len(values), - percentiles=Percentiles.from_values(values), - ) - - @staticmethod - def from_timestamped_values( - values: List[Tuple[float, float]], - ) -> "DistributionSummary": - """ - Calculate a distribution summary from a list of timestamped values. - Specifically, this calculates the statistics assuming a piecewise - continuous distribution of values over time. - For example, rather than finding the average concurrency of requests - over a given time period, this will calculate that along with other - statistics such as the variance and percentiles. - If the list is empty, all values are set to 0. - If the list contains only one value, all values are set to that value. - Note, since this is calculating statistics over time, the values - should contain the entire time range. Generally, this means the first - value should be the start time with a measurement of 0. - - :param values: A list of timestamped numerical values of the form - (timestamp, value). - :return: An instance of DistributionSummary with calculated values. - """ - - if not values: - return DistributionSummary( - mean=0.0, - median=0.0, - variance=0.0, - std_dev=0.0, - min=0.0, - max=0.0, - count=0, - percentiles=Percentiles.from_values([]), - ) - - if len(values) == 1: - return DistributionSummary( - mean=values[0][1], - median=values[0][1], - variance=0.0, - std_dev=0.0, - min=values[0][1], - max=values[0][1], - count=1, - percentiles=Percentiles.from_values([values[0][1]]), - ) - - # ensure values are sorted and piecewise continuous - # (combine any values at the same time) - tmp_values = sorted(values, key=lambda x: x[0]) - values = [] - epsilon = 1e-6 - - for val in tmp_values: - if values and abs(values[-1][0] - val[0]) < epsilon: - values[-1] = (val[0], val[1] + values[-1][1]) - else: - values.append(val) - - duration = values[-1][0] - values[0][0] - - # mean calculations - integral = sum( - (values[ind + 1][0] - values[ind][0]) * values[ind][1] - for ind in range(len(values) - 1) - ) - mean = integral / duration if duration > 0 else 0.0 - - # variance calculations - variance = ( - sum( - (values[ind + 1][0] - values[ind][0]) * (values[ind][1] - mean) ** 2 - for ind in range(len(values) - 1) - ) - / duration - if duration > 0 - else 0.0 - ) - - # percentile calculations - value_durations_dict = defaultdict(float) - for ind in range(len(values) - 1): - value_durations_dict[values[ind][1]] += values[ind + 1][0] - values[ind][0] - value_durations = sorted( - [(duration, value) for value, duration in value_durations_dict.items()], - key=lambda x: x[0], - ) - - def _get_percentile(percentile: float) -> float: - target_duration = percentile / 100 * duration - cumulative_duration = 0.0 - for dur, val in value_durations: - cumulative_duration += dur - if cumulative_duration >= target_duration: - return val - return value_durations[-1][1] - - return DistributionSummary( - mean=mean, - median=_get_percentile(50.0), - variance=variance, - std_dev=math.sqrt(variance), - min=min([meas[1] for meas in values]), - max=max([meas[1] for meas in values]), - count=len(values), - percentiles=Percentiles( - p001=_get_percentile(0.1), - p01=_get_percentile(1.0), - p05=_get_percentile(5.0), - p10=_get_percentile(10.0), - p25=_get_percentile(25.0), - p75=_get_percentile(75.0), - p90=_get_percentile(90.0), - p95=_get_percentile(95.0), - p99=_get_percentile(99.0), - p999=_get_percentile(99.9), - ), - ) - - @staticmethod - def from_timestamped_values_per_frequency( - values: List[Tuple[float, float]], - frequency: float, - ) -> "DistributionSummary": - """ - Calculate a distribution summary from a list of timestamped values - at a given frequency. - Specifically, this calculates the statistics assuming a piecewise - continuous distribution of values over time and then samples at - the given frequency from that distribution. - For example, rather than finding the average requests per second - over a given time period, this will calculate that along with other - statistics such as the variance and percentiles. - If the list is empty, all values are set to 0. - If the list contains only one value, all values are set to that value. - Note, since this is calculating statistics over time, the values - should contain the entire time range. Generally, this means the first - value should be the start time with a measurement of 0. - - :param values: A list of timestamped numerical values of the form - (timestamp, value). - :param frequency: The frequency to sample the distribution at - represented in the same units as the timestamps. - :return: An instance of DistributionSummary with calculated values. - """ - values.sort(key=lambda x: x[0]) - samples = [] - min_time = values[0][0] - max_time = values[-1][0] + frequency - - for time_iter in np.arange( - min_time, - max_time, - frequency, - ): - count = 0 - while values and values[0][0] <= time_iter: - count += values[0][1] - values.pop(0) - samples.append((time_iter, count)) - - return DistributionSummary.from_timestamped_values(samples) - - @staticmethod - def from_timestamped_interval_values( - values: List[Tuple[float, float, float]], - ) -> "DistributionSummary": - """ - Calculate a distribution summary from a list of timestamped interval values, - that may or may note be overlapping in ranges. - Specifically, this calculates the statistics assuming a piecewise - continuous distribution of values over time. - For example, rather than finding the average concurrency of overlapping requests - over a given time period, this will calculate that along with other - statistics such as the variance and percentiles. - If the list is empty, all values are set to 0. - If the list contains only one value, all values are set to that value. - Note, since this is calculating statistics over time, the values - should contain the entire time range. - - :param values: A list of timestamped numerical values of the form - (start_time, end_time, value). - :return: An instance of DistributionSummary with calculated values. - """ - events_dict = defaultdict(int) - for start, end, count in values: - events_dict[start] += count - events_dict[end] -= count - - timestamped_values = [] - current_value = 0 - - for time, delta in sorted(events_dict.items()): - current_value += delta - timestamped_values.append((time, current_value)) - - return DistributionSummary.from_timestamped_values( - timestamped_values, - ) - - -class StatusDistributionSummary(Serializable): - """ - A serializable model representing distribution summary statistics - based on groupings of status (e.g., completed, errored) for a given - distribution of numerical values. - Handles the total, completed, and errored distributions where the total - is the combination of the completed and errored distributions. - """ - - total: DistributionSummary = Field( - description="The distribution summary for all statuses (errored, completed).", - ) - completed: DistributionSummary = Field( - description=( - "The distribution summary for completed statuses " - "(e.g., successful requests)." - ) - ) - errored: DistributionSummary = Field( - description=( - "The distribution summary for errored statuses " "(e.g., failed requests)." - ) - ) - - @staticmethod - def from_values( - completed_values: List[float], - errored_values: List[float], - ) -> "StatusDistributionSummary": - """ - Calculate distribution summaries from a list of values for - completed, errored, and the total combination of both. - If the lists are empty, all values are set to 0. - - :param completed_values: A list of numerical values for completed statuses. - :param errored_values: A list of numerical values for errored statuses. - :return: An instance of StatusDistributionSummary with calculated values. - """ - return StatusDistributionSummary( - total=DistributionSummary.from_values( - completed_values + errored_values, - ), - completed=DistributionSummary.from_values(completed_values), - errored=DistributionSummary.from_values(errored_values), - ) - - @staticmethod - def from_timestamped_values( - completed_values: List[Tuple[float, float]], - errored_values: List[Tuple[float, float]], - ) -> "StatusDistributionSummary": - """ - Calculate distribution summaries from a list of timestamped values for - completed, errored, and the total combination of both. - Specifically, this calculates the statistics assuming a piecewise - continuous distribution of values over time. - For example, rather than finding the average concurrency of requests - over a given time period, this will calculate that along with other - statistics such as the variance and percentiles. - If the lists are empty, all values are set to 0. - If the lists contain only one value, all values are set to that value. - Note, since this is calculating statistics over time, the values - should contain the entire time range. Generally, this means the first - value should be the start time with a measurement of 0. - - :param completed_values: A list of timestamped numerical values for - completed statuses. - :param errored_values: A list of timestamped numerical values for - errored statuses. - :return: An instance of StatusDistributionSummary with calculated values. - """ - return StatusDistributionSummary( - total=DistributionSummary.from_timestamped_values( - completed_values + errored_values, - ), - completed=DistributionSummary.from_timestamped_values( - completed_values, - ), - errored=DistributionSummary.from_timestamped_values( - errored_values, - ), - ) - - @staticmethod - def from_timestamped_values_per_frequency( - completed_values: List[Tuple[float, float]], - errored_values: List[Tuple[float, float]], - frequency: float, - ) -> "StatusDistributionSummary": - """ - Calculate distribution summaries from a list of timestamped values for - completed, errored, and the total combination of both at a given frequency. - Specifically, this calculates the statistics assuming a piecewise - continuous distribution of values over time and then samples at - the given frequency from that distribution. - For example, rather than finding the average requests per second - over a given time period, this will calculate that along with other - statistics such as the variance and percentiles. - If the lists are empty, all values are set to 0. - If the lists contain only one value, all values are set to that value. - Note, since this is calculating statistics over time, the values - should contain the entire time range. Generally, this means the first - value should be the start time with a measurement of 0. - - :param completed_values: A list of timestamped numerical values for - completed statuses. - :param errored_values: A list of timestamped numerical values for - errored statuses. - :param frequency: The frequency to sample the distribution at - represented in the same units as the timestamps. - :return: An instance of StatusDistributionSummary with calculated values. - """ - return StatusDistributionSummary( - total=DistributionSummary.from_timestamped_values_per_frequency( - completed_values + errored_values, - frequency, - ), - completed=DistributionSummary.from_timestamped_values_per_frequency( - completed_values, - frequency, - ), - errored=DistributionSummary.from_timestamped_values_per_frequency( - errored_values, - frequency, - ), - ) - - @staticmethod - def from_timestamped_interval_values( - completed_values: List[Tuple[float, float, float]], - errored_values: List[Tuple[float, float, float]], - ) -> "StatusDistributionSummary": - """ - Calculate distribution summaries from a list of timestamped interval values for - completed, errored, and the total combination of both. - Specifically, this calculates the statistics assuming a piecewise - continuous distribution of values over time. - For example, rather than finding the average concurrency of overlapping requests - over a given time period, this will calculate that along with other - statistics such as the variance and percentiles. - If the lists are empty, all values are set to 0. - If the lists contain only one value, all values are set to that value. - Note, since this is calculating statistics over time, the values - should contain the entire time range. - - :param completed_values: A list of timestamped numerical values for - completed statuses. - :param errored_values: A list of timestamped numerical values for - errored statuses. - :return: An instance of StatusDistributionSummary with calculated values. - """ - return StatusDistributionSummary( - total=DistributionSummary.from_timestamped_interval_values( - completed_values + errored_values, - ), - completed=DistributionSummary.from_timestamped_interval_values( - completed_values, - ), - errored=DistributionSummary.from_timestamped_interval_values( - errored_values, - ), - ) diff --git a/src/guidellm/objects/statistics.py b/src/guidellm/objects/statistics.py new file mode 100644 index 00000000..4b2d3465 --- /dev/null +++ b/src/guidellm/objects/statistics.py @@ -0,0 +1,516 @@ +import math +import time as timer +from collections import defaultdict +from typing import List, Literal, Optional, Tuple + +import numpy as np +from pydantic import Field, computed_field + +from guidellm.objects import Serializable + +__all__ = [ + "Percentiles", + "DistributionSummary", + "StatusDistributionSummary", + "RunningStats", +] + + +class Percentiles(Serializable): + """ + A serializable model representing percentiles of a distribution. + """ + + p001: float = Field( + description="The 0.1th percentile of the distribution.", + ) + p01: float = Field( + description="The 1st percentile of the distribution.", + ) + p05: float = Field( + description="The 5th percentile of the distribution.", + ) + p10: float = Field( + description="The 10th percentile of the distribution.", + ) + p25: float = Field( + description="The 25th percentile of the distribution.", + ) + p75: float = Field( + description="The 75th percentile of the distribution.", + ) + p90: float = Field( + description="The 90th percentile of the distribution.", + ) + p95: float = Field( + description="The 95th percentile of the distribution.", + ) + p99: float = Field( + description="The 99th percentile of the distribution.", + ) + p999: float = Field( + description="The 99.9th percentile of the distribution.", + ) + + +class DistributionSummary(Serializable): + """ + A serializable model representing a statistical summary for a given + distribution of numerical values. + """ + + mean: float = Field( + description="The mean/average of the distribution.", + ) + median: float = Field( + description="The median of the distribution.", + ) + mode: float = Field( + description="The mode of the distribution.", + ) + variance: float = Field( + description="The variance of the distribution.", + ) + std_dev: float = Field( + description="The standard deviation of the distribution.", + ) + min: float = Field( + description="The minimum value of the distribution.", + ) + max: float = Field( + description="The maximum value of the distribution.", + ) + count: int = Field( + description="The number of values in the distribution.", + ) + percentiles: Percentiles = Field( + description="The percentiles of the distribution.", + ) + cumulative_distribution_function: Optional[List[Tuple[float, float]]] = Field( + description=("The cumulative distribution function (CDF) of the distribution."), + default=None, + ) + + @staticmethod + def from_distribution_function( + distribution: List[Tuple[float, float]], + include_cdf: bool = False, + ) -> "DistributionSummary": + """ + Calculate a distribution summary from a values or + probability distribution function (PDF). + For a PDF, it is expected to be a list of tuples where each tuple + contains a value and its probability. + The probabilities across all elements should be normalized (sum to 1). + If the PDF is not normalized, it will be normalized. + The values distribution function is a list of tuples where each tuple + contains a value and some weighting for that value. + The weightings will be normalized to a probability distribution function. + + :param pdf: A list of tuples representing the PDF. + Each tuple contains a value and its probability. + :param include_cdf: Whether to include the cumulative distribution function + in the output DistributionSummary. + :return: An instance of DistributionSummary with calculated values. + """ + values, weights = zip(*distribution) if distribution else ([], []) + values = np.array(values) + weights = np.array(weights) + + # create the PDF + probabilities = weights / np.sum(weights) + pdf = np.column_stack((values, probabilities)) + pdf = pdf[np.argsort(pdf[:, 0])] + values = pdf[:, 0] + probabilities = pdf[:, 1] + + # calculate the CDF + cumulative_probabilities = np.cumsum(probabilities) + cdf = np.column_stack((values, cumulative_probabilities)) + + # calculate statistics + mean = np.sum(values * probabilities).item() + median = cdf[np.argmax(cdf[:, 1] >= 0.5), 0].item() if len(cdf) > 0 else 0 + mode = values[np.argmax(probabilities)].item() if len(values) > 0 else 0 + variance = np.sum((values - mean) ** 2 * probabilities).item() + std_dev = math.sqrt(variance) + minimum = values[0].item() if len(values) > 0 else 0 + maximum = values[-1].item() if len(values) > 0 else 0 + count = len(values) + + return DistributionSummary( + mean=mean, + median=median, + mode=mode, + variance=variance, + std_dev=std_dev, + min=minimum, + max=maximum, + count=count, + percentiles=( + Percentiles( + p001=cdf[np.argmax(cdf[:, 1] >= 0.001), 0].item(), # noqa: PLR2004 + p01=cdf[np.argmax(cdf[:, 1] >= 0.01), 0].item(), # noqa: PLR2004 + p05=cdf[np.argmax(cdf[:, 1] >= 0.05), 0].item(), # noqa: PLR2004 + p10=cdf[np.argmax(cdf[:, 1] >= 0.1), 0].item(), # noqa: PLR2004 + p25=cdf[np.argmax(cdf[:, 1] >= 0.25), 0].item(), # noqa: PLR2004 + p75=cdf[np.argmax(cdf[:, 1] >= 0.75), 0].item(), # noqa: PLR2004 + p90=cdf[np.argmax(cdf[:, 1] >= 0.9), 0].item(), # noqa: PLR2004 + p95=cdf[np.argmax(cdf[:, 1] >= 0.95), 0].item(), # noqa: PLR2004 + p99=cdf[np.argmax(cdf[:, 1] >= 0.99), 0].item(), # noqa: PLR2004 + p999=cdf[np.argmax(cdf[:, 1] >= 0.999), 0].item(), # noqa: PLR2004 + ) + if len(cdf) > 0 + else Percentiles( + p001=0, + p01=0, + p05=0, + p10=0, + p25=0, + p75=0, + p90=0, + p95=0, + p99=0, + p999=0, + ) + ), + cumulative_distribution_function=cdf.tolist() if include_cdf else None, + ) + + @staticmethod + def from_values( + values: List[float], + weights: Optional[List[float]] = None, + include_cdf: bool = False, + ) -> "DistributionSummary": + """ + Calculate a distribution summary from a list of values. + If the list is empty, all stats are set to 0. + If weights are provided, they are used to weight the values + so that the probabilities are shifted accordingly and larger + weights are given more importance / weight in the distribution. + If the weights are not provided, all values are treated equally. + + :param values: A list of numerical values. + :param weights: A list of weights for each value. + If None, all values are treated equally. + :param include_cdf: Whether to include the cumulative distribution function + in the output DistributionSummary. + :return: An instance of DistributionSummary with calculated values. + """ + if weights is None: + weights = [1.0] * len(values) + + if len(values) != len(weights): + raise ValueError( + "The length of values and weights must be the same.", + ) + + return DistributionSummary.from_distribution_function( + distribution=list(zip(values, weights)), + include_cdf=include_cdf, + ) + + @staticmethod + def from_request_times( + requests: List[Tuple[float, float]], + distribution_type: Literal["concurrency", "rate"], + include_cdf: bool = False, + epsilon: float = 1e-6, + ) -> "DistributionSummary": + if distribution_type == "concurrency": + # convert to delta changes based on when requests were running + time_deltas = defaultdict(int) + for start, end in requests: + time_deltas[start] += 1 + time_deltas[end] -= 1 + + # convert to the events over time measuring concurrency changes + events = [] + active = 0 + + for time, delta in sorted(time_deltas.items()): + active += delta + events.append((time, active)) + elif distribution_type == "rate": + # convert to events for when requests finished + global_start = min(start for start, _ in requests) if requests else 0 + events = [(global_start, 1)] + [(end, 1) for _, end in requests] + + # combine any events that are very close together + flattened_events = [] + for time, val in sorted(events): + last_time, last_val = ( + flattened_events[-1] if flattened_events else (None, None) + ) + + if last_time is not None and abs(last_time - time) <= epsilon: + flattened_events[-1] = (last_time, last_val + val) + else: + flattened_events.append((time, val)) + + # convert to value distribution function + distribution = defaultdict(float) + + for ind in range(len(flattened_events) - 1): + start_time, value = flattened_events[ind] + end_time, _ = flattened_events[ind + 1] + duration = end_time - start_time + + if distribution_type == "concurrency": + # weight the concurrency value by the duration + distribution[value] += duration + elif distribution_type == "rate": + # weight the rate value by the duration + rate = value / duration + distribution[rate] += duration + + distribution = sorted(distribution.items()) + + return DistributionSummary.from_distribution_function( + distribution=distribution, + include_cdf=include_cdf, + ) + + @staticmethod + def from_iterable_request_times( + requests: List[Tuple[float, float]], + first_iter_times: List[float], + iter_counts: List[int], + first_iter_counts: Optional[List[int]] = None, + include_cdf: bool = False, + epsilon: float = 1e-6, + ) -> "DistributionSummary": + if first_iter_counts is None: + first_iter_counts = [1] * len(requests) + + if ( + len(requests) != len(first_iter_times) + or len(requests) != len(iter_counts) + or len(requests) != len(first_iter_counts) + ): + raise ValueError( + "requests, first_iter_times, iter_counts, and first_iter_counts must" + "be the same length." + f"Given {len(requests)}, {len(first_iter_times)}, {len(iter_counts)}, " + f"{len(first_iter_counts)}", + ) + + # first break up the requests into individual iterable events + events = defaultdict(int) + global_start = min(start for start, _ in requests) if requests else 0 + global_end = max(end for _, end in requests) if requests else 0 + events[global_start] = 0 + events[global_end] = 0 + + for (_, end), first_iter, first_iter_count, total_count in zip( + requests, first_iter_times, first_iter_counts, iter_counts + ): + events[first_iter] += first_iter_count + + if total_count > 1: + iter_latency = (end - first_iter) / (total_count - 1) + for ind in range(1, total_count): + events[first_iter + ind * iter_latency] += 1 + + # combine any events that are very close together + flattened_events = [] + + for time, count in sorted(events.items()): + last_time, last_count = ( + flattened_events[-1] if flattened_events else (None, None) + ) + + if last_time is not None and abs(last_time - time) <= epsilon: + flattened_events[-1] = (last_time, last_count + count) + else: + flattened_events.append((time, count)) + + # convert to value distribution function + distribution = defaultdict(float) + + for ind in range(len(flattened_events) - 1): + start_time, count = flattened_events[ind] + end_time, _ = flattened_events[ind + 1] + duration = end_time - start_time + rate = count / duration + distribution[rate] += duration + + distribution = sorted(distribution.items()) + + return DistributionSummary.from_distribution_function( + distribution=distribution, + include_cdf=include_cdf, + ) + + +class StatusDistributionSummary(Serializable): + """ + A serializable model representing distribution summary statistics + based on groupings of status (e.g., completed, errored) for a given + distribution of numerical values. + Handles the total, completed, and errored distributions where the total + is the combination of the completed and errored distributions. + """ + + total: DistributionSummary = Field( + description="The distribution summary for all statuses (errored, completed).", + ) + completed: DistributionSummary = Field( + description=( + "The distribution summary for completed statuses " + "(e.g., successful requests)." + ) + ) + errored: DistributionSummary = Field( + description=( + "The distribution summary for errored statuses " "(e.g., failed requests)." + ) + ) + + @staticmethod + def from_values( + completed_values: List[float], + errored_values: List[float], + completed_weights: Optional[List[float]] = None, + errored_weights: Optional[List[float]] = None, + include_cdf: bool = False, + ) -> "StatusDistributionSummary": + if completed_weights is None: + completed_weights = [1.0] * len(completed_values) + + if errored_weights is None: + errored_weights = [1.0] * len(errored_values) + + return StatusDistributionSummary( + total=DistributionSummary.from_values( + values=[*completed_values, *errored_values], + weights=[*completed_weights, *errored_weights], + include_cdf=include_cdf, + ), + completed=DistributionSummary.from_values( + values=completed_values, + weights=completed_weights, + include_cdf=include_cdf, + ), + errored=DistributionSummary.from_values( + values=errored_values, + weights=errored_weights, + include_cdf=include_cdf, + ), + ) + + @staticmethod + def from_request_times( + completed_requests: List[Tuple[float, float]], + errored_requests: List[Tuple[float, float]], + distribution_type: Literal["concurrency", "rate"], + include_cdf: bool = False, + epsilon: float = 1e-6, + ) -> "StatusDistributionSummary": + return StatusDistributionSummary( + total=DistributionSummary.from_request_times( + requests=[*completed_requests, *errored_requests], + distribution_type=distribution_type, + include_cdf=include_cdf, + epsilon=epsilon, + ), + completed=DistributionSummary.from_request_times( + requests=completed_requests, + distribution_type=distribution_type, + include_cdf=include_cdf, + epsilon=epsilon, + ), + errored=DistributionSummary.from_request_times( + requests=errored_requests, + distribution_type=distribution_type, + include_cdf=include_cdf, + epsilon=epsilon, + ), + ) + + @staticmethod + def from_iterable_request_times( + completed_requests: List[Tuple[float, float]], + errored_requests: List[Tuple[float, float]], + completed_first_iter_times: List[float], + errored_first_iter_times: List[float], + completed_iter_counts: List[int], + errored_iter_counts: List[int], + completed_first_iter_counts: Optional[List[int]] = None, + errored_first_iter_counts: Optional[List[int]] = None, + include_cdf: bool = False, + epsilon: float = 1e-6, + ) -> "StatusDistributionSummary": + if completed_first_iter_counts is None: + completed_first_iter_counts = [1] * len(completed_requests) + + if errored_first_iter_counts is None: + errored_first_iter_counts = [1] * len(errored_requests) + + return StatusDistributionSummary( + total=DistributionSummary.from_iterable_request_times( + requests=[*completed_requests, *errored_requests], + first_iter_times=[ + *completed_first_iter_times, + *errored_first_iter_times, + ], + iter_counts=[*completed_iter_counts, *errored_iter_counts], + first_iter_counts=[ + *completed_first_iter_counts, + *errored_first_iter_counts, + ], + include_cdf=include_cdf, + epsilon=epsilon, + ), + completed=DistributionSummary.from_iterable_request_times( + requests=completed_requests, + first_iter_times=completed_first_iter_times, + iter_counts=completed_iter_counts, + first_iter_counts=completed_first_iter_counts, + include_cdf=include_cdf, + epsilon=epsilon, + ), + errored=DistributionSummary.from_iterable_request_times( + requests=errored_requests, + first_iter_times=errored_first_iter_times, + iter_counts=errored_iter_counts, + first_iter_counts=errored_first_iter_counts, + include_cdf=include_cdf, + epsilon=epsilon, + ), + ) + + +class RunningStats(Serializable): + count: int = Field( + default=0, + ) + total: float = Field( + default=0.0, + ) + start_time: float = Field( + default=timer.time, + ) + + @computed_field + @property + def mean(self) -> float: + if self.count == 0: + return 0.0 + return self.total / self.count + + @computed_field + @property + def rate(self) -> float: + if self.count == 0: + return 0.0 + return self.total / (timer.time() - self.start_time) + + def update(self, value: float, count: int = 1) -> None: + """ + Update the running statistics with a new value. + :param value: The new value to add to the running statistics. + """ + self.count += count + self.total += value diff --git a/src/guidellm/objects/test.py b/src/guidellm/objects/test.py new file mode 100644 index 00000000..019488ea --- /dev/null +++ b/src/guidellm/objects/test.py @@ -0,0 +1,364 @@ +from typing import List, Tuple + +from guidellm.objects.statistics import DistributionSummary + + +def generate_stats_outputs_tokens_per_seconds( + requests: List[Tuple[float, float]], + first_token_times: List[float], + output_token_counts: List[int], + epsilon: float = 1e-6, +): + distribution = DistributionSummary.from_iterable_request_times( + requests=requests, + first_iter_times=first_token_times, + iter_counts=output_token_counts, + epsilon=epsilon, + ) + print(distribution) + + +def generate_stats_inter_token_latencies( + latencies: List[float], + weights: List[float] = None, + epsilon: float = 1e-6, +): + distribution = DistributionSummary.from_values(latencies, weights) + print(distribution) + + +def generate_stats_requests_per_second( + requests: List[Tuple[float, float]], + epsilon: float = 1e-6, +) -> List[Tuple[float, float]]: + distribution = DistributionSummary.from_request_times( + requests, "rate", epsilon=epsilon + ) + print(distribution) + + +def generate_stats_concurrent_requests( + requests=List[Tuple[float, float]], + epsilon: float = 1e-6, +): + distribution = DistributionSummary.from_request_times( + requests, "concurrency", epsilon + ) + print(distribution) + + +# Example Usage + +request_times = [ + (1743163300.403223, 1743163301.9923513), + (1743163300.4043713, 1743163302.015711), + (1743163300.407332, 1743163302.0157585), + (1743163300.4105933, 1743163302.0158577), + (1743163300.4118826, 1743163302.015753), + (1743163302.0636709, 1743163303.6263561), + (1743163302.0829957, 1743163303.64966), + (1743163302.0883937, 1743163303.6496801), + (1743163302.0896976, 1743163303.6497076), + (1743163302.0892832, 1743163303.649676), + (1743163303.701261, 1743163305.26519), + (1743163303.7232263, 1743163305.2885003), + (1743163303.7202537, 1743163305.2885823), + (1743163303.7239172, 1743163305.2885487), + (1743163303.7244577, 1743163305.288734), + (1743163305.3386924, 1743163306.9014742), + (1743163305.3575644, 1743163306.9247215), + (1743163305.364691, 1743163306.9247289), + (1743163305.3667543, 1743163306.9247417), + (1743163305.3670223, 1743163306.9247096), + (1743163306.972128, 1743163308.5344012), + (1743163306.997972, 1743163308.5577517), + (1743163306.9919195, 1743163308.5577645), + (1743163306.998968, 1743163308.5578134), + (1743163306.9986732, 1743163308.5578017), + (1743163308.6057122, 1743163310.172029), + (1743163308.6297, 1743163310.1953459), + (1743163308.6132185, 1743163310.1953993), + (1743163308.630173, 1743163310.195401), + (1743163308.6345012, 1743163310.1953926), + (1743163310.2397547, 1743163311.801659), + (1743163310.2674718, 1743163311.824895), + (1743163310.2598615, 1743163311.8248944), + (1743163310.2720146, 1743163311.824908), + (1743163310.2664378, 1743163311.8248801), + (1743163311.8722754, 1743163313.4339283), + (1743163311.900499, 1743163313.457236), + (1743163311.895735, 1743163313.4572852), + (1743163311.8999357, 1743163313.457477), + (1743163311.8968449, 1743163313.4572716), + (1743163313.5064678, 1743163315.068746), + (1743163313.5301821, 1743163315.0921736), + (1743163313.5289028, 1743163315.092185), + (1743163313.5287688, 1743163315.0922654), + (1743163313.5301483, 1743163315.092442), + (1743163315.1421392, 1743163316.705413), + (1743163315.1619313, 1743163316.7287838), + (1743163315.1561904, 1743163316.7287745), + (1743163315.1638331, 1743163316.7288203), + (1743163315.1655514, 1743163316.7288005), + (1743163316.7788277, 1743163318.340335), + (1743163316.8018036, 1743163318.363607), + (1743163316.8013813, 1743163318.3635497), + (1743163316.7962375, 1743163318.3637505), + (1743163316.8082492, 1743163318.3635893), + (1743163318.4177032, 1743163319.9805896), + (1743163318.4415567, 1743163320.0040746), + (1743163318.438249, 1743163320.0040603), + (1743163318.4454482, 1743163320.0039873), + (1743163318.435946, 1743163320.0040472), + (1743163320.059263, 1743163321.6231477), + (1743163320.0803068, 1743163321.646229), + (1743163320.077479, 1743163321.6461928), + (1743163320.077346, 1743163321.646171), + (1743163320.0767386, 1743163321.6462295), + (1743163321.6983657, 1743163323.262142), + (1743163321.720098, 1743163323.2855532), + (1743163321.7210836, 1743163323.2855496), + (1743163321.7272742, 1743163323.285537), + (1743163321.720461, 1743163323.285713), + (1743163323.3382583, 1743163324.9130867), + (1743163323.3578243, 1743163324.9363666), + (1743163323.3580399, 1743163324.9365273), + (1743163323.3582692, 1743163324.9365368), + (1743163323.3700652, 1743163324.9365063), + (1743163324.9848232, 1743163326.548244), + (1743163325.0123115, 1743163326.5716817), + (1743163325.0087984, 1743163326.57175), + (1743163325.0095794, 1743163326.571734), + (1743163325.0168707, 1743163326.5717053), + (1743163326.61861, 1743163328.1839957), + (1743163326.633079, 1743163328.2074084), + (1743163326.6254075, 1743163328.207643), + (1743163326.642937, 1743163328.207623), + (1743163326.628424, 1743163328.2074928), + (1743163328.2542157, 1743163329.8191519), + (1743163328.2807877, 1743163329.842159), + (1743163328.2869048, 1743163329.8421898), + (1743163328.2784348, 1743163329.8422613), + (1743163328.3039563, 1743163329.8646753), + (1743163329.8803456, 1743163331.4419262), + (1743163329.8897858, 1743163331.4652398), + (1743163329.9109275, 1743163331.4652941), + (1743163329.8975961, 1743163331.4653), + (1743163329.8834872, 1743163331.4652634), + (1743163331.5126238, 1743163333.0786362), + (1743163331.5387557, 1743163333.1019342), + (1743163331.5308228, 1743163333.1019886), + (1743163331.5362875, 1743163333.1019595), + (1743163331.5372543, 1743163333.1019611), +] +first_token_times = [ + 1743163300.4774668, + 1743163300.5430598, + 1743163300.5430408, + 1743163300.5430439, + 1743163300.5430408, + 1743163302.1260648, + 1743163302.1790159, + 1743163302.1789834, + 1743163302.178902, + 1743163302.1788735, + 1743163303.7622797, + 1743163303.8150222, + 1743163303.8150568, + 1743163303.815119, + 1743163303.8151877, + 1743163305.400849, + 1743163305.4539392, + 1743163305.4539232, + 1743163305.4539037, + 1743163305.4538555, + 1743163307.0341141, + 1743163307.0872338, + 1743163307.0871842, + 1743163307.0872376, + 1743163307.0871847, + 1743163308.6717093, + 1743163308.7247245, + 1743163308.7246864, + 1743163308.7246675, + 1743163308.7246442, + 1743163310.3007226, + 1743163310.3539567, + 1743163310.3539433, + 1743163310.3539903, + 1743163310.353892, + 1743163311.9333777, + 1743163311.9865408, + 1743163311.9865537, + 1743163311.9865468, + 1743163311.9865608, + 1743163313.5681715, + 1743163313.6213248, + 1743163313.6213076, + 1743163313.6213694, + 1743163313.6212204, + 1743163315.2052338, + 1743163315.257879, + 1743163315.2578883, + 1743163315.2578585, + 1743163315.2578545, + 1743163316.839769, + 1743163316.8927107, + 1743163316.8927133, + 1743163316.8925958, + 1743163316.8925962, + 1743163318.4791622, + 1743163318.5325277, + 1743163318.53261, + 1743163318.5324767, + 1743163318.5324028, + 1743163320.1203272, + 1743163320.1740332, + 1743163320.1739714, + 1743163320.174067, + 1743163320.174065, + 1743163321.759952, + 1743163321.813341, + 1743163321.813346, + 1743163321.8132503, + 1743163321.8133996, + 1743163323.410784, + 1743163323.464367, + 1743163323.4643219, + 1743163323.4643698, + 1743163323.4643369, + 1743163325.0459666, + 1743163325.0997808, + 1743163325.0997987, + 1743163325.099642, + 1743163325.0997283, + 1743163326.681901, + 1743163326.7354028, + 1743163326.7354014, + 1743163326.735491, + 1743163326.7354689, + 1743163328.3170884, + 1743163328.3714006, + 1743163328.3715012, + 1743163328.3715947, + 1743163328.3944945, + 1743163329.9375503, + 1743163329.9934423, + 1743163329.9931898, + 1743163329.9932914, + 1743163329.9932675, + 1743163331.5760312, + 1743163331.629882, + 1743163331.6298609, + 1743163331.6300056, + 1743163331.6299996, +] +output_tokens = [ + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, + 64, +] + + +generate_stats_outputs_tokens_per_seconds( + request_times, + first_token_times, + output_tokens, +) diff --git a/src/guidellm/request/__init__.py b/src/guidellm/request/__init__.py index 10ef0be7..b0420694 100644 --- a/src/guidellm/request/__init__.py +++ b/src/guidellm/request/__init__.py @@ -1,7 +1,13 @@ -from .loader import RequestLoader +from .loader import ( + GenerativeRequestLoader, + GenerativeRequestLoaderDescription, + RequestLoader, +) from .request import GenerationRequest __all__ = [ - "GenerationRequest", "RequestLoader", + "GenerativeRequestLoaderDescription", + "GenerativeRequestLoader", + "GenerationRequest", ] diff --git a/src/guidellm/request/loader.py b/src/guidellm/request/loader.py index e935d5e2..768ab150 100644 --- a/src/guidellm/request/loader.py +++ b/src/guidellm/request/loader.py @@ -1,15 +1,267 @@ +from abc import abstractmethod from pathlib import Path -from typing import Any, Callable, Dict, Optional, Union +from typing import ( + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + Literal, + Optional, + Union, +) -from datasets import Dataset, IterableDataset +from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict from transformers import PreTrainedTokenizer +from guidellm.dataset import ColumnInputTypes, load_dataset +from guidellm.objects import Serializable +from guidellm.request.request import GenerationRequest + +__all__ = [ + "RequestLoader", + "GenerativeRequestLoaderDescription", + "GenerativeRequestLoader", +] + + +class RequestLoader(Iterable): + @abstractmethod + def __iter__(self): ... + + @abstractmethod + def __len__(self): ... + + @property + @abstractmethod + def description(self) -> Serializable: ... + + +class GenerativeRequestLoaderDescription(Serializable): + data: str + data_args: Optional[Dict[str, Any]] + processor: str + processor_args: Optional[Dict[str, Any]] + + +class GenerativeRequestLoader(RequestLoader): + DEFAULT_PROMPT_COLUMNS = [ + "prompt", + "prompts", + "instruction", + "instructions", + "question", + "questions", + "input", + "inputs", + "context", + "content", + "conversation", + "conversations", + "text", + ] -class RequestLoader: def __init__( self, - dataset: Union[Dataset, IterableDataset], + data: Union[ + str, + Path, + Iterable[Union[str, Dict[str, Any]]], + Dataset, + DatasetDict, + IterableDataset, + IterableDatasetDict, + ], + data_args: Optional[Dict[str, Any]], processor: Optional[Union[str, Path, PreTrainedTokenizer, Callable]], processor_args: Optional[Dict[str, Any]], + shuffle: bool = True, + iter_type: Literal["finite", "infinite"] = "finite", + random_seed: int = 42, ): - pass + self.data = data + self.data_args = data_args + dataset, args_column_mappings = load_dataset( + data, + data_args, + processor, + processor_args, + random_seed, + ) + self.dataset = dataset + self.processor = processor + self.processor_args = processor_args + self.shuffle = shuffle + self.iter_type = iter_type + self.random_seed = random_seed + + self.column_mappings = self._create_column_mappings(args_column_mappings) + self.preserve_iter_state = iter_type == "infinite" # ensure no caching requests + self._preserved_iter = None + + def __iter__(self) -> Iterator[GenerationRequest]: + scope_create_count = 0 + + while (dataset_iter := self._get_dataset_iter(scope_create_count)) is not None: + scope_create_count += 1 + + for item in dataset_iter: + yield self._create_request(item) + + self._preserved_iter = None + + def __len__(self) -> int: + if self.iter_type == "finite": + try: + return len(self.dataset) + except Exception: + pass + + try: + dataset_size = self.dataset.info.dataset_size + if dataset_size is not None: + return dataset_size + except Exception: + pass + + raise ValueError(f"Unable to determine length of dataset: {self.data}") + + @property + def description(self) -> GenerativeRequestLoaderDescription: + return GenerativeRequestLoaderDescription( + data=str(self.data), + data_args=self.data_args, + processor=str(self.processor), + processor_args=self.processor_args, + ) + + def _create_column_mappings( + self, + args_column_mappings: Dict[ColumnInputTypes, str], + ) -> Dict[ColumnInputTypes, str]: + column_mappings = {} + + if "text_column" in args_column_mappings: + column_mappings["prompt_column"] = args_column_mappings["text_column"] + else: + column_mappings["prompt_column"] = self._extract_text_column() + + if "prompt_tokens_count_column" in args_column_mappings: + column_mappings["prompt_tokens_count_column"] = args_column_mappings[ + "prompt_tokens_count_column" + ] + elif prompt_tokens_count_column := self._extract_prompt_tokens_count_column(): + column_mappings["prompt_tokens_count_column"] = prompt_tokens_count_column + + if "output_tokens_count_column" in args_column_mappings: + column_mappings["output_tokens_count_column"] = args_column_mappings[ + "output_tokens_count_column" + ] + elif output_tokens_count_column := self._extract_output_tokens_count_column(): + column_mappings["output_tokens_count_column"] = output_tokens_count_column + + return column_mappings + + def _extract_text_column(self) -> str: + column_names = self._dataset_columns( + err_msg=( + "Unable to determine text column from dataset and it is required. " + "To specify the text column, set the 'text_column' key in the " + "'data_args' dictionary." + ) + ) + + if len(column_names) == 1: + return column_names[0] + + for def_column in self.DEFAULT_PROMPT_COLUMNS: + if def_column in column_names: + return def_column + + raise ValueError( + f"Unable to determine text column from dataset columns: {column_names}. " + "To specify the text column, set the 'text_column' key in the " + "'data_args' dictionary." + ) + + def _extract_prompt_tokens_count_column(self) -> Optional[str]: + column_names = self._dataset_columns() + + if column_names and "prompt_tokens_count" in column_names: + return "prompt_tokens_count" + + if column_names and "prompt_tokens" in column_names: + return "prompt_tokens" + + return None + + def _extract_output_tokens_count_column(self) -> Optional[str]: + column_names = self._dataset_columns() + + if column_names and "output_tokens_count" in column_names: + return "output_tokens_count" + + if column_names and "output_tokens" in column_names: + return "output_tokens" + + return None + + def _dataset_columns(self, err_msg: Optional[str] = None) -> Optional[List[str]]: + try: + column_names = self.dataset.column_names + + if not column_names and err_msg: + raise ValueError(f"No column names found in dataset: {self.data}") + except Exception as err: + if err_msg: + raise ValueError(err_msg) from err + + column_names = None + + return column_names + + def _get_dataset_iter( + self, scope_create_count: int + ) -> Optional[Iterator[Dict[str, Any]]]: + if scope_create_count > 0 and self.iter_type != "infinite": + return None + + if self.preserve_iter_state and self._preserved_iter is not None: + return self._preserved_iter + + dataset = ( + self.dataset + if not self.shuffle + else self.dataset.shuffle(seed=self.random_seed) + ) + + dataset_iter = iter(dataset) + + if self.preserve_iter_state: + self._preserved_iter = dataset_iter + + return dataset_iter + + def _create_request(self, item: Dict[str, Any]) -> GenerationRequest: + prompt_tokens = ( + item[self.column_mappings["prompt_tokens_count_column"]] + if "prompt_tokens_count_column" in self.column_mappings + else None + ) + output_tokens = ( + item[self.column_mappings["output_tokens_count_column"]] + if "output_tokens_count_column" in self.column_mappings + else None + ) + + return GenerationRequest( + request_type="text_completions", + content=item[self.column_mappings["prompt_column"]], + stats=( + {"prompt_tokens": prompt_tokens} if prompt_tokens is not None else {} + ), + constraints=( + {"output_tokens": output_tokens} if output_tokens is not None else {} + ), + ) diff --git a/src/guidellm/request/request.py b/src/guidellm/request/request.py index a11cffad..400c15c1 100644 --- a/src/guidellm/request/request.py +++ b/src/guidellm/request/request.py @@ -37,12 +37,12 @@ class GenerationRequest(Serializable): default_factory=lambda: str(uuid.uuid4()), description="The unique identifier for the request.", ) - request_type: Literal["text", "chat"] = Field( - default="text", + request_type: Literal["text_completions", "chat_completions"] = Field( + default="text_completions", description=( "The type of request (e.g., text, chat). " - "If request_type is 'text', resolved by backend.text_completions. " - "If request_type is 'chat', resolved by backend.chat_completions." + "If request_type='text_completions', resolved by backend.text_completions. " + "If request_typ='chat_completions', resolved by backend.chat_completions." ), ) content: Any = Field( diff --git a/src/guidellm/scheduler/result.py b/src/guidellm/scheduler/result.py index 41edc8c1..c86a655a 100644 --- a/src/guidellm/scheduler/result.py +++ b/src/guidellm/scheduler/result.py @@ -77,7 +77,7 @@ class SchedulerRequestInfo(Serializable): process_id: int = -1 -class SchedulerResult(Generic[REQ, RES], Serializable): +class SchedulerResult(Serializable, Generic[REQ, RES]): """ The yielded, iterative result for a scheduler run. These are triggered on the start and end of the run, diff --git a/src/guidellm/scheduler/scheduler.py b/src/guidellm/scheduler/scheduler.py index 5018f2ed..32efd091 100644 --- a/src/guidellm/scheduler/scheduler.py +++ b/src/guidellm/scheduler/scheduler.py @@ -1,9 +1,9 @@ import asyncio -import concurrent.futures import math import multiprocessing import multiprocessing.queues import time +from concurrent.futures import ProcessPoolExecutor from typing import ( Any, AsyncGenerator, @@ -113,10 +113,7 @@ async def run( if max_duration is not None and max_duration < 0: raise ValueError(f"Invalid max_duration: {max_duration}") - with ( - multiprocessing.Manager() as manager, - concurrent.futures.ProcessPoolExecutor() as executor, - ): + with multiprocessing.Manager() as manager, ProcessPoolExecutor() as executor: futures, requests_queue, responses_queue = await self._start_processes( manager, executor, scheduling_strategy ) @@ -131,32 +128,41 @@ async def run( run_info=run_info, ) - while True: - if ( - requests_iter is None - and run_info.completed_requests >= run_info.created_requests - ): - # we've exhausted all requests we've wanted to run - # and yielded all responses - break - - requests_iter = self._add_requests( - requests_iter, - times_iter, - requests_queue, - run_info, - ) - await asyncio.sleep(0) # enable requests to start + try: + while True: + # check errors and raise them + for future in futures: + if future.done() and (err := future.exception()) is not None: + raise err + + if ( + requests_iter is None + and run_info.completed_requests >= run_info.created_requests + ): + # we've exhausted all requests we've wanted to run + # and yielded all responses + break + + requests_iter = self._add_requests( + requests_iter, + times_iter, + requests_queue, + run_info, + ) + await asyncio.sleep(0) # enable requests to start - iter_result = self._check_result_ready( - responses_queue, - run_info, - ) - if iter_result is not None: - yield iter_result + iter_result = self._check_result_ready( + responses_queue, + run_info, + ) + if iter_result is not None: + yield iter_result - # yield control to the event loop - await asyncio.sleep(settings.default_async_loop_sleep) + # yield control to the event loop + await asyncio.sleep(settings.default_async_loop_sleep) + except Exception as err: + print(err) + raise RuntimeError(f"Scheduler run failed: {err}") from err yield SchedulerResult( type_="run_complete", @@ -171,7 +177,7 @@ async def run( async def _start_processes( self, manager, - executor: concurrent.futures.ProcessPoolExecutor, + executor: ProcessPoolExecutor, scheduling_strategy: SchedulingStrategy, ) -> Tuple[ List[asyncio.Future], @@ -183,12 +189,12 @@ async def _start_processes( ) responses_queue = manager.Queue() per_process_requests_limit = scheduling_strategy.processing_requests_limit // ( - scheduling_strategy.num_processes + scheduling_strategy.processes_limit ) futures = [] loop = asyncio.get_event_loop() - for process_id in range(scheduling_strategy.num_processes): + for process_id in range(scheduling_strategy.processes_limit): if scheduling_strategy.processing_mode == "sync": futures.append( loop.run_in_executor( @@ -238,7 +244,7 @@ def _run_setup( iter_length = len(self.request_loader) if 0 < iter_length < end_number: end_number = iter_length - except TypeError: + except Exception: pass if end_number == math.inf and end_time is None: @@ -272,10 +278,10 @@ def _add_requests( not requests_queue.full() and added_count < settings.max_add_requests_per_loop ): - if run_info.queued_requests >= run_info.end_number: + if run_info.created_requests >= run_info.end_number: raise StopIteration - if request_time := next(times_iter) >= run_info.end_time: + if (request_time := next(times_iter)) >= run_info.end_time: raise StopIteration request = next(requests_iter) @@ -283,6 +289,7 @@ def _add_requests( request=request, start_time=request_time, timeout_time=run_info.end_time, + queued_time=time.time(), ) requests_queue.put(work_req) diff --git a/src/guidellm/scheduler/strategy.py b/src/guidellm/scheduler/strategy.py index 4c1b37e9..b7f1d0bc 100644 --- a/src/guidellm/scheduler/strategy.py +++ b/src/guidellm/scheduler/strategy.py @@ -440,8 +440,8 @@ def request_times(self) -> Generator[float, None, None]: # handle bursts first to get to the desired rate if self.initial_burst is not None: - # calcualte total burst count based on sending initial at rate - # plus any within the time to ramp up + # send an initial burst equal to the rate + # to reach the target rate burst_count = math.floor(self.rate) for _ in range(burst_count): yield start_time @@ -490,6 +490,10 @@ class AsyncPoissonStrategy(ThroughputStrategy): "to reach target rate. False to not send an initial burst." ), ) + random_seed: int = Field( + default=42, + description=("The random seed to use for the Poisson distribution. "), + ) def request_times(self) -> Generator[float, None, None]: """ @@ -504,15 +508,18 @@ def request_times(self) -> Generator[float, None, None]: start_time = time.time() if self.initial_burst is not None: - # calcualte total burst count based on sending initial at rate - # plus any within the time to ramp up + # send an initial burst equal to the rate + # to reach the target rate burst_count = math.floor(self.rate) for _ in range(burst_count): yield start_time else: yield start_time + # set the random seed for reproducibility + rand = random.Random(self.random_seed) + while True: - inter_arrival_time = random.expovariate(self.rate) + inter_arrival_time = rand.expovariate(self.rate) start_time += inter_arrival_time yield start_time diff --git a/src/guidellm/scheduler/worker.py b/src/guidellm/scheduler/worker.py index 3416a02a..a633f33b 100644 --- a/src/guidellm/scheduler/worker.py +++ b/src/guidellm/scheduler/worker.py @@ -56,7 +56,7 @@ class WorkerProcessResult(Generic[REQ, RES]): info: SchedulerRequestInfo -class RequestsWorker(Generic[REQ, RES], ABC): +class RequestsWorker(ABC, Generic[REQ, RES]): """ An abstract base class for a worker that processes requests. This class defines the interface for a worker that can resolve requests @@ -256,7 +256,6 @@ class GenerativeRequestsWorker(RequestsWorker[GenerationRequest, ResponseSummary def __init__(self, backend: Backend): self.backend = backend - self.backend.validate() @property def description(self) -> Serializable: @@ -334,7 +333,7 @@ def _create_request_func_kwargs( ] request_kwargs: Dict[str, Any] - if request.request_type == "text": + if request.request_type == "text_completions": request_func = self.backend.text_completions request_kwargs = { "prompt": request.content, @@ -343,7 +342,7 @@ def _create_request_func_kwargs( "output_token_count": request.constraints.get("output_tokens", None), **request.params, } - elif request.request_type == "chat": + elif request.request_type == "chat_completions": request_func = self.backend.chat_completions request_kwargs = { "content": request.content, diff --git a/src/guidellm/utils/__init__.py b/src/guidellm/utils/__init__.py index 2fdd8ca8..a0e12b81 100644 --- a/src/guidellm/utils/__init__.py +++ b/src/guidellm/utils/__init__.py @@ -1,40 +1,14 @@ -from .injector import create_report, inject_data -from .progress import BenchmarkReportProgress -from .text import ( - clean_text, - filter_text, - is_path, - is_path_like, - is_url, - load_text, - load_text_lines, - parse_text_objects, - split_lines_by_punctuation, - split_text, -) -from .transformers import ( - load_transformers_dataset, - resolve_transformers_dataset, - resolve_transformers_dataset_column, - resolve_transformers_dataset_split, +from .hf_transformers import ( + check_load_processor, ) +from .random import IntegerRangeSampler +from .text import EndlessTextCreator, clean_text, filter_text, load_text, split_text __all__ = [ - "BenchmarkReportProgress", - "clean_text", - "create_report", + "check_load_processor", "filter_text", - "inject_data", - "is_path", - "is_path_like", - "is_url", - "load_text", - "load_text_lines", - "load_transformers_dataset", - "parse_text_objects", - "resolve_transformers_dataset", - "resolve_transformers_dataset_column", - "resolve_transformers_dataset_split", - "split_lines_by_punctuation", + "clean_text", "split_text", + "load_text", + "EndlessTextCreator", ] diff --git a/src/guidellm/utils/cli_params.py b/src/guidellm/utils/cli_params.py deleted file mode 100644 index 4e8800d2..00000000 --- a/src/guidellm/utils/cli_params.py +++ /dev/null @@ -1,34 +0,0 @@ -""" -This module includes custom CLI parameters for the `click` package. -""" - -from typing import Any, Optional - -from click import Context, Parameter, ParamType - -__all__ = ["MAX_REQUESTS"] - - -class MaxRequestsType(ParamType): - """ - Catch the `dataset` string parameter to determine the behavior of the Scheduler. - """ - - name = "max_requests" - - def convert( - self, value: Any, param: Optional[Parameter], ctx: Optional[Context] - ) -> Any: - if isinstance(value, int): - return value - - try: - return int(value) - except ValueError: - if value == "dataset": - return value - else: - self.fail(f"{value} is not a valid integer or 'dataset'", param, ctx) - - -MAX_REQUESTS = MaxRequestsType() diff --git a/src/guidellm/utils/hf_transformers.py b/src/guidellm/utils/hf_transformers.py new file mode 100644 index 00000000..b5b0f4db --- /dev/null +++ b/src/guidellm/utils/hf_transformers.py @@ -0,0 +1,35 @@ +from pathlib import Path +from typing import Any, Dict, Optional, Union + +from transformers import AutoTokenizer, PreTrainedTokenizerBase + +__all__ = [ + "check_load_processor", +] + + +def check_load_processor( + processor: Optional[Union[str, Path, PreTrainedTokenizerBase]], + processor_args: Optional[Dict[str, Any]], + error_msg: str, +) -> PreTrainedTokenizerBase: + if processor is None: + raise ValueError(f"Processor/Tokenizer is required for {error_msg}.") + + try: + if isinstance(processor, (str, Path)): + loaded = AutoTokenizer.from_pretrained( + processor, + **(processor_args or {}), + ) + else: + loaded = processor + except Exception as err: + raise ValueError( + f"Failed to load processor/Tokenizer for {error_msg}." + ) from err + + if not isinstance(loaded, PreTrainedTokenizerBase): + raise ValueError(f"Invalid processor/Tokenizer for {error_msg}.") + + return loaded diff --git a/src/guidellm/utils/progress.py b/src/guidellm/utils/progress.py deleted file mode 100644 index a1e1e798..00000000 --- a/src/guidellm/utils/progress.py +++ /dev/null @@ -1,199 +0,0 @@ -from datetime import datetime -from typing import List - -from loguru import logger -from rich.console import Group -from rich.live import Live -from rich.panel import Panel -from rich.progress import ( - BarColumn, - Progress, - SpinnerColumn, - TaskID, - TaskProgressColumn, - TextColumn, - TimeElapsedColumn, - TimeRemainingColumn, -) - -__all__ = ["BenchmarkReportProgress"] - - -class BenchmarkReportProgress: - """ - Manages the progress display for benchmarks and report generation using Rich. - - This class provides a visual representation of the benchmarking process - and report generation using Rich's progress bars and panels. - """ - - def __init__(self): - """ - Initialize the BenchmarkReportProgress with default settings. - - This method sets up the progress displays for both individual benchmarks - and the overall report, as well as initializing internal task management - structures. - """ - logger.info("Initializing BenchmarkReportProgress instance") - - self.benchmarks_progress = Progress( - TextColumn("[{task.fields[start_time_str]}]"), - SpinnerColumn(), - TaskProgressColumn(), - TextColumn("{task.description}"), - TextColumn(" "), - TextColumn( - "[bold cyan]({task.fields[req_per_sec]} req/sec avg)[/bold cyan]" - ), - ) - self.benchmarks_panel = Panel( - self.benchmarks_progress, - title="Benchmarks", - title_align="left", - expand=True, - ) - self.report_progress = Progress( - SpinnerColumn(), - TextColumn("Generating report..."), - BarColumn(bar_width=None), - TextColumn( - "({task.fields[completed_benchmarks]}/{task.fields[total_benchmarks]})" - ), - TextColumn("["), - TimeElapsedColumn(), - TextColumn("<"), - TimeRemainingColumn(), - TextColumn("]"), - ) - self.render_group = Group(self.benchmarks_panel, self.report_progress) - self.live = Live(self.render_group, redirect_stdout=True, redirect_stderr=True) - - self.report_task: TaskID = None # type: ignore # noqa: PGH003 - self.benchmark_tasks: List[TaskID] = [] - self.benchmark_tasks_started: List[bool] = [] - self.benchmark_tasks_completed: List[bool] = [] - self.benchmark_tasks_progress: List[float] = [] - - def start(self, task_descriptions: List[str]) -> None: - """ - Starts the live progress display and initializes benchmark tasks. - - :param task_descriptions: List of descriptions for each benchmark task. - :type task_descriptions: List[str] - """ - logger.info( - "Starting BenchmarkReportProgress with task descriptions: {}", - task_descriptions, - ) - self.live.start() - - for task_description in task_descriptions: - logger.debug("Adding task with description: {}", task_description) - task_id = self.benchmarks_progress.add_task( - task_description, - start=False, - total=None, - start_time_str="--:--:--", - req_per_sec="#.##", - ) - self.benchmark_tasks.append(task_id) - self.benchmark_tasks_started.append(False) - self.benchmark_tasks_completed.append(False) - self.benchmark_tasks_progress.append(0) - - self.report_task = self.report_progress.add_task( - "", - total=len(self.benchmark_tasks) * 100, # 100 points per report - completed_benchmarks=0, - total_benchmarks=len(task_descriptions), - ) - logger.info("Initialized {} benchmark tasks", len(task_descriptions)) - - def update_benchmark( - self, - index: int, - description: str, - completed: bool, - completed_count: int, - completed_total: int, - start_time: float, - req_per_sec: float, - ) -> None: - """ - Updates the progress of a specific benchmark task. - - :param index: Index of the benchmark task to update. - :type index: int - :param description: Description of the current benchmark task. - :type description: str - :param completed: Flag indicating if the benchmark is completed. - :type completed: bool - :param completed_count: Number of completed operations for the task. - :type completed_count: int - :param completed_total: Total number of operations for the task. - :type completed_total: int - :param start_time: Start time of the benchmark in timestamp format. - :type start_time: float - :param req_per_sec: Average requests per second. - :type req_per_sec: float - :raises ValueError: If trying to update a completed benchmark. - """ - - if self.benchmark_tasks_completed[index]: - err = ValueError(f"Benchmark {index} already completed") - logger.error("Error updating benchmark: {}", err) - raise err - - if not self.benchmark_tasks_started[index]: - self.benchmark_tasks_started[index] = True - self.benchmarks_progress.start_task(self.benchmark_tasks[index]) - logger.info("Starting benchmark task at index {}", index) - - if completed: - self.benchmark_tasks_completed[index] = True - self.benchmark_tasks_progress[index] = 100 - self.benchmarks_progress.stop_task(self.benchmark_tasks[index]) - logger.info("Completed benchmark task at index {}", index) - - self.benchmark_tasks_progress[index] = completed_count / completed_total * 100 - self.benchmarks_progress.update( - self.benchmark_tasks[index], - description=description, - total=completed_total, - completed=completed_count if not completed else completed_total, - req_per_sec=(f"{req_per_sec:.2f}" if req_per_sec else "#.##"), - start_time_str=( - datetime.fromtimestamp(start_time).strftime("%H:%M:%S") - if start_time - else "--:--:--" - ), - ) - logger.debug( - "Updated benchmark task at index {}: {}% complete", - index, - self.benchmark_tasks_progress[index], - ) - self.report_progress.update( - self.report_task, - total=len(self.benchmark_tasks) * 100, - completed=sum(self.benchmark_tasks_progress), - completed_benchmarks=sum(self.benchmark_tasks_completed), - total_benchmarks=len(self.benchmark_tasks), - ) - - def finish(self) -> None: - """ - Marks the overall report task as finished and stops the live display. - """ - logger.info("Finishing BenchmarkReportProgress") - self.report_progress.update( - self.report_task, - total=len(self.benchmark_tasks) * 100, - completed=len(self.benchmark_tasks) * 100, - completed_benchmarks=len(self.benchmark_tasks), - total_benchmarks=len(self.benchmark_tasks), - ) - self.report_progress.stop_task(self.report_task) - self.live.stop() - logger.info("BenchmarkReportProgress finished and live display stopped") diff --git a/src/guidellm/utils/random.py b/src/guidellm/utils/random.py new file mode 100644 index 00000000..17873d12 --- /dev/null +++ b/src/guidellm/utils/random.py @@ -0,0 +1,42 @@ +import random +from typing import Iterator, Optional + +__all__ = ["IntegerRangeSampler"] + + +class IntegerRangeSampler: + def __init__( + self, + average: int, + variance: Optional[int], + min_value: Optional[int], + max_value: Optional[int], + random_seed: int, + ): + self.average = average + self.variance = variance + self.min_value = min_value + self.max_value = max_value + self.seed = random_seed + self.rng = random.Random(random_seed) + + def __iter__(self) -> Iterator[int]: + calc_min = self.min_value + if not calc_min: + calc_min = max( + 0, self.average - 5 * self.variance if self.variance else self.average + ) + calc_max = self.max_value + if not calc_max: + calc_max = ( + self.average + 5 * self.variance if self.variance else self.average + ) + + while True: + if calc_min == calc_max: + yield calc_min + elif not self.variance: + yield self.rng.randint(calc_min, calc_max + 1) + else: + rand = self.rng.gauss(self.average, self.variance) + yield round(max(calc_min, min(calc_max, rand))) diff --git a/src/guidellm/utils/text.py b/src/guidellm/utils/text.py index f8c5038c..6a3b6042 100644 --- a/src/guidellm/utils/text.py +++ b/src/guidellm/utils/text.py @@ -1,60 +1,22 @@ -import csv -import json import re from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union -from urllib.parse import urlparse +from typing import List, Optional, Union import ftfy -import requests -import yaml +import httpx from loguru import logger from guidellm.config import settings __all__ = [ - "clean_text", "filter_text", - "is_path", - "is_path_like", - "is_url", - "load_text", - "load_text_lines", - "parse_text_objects", - "split_lines_by_punctuation", + "clean_text", "split_text", + "load_text", + "EndlessTextCreator", ] - -NAME_TITLES = [ - "Mr.", - "Mrs.", - "Ms.", - "Dr.", - "Prof.", - "Jr.", - "Sr.", - "St.", - "Lt.", - "Col.", - "Gen.", - "Rep.", - "Sen.", - "Gov.", - "Pres.", -] -SENTENCE_REGEX = r'[^.!?]*[.!?]["\']?\s*(?=[A-Z])' -MAX_EXTENSION_LENGTH = 8 MAX_PATH_LENGTH = 4096 -EXTENSION_TYPES = { - "csv": "csv", - "jsonl": "jsonl", - "json": "json", - "yaml": "yaml", - "yml": "yaml", - "txt": "txt", - "text": "txt", -} def filter_text( @@ -95,216 +57,17 @@ def filter_text( return text -def clean_text( - text: str, - fix_encoding: bool = True, - clean_whitespace: bool = False, - remove_empty_lines: bool = False, - force_new_line_punctuation: bool = False, -) -> str: - """ - Clean text by fixing encoding, cleaning whitespace, removing empty lines, - and forcing new line punctuation - - :param text: the text to clean - :param fix_encoding: True to fix the encoding of the text, False to leave as is - :param clean_whitespace: True to clean the whitespace in the text - (remove extra spaces, tabs, etc), False to leave as is - :param remove_empty_lines: True to remove empty lines from the text - (lines with only whitespace), False to leave as is - :param force_new_line_punctuation: True to force new lines at punctuation - (line ends in a period, exclamation point, or question mark), - False to leave as is - :return: The cleaned text - """ - - if fix_encoding: - text = ftfy.fix_text(text) - - if clean_whitespace: - text = "\n".join( - [re.sub(r"\s+", " ", line).strip() for line in text.splitlines()] - ) - - if remove_empty_lines: - text = "\n".join([line for line in text.splitlines() if line.strip()]) - - if force_new_line_punctuation: - # first remove any existing new lines - text = " ".join(line for line in text.splitlines() if line.strip()) - lines = split_lines_by_punctuation(text) - text = "\n".join(lines) - - return text - - -def split_lines_by_punctuation(text: str) -> List[str]: - """ - Split text into lines based on punctuation - - :param text: the text to split - :return: the list of lines - """ - - lines = [] - current_line = "" - skip_next = False - - for index, char in enumerate(text): - if skip_next: - skip_next = False - continue - - current_line += char - - if char not in [".", "!", "?"]: - # must match end of sentence punctuation - continue - - # if this is the character for a title, don't split - if any(current_line.endswith(title) for title in NAME_TITLES): - continue - - char_next_1 = text[index + 1] if index + 1 < len(text) else None - char_next_2 = text[index + 2] if index + 2 < len(text) else None - char_next_3 = text[index + 3] if index + 3 < len(text) else None - - next_is_space = char_next_1 and char_next_1.isspace() - next_is_quote_and_space = char_next_1 in ["'", '"'] and char_next_2 == " " - - # next character must be a space or a quote, otherwise skip - if not next_is_space and not next_is_quote_and_space: - continue - - # after this, next character must be an upper case letter - upper_char = char_next_3 if next_is_quote_and_space else char_next_2 - next_is_upper = upper_char and ( - upper_char.isupper() or upper_char in ["'", '"'] - ) +def clean_text(text: str) -> str: + return re.sub(r"\s+", " ", ftfy.fix_text(text)).strip() - if not next_is_upper: - continue - # if next char is a quote, add it and skip next - if next_is_quote_and_space: - current_line += text[index + 1] - skip_next = True +def split_text(text: str, split_punctuation: bool = False) -> List[str]: + text = clean_text(text) - lines.append(current_line.strip()) - current_line = "" + if split_punctuation: + return re.findall(r"[\w]+|[.,!?;]", text) - if current_line: - lines.append(current_line.strip()) - - return lines - - -def is_url(url: str) -> bool: - """ - Check if a string is a URL - - :param url: the string to check - :return: True if the string is a URL, False if not - """ - try: - result = urlparse(url) - return all([result.scheme, result.netloc]) - except Exception: # noqa: BLE001 - return False - - -def is_path(path: Any) -> bool: - """ - Check if a string is a path - - :param path: the string to check - :return: True if the string is a path, False if not - """ - if not isinstance(path, (str, Path)): - return False - - if isinstance(path, str): - path = Path(path) - - return path.exists() - - -def is_path_like(path: Any, enforce_file: bool = False) -> bool: - """ - Check if a string has a path like structure where it doesn't need to exist - - :param path: the string to check - :param enforce_file: True if the path should be a file, False if not - :return: True if the string is path like, False if not - """ - # if path isn't a str or Path, it's not a path - if not isinstance(path, (str, Path)): - return False - - if isinstance(path, Path): - path = str(path) - - # if text is too long, it's not a path (4096 for most linux setups) - if len(path) > MAX_PATH_LENGTH: - return False - - # if it starts with a URL scheme, it's not a path - if path.startswith(("http", "ftp")): - return False - - test_path = Path(path) - - # if it's supposed to be a file and there's no extension or - # the extension is too long, it's not a path - return not enforce_file or ( - bool(test_path.suffix) and len(test_path.suffix) <= MAX_EXTENSION_LENGTH - ) - - -def split_text(text: str) -> Tuple[List[str], List[str], List[int]]: - """ - Split text into words / tokens, the white space separators between words, - and the indices for each new line - - :param text: the text to split - :return: the words, the white space separators, and the new line indices - """ - if not text or not text.strip(): - return [], [], [] - - text = text.strip() - tokens = [] # type: List[str] - separators = [] # type: List[str] - new_lines = [0] - buffer = text[0] - is_token = not text[0].isspace() - - for char in text[1:]: - char_whitespace = char.isspace() - - if char == "\n": - new_lines.append(len(tokens) + 1) - - if char_whitespace and is_token: - tokens.append(buffer) - buffer = char - is_token = False - elif char_whitespace: - buffer += char - elif not char_whitespace and not is_token: - separators.append(buffer) - buffer = char - is_token = True - else: - buffer += char - - if buffer and is_token: - tokens.append(buffer) - separators.append(" ") - elif buffer: - separators.append(buffer) - - return tokens, separators, new_lines + return text.split() def load_text(data: Union[str, Path], encoding: Optional[str] = None) -> str: @@ -324,132 +87,62 @@ def load_text(data: Union[str, Path], encoding: Optional[str] = None) -> str: return "" # check URLs - if isinstance(data, str) and data.startswith("http"): - response = requests.get(data, timeout=settings.request_timeout) - response.raise_for_status() - return response.text - - # check raw text - if isinstance(data, str) and not is_path_like(data, enforce_file=True): + if isinstance(data, str) and data.strip().startswith(("http", "ftp")): + with httpx.Client(timeout=settings.request_timeout) as client: + response = client.get(data.strip()) + response.raise_for_status() + return response.text + + # check if it's raw text by not being a path + if isinstance(data, str) and ( + len(data) > MAX_PATH_LENGTH or not Path(data).exists() + ): return data # assume local file if not isinstance(data, Path): data = Path(data) - if not data.exists(): + if not data.exists() or not data.is_file(): raise FileNotFoundError(f"File not found: {data}") - if not data.is_file(): - raise IsADirectoryError(f"Path is a directory: {data}") - return data.read_text(encoding=encoding) -def parse_text_objects(data: str, format_: str = "txt") -> List[Dict]: +def is_puncutation(text: str) -> bool: """ - Parse text data into a list of dictionaries based on the format given - (csv, jsonl, json, yaml, txt). - - :param data: the text data to parse - :param format_: the format of the data to parse: - 'csv', 'jsonl', 'json', 'yaml', 'txt' - :return: the list of dictionaries parsed from the data, if text - then each line is a dictionary with a single key 'text' - """ - if not isinstance(data, str): - raise ValueError(f"Unsupported data given of type: {type(data)}") - - if format_ == "csv": - reader = csv.DictReader(data.splitlines()) - columns = reader.fieldnames - return [{col: row[col] for col in columns} for row in reader] # type: ignore # noqa: PGH003 - - if format_ == "jsonl": - return [json.loads(line) for line in data.splitlines() if line] - - if format_ in ("json", "yaml"): - data = json.loads(data) if format_ == "json" else yaml.safe_load(data) - - if not data: - return [] - - if isinstance(data, dict) and len(data) == 1: - logger.debug("Getting first value from JSON/YAML object: {}", data) - data = list(data.values())[0] - elif isinstance(data, dict): - logger.debug("Converting JSON/YAML object to list: {}", data) - data = list(data.values()) - - if not isinstance(data, list) or not isinstance(data[0], dict): - raise ValueError(f"Unsupported data structure given: {data}") - - return data - - if format_ == "txt": - return [{"text": line} for line in data.splitlines() if line] + Check if the text is a punctuation - raise ValueError(f"Unsupported format given: {format_}") - - -def load_text_lines( - data: Union[str, Path, List[Dict]], - format_: Optional[str] = None, - filters: Optional[List[str]] = None, - encoding: Optional[str] = None, -) -> List[str]: + :param text: the text to check + :type text: str + :return: True if the text is a punctuation, False otherwise + :rtype: bool """ - Load text lines from a file or data object with optional filtering and formatting. - - - :param data: the data to load the text lines from - :param format_: the format of the data to load, if not provided will be inferred. - Supported formats: 'csv', 'jsonl', 'json', 'yaml', 'txt' - :param filters: the keys to filter the data by when loading in order of preference. - If not provided, will use the first key in the data object. - :param encoding: the encoding to use when reading the file - :return: the list of text lines - """ - logger.debug( - "Loading text lines with format {}, filters {}, encoding {} for data: {}", - format_, - filters, - encoding, - data, - ) - - if not data: - return [] - - if not format_ and isinstance(data, (str, Path)) and "." in str(data): - extension = str(data).split(".")[-1] - format_ = EXTENSION_TYPES.get(extension, "txt") - elif not format_: - format_ = "txt" + return len(text) == 1 and not text.isalnum() and not text.isspace() - # load the data if it's a path or URL - if isinstance(data, (Path, str)): - data = load_text(data, encoding=encoding) - data = clean_text(data) - # parse the data into a list of dictionaries based on the format - if isinstance(data, str): - data = parse_text_objects(data, format_) +class EndlessTextCreator: + def __init__( + self, + data: Union[str, Path], + filter_start: Optional[Union[str, int]] = None, + filter_end: Optional[Union[str, int]] = None, + ): + self.data = data + self.text = load_text(data) + self.filtered_text = filter_text(self.text, filter_start, filter_end) + self.words = split_text(self.filtered_text, split_punctuation=True) - if not isinstance(data, list): - raise ValueError(f"Unsupported data given of type: {type(data)}") + def create_text(self, start: int, length: int) -> str: + text = "" - if not isinstance(data[0], dict): - raise ValueError(f"Unsupported data item type given: {type(data[0])}") + for counter in range(length): + index = (start + counter) % len(self.words) + add_word = self.words[index] - # grab the first available filter key to use if preference order as provided - filter_ = list(data[0].keys())[0] - for filt in filters or []: - if filt not in data[0]: - continue + if counter != 0 and not is_puncutation(add_word): + text += " " - filter_ = filt - break + text += add_word - # extract the lines from the data - return [row[filter_] for row in data] if filter_ else [str(row) for row in data] + return text diff --git a/src/guidellm/utils/transformers.py b/src/guidellm/utils/transformers.py deleted file mode 100644 index 54057299..00000000 --- a/src/guidellm/utils/transformers.py +++ /dev/null @@ -1,151 +0,0 @@ -from pathlib import Path -from typing import List, Optional, Union - -from datasets import ( # type: ignore # noqa: PGH003 - Dataset, - DatasetDict, - IterableDataset, - IterableDatasetDict, - load_dataset, -) -from loguru import logger - -from guidellm.config import settings - -__all__ = [ - "load_transformers_dataset", - "resolve_transformers_dataset", - "resolve_transformers_dataset_column", - "resolve_transformers_dataset_split", -] - - -def load_transformers_dataset( - dataset: Union[ - str, Path, DatasetDict, Dataset, IterableDatasetDict, IterableDataset - ], - split: Optional[str] = None, - preferred_splits: Optional[List[str]] = settings.dataset.preferred_data_splits, - **kwargs, -) -> Union[Dataset, IterableDataset]: - """ - Load a dataset from a file or a script and resolve the preferred split. - - :param dataset: the dataset file or script to load - :param split: the dataset split to use - (overrides preferred_splits, must be in dataset) - :param preferred_splits: the preferred dataset splits to use - :param kwargs: additional keyword arguments to pass to the dataset loader - :return: the loaded dataset - """ - dataset = resolve_transformers_dataset(dataset, **kwargs) - - return resolve_transformers_dataset_split(dataset, split, preferred_splits) - - -def resolve_transformers_dataset( - dataset: Union[ - str, Path, DatasetDict, Dataset, IterableDatasetDict, IterableDataset - ], - **kwargs, -) -> Union[DatasetDict, Dataset, IterableDatasetDict, IterableDataset]: - """ - Resolve the dataset from a file (csv, json, script) or a dataset name. - - :param dataset: the dataset file or script to load - :param kwargs: additional keyword arguments to pass to the dataset loader - :return: the loaded dataset - """ - if isinstance( - dataset, (DatasetDict, Dataset, IterableDatasetDict, IterableDataset) - ): - return dataset - - if not isinstance(dataset, (str, Path)): - raise ValueError(f"Invalid dataset type: {type(dataset)}") - - dataset = str(dataset) - - if dataset.endswith((".csv", ".json")): - logger.debug("Loading dataset from local path: {}", dataset) - extension = dataset.split(".")[-1] - - return load_dataset(extension, data_files=dataset, **kwargs) - - if dataset.endswith(".py"): - logger.debug("Loading dataset from local script: {}", dataset) - - return load_dataset(dataset, **kwargs) - - logger.debug("Loading dataset: {}", dataset) - - return load_dataset(dataset, **kwargs) - - -def resolve_transformers_dataset_split( - dataset: Union[DatasetDict, Dataset, IterableDatasetDict, IterableDataset], - split: Optional[str] = None, - preferred_splits: Optional[List[str]] = settings.dataset.preferred_data_splits, -) -> Union[Dataset, IterableDataset]: - """ - Resolve the preferred split from a dataset dictionary. - - :param dataset: the dataset to resolve the split from - :param split: the dataset split to use - (overrides preferred_splits, must be in dataset) - :param preferred_splits: the preferred dataset splits to use - :return: the resolved dataset split - """ - if not isinstance(dataset, (DatasetDict, IterableDatasetDict)): - logger.debug("Dataset is not a dictionary, using default split") - return dataset - - if split: - if split not in dataset: - raise ValueError(f"Split '{split}' not found in dataset") - - return dataset[split] - - if preferred_splits: - for spl in preferred_splits: - if spl not in dataset: - continue - return dataset[spl] - - return list(dataset.values())[0] - - -def resolve_transformers_dataset_column( - dataset: Union[Dataset, IterableDataset], - column: Optional[str] = None, - preferred_columns: Optional[List[str]] = settings.dataset.preferred_data_columns, -) -> str: - """ - Resolve the preferred column from a dataset. - - :param dataset: the dataset to resolve the column from - :param column: the dataset column to use - (overrides preferred_columns, must be in dataset) - :param preferred_columns: the preferred dataset columns to use - :return: the resolved dataset column - """ - column_names = dataset.column_names - - if not column_names: - # grab from the first item - first_item = next(iter(dataset)) - column_names = list(first_item.keys()) - - if column: - if column not in column_names: - raise ValueError(f"Column '{column}' not found in dataset") - - return column - - if preferred_columns: - for col in preferred_columns: - if col not in column_names: - continue - return col - - return list(column_names)[0] diff --git a/tests/unit/core/test_distribution.py b/tests/unit/core/test_distribution.py deleted file mode 100644 index 95b7e923..00000000 --- a/tests/unit/core/test_distribution.py +++ /dev/null @@ -1,107 +0,0 @@ -import pytest - -from guidellm.core import Distribution - - -@pytest.mark.smoke() -def test_distribution_initialization(): - data = [1, 2, 3, 4, 5] - dist = Distribution(data=data) - assert dist.data == data - - -@pytest.mark.smoke() -def test_distribution_statistics(): - data = [1, 2, 3, 4, 5] - dist = Distribution(data=data) - assert dist.mean == 3.0 - assert dist.median == 3.0 - assert dist.variance == 2.0 - assert dist.std_deviation == pytest.approx(1.414213, rel=1e-5) - assert dist.min == 1 - assert dist.max == 5 - assert dist.range == 4 - assert dist.percentile(50) == 3.0 - assert dist.percentiles([25, 50, 75]) == pytest.approx([2.0, 3.0, 4.0]) - - -@pytest.mark.smoke() -def test_distribution_no_data(): - dist = Distribution(data=[]) - assert dist.mean == 0.0 - assert dist.median == 0.0 - assert dist.variance == 0.0 - assert dist.std_deviation == 0.0 - assert dist.min == 0.0 - assert dist.max == 0.0 - assert dist.range == 0.0 - assert dist.percentile(50) == 0.0 - assert dist.percentiles([25, 50, 75]) == [0.0, 0.0, 0.0] - - -@pytest.mark.sanity() -def test_distribution_add_data(): - data = [1, 2, 3, 4, 5] - dist = Distribution(data=data) - new_data = [6, 7, 8] - dist.add_data(new_data) - - assert dist.data == data + new_data - - -@pytest.mark.sanity() -def test_distribution_remove_data(): - data = [1, 2, 3, 4, 5] - dist = Distribution(data=data) - remove_data = [2, 4] - dist.remove_data(remove_data) - assert dist.data == [1, 3, 5] - - -@pytest.mark.regression() -def test_distribution_str(): - data = [1, 2, 3, 4, 5] - dist = Distribution(data=data) - assert "Distribution({" in str(dist) - assert "'mean': 3.0" in str(dist) - assert "'median': 3.0" in str(dist) - assert "'variance': 2.0" in str(dist) - assert "'percentile_indices': [10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 99]" in str( - dist - ) - assert ( - "'percentile_values': [1.4, 1.8, 2.2, 2.6, 3.0, 3.4, 3.8, 4.2, 4.6, 4.8, 4.96]" - in str(dist) - ) - assert "'min': 1" in str(dist) - assert "'max': 5" in str(dist) - assert "'range': 4" in str(dist) - - -@pytest.mark.regression() -def test_distribution_repr(): - data = [1, 2, 3, 4, 5] - dist = Distribution(data=data) - assert repr(dist) == f"Distribution(data={dist.data})" - - -@pytest.mark.regression() -def test_distribution_json(): - data = [1, 2, 3, 4, 5] - dist = Distribution(data=data) - json_str = dist.to_json() - assert f'"data":[{dist.data[0]}' in json_str - - dist_restored = Distribution.from_json(json_str) - assert dist_restored.data == data - - -@pytest.mark.regression() -def test_distribution_yaml(): - data = [1, 2, 3, 4, 5] - dist = Distribution(data=data) - yaml_str = dist.to_yaml() - assert f"data:\n- {dist.data[0]}" in yaml_str - - dist_restored = Distribution.from_yaml(yaml_str) - assert dist_restored.data == data diff --git a/tests/unit/objects/test_distribution.py b/tests/unit/objects/test_distribution.py new file mode 100644 index 00000000..e721e0c1 --- /dev/null +++ b/tests/unit/objects/test_distribution.py @@ -0,0 +1,337 @@ +import math + +import numpy as np +import pytest + +from guidellm.objects import DistributionSummary, Percentiles + + +def create_default_percentiles() -> Percentiles: + return Percentiles( + p001=0.1, + p01=1.0, + p05=5.0, + p10=10.0, + p25=25.0, + p75=75.0, + p90=90.0, + p95=95.0, + p99=99.0, + p999=99.9, + ) + + +def create_default_distribution_summary() -> DistributionSummary: + return DistributionSummary( + mean=50.0, + median=50.0, + variance=835, + std_dev=math.sqrt(835), + min=0.0, + max=100.0, + count=1001, + percentiles=create_default_percentiles(), + ) + + +@pytest.mark.smoke() +def test_percentiles_initialization(): + percentiles = create_default_percentiles() + assert percentiles.p001 == 0.1 + assert percentiles.p01 == 1.0 + assert percentiles.p05 == 5.0 + assert percentiles.p10 == 10.0 + assert percentiles.p25 == 25.0 + assert percentiles.p75 == 75.0 + assert percentiles.p90 == 90.0 + assert percentiles.p95 == 95.0 + assert percentiles.p99 == 99.0 + assert percentiles.p999 == 99.9 + + +@pytest.mark.smoke() +def test_percentiles_invalid_initialization(): + test_kwargs = { + "p001": 0.1, + "p01": 1.0, + "p05": 5.0, + "p10": 10.0, + "p25": 25.0, + "p75": 75.0, + "p90": 90.0, + "p95": 95.0, + "p99": 99.0, + "p999": 99.9, + } + test_missing_keys = list(test_kwargs.keys()) + + for missing_key in test_missing_keys: + kwargs = {key: val for key, val in test_kwargs.items() if key != missing_key} + with pytest.raises(ValueError): + Percentiles(**kwargs) + + +@pytest.mark.smoke() +def test_percentiles_from_values(): + values = [val / 10 for val in range(1001)] + percentiles = Percentiles.from_values(values) + true_percentiles = np.percentile( + values, [0.1, 1.0, 5.0, 10.0, 25.0, 75.0, 90.0, 95.0, 99.0, 99.9] + ) + assert percentiles.p001 == pytest.approx(true_percentiles[0]) + assert percentiles.p01 == pytest.approx(true_percentiles[1]) + assert percentiles.p05 == pytest.approx(true_percentiles[2]) + assert percentiles.p10 == pytest.approx(true_percentiles[3]) + assert percentiles.p25 == pytest.approx(true_percentiles[4]) + assert percentiles.p75 == pytest.approx(true_percentiles[5]) + assert percentiles.p90 == pytest.approx(true_percentiles[6]) + assert percentiles.p95 == pytest.approx(true_percentiles[7]) + assert percentiles.p99 == pytest.approx(true_percentiles[8]) + assert percentiles.p999 == pytest.approx(true_percentiles[9]) + + +@pytest.mark.smoke() +def test_percentiles_marshalling(): + percentiles = create_default_percentiles() + serialized = percentiles.model_dump() + deserialized = Percentiles.model_validate(serialized) + + for key, value in vars(percentiles).items(): + assert getattr(deserialized, key) == value + + +@pytest.mark.smoke() +def test_distribution_summary_initialization(): + distribution_summary = DistributionSummary( + mean=50.0, + median=50.0, + variance=835, + std_dev=math.sqrt(835), + min=0.0, + max=100.0, + count=1001, + percentiles=create_default_percentiles(), + ) + assert distribution_summary.mean == 50.0 + assert distribution_summary.median == 50.0 + assert distribution_summary.variance == 835 + assert distribution_summary.std_dev == math.sqrt(835) + assert distribution_summary.min == 0.0 + assert distribution_summary.max == 100.0 + assert distribution_summary.count == 1001 + assert distribution_summary.percentiles.p001 == 0.1 + assert distribution_summary.percentiles.p01 == 1.0 + assert distribution_summary.percentiles.p05 == 5.0 + assert distribution_summary.percentiles.p10 == 10.0 + assert distribution_summary.percentiles.p25 == 25.0 + assert distribution_summary.percentiles.p75 == 75.0 + assert distribution_summary.percentiles.p90 == 90.0 + assert distribution_summary.percentiles.p95 == 95.0 + assert distribution_summary.percentiles.p99 == 99.0 + assert distribution_summary.percentiles.p999 == 99.9 + + +@pytest.mark.smoke() +def test_distribution_summary_invalid_initialization(): + test_kwargs = { + "mean": 50.0, + "median": 50.0, + "variance": 835, + "std_dev": math.sqrt(835), + "min": 0.0, + "max": 100.0, + "count": 1001, + "percentiles": create_default_percentiles(), + } + test_missing_keys = list(test_kwargs.keys()) + for missing_key in test_missing_keys: + kwargs = {key: val for key, val in test_kwargs.items() if key != missing_key} + with pytest.raises(ValueError): + DistributionSummary(**kwargs) + + +@pytest.mark.smoke() +def test_distribution_summary_from_values(): + values = [val / 10 for val in range(1001)] + distribution_summary = DistributionSummary.from_values(values) + assert distribution_summary.mean == pytest.approx(np.mean(values)) + assert distribution_summary.median == pytest.approx(np.median(values)) + assert distribution_summary.variance == pytest.approx(np.var(values, ddof=1)) + assert distribution_summary.std_dev == pytest.approx(np.std(values, ddof=1)) + assert distribution_summary.min == min(values) + assert distribution_summary.max == max(values) + assert distribution_summary.count == len(values) + assert distribution_summary.percentiles.p001 == pytest.approx( + np.percentile(values, 0.1) + ) + assert distribution_summary.percentiles.p01 == pytest.approx( + np.percentile(values, 1.0) + ) + assert distribution_summary.percentiles.p05 == pytest.approx( + np.percentile(values, 5.0) + ) + assert distribution_summary.percentiles.p10 == pytest.approx( + np.percentile(values, 10.0) + ) + assert distribution_summary.percentiles.p25 == pytest.approx( + np.percentile(values, 25.0) + ) + assert distribution_summary.percentiles.p75 == pytest.approx( + np.percentile(values, 75.0) + ) + assert distribution_summary.percentiles.p90 == pytest.approx( + np.percentile(values, 90.0) + ) + assert distribution_summary.percentiles.p95 == pytest.approx( + np.percentile(values, 95.0) + ) + assert distribution_summary.percentiles.p99 == pytest.approx( + np.percentile(values, 99.0) + ) + assert distribution_summary.percentiles.p999 == pytest.approx( + np.percentile(values, 99.9) + ) + + +@pytest.mark.smoke() +def test_distribution_summary_from_time_measurements_count(): + # create bimodal distribution to test count comes out to average + # ie, 1 is active for 50 seconds, 2 is active for 100 seconds + values = [(val / 10, 1) for val in range(500)] + values += [(val / 5 + 50, 2) for val in range(500)] + distribution_summary = DistributionSummary.from_time_measurements( + values, time_weighting="count" + ) + assert distribution_summary.mean == pytest.approx( + (1 * 50 + 2 * 100) / 150, abs=0.001 + ) + assert distribution_summary.median == pytest.approx(2) + assert distribution_summary.variance == pytest.approx(0.2223, abs=0.001) + assert distribution_summary.std_dev == pytest.approx(0.4715, abs=0.001) + assert distribution_summary.min == 1 + assert distribution_summary.max == 2 + assert distribution_summary.count == pytest.approx(100000, abs=1000) + assert distribution_summary.percentiles.p001 == pytest.approx(1) + assert distribution_summary.percentiles.p01 == pytest.approx(1) + assert distribution_summary.percentiles.p05 == pytest.approx(1) + assert distribution_summary.percentiles.p10 == pytest.approx(1) + assert distribution_summary.percentiles.p25 == pytest.approx(1) + assert distribution_summary.percentiles.p75 == pytest.approx(2) + assert distribution_summary.percentiles.p90 == pytest.approx(2) + assert distribution_summary.percentiles.p95 == pytest.approx(2) + assert distribution_summary.percentiles.p99 == pytest.approx(2) + assert distribution_summary.percentiles.p999 == pytest.approx(2) + + +@pytest.mark.smoke() +def test_distribution_summary_from_time_measurements_multiply(): + # create consistent timestamped values matching a rate of 10 per second + values = [(val / 10, 1) for val in range(1001)] + distribution_summary = DistributionSummary.from_time_measurements( + values, time_weighting="multiply" + ) + assert distribution_summary.mean == pytest.approx(0.1) + assert distribution_summary.median == pytest.approx(0.1) + assert distribution_summary.variance == pytest.approx(0) + assert distribution_summary.std_dev == pytest.approx(0) + assert distribution_summary.min == pytest.approx(0.1) + assert distribution_summary.max == pytest.approx(0.1) + assert distribution_summary.count == len(values) - 1 + assert distribution_summary.percentiles.p001 == pytest.approx(0.1) + assert distribution_summary.percentiles.p01 == pytest.approx(0.1) + assert distribution_summary.percentiles.p05 == pytest.approx(0.1) + assert distribution_summary.percentiles.p10 == pytest.approx(0.1) + assert distribution_summary.percentiles.p25 == pytest.approx(0.1) + assert distribution_summary.percentiles.p75 == pytest.approx(0.1) + assert distribution_summary.percentiles.p90 == pytest.approx(0.1) + assert distribution_summary.percentiles.p95 == pytest.approx(0.1) + assert distribution_summary.percentiles.p99 == pytest.approx(0.1) + assert distribution_summary.percentiles.p999 == pytest.approx(0.1) + + +@pytest.mark.smoke() +def test_distribution_summary_from_time_measurements_divide(): + # create consistent timestamped values matching a rate of 10 per second + values = [(val / 10, 1) for val in range(1001)] + distribution_summary = DistributionSummary.from_time_measurements( + values, time_weighting="divide" + ) + assert distribution_summary.mean == pytest.approx(10.0) + assert distribution_summary.median == pytest.approx(10.0) + assert distribution_summary.variance == pytest.approx(0) + assert distribution_summary.std_dev == pytest.approx(0) + assert distribution_summary.min == pytest.approx(10.0) + assert distribution_summary.max == pytest.approx(10.0) + assert distribution_summary.count == len(values) - 1 + assert distribution_summary.percentiles.p001 == pytest.approx(10.0) + assert distribution_summary.percentiles.p01 == pytest.approx(10.0) + assert distribution_summary.percentiles.p05 == pytest.approx(10.0) + assert distribution_summary.percentiles.p10 == pytest.approx(10.0) + assert distribution_summary.percentiles.p25 == pytest.approx(10.0) + assert distribution_summary.percentiles.p75 == pytest.approx(10.0) + assert distribution_summary.percentiles.p90 == pytest.approx(10.0) + assert distribution_summary.percentiles.p95 == pytest.approx(10.0) + assert distribution_summary.percentiles.p99 == pytest.approx(10.0) + assert distribution_summary.percentiles.p999 == pytest.approx(10.0) + + +@pytest.mark.smoke() +def test_distribution_summary_from_time_ranges_count(): + # create consistent time ranges representing 10 concurrent requests constant + values = [(val / 10, val / 10 + 1, 1) for val in range(10001)] + distribution_summary = DistributionSummary.from_time_ranges( + values, time_weighting="count" + ) + assert distribution_summary.mean == pytest.approx(10.0, abs=0.01) + assert distribution_summary.median == pytest.approx(10.0) + assert distribution_summary.variance == pytest.approx(0, abs=0.1) + assert distribution_summary.std_dev == pytest.approx(0, abs=0.3) + assert distribution_summary.min == pytest.approx(1) + assert distribution_summary.max == pytest.approx(10.0) + assert distribution_summary.count == pytest.approx(100000, abs=1000) + assert distribution_summary.percentiles.p001 == pytest.approx(10, abs=5) + assert distribution_summary.percentiles.p01 == pytest.approx(10) + assert distribution_summary.percentiles.p05 == pytest.approx(10) + assert distribution_summary.percentiles.p10 == pytest.approx(10) + assert distribution_summary.percentiles.p25 == pytest.approx(10) + assert distribution_summary.percentiles.p75 == pytest.approx(10) + assert distribution_summary.percentiles.p90 == pytest.approx(10) + assert distribution_summary.percentiles.p95 == pytest.approx(10) + assert distribution_summary.percentiles.p99 == pytest.approx(10) + assert distribution_summary.percentiles.p999 == pytest.approx(10) + + +@pytest.mark.smoke() +def test_distribution_summary_from_time_ranges_multiply(): + # create consistent time ranges representing 10 concurrent requests constant + values = [(val / 10, val / 10 + 1, 1) for val in range(10001)] + distribution_summary = DistributionSummary.from_time_ranges( + values, time_weighting="multiply" + ) + assert distribution_summary.mean == pytest.approx(1.0, abs=0.01) + assert distribution_summary.median == pytest.approx(1.0) + assert distribution_summary.variance == pytest.approx(0, abs=0.1) + assert distribution_summary.std_dev == pytest.approx(0, abs=0.3) + assert distribution_summary.min == pytest.approx(0.1) + assert distribution_summary.max == pytest.approx(1.0) + assert distribution_summary.count == pytest.approx(10000, abs=10) + assert distribution_summary.percentiles.p001 == pytest.approx(1.0, abs=0.5) + assert distribution_summary.percentiles.p01 == pytest.approx(1.0) + assert distribution_summary.percentiles.p05 == pytest.approx(1.0) + assert distribution_summary.percentiles.p10 == pytest.approx(1.0) + assert distribution_summary.percentiles.p25 == pytest.approx(1.0) + assert distribution_summary.percentiles.p75 == pytest.approx(1.0) + assert distribution_summary.percentiles.p90 == pytest.approx(1.0) + assert distribution_summary.percentiles.p95 == pytest.approx(1.0) + assert distribution_summary.percentiles.p99 == pytest.approx(1.0) + assert distribution_summary.percentiles.p999 == pytest.approx(1.0) + + +@pytest.mark.smoke() +def test_distribution_summary_marshalling(): + distribution_summary = create_default_distribution_summary() + serialized = distribution_summary.model_dump() + deserialized = DistributionSummary.model_validate(serialized) + + for key, value in vars(distribution_summary).items(): + assert getattr(deserialized, key) == value diff --git a/tests/unit/core/test_serializable.py b/tests/unit/objects/test_serializable.py similarity index 100% rename from tests/unit/core/test_serializable.py rename to tests/unit/objects/test_serializable.py diff --git a/tests/unit/utils/test_transformers.py b/tests/unit/utils/test_transformers.py index 5153da3f..92d1e8b0 100644 --- a/tests/unit/utils/test_transformers.py +++ b/tests/unit/utils/test_transformers.py @@ -8,7 +8,7 @@ IterableDatasetDict, ) -from guidellm.utils.transformers import ( +from guidellm.utils.hf_transformers import ( load_transformers_dataset, resolve_transformers_dataset, resolve_transformers_dataset_column, From cfdc2ed76151df9a5a0f702dffc0c41324268441 Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Wed, 2 Apr 2025 16:15:41 +0000 Subject: [PATCH 14/43] Working clis and entrypoints --- plot.png | Bin 43285 -> 0 bytes pyproject.toml | 2 +- src/guidellm/__init__.py | 16 +- src/guidellm/__main__.py | 270 ++++++++++++++++ src/guidellm/backend/backend.py | 18 +- src/guidellm/backend/openai.py | 169 +++++----- src/guidellm/backend/response.py | 9 +- src/guidellm/benchmark/__init__.py | 2 + src/guidellm/benchmark/aggregator.py | 444 +++++++++++++++----------- src/guidellm/benchmark/benchmark.py | 133 +++++--- src/guidellm/benchmark/benchmarker.py | 3 + src/guidellm/benchmark/entrypoints.py | 41 ++- src/guidellm/benchmark/output.py | 304 ++++++++++++++++++ src/guidellm/benchmark/profile.py | 15 +- src/guidellm/benchmark/progress.py | 255 ++++++++++----- src/guidellm/benchmark/test.py | 27 +- src/guidellm/config.py | 2 +- src/guidellm/main.py | 346 -------------------- src/guidellm/objects/__init__.py | 2 + src/guidellm/objects/serializable.py | 2 +- src/guidellm/objects/statistics.py | 59 +++- src/guidellm/objects/test.py | 364 --------------------- src/guidellm/request/loader.py | 27 +- src/guidellm/scheduler/__init__.py | 2 + src/guidellm/scheduler/result.py | 4 +- src/guidellm/scheduler/scheduler.py | 7 +- src/guidellm/scheduler/strategy.py | 83 ++--- src/guidellm/scheduler/worker.py | 141 +++++--- 28 files changed, 1498 insertions(+), 1249 deletions(-) delete mode 100644 plot.png create mode 100644 src/guidellm/__main__.py create mode 100644 src/guidellm/benchmark/output.py delete mode 100644 src/guidellm/main.py delete mode 100644 src/guidellm/objects/test.py diff --git a/plot.png b/plot.png deleted file mode 100644 index 17ce9190d0742f6cfca9a79e71905b8f939711fc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 43285 zcmdqJ2UL~mmMw|_EyYkq1QkjJ$zVbxn@|Cj%qAlSkeorXRVoHh6iF%wC^-s}H%So? z5K*FlpKI4CV2d}Ax9-pP@LIT3@6L-1l{Dxb>&mj&03jOl0G}~Qi=$3yCJ9nFQ`8S0Fxht1`yB)dZ z%8I4maw~ts(2$=i2CG$;e>=p#l6(0#KgJb1mw&tX_`h+vlN<>u+>A%AKd49%upQJ; z<=$!Bls)h1By80kv}v!$D8Lh zqUEx7(;l;;1(^V$(B7(O!w-Ma^48`YDD+^a*3ad$GdMNsEie{1$q@P#D6NAQS0nY=!uTcj>0Qk0s=KR_)jD{Oq$r; zx^=TAPBV^|ma=Qb&f=uZ4;LHr43eXC@?6R7KVcH>?H$oDY5nw2Eyd`CLh1{FL&Xla z?Cd!H_~YbWA**kn9&Yn~`LefiUa(@Yfy%G@WmouS=h^W-y-9zKDV1acZ-q%E4N5D2 z^M;DBOA4V<-j&f5K6UlR+u}oV!KZ2Oxw;y(e|{!p-g-AS@NLs9Bfmk!wZbR%qoo4Y zBke_wv(#Jl^kTYS{W&C~lw)76*|aA@?(I3vX*L=f-hGD;%Xx^+MD$k2az$4~6?=2_ zRjZp;CFxg;m5DE=Wn^*{7!9ej+LduUJaGQW%`bk3XNC)z0}@5`j$AXgW9}5P|7lUe z!y9W{;={v;DOgyTHTA7FPuRwGx!aqf_1&^elubZjcGte6M-@ynhh5c@4Op3(nW>8k z^Aq)^Db7y{46J)A<*cldI={Sl8xs@bILlLfLh#rz#Q-7e^6u|%P)cueiQh{qRH{RE$5GucPg7GikFv-%EZ&pXidz{a>cCS$7PH7Jm2Zmw&QH@J+(C zLe3K>R7zj+bB)+bFdWc|`)Ke=)FDN>th1+qnmU-?#Y8>s;^IQDWZCG$qkhb$UuC?n zMn)-0F+tsC_(#&Qti^>{A=^QJ`^iD$n%6!)VG$AR13wbZ1PI$!ui7l~hpDOQKm(QH zwv(qkM)kc{K!9=T<;Gi_TwK!kH#1jb=Z*i$+icid899TyRHE7#Qx_x{*vJ!RbiH=) z^7D(ZAMX_${;YFwwaV=f>QJ`2K!eqD7X9YjKgpUU#?U zv9ZkS%$qlFw!7uk7eN%^pt*z}YCT~~1oH_Z($jA&l;a*-$KXqH>Khw}Exmj!~C}i|h_0cksx7cANB_-028Trfm;&XWO z4(%~hS|0IRzk--t=FIsGDlc@D_)?M$sw#0mEBVyP<1smn z$tguEU-IUnC||tjzA)3rM-`(lGj0ByKgy%Ry)!_}NsZa@*PjE&8W4;1^ISJ3s&LQr zzb_jxzSi>b9{%HzhX{A5#V-*a2L)X(l7K%G)!W8=u=WX-RZJn!evucbPhn8bx$dKGr? z(#y{V$;C}g+H!JoDRx!s?KifIGC%3uYS$HDOK$u##u?-H4NK+O(Do^0Id*RDizFHu zr#6JG-@zH7Tl&&|yo%yB=E$v{a`D554-yMj1FcP6(&Kyi4ZJoni9}3~cG2t4(o;vj z3MV^GkE~o=LzC-0)*Bs<-KStblr_1>`FF(}XQz$3k6-j`n|vm+(@o-9fqQ~epupv+ zfz&4K59aBqsn58}x91*v>g<|n^ANYf=b|h+iq~)0P}yGed?1DLZT8%$A#-|%`SF^n z3~NI&w`A54R!2)K6#XY-zXk|Ln>1u>nk2i2{}B)4+I4T?OX2AZ{M{m;?WMSDD2zmC#gZxe!M?BLk%pCtyPvQ zthO61;l-Hg=;)*j2fq)~X75fp3vy;*1J(xzu$Cd7_-(gk4 zliE%f>2~y-4GRZ{q_FLv;`QqxnzKkE;XAmLZ|2@!G5wLN$FX?=K||YO)^n`+&SnF| zx|u=hP!gtif2?INMmrKYYi4e=%q@#IjtPNr)x(DmJ-?ReKkxdPZrS@I;cXNpK91ad zLx%Npon5`

B=0+BQl^Naz=PagYs45<`ALL1+CS-WD7F+Fr_OX6$>Kd7IVTlsVp4 zd+Ed!Hj5#qjZ8?R&WtfP8g+v_`@?mIJvoo|5b|LEbv?+7rJ$%Zv0 z3uUACnVs4B`1q2I>y??Er-H@iew`7W>3-Y$E%=T>P27OlNIxyz&Iqk6dmFvt08-9AQ zr7pudn#as5q`%+Dy8nlqda6m)a~ZM7o{I3kXXlx|FU-%<`I`n|K`;tgMX>bTqt|*T z%7!P-VzTevw^!U*BbpMQqRz(3DuLk5H^Sziudi?8L6?`*`njk`uJFlTrC2roECeNk zrtGYPGOzuIES{}cwWd2V{I&c*UGjJ_SA6$$SD-IKuv4$nH^ZPmuw%ZHfQ~hB6CjJk z;)VGkm%cdaU@`q5Hih=1M~~!z7x?W)uJfDxR4Egk^j{baT+|quN-?T^vy(?{`}DNg z*sO-%eJ1-0o^2(ZiM}QA#z@Og3qld1REM zSN6KNw6v;(FDC|pttZr1s~p2-jUW<5;zNfwWod&mi-|PdxAMJKm!xmdn3)dH8vbw_ zTXA0A83ea?vmW;5jj8q%{gELdA&x8ZKHOQGY~80!72L^o(Ju)2g8$~XlL}!MzbnXy zh9Z+jVt9Q;W#n{*1nr!sJLs(kf2!h#QNY9>BNoG}nQ?Qpv()oz;QTAwf}#T+B3Fec zYzFFTQ_Qi{8k{B@460*ZkjPzcLXG{FaS&gEz0jTB^}6C)@ENlCCC;7O>9COWrqU+3 z^+{~b{DitFA_Q5NT2A9iH*|HQF!4gpbGM%LBBfl+9`=2^nZ@lY(u4g#a?M!xTaR8c z6o0N-Yt`g5S!-;HENxH~KFkor(%PS~67GIdkQ%Sn{Ob#pTE#h%Opv!_jimtJN z`kJTLMHc5qZTk4sQ@%XCo;g-7Z37f#D}X{~n53yX76woEf%???It2EcDzVCK+KV&3 zu>)UT9G&iU_wW#4VZe}=P#my*A7gK*&!RXnxNt;lan4d*eRg8t@x(xVeYT90)Dad2 z+*U$zettd!{i0-qO2V7(sisYiL;+~O|Rd$Q9Fb8mOi~c zhrs2kv#VgU_?%f0@(pt_W6!Az@*RxL`rf;=E}dFwk$m{dVYbpqMdb z7MId)c6M}hbnh&E)m1-{?}44q+mwN|fVp?0=i?nyp~pJ9c_~mW#$*_Ort84jN6_PNVZq zS+^Vsq`rIa9s?>Ls-}ZnQ!2SMA78xV5E}}ND4hfq_d9DhGh)C|@E-cv(C`?r=g0Ug zqSnUEn>`!{Xa`bRuyaX%IOF;3DPxbe%_fmU<*Q}x_2DQ+&+`FqtdxOK_%V-+&4EN!u=jm4?)bRAg}dqqdI z+~}FlpW^Q_jz*z(&fY#Fb{>@&mT8GEUurUv#@7hNNY8X- zks4@U(G=3WkujUh>tm-+pLTC z8d8I5Mzme#Z(K;6&G{Z$P(-hmW)_S$Adx;&+#Dz7|0iwt9DfBec$6kJ1w{rcN)@Mx zTD>UM_hFhj2zvESIiLa3snq(fUy3mMWuFhZJL)|@d-iO^(n9O2jC|`bltm#aM6=Pl z#a|fVw+gUZ2_Q--QeI-+x^>eAF740G_jhjIC35SXh=@oMlJ`|HIth9JajZ0|!X!PP zR*YrWSHT-hD{uq#nsyuVzJ&)mM|F4KWN_7)ERseN@_zR0*^pgNQ%+8U|0d)56mP4A z?S_&g!^6*5VzsjBT2zhmJ32eV)-pTFYC8_!YiPK;lUpSSE0~LgrQNgsom#Bgd)>zk zZLKAIkJ=InncmN4<0>g_z9BN=^NN6Z=H}+p)jUP9RInA%h!>*cl?pb`-8L~Ey8vAN z)s~YA+|^*1Us0jZ($ZqXFXZ|9HSoS7p!~|&W5D$o#`WvgeShmA7KL)w5M@|?#=g9J z@gL}%W_rSH#1EX;+uvE`cm}vgsmiPFH);tIxAocznZmyR@{l zdYf`wp3N41`t&2~NrvrETu@LDmr%CXBk>dzMoF*an+tZEDuPOIo2zrcG>$yZk+NSE z@LJTN?y(FnlS>%b0wi+#sh0c3@$n+83=WcAf$_h7nE|X8RCf*x#GYreJBw}i4eZUp zjJAQh#?!V!kCME+15J;O{O1MOOlRpA#kO1ff7#VvSI9DWI|RTb8tl->uV1pLT7V(E zhBh%XF)>9cMf+&hV^JyrdONx##pf*8*N@O46aUeaz5BRP1U4!=KfltSfBres73gwI zzkI1WM2(%YTTxI*X`R&p9dK3EphfDZPiLzl|ieN*wHazSE z#EAlw%qFN}htR{>d&%Bbht|`uDy3W8kepr|?Yc;xQ#gal#UQzJw!XWqO`*Lc?GZLZ zPNg(oRtUnG9PmoYM9H*z{d+wb9@GNE!@~v*8Od9>ZA-pZxQW`c4*8M|1@4m;madDZ zwZ1ugy=R?0J4j_bVHWh-Wx=k&yl^i;4wYFBllz;V#UWPazkD~}+Lo-s9i^SKuw$>b zR6eu1_ME-fh=<+!3arZT$2q+G3t#9Se*E~+KT*7AZq&S`;pwQxl%r-^j&zlZ2K_>5 z`qYz$tmeIy9Fo(k65c#{&TvO_7R0I~HsSQazDz$~UrOZ9lnD@#mHvX3W0eXr9Q*g5 z784UgSxn%OwCPdD1k3{|>0B13!xIw|lYrP9x5fK@LS7rMH_fSSG#Y)9E)yt53fVks z-pui&kd;G6taQ2$1E3PJ2e>3Xd#0Cb<6a@f=Lav9Bd(0ndP{6!WQ+nOWUw$dO~5~j zxQ?D4)#iYaM{k1!THM%fl(iL8xpyeZIiGtnH8WFuHDSBZH%_U~ihkmYa|#R$46K{3 zjT%t>8EJx8I4N;4SnNMaV^#fHvvV#yS#+lFFtwMQ@a{sjqh}EkQrp3)_>ENbxV1hMr9Q7-y&4GPDW5ifkbm{N%fcx0{X)B8 z29A6suStL8QA6z7Isgsj7?ohur~1RK`5B-cQ9f($=y6(XrS=M)(GjmGXLR$j5|P`! z(=ETn=8^D%#W|j@^5u-tO3`76*gYtBZCd1kK3FC3MDGWT;!2)Prn(*8wf-aP-P{{# zAeyW|zd8n54c5LdOqn|k+*OkL7_{6a-0!=9!=sqD2-BwQmm+qFB>V^uWhxlgC2^Q} zesfE;!>qHiv2{`3N=t5^=#hxm%4#1Ri~}cofo>j-R?s<|}g#!7J)%g#eGa&O=izXPk<1*_;#O7DuF0V|t)MGt(aA(hxTuSSFhg5%G zg^aj&l|dDjC!zlOkdJwP8M(Lm+sP$dl_0f)v=6r&c#)k*jD)y5_n0w74B#7B3@OI^ z#F})(sz;gP=O4|yZbQ}+9!xt!A`xN8fz4yJa~f^dy?tMHBK?*l@U-*dye)P32AG_^ zPh|KOUhE7{K@AZf@`g!UuA^amM1F;(d0@O-p_b=pV0>m!Z|g7xq%({6)Ynpa%Yt2B zN=q+c_gG1Z9%5d}-F)OHh--{)k!|Pq-KLrQ-16UmldO*u6%%VTiE{;}KSQ<);TY@L z_74sYPK(jn^~I*HpG(rKj#0I-6?k#@nkxXFuW1u29yovHoulI7;-m_CDejU@(so2d zM9D%@8!$jQDX- zXM@FUpP%d%Zf|RQyW1q~Ls8X=+|c=^#rZ%^<(LTM_Fgdc(>`nm4;rAxLJ64A(bd&e zIqCaXyXIG?mTU_Pi}n6Y&8IO?h*&n)f>*8Gbh9?`#`Mh2aVIVrz<=xZ&s(w(Xn`UE zN1S$nD48s3XwZa=Q7RyIbVFMr?{n^6hbM10zXHX5P%_25B(#7%pWn1mg!+3Ar~o+& zi^PVhM66>%6(qiYZ)~jXJROSiaWmC;$KPuK=uaQtWpk}G(0QiFX0osqBpronGcdL= zJJg3F)TV@&$w5-WB^wJ;mtG|hoFw6e%*@`MK66G|u`hpQ@R0NH6c>Uo2t51A2HU+0 zV!C>IhEu;3tv@iUlSwgF_aw>5m9LbH5lR$MKVyht78YOn*jvJ$X-vh9!-37$YJk29=#ENTmDKqSO!Qb`Jz1Aj{xE874k z@UMtNypoLK&W;Hw^AJVYAC4Or(002?9|kQ-K4QGSXXNv1MBbL0`w9U<6| zjv8#%_~qo}ym|8`>3Y#tpy+)4;KwK=?Z>)@N z>A}5LqJZ1JV@D{8J}0On8F=Z~_0PXc`Z{lH9|r0AV9NobUx3_-MDd3asr5(0m;eRw=HFHu1mb`y`BfQX*h^#gZY_pf}cs5>NK2B55AbSZV=0D``*2! zb96KsZ?L*ZbwMZ-)E`=h4e z-*oWPn{3k#&?F9uk@Bn?nZ1d+Fp1z!O>u@NZ(E6cCZu4^yiWXW+mTLg5)P^lfI{(=esO(`aUo%cj|P?pHPrcGizYDpIm-|fNF z4OCCJKbL6-j>lfwE9R(zO0ODaJycKy9*`er`B4feqsUB#yfM6$nJsQ*sO9@Jx&DtV z46o&#aG9+J*g8m$?GmNio*i>`qYg6we;`<>Cl0Om#J2t>xx^1pn7f^won29YeNVV{{u<)cw8^Vilv%Kt#v&u151hlyY((kU z1`+4j6J~Lf)KWLoRsG#5we(GiwTCv4Pn<(6BLu>3&yfaBK`yS62!QIAwzg}aV#M!> z-M1StX?r!vjxlW+8Zu2nIqr8z=Pz?i&;7o=qV|=yS|75xL+;s)V(2KUs>evWd4Bwc z%Gt(zs)?UW8r(c=LC+NAZ5B6zxK)OV(=|Rm-n`y8mGE83>Sf;tf>9kCSH`N>nAzHv z+18upO*I*Z@nx1yg>IjiDSrO)r2&Ms(g+aua>)itZQsu`^8?R&4|S=0OW8B;SO!ug zbE3}h92V|I85x0}}h%Ta9#!b10w=-}=?;n5t|7 z^41(G$_Ym%-fZKH13>$(fq{XC!5H2zRQX|Cz!xJr)Y8+Vzk1CYM%zZa@WbGpsr9b( zw)Hs+Gtv7vnhe*`NMWze>uz8?_W8F>n~oz-K9=(4WI#-c9)>c`@Z!ab6lcA!{)g9W z*mY%UYHAnu&5^;JMdvo`SyvV?XJ4!1gL3ZBW(}Q~Q4jR){&64(R3j8l;VRdB|7uJHPsZf<>(m*~Ees5*&7;#;P>cYJhmmM+ zD#oaMhklnZ{AruaF5ojBvjDnDbk6am-W8TpIj%BF_f9wWvwnwXf_#H*h#^udT(4K+5T zWTC)rLmW-&Q>Z*_p#uf@>MaL{-5C0!)P9Ut0Yb3JPsh?zczuO#vC#70u97nq*0UE|c3KWT$Lp`r?_tW?bZ^eHW4dxamQcqj;l z!lBHR0s!Y&qyv4VhB)5*P7DwM8sp{Cd|R|u(A+kJ4(hQmw74J>pEdC^a?qVuk}Tfr zxsj5qGWvf-Z`QlAf;|N_>Sf4ImH0HEn4Z94Csjn-{R2PK6-aFdf5$O|P>Szr;?Jqs z0LM-~SaR{wKyhtw0Tqy!cI@2Q+3TtC^=iWB`g(OyhlxrIdPmC#(Iey|xvKv#{@{Vg zpJ-_Q$slxCQhE!xs@~7f4n!$NZncHUCdmEphBF}d6ahrqI}5s8Xmv7I)10ie4lR^k zts;RMYD-m|W_{HST#24&j;Q;2D84#s?!NdwZ$`c?S9ZuD$*}}1o`OhTn_opkbGzKC z_0u}?vExj)uaP&hQ{%>NJ*0_#Fmm}$yhi$wRk+<^c3wBGJ#DNsG-ja%(idBoZuZ~Y z?mzrYewf3HECK?Q+S=MU-c{_+v7+NY^8~{0WHyaw&Se3(M4}A={D5`HOA-qg^~y7B zx@c*3Nx^jh-DoP#V#At87UKOcupSlrND!&czNBQNZTXtD> zeqkU42?|CtYr&H*+J}bXVPgUAvm_ZXx`bSrcIp8$&BZ_1*v@;VS*j= zij&{4+MVi%`_c#{pJF8o0;Vhc&(7XH3Lhj=4>ky!WL=v1TjO+8-ypy%!N@O_0-nzy z3lsB_dJ`QpzJkYWk%NQdCFo-<$Km|_HALhq4G^xGmT`T~t(l>S+9K!8EKwh;a$H=H z-J}sEF_d!fU?gL6ab+~oX%+?X!ncC$kZkA%Jq_h>(v=T?aV5&&u{Bg-IlEo56<83X z3YkQvOk~uZ$g?$C4yAErsEkNi;RKS<9=?Von*lYTgB}FKls;Re33frj+QIK}n$(q( z^FRnuh+i3~=pJc2#}31)ZS}vCv(nLtaJ--}=kW#6AcP5e92;_3R41xD;lCiO^}W!0 zoqlze{|*L;KvXMh2HYR}3T&&~8^a?b@>i||5m>pANkj!4?+n0A|BPb1mL_;SgCFmL zQJ91RtCT=;2Z^83*VWPz3}*uMypFA|2DBp}AUCCoeBa+YJ39@^LnO&4O^Wu4#1gAw1Nkym~ zfgC)Ew2GSYyG>YazTUy5Dgy8Y{*IL^R}!}CYL)Z8yhD{(5=0hkavXh(R1tz3k5qlH zON>+`$Mg}>l~}1LxH}@LiUPRceO|_PjiXbz4=|G#Fw+%8IR&kul{i6043WGV5y?oe zgkV|JA&iHv+#a}vI2?ldsCI8&=tMo%>RypsvY-W%jO*<%AXp+c)WE$M=eSK_8{dV= z1ja!zfSChyq7{VBt?u!$G zz9Jj)(qihMQN;K6_e)RM^(Pc2!Kwww5p#A{Qbwi*q=R+EB>_$4OC)+`+KnEA{zwd0 z_9$4ZV3=?Mj`R$H6D1m{0-msP;`RciC_48mFGYGQJqmuHXv2?p*UdmC3x&gJls*8F;R)Xk#QPB5M=*788LQOvy$sL^Eo_Z65XsA`y=f z(V~=R@%KN5z9dzI%S5AWjso(7zT*G@dzL^mx>goqs*$>QvC?(p?lIy78vL1_QgW0o z+6guLBh>vtr|uZ()ghRwr>CbACfbKv^$p}m!?Et~#9Tpqa`bjND5@~E$;V87o~Tb5 zMPYG%#flX(AQSe8iDNbn@olEiH-B!;ciXXhcl3*+dbpDEOCjqhQq;WWyO) zYuL>%0T#YIVg9BX8pwkOze&pDBIDT9y-kY?sdcHQ@gV0*3JcG}%SPf2izIlVBx3-( z%Ya1HU~t&T{v$lpI0*xoJ*E4b_8v3(aR|0TBCeCb09hl+yzLzEQjobd$)46#ije)# zau{5s987b{NMu$G)-{oKqn#3Q^H(v)?;Qr|63tppLv@B)Zx;oLmZb;xr3{`*%dTSw z?dsJRJR!kGK>}0UO9`>sEBqp|*E_~Q zzT$ivrN6vDG_qLCH#&&3qbkFhKcJ{suD8g@&LKV8@cqk?UuFt*k=)Zhla;UEvRNOR zPuh_8)T2tF%UxqYMl{wX(Ngh4g{G~Bwr0*A!C;Lu#rzi52e~MEiDeu?iAp@ZirzV? zzQ((EN?tNl`mU{)=vCqv>+RY1tGs41A@{bqZFBdP8HTDjHLb_1xVQ!seVzn-sT@4OW;Hc*u_{VUt3j<& zK02dn%t=0@buyfz=4q7jFE4FYK3e4zPLWTPK@*>kuTR%o`ca<8+kUU%DJuK?-G(pK z^-X`9#mp-u{k?_BBaSP~`ae$k_c<<3YFQ;8x^bKTM)CHIP|r{$`?8P{7v8>aQ10Uq z4aKw<5xqD#Ii*mN$FS(nO$`%zq0xEP81JkALiOBAb9>Zry2E#5Vxk&6z4xnE(QBD* zorJd}z1`kU&tCJ^Lkb{$Kd(4<{~cGt{(4?dR+HAK0GN$~6x5A)o%?M#JPuYJ8%(F4+_7_>?Q(_c_mxQ}~VRDtBs0mP7rG0uq7%RI~F+gIV52 zxGGb}*R5ZlSZb)OGH~w&Sd2z^sbF)y0IP!sz*AL>>QSoLy@5&jVgy_~#rDXdDib2G z!Nie-m*vC#v~F36d$fWY*#?xv0^X+BcK0t=*XKvfu+Drq?`U-5WPJxcXe~guqVi_3Hsm z&{VOnWeozLX3EeKCwA;w!3F}(Q1*@dVy{y#fndnlGq<;Gq`iUv9XpZOhnL`=~~f*|H^;wRW&<(!atv=BB)g)5(x_R!zA2bG7% z_l?O4k{qMwQ41lKE;y!;abyt$$3T$ufLrGpapq$c6%}pj>rk`C`o#iMgmdKGz=jVW zSy)-2R$jIzX&F*xta|EYkZg)Dp$OSnl{OvdnRAI^eihi9IPd%EMgN|FqOTlDGI9F` z<|^#kO4{8oZhRdW#NedXPPbcmK&rZVdL1V_8<$bT8}^&C9P~z%@*7pMUQQZmTIRF= z*ym(wXC}i!90$@}jRq)Z9^A;F;o{-B1ay)jV>`Nf_39r7GqY5rq#orh0OqC)y?yhB zxQ;kjSkA(wNAjf=$~G&a^;q_=cDRf<*4F+tv>K1lR3JuF+CZR`xN;8n0eiGY&xqO2GaD z2X@fYhjS`L-2?%e0s5Fvbv7R*c^FvUORq&#QTlxm*jx=v5{ox=1I5)Ko{%}JS5 zU`kXtBoM_Arya_(Bt0Srb+rFjTZ!)DOg=y@}P!olCdP*`- z@+(f>8{PJGDDc+ieLHU(mcuUP_2NY&aDf3fGx6nm!x|qd?dx;vm7QLf^Xx#XZzgxt zA4s$1*f`^G%%E+A0zO9$R%61nqv0YN(jaLQX|(U-0p~4^uYHOFB*mm53KR&Y^#b5r z6#z^J)E7kL8#iv~11?}5a*T!!kSz+vfCq!VWJfseIAzuVR>a1)SF@W)-&hlXbi#ma zNzpxcH9Q+=xH!Rq+OhkD|)Rt^nT+4wVPIF{uzR_Z1+Oc5L1H2HRMcAX?D8 zkH#%GZrW5$2x)lZ?d&1vOFEMVZqAzlikx^2~Y@qrTm@E_>e&`yM?x1NLGuUXA#)`O>JQ zCv9qW*)%6?`p;1#j-i=;(7Qm5S~hgBoW|xRf899-AYR_ZHj{-43zXKDQRg1gics|V-Z!QJdxRhs-h`|P!Lnr zby8u9=THdSK5dt!IDmSF0tea-SO5t4fECUjgyA^o-v}fPVm@e~rjqSF4?2xw#FrafwkuN9$z#PXyy)g)JRPlo|K&7uxiT8$>QVj11RI zjC=2ug&j}A$oi_**Zx#4{0q*A^Ztp+~85wNDMa~IY|nt89kxNo*@GV4y=23`v@oGFgz?^zWE6b!97PeoI$};PKY^h3e=1}eky8e zFG+cdiKD-Dj-r!upJK2_R*idS2`ihqDW+#;a>!I*a}TyR*dDe3LCXT2+a28-Vvt;g z2kZ>Q2D#>yKA-J2Gn&%eSassp+-MG4$6c+(-qfwJ>f7@hoFxs|d&NeoxlCN%bH=u% zDZl2UdE%5TEWvELIO;in&N^cuu`D;Eyp6RY;7?Y`goFfQ>oCdaIe72>ebQzlH@TD)^MUHY;n?M zF&s54)kht``R0(>m9!`;6()Z@kGbrHZ|dT8#r(ATDK`^I(cay6k}+ z1Rlz0qwr*AKujn)DGR^{)sSLAXK@o05$HJbkWBu&cZVdqhE^wAc3p;zNdi?J(WZ#i zn#hQu7BBvwX)e}>N11rWDZVS7Q+vW*!F(YD4$@%DmMz2+uUU^_&=As784Nu*a8k_t zd$7&EJJeSa_=Qud-)uNT;#sPb0!OA`-$}~O&kmFzSSB-YSg>GP*;-x#3~71s0K}HR?t`oc1O3|cM$egXfz?} zmM4+?@MNOU1c>HEdc}x=2RM&?grV=N^J~V;cc=B@KMMZjvd@V2zyE51w|(TifZKNE z_lY)p#mtL}Fem>b@E!PdF-96o@xKJVPhNF6-r{e4EgRn7CgtJ#+q*{(Xy|gi&)%j@ zYfRpGg=Evz1{?M438=OPU>t~~xoOXdAks7wJBoy82%{|_8vtQLf$ewL1;%Z7clr)z zr1iXk&~xv%mCtKOCM+m5y8In2En(QHBRoTqqUU9LugvH(xb9<)4*$xKUKePuws^fO z;;(S(U|M#Dv!I2T?Ejf{=A@_I=k|6+8~$l~dZ&grpX(PAp1+9FD*B(57ynvowsWsm zIe6+eZL`$=+?7}#bni4i)Rrma(JQl4e_P=b4uSd8W#25&FR~UC`^@VLZhKY)ZD=`4 zn`=Eaf0Q|JM1tBRD>n=EAew$?g67If=zGueugbywhsxkaf?0-`>`PGA_-_;d2wGje z7QeJp@2|NzJbB-(ax=Kr`XySM5ZYknNlX<&^DTJkVitH~VD^Q?-%u6WoKtXCfJ6Q^ z|G^AN4#idY+(R_}Ui=6J0cG;H+V`|g$?J2y70~1Mk?T3fP*0rjnP|h(!E3Y==|__? ziXBEQSJ5=9okz>@!PddSuP#9=npeKRW>Otw*Cf0lVQ@oiT0oqa(Rb=EPuUaUOwSs|Om=3h}P^N=E{+?M*e!xXbTt0MpT z58IhqqyJ!z=BDLNy|8{H0mMq`u@Z8PzrVu5_B}hVP-RPErYsk)km8^Tt%L?k?sbT8AR)ycU@7US?rug3s$eWLUC5@JC$a;TRZh&!0-|YU(`pv@~ zU$~(3{JzwC?)Mr|NkyrqNqx(BRC*d3_GRnP?}&%M&;Jh%?N{v6UH3^MNJ$MpEcAYE zn3lF5xAam(LzCQ)te+Cqlb>(lHj92mHdGSdFpOa9n=?D` z`JFWE=X#~EhH?oWc%R$h8UiG3EG@;C^#*#8^xrQ(3E)1F$tj6AVm6LQgI73|;;&m9 z{PQ2~-Jjz8T*xWCqJ+m2D-k2lQ<{i((*1pnpAFntoz`jGe3kguo2q_l3l0@XQ(k^K z33GSstqdKw!}{d4Gb;S}9fV7I1Z28@8jy&0e+LCq2ooJw70<@YF*INCAu7!q^;Brf9Z?#F~qSRdLq|N1>o9 zXfptGMCKTqnYuhEM2n-jBDGxc@|>rq>rBQ3zbaX`f`)z7-!ImCmV5g@Q*E|5BX4mo z472R3C5QQD{rly@%RBdId517#9_;^KCQ+0`q2z!50|t0U`!(;-+Z{V4Pl+jGntgOt zaShtPuQ6z{GhgDqkcJPv0sbv+hz)u`)NZcjm(}ESPQCmdelIiY=Rn;mrx@}(rdmT0 zLfAhBR(JBZ@{oth(7>)V6*4(o1oH%- zYi$9cnEaR>S6HKh)*NN9ZzyTKo<0piAsU71nz*9!7C>(i?U5h73G=Y5l2)+{n*r4R zHXVJlQ1A8OvWSvFu0Y}3RU5F6y#BGqb08&#yQPGPu zVQPYggWoO_usaw8etldMc(8H!jtpSS2h*dWzlR(+f*;=J{~!+`7N#5*5D-9-&Iu-| zMF#CcNyj?c+u8s4BR5VI zG!eL}O*N=((yuR4wCp6{2!XbA^I5OoF?~;{xketSQ0PZ5EMK5`Lm1t=xVzB7ZbBn^ zDQp|Zy_iKi8j7Z|(u2k8Fsq5WM)x&MFMS=@De}0fB!z|R@QrMcMkKpDYXae7LZ51D84%GV18CKk(Qu|A0Yk0=p3ph z<=2-lU*w~(XM(Qay9k9%{Eh0VU#mevMMDUIE&gMR5WqSHNgq6C%ED&Rv8QM&h{T7Fe5D5U>xBaOkc5+6rhwi6XEvo00= z0K2-W9%vBaICMxBDj;XW9V~NpTKdK+RZpDK5lOlt0sU)mOM&r=C|#`JEbjetEz?sL zS{!WxTMuaymP$6kNS(iEy9S-n|3Oi-?SG{xqUq zDLVO6fs-Hix4pni?}}tOY1W0=@_xy2lb?R^v1objg|7K5)di%*R%L|YD<=0Es_dXB zZ>q6t{&TGt_%ImK;Q@l^b=gC7LM7Pu_a$1Jt7Kvx#%6z?RpKeO%~x`U>2l+#@iBOJzR7OrNh(L}@7N2HD1y8C_prGYJ?F$`)gS?VQHWca!gUyP>tlDhWQ-?Nj zr;DvyQTd6cgVedOHDxm2kP2vH)?sP0E}fH86NJ@N`dU<-NWer;fi>VnlV-qq_gy-D z%zO97lG9DFa-^iB%29Hn_lg|`#`?cv%6*<2-UM*7*dEyNPkCSo-83_ja+Mrz0y;}d zQc@1)<&lP}>5oU_pqy|3s6RV!z86F!u{}doL(;2V!=A&{&E)0cUczFP! zCr)eW5f6p_`dyoLTIB}Em2?Stq&|fDP*rSRGvbQ?obg;PoHzM)+*3nZw6Nu3Z^abW ztY3av8OcI-On;v#->mSyHUAze_zXrOTe7f~6ZK_x%6XQAN1c|LWx`O87TOmM>H)JdzQK^Vi7B z-$CGcdrL1#a<@Om)Op9)4XDea{ZSe2u@BU$*MM{-H%lat1dSO}|tUU5vD6jaFDLBA5Tb0ofE#L3O=~$+PY4 z*OYEOFxB@V*ZBJ@+6Q&zG^`@>ZrBAYmQ}z-jo0S3gFui$TFYnOB&)tzZFK| zcXm$h&&&17zYfURd$jspfeiL~*WT3e|Eg%-G`@qWwd4f_h;QHWIujKrnSK`)my^1` zSIN?xAK4+veBC}WA;4Duor9;AKl;D7IR5V`;&NHg;ylp1^&;zkI=#6pa0ZQ$)*fgz z*47+h&HveK=d!@~91qib14I$RUu!u}-`Fc?@u})Yy5hfXbX)4gN#yo8Ne5 zCerKI#UrhpSwg873t+GUS6Y{D8BR_V0;(YA1p(8!5;_r5bXvwIc~llIFRp01m`YSe z7u>jmb$0k~KtBPkAPR!WNwkGkp-HnU%fWWzu0tn5XNMDqhH*oNlD5;hcXSC55j+3V z>O7cW)gw<%@PQ*%`R2_C!j_NsRGcSAD~;j7F-I6es^_tZK{8tnAna+zG4E7{njpF?6e=~!sCzxuTLDO9RmZ#>ORk( zzePKLB;a)%CwQRTj-`>*=!YzfUZo8&F!Ay{K8?8{7@0H`{rK_Y^z7vqM?$@# z5Eu{9FXaUK8JzQjBY0HN?GH*iRB0<7s%Pn;(!SmE-Dt5+=fgjvf+btWU3*O7j^$91 z{9jete~_xzEEtCK_|H!0N3^HqIvh?ugi_nx?O%#+tk6>rNYS#yYBZBxnK<4LNz0|T zu=XuhNmmxPkiXSRUbfXpp@h<+X+DiZ7|gBLg@If7x4}09v5PC~BPsUL){GIPxqaZD zt2T2|Dc+_d9TTWR6yO0P?hRFLTJ80|PoK&@@=n(pmr@R&K=h~ZEss^kWTOKlV^&F0 z$QZgEezBa6y*;1xm@#p?g6C&C}CmA|=8yJiJ0) z>wf-gwRusOsE=^5u}TI92OHEThPrWC1YpCKc2}Iv^PLMp8Jj}(@!u52$6h>@{^0?Z zK^RRa=%pfuMkO1BLUi0w=ThX!N{meS8^H4$NC|a3%&-!`Y8pch@%l>L2RXSf9NzN5GsI@&+}QUZ^|1lV z)NM_r{jgK0Zl|C-*xSbVZSh2WR~e ztB_70EF=LZ%s&odsLWW2#}E%yTJsv=1v*+<(0;6V%R`#CiGGe#LEv@!k-T>GYIauE z{1ze9tP;J8Fgg~4M^=tjA_obg%kls^D(~0(|8r4LAcyN~z*3SGhJIQ3`c%`@WE>i) z3Fl)4^zy-vAQEXbqAzO3<6y5;A|5G1$u?zrBsCw@K+pSy(>h2bUF|I`!B{OSpb4zyym9$BI7Ldz%4A^C6S5zR z(40kqv}4zg+}!=+@sM*Ng2Y2y>+?}JkwZt-79O_Xb0?P9C z$4dc1RaLCGq~GhFmE1((H>#D#_J8kG?6+L+>J#y^<{`|RILwUnh1b?$VBS7>@Zj_$ zag$-x^}IJ>gB!u7XuTY3^)hIrJ+TFhr z8G>a`{HdgMnRpFxDjo42;ANkX(ePFozeA^RGMyace8hubc}j?dM+(xM6;BKb({AJu?VCtmWnswI5A=f zCFeLoDkq9MX%at4YrDJ>BmmNpLI7IvF7AOAv9AmB?Zy5%mD^Gnbu z_1e_#^5rLVB#oGff&ubG0sC|%9|NT7YuPDiSloW=IO$X_50ws3Wx}{uEZNzQ8~q4E z5!M}ipD8im@B*=V;-tXD=n{ljGlo=YTdcH``YPFu>7<_y+`}bsDF*c^vBac?!j|wl z`HhXJICt#X!;P~wvB~w}OTIU;h_yD379Mt(q2VqdXozrD<#}XpFfIk}=NDvTDDW1r z1_TaqB9eGkqFoLlE>flbKM7zk0#>8tHxlNkZuG={hgi)Ie-I3Jqa&NZF6FLUhCy#| zEE@_-PZn}66&a5moehr4Ttgf)W0i0=;x zADwhsfCpJI*U0FuS-$sfP+`t9J#3QYgj6AJT7%?X^f6V#)TiD)aQ#0BAJc$1+qY~9 zW_x|>G#tg4?h5Qax_}d6I7O1}vi~~=4UT_Hqa#{j$VN$O!Q#463WmsdRtP>j5_q?6 z-HMYgoEmPmJ(g&^j9rcHxWFsdtl6I?DjdXb&C`cvfEC9`Z zsB$+g(G3Mp_NupMNE!TxynyELJQhSv+Y&?nd_Bx9us~5_eno{K@+)Q8TZ4s?#!(ZB zSheGLLIcEY+PI_W+q`l8`VV~H@coKD$TZ7@fzSMlEB_#Xg`EO1X^zX2b{)&17k$7m z)r1q5_T^T^_S zBR2mb`9?bBNHadVf9mQ0_62BZX^Sc<-nXx{X84cfheKkJl6LKWBqmlTk;ci&o zVN_GTgg?rN44-|LO8n$dkx5gnJS;8bC^6hiw9DdxX9y0zz%nPjO`a^I%L09GDFhmj zEP#MTRubtM?P9uq_@AkZzp=E-;(RJB=Of8YF!PXJkw?j~==tQtNh@G7m0|Kj#0vv{ ztvzTXZq@zt=g$S(BhE;=$@b$0qBsa?$vpO6mmDDw^JCUJ?B7u#2fyR2P;%&yJ^nvX z2Xi_x-lqSk-woeaL#+lqJmQE&aHFGyNd7UKF2BsiD}g-9vwO$wxzcG#Xc;I+%;8u% zhS@9_F|T668YKleNsY+qol&Isow&+Csun1*X1FgCERYlSH61B&S$s_Nm{+V#7TBxC z3AO#>l`MO#EmHL+>mP&ibD|AMe95z*GavY|-`=G>2Ys@o#z>^7QG`{p0Ud zxR(wYm>e*o^xBbC`!9Geb2^)`v?}oCYTcM^P24pVogx{$Dw5$5y1y zefOAW`$SV$@V_bO(={^)r6E;<-iyi*$!&xbhSjHrxKrUT`&M-WksxjR5;@}c34RH} zm~CV@a+WMdmL6g!yyhVq1~QyX7jGQ2z+8U+omU3Hb0T(IS9jv-#s0)tgrj?9(IFoj z3+^$a#dY^Fv^0@TA*(Amu=OKJ+oXXX=b(7Mc+nG6iooZ^fMN_zN9*hfbZp?5japie zIy8QFDptk-WRei{=<#DEGy#!QP=R|(RbPS2N$VsX)r3ONp2&BjkH!&oyb()9oWIRF z9dLJ~V~SWJ$ze}8RyG<$Drv4KJ`Y0t7ojy)m5S(-*6F@iSedlwLC-Lj4M80*9XjUotr# z2R1nJmxb|om-Gw~43V$7{cX~DLk{M{87<{xq~SOuZgT`Wa)!tt+EDd|P?$AG*`iV- z4KT#mOv-S^}_Xr;EjTT$qYr{Ksb;e?{O@H7|xQj)NE%a9}2B2qO3k z?0k~C7-Gp1Kk@&x_a)$5=k2?{>TQ~#Wu{Fvqy^cs6j@rd7ozNxrHDui*+Qb4GKChq z5S5)Qkt|uJMfPOh6(M_3gmCWX$IRQzf8O{1Kj-?NbDeWt$928)3cv09{d}I!^W4vU z-_M(aimdSzlA%BY=?y>@4oEVo&dW2ejK!Ocl_ec1)MWbp6P?=^yIdxC%~K1Hn*GeI z`aIlUAWu5h1E}N8ZO97W0e2A?$}%V#iws{FbXFUfDO%=6#(UMCn?l> zIQCFS&qcuE0nlu@;#@(q)RlZvt5wh@m5<}arJnC&^&K75;Yi@g)|Nb-SbEYeJ4y>` z7SToQ!qZ!G{=?4BE+5GY(&ORY58B?}9vxO%)4V~`1z)De6d%R2kVx!g?KT83iUhQz zyJenY0&*69onF_h*L2-u*5`2rEb+(!T0YR@7V3a;D~2pBJsFBWvZqMyTf11^k2U_^ z9Rcaxvh@Qb>W8Qan$eqFq>T%581zF6lK7}nrV@DWssLnyDQW*unPgq4q9^h0khg@P zy|yLqf|MDc1xTKpKSbVkVC#Btk>e;81Gs!vShUBnr;2N_q!{@(r>gr#-CpY+F#D*| zN5fYrYBoMNPxwqtSm06W%f+-}L7bz`9+XW?Eo9{>JjH<=;JAD|t2EuqV=n}4S!U^y z>CkKaQ~*!@W4O-tv(Uei?X>$R$O$^&xa9#-`Ki>_#rRw$TItw_TbJTc_<7c!JO1H; z7s!NT$X!{fFR-YheBOGIkVwU3-T3f~-g!QHr`}IK-hm0CaUC1{Mz+nb!$_ z#%qAXfP|5da+QKS4-p~%p-~Q-`n#}Xf5v)b`&t@u|U~fLrwd>%9t8eY8>#JqaOim|F z5gT>inBj;F{+ZNmv_3Chym)lDFHj#J@qieK;uZ^O`^LfcqZ2e>i)~Ca?oAS-OJ-vD z2H9jCI-OCWFA+&yCab9Mq6=y;Z-4`bv?=xP3>D*aGd{sH zi4P9DlX{QZC$@=K=5fX%G1KtDVfSVoLSD`d&NxIa+FZr8T)3%z>!5_BDuf!mU?VRC z(-=a9NF2#zWi?f7VSz}(qzspzA02FTi%ApR(2t|_SPvq=o7(^~BcY!%g_|JS*FNaF zhT`VZff_4B*FKbPZci;f8BS|nk4!krlD7%CluFor+B&F^olGRxW<$qnQ5l90lplhO zK?e*{RS*J5--#IuX=z+#x!?3|0H>v z(>&J84DO_F-wb=n5paAf=9?XKqWl^ROcOA^L8V9@q+}n?24|Sho&O8;8Ji(ev4dW%0*5szUSowK z%Uj78i(e9tHzchpqs(@FJ|22tO<=qjz!J-c#v;gUw#ne?K zI|tON(S!-%ipC1PYz19LH% *$(r;%^(srS_ee#$F91Mh#`Qb58}aX7^);%(`GKB zEf6g);`Q%vS1e4z?{InOuc>Orf|P?ufTTHz)(9(RMN}A*mM9u*b&`&=^wdFOq8yda z{s&O|!?3p&Y#f&3f{RimbkfkFNnZyK$nhWaG~ZDsU2fXA=>J>oZByG*7ReQ;7FV z*Prk#q$5xdY$?6B%LPPX8S?d^f}I?9T@O-jW`alyL6dssAavX7WbT|9?OQr#HI}hR zDQ?+Yoa2UthO;zD388!T?M(9WANHpHlim1mJ3KKjP9;ons37xmb91ZC!@-A4<|niF zXLu|*x8v(V0rG=WWO(b7{xhzcsqQulXFN#|Dkt;FAZk`)ahPD=6)S?XTS*X&XfI^L zJca3?f!ZrqKjA$Lh;BN+oVa3)v<(Oh|EC!{=;<2pQuvAWr5G0gTwO>Wrd~(Av0)ZY zwmoEe-q_ffY`} z!#|Xg914JBB#5X=PHDtbpqh0m&QC%QdqG^lG07#14~*M#lZ!N};Io?L*|(@ z`QcP|?)+uds#V%LR|+2dXWe^dTa1r713VPuK=Y&~^H-)a_5zXZsnqHWP-eEQ;&*klx_WGQ4wfoW^ z)bKJYHRO6B$pAx49o-l>K;F&G-Fpa$wLfV-IXU~x0>6AYn?CMN?XT&h@-+Sg2O>G2 zc&1T!^+YQPC}m38Y@e$vtjha##&bo2w&@`DezC)7ea|mU4o29Nt7+^+dfyzTXAap0 zl24`(oo|Be!V6-$`X2mhS24xZCqiNptV22hM*AFDa$D43_k}S~yF&08y!8>Ef@ADk z+$6d)Z7eK2va+&92798BAf!Z~f@V~$*KtBw_BU|p;lqb-jkI6Zm^E*CIUpX^3?2(K zWKeBCa&Vk<7)noZ#(O<*6x_*J0tT`@_esq1hV2gxz(KxRQ;Wa{1Veh)&YhClYd}Pd z&Ek5Xm6#LjsC$+i=g5b#B5Kk3&Q!!|WUwcO@Q(1cwPSt5QcZXCoBnI)COm#2JEJO% zq-Eq|hc*{*j6|-$d2!?~izV)WBn=QmLeY)Zt}X$D?kaYaz1cI@tzZ8cE@tEeLKQOJ zagX;JF0K?}8Z)Za89LF4@n-^A5-Op(ojs_F-2Lk&6qlO^+p3KCM{`d~iiQt-@xq-583OpBNJ#cEWPr!kf5S~UedE-k63{ZnU=N?KT;8GU?<+7p&BwM zbTGJ~+rm*NJ;|gXXJZf;Eu=A-h-uCwvZzdVI}o-5mGx+RkrWwOpBhG}0TC}g1tEPJ zlRss)cJd^8k3x?=61q7kJe*|F180mgl@HlmWci5QL(NUTI)Atv=t8a;8WKW|o_2VD zkSU(e`-~|cA?U(%p}_~LYC(W90$FzJ;kkfC7#?Khh9rVB{Ddi-i3uS@A&9m`Ic$CC zzz0+&V3~mu-Wc|xR9>re;$gpcSYe~{5gipF5@{IqBhw=A`lyKCn>y9v3FqN3uhWRJkW2L$9{ zdwfGp&dmNy{xm?sOyN2~ZZJZr)bnNTbm3FE>Z+tp(fvA2wlvJagxHlJj)6zc&oB7? zTK58grL2=gpA(ol!B-~>jVhXUU>J4R&X_(u>Da5~&RyqF!w@(36THC|+sYtED8(lc z4lkGgy?@d?@IZM|7V&3>#^L$cF*bSU_uttRNz+ykKfHs2^lnxCEUNztTA?SKf+Wqi zqXxA;s58m44-|gl&b7mRS1Ha}t7FE&^$jTot7(f&u8eBjqPqokUI}le^VTn zk4$CI@P%*Uz~}w&*+Gy-0&&u`ps2`;i)I8O>mg6e%+HWJ!v|F@}k^ZCJHW z?FCL%4k{*CN7gUsM#nNKBZuc>2jx~7C4B8Gymzno0?fA~K?x?bnRp!`0~gAXDpO1I z+!(e_49dxx*poC4(6&DWU|U;xV$`*kZ)>2*Pe2nCx-(^nuz=H&%!n|fj<+)Tzt$K= zeTe>C;-M_`hul5mGQ|EO6A}4=lQZbtN%B}kqlgMiESZJ2WFw~vkr}bA(2zoXy$_1s zhYH>2k!2!dk$NyFuyT@h7JBrkbik~FReT@@QVKwIyyw)_XT=$VUFDL|2s$#m?mAvs zd~!j{27q~s?H!BTCW&ZZ-R3?P?0cFIPY)imS&)G-$vq-hQApQ1oL+JThGm~&ED{X` zY%E{?3&<5H@tUL8@1bZoc6x67U+bNf*$87IS5dgbG%Hu6TiBOQcIu6$NAOpi&5zo(;JBgtna2~2ZV7*Cb1=mjzRv;S zwRZwSXY9+Xe|!f~hj#`^#bL+*4#ldbhd7tXYlXI%_5mm?rifiB)c`7y;;sPyL#k;V zxEC~xiCVg>L<<#}#k}=wqX3=I+f#efpWd4$58r;?7uWDwg-mEV`vsBQ0EsS2sfU*l z^!}lmL>)_=WLeP&LF|**={|k15=8y}r{7j1+TMpao^Sp7^-T{+GLB4*nwC?q4`(R6 z(1A)fb}3~PrD`~a#K9&fj{|qw;{*ev4k9lC9c^yBYS+}O!aI`!E;x5I4GN&AJBnGP zEqmw@sJ4V+aka9oCe!{mXuWPv&DlHggD6mk zYM73LzAPRT=8p4K{x_|5-_rlUzYoM|LsXKxD*yrE0K#* zp$gWE0E{a^RwL&K$kbSRvS4&hP&a<2a9iv(0#uK;FiKG}s75i(19=7-kRMspL9=BX ztNs^S+#_j>f0->>R6By3L+uHe69;1;-!?+`Kx0J27CIC}WC?(y1R%?GiQq-d{Jt&z z*Xev~XQlyS0f~_?bCSeJHr0;g5_s(z6BtGAz#I(#i@Z;8d#oGT1ewjU&2~vBEaj)3 z`kw+rKmP1|Fq6>C=)TDF7ctxL#Yp^oaW#~P3=QW=r1WnDBPugBF|iL7jS@}*9eex- zQz@7=rgF^qE-x>yQ0RB6fjct}*!?R3sxN=`IuGCxc`skS9K`U*Y5g7T%FZZJBev6By_>>MOJ{B6@Ts|LRtZk&0ymo#&9*iIR4916lHfv1a|WR=k%;*v!+ z0H3gZ9qdacpI`2|7GdPhgB+&m$OI;7zDLtS1H=$=HKl18Gz1dazg%kn*KbV%%Iyc$AK4ma zkT(3}5kMm4%rr{d0F(eSC54)_wzj^$e&N!kk+3jdb+QEwIpQwE>^4r-Z`Z@liMsTm zl&Gk$6D@tuIGTCRNW_`&UtrtmTKZ7`yXcQDWkvhSN}a$74Z{Vew*Ir~(t*jUw*6~@l4A@O_3KMpV>`Xw4B3)LzO zuYr}8ia{0yX)4RP`A7UzL#g{ZTy>m(=7dA2q0|ZcP&kia9ym+KHEmJIs&|GWWuq({Pd{QXc-6hG#>C(Fv>XG*C3TW#JL0hR95GT-j2+@qV_@(9jNe&8wiV z(rs!;)d!eE-0e53H|$i}wuN7i|A94!pnpj23Qq21tDfxxz>l9Akl-Mj+Ze2N8Z<#< zTd+n+0u9(CNw_-`h}NQoUx((N2N6+COUth}rDZNYcMYd-#wYh^#cjQ(HiQn{h_v?a znV9fvl-b>TD*5;*7PMXXQl2!t~oI;iU&wuCCslQs<9ym#@OCobHW{(bGAk#QO?EM%@@L&LU( z#)VIdq^+as?`FhDs2VIjrEa8n!DSUr3%9GZS{Z_oPo5fgzTBwR@ZzG^NfnvKyjDOu zV$+!G@33r*?H<(wss~%FS4Rd%sN!GCgqf_Esb9~MJ1(RvRff|{`^n_xc7HnK+_EIH z3pmTXa8BIZ$#axcZ{9c|e&Vt0HI{xQ{koeuvk9hW%p`D26XH&VjxmF%26e4+xjxR- zHji)1k<*9rwrP}Kdshf=kh5@pUeHw6aCh^%+&%a{TJMDCrx`@Fl~51=}Z|sNHDep_t=rZE+z2x6>uTn=h&U!|KC(H}h}k z_5RO&=dE$Xm8)a)(#xdteMh`Bmay$XvuqjCXl8kN+`~^L*ruvo-o3JId1-C^4!U@D z6H7k3xScs^hFGm6LQ@17-d^}SM8dA>$TQA^Pwy$aX{BiSNzD|x2F{Hb4-285@&7j7 zaT~UCC~Z?RajSC+xp+?*|1xl=>qn`2SS&zxKZ>RHvi zMI7^m9Z%ah4d+Vo(6&D?0 z{DSCi>N~*OJOpe6v5|d(ESf@XT^+H%R2C(Dbk<|9l^FlvqIgHhCx|U;6}`AIxj@o$ zA>XG_Tb*W3E^-(g1y4iLaj%yd-5;&21 z-nkPC(NQ!YBnZ@B10s#;nE+5@3JZ;80AmT|f&Pjnq*MQ5$QDo#X&5pv#**?Wq4e%` zgt8|kB_$}*K2gjq&ACh3^3B zG(ha7W)lt@8jcp3Jb}Imnc6d_mIJ6FdLm%`42%MIFD}lc42+bb$bX6F02V+j&G0q` zear2v5nevxMq$8o+L|vFr5f8ZC!=xLuQRg^!HJ}ca3guVKE&usZ)3(Gq>!LOn_KZ( zSwm@Lf{*B3b3=`ov@}>D`dP%S*2-*t0Rf}|{$gld*i)+xASFEAMvm(x!r4v1SLC<5 zYoUqK4M0Mqa^%Qe8oE$->EcDQ)1-lIB$Rs=w0DN2lKS~^Mck1Kap4&6q-XL#@KcOAV-a>~e(HAZ4XM$Rj|zr7jQ}wN|g!dh1DTAWs&wD!O}+ z^}36)B_FUwH;x!S@NprUYK)T;1LhmXNQWRQ$_RdW2hQ$jmxs$t48Ot%`i7aCz1ItB zWMi29pK?gYiP5HtOn3su*N&rgHQ3{IQv9=26)0$ypyZAYz#nELaPN%6qBaah=jrzK zeDyDe<|9LV{+83Zece=oxKma`?DUgthM0-h`W>&<8mJb}4&2B!esN&}?L)X(^8*4P zIyw3}BuYY;sT2Jd%P&Ly>tAiUj}BT?Ow2x9C7LRRJBenJ*k;V4nmbB zZMQSpdi!m-lTKCgzC_{_|8NGk)uHI5*fB=1bk34M-i<1l1>0c~DqN@3Qg}et&aOt$ z%kxfb*?8xA|MvZkBOWI<_!A8ojhn7~fw*!t4V_T9kiG}%YX1Ib2yxJmrbap#tAQZN zgwre?)cLg;_K$Zo;Rgci)|HM&JlLBEHx_E>quFU@dDBlXf;~VuV2%yL=uEV0(Y5s>mMR{Jgg6>#H2L(8iPM0I zXM%jPgHN>-&@HVeF;QIJ_jo^g%`V`f3^V~yf{U5y#(_;Bbh3*>#A{}jLGqxqzZ z7R?=1gt!eJ8$GXNM&7S}@$x0j5&(0l3{{F1I5x`+u%`X)oNfN0;Y`6=8r4SyFKD7t z-DO6JYFcyouxK#)^QL8TwMk+P8PSkl9HVpT!-5*SajZ*kLK$=hSoc}MxssR!b4O`v zGzJeHcYWn0IMn2U>Jd)tRj5-qYKQuo;y`aYuD1Xsi|ItZ4E+E`!p>XTGhYh;7N~jq z@HrOzZa)2*6C2hp4Gi%JPdx{P#IY}SXZAG7?>g(W|I*+Lr_O*)I%Zw=!_>tMu`>nm8F+o5v0leh2w%ev8FDUXA-9ha>APy8M z!hL2-qT@-r7a^TA8kUE{sf=^t%N|nTV;~RUCUuxhJ6PYwG=Vi?8Tih6Kquoq=ZYdTHv%Ji2rqnn%csez7R0Fl-~;6lM}_*DEr z)I9&9oKU3Z!Flp0e!hoby=bwQ`o_B7c=(V-Q+~EljA5*5KiGXsbT{ChGj^PiBcR`| zasGVfj#2R{?bQ%j>IM)~0et6-S_cR;=Bcr0qWuQu^!+mPK$4}to-uFJ3faMCIdHqI zpaqk2CyoWa4z5WJc!jAHy&t~f098WHLGDVReh~p+q^~=vQw&`_CNsvqZ?pZhE(j;n z!AkE*EGEGgu|CoD@!$d^sfdHF45}P;gK4H1c(Dwl(;*8X8X}=5)OfZ;46Pj91o2-4 z+J&;mSbf1ZP=KywZM#WRBU}s>SGZ}Gl$eh=vffrmR z?@R7t)Z%5tHldDg;bxHYs%S(b*v@J|1Zd|Y+N|j~vCSBc%@iprFmHxb=Cm3yY)076 z6lpC(NPIHieeHYM-_V2onx+>lgc@b8%&edGl?T~j?eIRiz!9T%j@J$Y6IPAqS$q4m z!kg`Mj?IxMfBH7Zmp{>a%a=DZKxnHOf08V416TncC+AD8xH%aSws0HhmeGhc;9z6Z z2tqV+aQUtP!O9j(vFZ8&RYi&$i65wYN6~0hR_M_RRbZ z@KpykQz<6c9&=Q%wN1t5uxfsO^*Gz(EJ)Tr0nP0~0Dg_`*aLLvfdk6JC9MKOXw~8d zVhS{uOPXcB756d?Mz8Un?#tt|y#)`?VV69B6qUr%==+jSF!p=xtaG3W&t24P8k@SC zR4$Y$k$Mw|VhPBf9F1pT_~Z)-ZY<580tug>X#~~ib6=1;9p*q~D`vCzoL9QTt_@Ec zG8tlXKBrC(+wUANT{OF=Y-P+gmo7Zj%_5`dk2ywG=HOC84iKLoI13Rp5=UZ{2`Azl zO~7HVjeXLeFZ}|dL-o*Vq`)2gV`x10$Y?CCi!zNCl&-MK%BqC121$%a8|c6hjWrA4 zIB;lFal;MK-S9luz*?tmL4(@{dk~zMAsaezHQSRXAAJuN4FzI{kx#FoTKNRPrxRnJ z=zH|?x&4bO4;$F_F4JW~siT5o0trW_$-9$&w1xW;X8uK%T4Nv8hA&iqX_VMOk3sW)!?59_2|=I1<+~dEIGMF%gqKKZUdHKd?NPt(q-W6V)c`U2+Yh5cv;1xLu=P) z6;Jwl35mx|mW!0(p-aBT&dC&+kOMy6f}qRab$F?PsO z5qykREfG{;MmN}l|KPl~31t069CHs|1Bt>a;fIKw-@wn`zwYq*FbHo!(C;uiCKvri zh?phun`|WfQOL!XpIYl_xy$03217gSjFIutn=P|%PgaSJzZqRA#wUh>1&u74fFzp_ za5?5%mLg#HbjfHSl~zImd>-srVpbt`Gg}j&a`#FsQbBtl2??|F(_&0L4Mmv&P_Cpsz4&o4PXkbjR+vv|&wcf&dv7kfj!F(_4 zYH#dm+$9a#cW8fL5T_7$cEgmh?%lnO-wvUV&8Wj3&JUOVRnD+6aZe=}2 z0gyb9GNt9V?|y=cq|+vAG!RD)d+(|>folVHVs0+daB>e(Q&kN?gh02dL*j)QQjG#I z0%&S(&&dt`3bfEp3bD^TQ3Lor1>%%)AFa_*Q6 z9E0j$2aHgZ(3$cAn+OGTWpxtFdMVV{9$ky8M8I>jEd2Z6+p>n6Xg~9&F`K>!(;8#D#;JytLwO@C64DNy%=G+AX$c zRkNB^PX-0C2F}v7ozQB=5Tm7F zj=R$wnH0(SZ<;}x=aEH%c>0q;;;$R&tpw*`UMsn?z?^%9EX_tJGO{pNb^44&K(nFn z6N|h+7QV^;%WY3&?y?Wr5*X!fC{khP(&~Oscyj%1_0UhOA?-MJ0?aaME8a5L<^btH zf?*`i&jhX*PCVon&22i&h>%%q&P0L!1ty#DojS)-11?!05kl7iq=_Zx_f6!VV^plx zLyMpztz>xq%ou3|gIin>8VQx_gHgsWA`-uC$brwGKHyo3l(V_T+?F=>oFZGcJ6l`Fa zS8$5MK{Vy&u>134&$L|qEtW{Mi<~Mx!hA%#evyyw1+_g$KM4jBelH3l*l>x|h2Nh^ zYVrX(za|*8RSb*o&gA-7W44(2B2&M@bEQ`IbC0uva6?~z0xb)sW1gAUojXEk3}`r? zQJQ?!f4)!GhETVrxIeum{Id9k^eI7NgwvNi6V#{g1l^c0@n665o{2`#9{5b$g0c}B zu~5t~(r_+6Orz4s5dtL+Ha==4tLjs(N=ODEnt#{T6%FDvPG0Fxpm+cJ-Mr;7keHxo zBB~CtOm_r1sp6qJGs7&OZ=VdJu2`Gah8*jN8L)d%Q=Su3N~pui9al)-*#G%6aW>(FY1~viH2wJY%;)XRRk=O^J2J~fa(z0l zoXfL)Coe8y7KwP$(%NcBRw0R~VAi8mx`3Vi#9&`jUuVg(=dd~|#P&8B|9Hc3iEJAX z70c>zycJ&@-H9*|OFUV(VL)bLKqd^?T^p#sgIgf_SdaF;IE`Y|LOU6ah(G^x@Djq!O+eHA1*r5oUQ7q2uTZ67w)2F^l94AifYC zXn!Sv2O6Q*(}q$&6by!S$c7S!mg2xMf!|YJe*QWz?!+k0rlLgau6q?AWq`1)07V9R zX)am``j>y0Ge;bQagv9IfxEe346`4UQ8jR0vqe%*Zy^tTq@l=y(SdUbV%`87{2(dz z2f|j|-I!6g3B`R*Sy>q45`hy zYaQR z$FOq4Enz9=bCE_1 z)}P)hwZJ2dum}TJQAmZv+zYcXBZrYK!w~9~NXG)=;+t>ZqgG z6~>O|Q8xOK-g4y~XjuF(_$|0NAtfKViYwQ^3pTy#mi|wl#N@pN6NYLaYP@w66u{H2 zQvTnkr>6QLakCEs-|;q9@Teq>8ElJ+jy)q;iMl}nr2Dbq;l?1Vt1rAzqZ*BmRAsF* zvcjZvec)h+seMnbbN1VFs?~&X2_cJrRD04@tBBqk9I$j+tTWjTEa>On!q&bc^WCQY z(6rThQJno~846dp%73qefnfW9{E+}BqX-8#><>to#k$&*9WJQ+f`XQk6Ii)Oi3-k- zcU8sFxq?n*w*x>f1PZEWszKAPI05fOXc zp;%ibOIZP#68>l!t{8}M9|F$iE-o%aoXFUZ4-45Q!2ay z$vZ~rYH-liu-cZX>iACn^PWHcZ#|HDOORL5HQ0D=1vmVMwGc0Ena z2pogXJ@5viX)^Cs8n&EwaJ@b9)oyfH-~2Mr05?tx|suzk1#-6~+uLUzawuQqcY zdl{0FqKS#gF=eBPLJ4AFC|uOX$H!?}Ad0Ea@WSKg;khEZ@dly`C3PV9#65INh}H$! zf(E70BuUte5LAr(Mpaq)sc%IyM2$4RiwazFy#c)NddIxYAz5QRrw7jpv_$hYhK*;G zEW9QBF;$YYKXhcJ-kG_>%41XeWBHuSJ&ZYRKrvOp&n8DQI*S0o+^2a}_wKD4L;73= z#>pqF;1p>=`B6|7J`)fGW(BF`tR&goh;%t_fe=inHC1E)bGfABrUEqRGGXkiUons}pcMgiZP}H8$;CmBogdD&qCYzvr04O^b zsWSO?z*}tXym|8=3K6lk`jMSI2Hgl{pfhjB)LnqV0lSt=B0S){)U+nnh`Gb_gIsE{ z%Q%^rTzodR-b%fwr85QDuvL_AEZ^+u{k*zR<^i4 zi#5=dDnX!Il6a6b79%)U9$9s%kFJK9GtD-hD69^I;oQq4n?7C81wNjSegPHqq1FvU zO4RK0!BzU+ThP7k`s{8YJ@m63`uDrZlJ>PBdhv`N>tpr&bJe27k0 zp7ra?ez|;E&Ay-piAy;>EAp@K+C#;6`o?TCHJ242FFlp{rI{7-&SMzXbc+0`3=(S! zX=qruq<^z-#k(q-3ehVG(Z#E8Y|@AKn*r8IAr59kgjxk~_@rPy7}!jwVqwjp8!&P9 zMTK09fq=K+<+cPgE??XT(HGoK9BGVPCY3JMCiczAA~ z&K%CESSjv{RH%M$RR0nXQAFFiPf7&Yb@I`bL#A4!AklEty05V({s!Db{2|3(>gebQ z+MIE3*2H<#=glR@(6a~w(w9?LSC?e``sow!Xw>wzV9=O~8|MaE zOG8~)iDMG~9f~Tp8K~rhOdP*F<0Sb$$Z3*EoExq8AzyioteseFWL66KJPijYC4JWx zpa$MTI-Mnp+1b0zTtRkKBJL}QcQhpsjTm&YB?@Ix@rteZ=POK3D2P}rsJX(xHNyc_ z1Hh#!2tUf95nHR5NpKb{Bt2=jlEaxpA&x;37$aN1|CSTL==!QG!X+y+CkZCD$_92xxuxr<4 z%Bny+Q-qwP5^hjTi61O7rWJpWYWm16;c4;Gbct+=RP@|_1XOYLS% z9e~G1F06^ka32O|0b3eD7$%SnkO2K0XFmBL0c(KOP2GWTa5YK}=lU=-1Btxglp_Kb zDq@TxFqPMMCsr>OX)woh68DSXU2^e5nL@pDEQ!p@$S#yLN9-~zt(Tn(#UT)h0FSv8 zN?ui0E?*X}|Kkp*gEX_%!wh+92Xc){G-SdFwL;2p3+w^}nNZv`VsE;uDkaK_` zqj;}?px{23z74jc;wN$Im@URS67G%ev4Dukho^y{w{M-XN^kmML+$O~2g`ndivsdi zarC?wEnN7X>|eN`JKqLvP${9*D04b6{(#1tnl>;kzwlEXfWWVDW5n$~8G}_@`*;g^z4xu-fg5SDb zz%j`(AGsiyeUacGl&Ae2Is_af@$}HZKp9R_B8!r%04ZGPDN9S0$DL8TYc3m}&xn5L zXCb_gNfkl8{R?xv0p3Nt*dzk-f9-ck6SMNOQm=jV+S-G+%aGlrxHES9;fwzR(jukg diff --git a/pyproject.toml b/pyproject.toml index c8ae6196..6145ec8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,7 @@ dev = [ [project.entry-points.console_scripts] -guidellm = "guidellm.main:generate_benchmark_report_cli" +guidellm = "guidellm.__main__:cli" guidellm-config = "guidellm.config:print_config" diff --git a/src/guidellm/__init__.py b/src/guidellm/__init__.py index 28c66117..fb531d49 100644 --- a/src/guidellm/__init__.py +++ b/src/guidellm/__init__.py @@ -6,14 +6,22 @@ # flake8: noqa import os -import transformers # type: ignore +import logging +import contextlib -os.environ["TOKENIZERS_PARALLELISM"] = "false" # Silence warnings for tokenizers -transformers.logging.set_verbosity_error() # Silence warnings for transformers +with open(os.devnull, "w") as devnull, contextlib.redirect_stderr( + devnull +), contextlib.redirect_stdout(devnull): + from transformers.utils import logging as hf_logging + + # Set the log level for the transformers library to ERROR + # to ignore None of PyTorch, TensorFlow found + os.environ["TOKENIZERS_PARALLELISM"] = "false" # Silence warnings for tokenizers + hf_logging.set_verbosity_error() + logging.getLogger("transformers").setLevel(logging.ERROR) from .config import settings from .logger import configure_logger, logger -# from .main import generate_benchmark_report __all__ = ["configure_logger", "logger", "settings", "generate_benchmark_report"] diff --git a/src/guidellm/__main__.py b/src/guidellm/__main__.py new file mode 100644 index 00000000..f0d40d30 --- /dev/null +++ b/src/guidellm/__main__.py @@ -0,0 +1,270 @@ +import asyncio +import json +from pathlib import Path +from typing import get_args + +import click + +from guidellm.backend import BackendType +from guidellm.benchmark import ProfileType, benchmark_generative_text +from guidellm.scheduler import StrategyType + +STRATEGY_PROFILE_CHOICES = set( + list(get_args(ProfileType)) + list(get_args(StrategyType)) +) + + +def parse_json(ctx, param, value): + if value is None: + return None + try: + return json.loads(value) + except json.JSONDecodeError as err: + raise click.BadParameter(f"{param.name} must be a valid JSON string.") from err + + +def parse_number_str(ctx, param, value): + if value is None: + return None + + values = value.split(",") if "," in value else [value] + + try: + return [int(val) if val.isdigit() else float(val) for val in values] + except ValueError as err: + raise click.BadParameter( + f"{param.name} must be a number or comma-separated list of numbers." + ) from err + + +@click.group() +def cli(): + pass + + +@cli.command() +@click.option( + "--target", + required=True, + type=str, + help="The target path for the backend to run benchmarks against. For example, http://localhost:8000", +) +@click.option( + "--backend-type", + type=click.Choice(list(get_args(BackendType))), + help=( + "The type of backend to use to run requests against. Defaults to 'openai_http'." + f" Supported types: {', '.join(get_args(BackendType))}" + ), + default="openai_http", +) +@click.option( + "--backend-args", + callback=parse_json, + default=None, + help=( + "A JSON string containing any arguments to pass to the backend as a " + "dict with **kwargs." + ), +) +@click.option( + "--model", + default=None, + type=str, + help=( + "The ID of the model to benchmark within the backend. " + "If None provided (default), then it will use the first model available." + ), +) +@click.option( + "--processor", + default=None, + type=str, + help=( + "The processor or tokenizer to use to calculate token counts for statistics " + "and synthetic data generation. If None provided (default), will load " + "using the model arg, if needed." + ), +) +@click.option( + "--processor-args", + default=None, + callback=parse_json, + help=( + "A JSON string containing any arguments to pass to the processor constructor " + "as a dict with **kwargs." + ), +) +@click.option( + "--data", + required=True, + type=str, + help=( + "The HuggingFace dataset ID, a path to a HuggingFace dataset, " + "a path to a data file csv, json, jsonl, or txt, " + "or a synthetic data config as a json or key=value string." + ), +) +@click.option( + "--data-args", + callback=parse_json, + help=( + "A JSON string containing any arguments to pass to the dataset creation " + "as a dict with **kwargs." + ), +) +@click.option( + "--data-sampler", + default=None, + type=click.Choice(["random"]), + help=( + "The data sampler type to use. 'random' will add a random shuffle on the data. " + "Defaults to None" + ), +) +@click.option( + "--rate-type", + required=True, + type=click.Choice(STRATEGY_PROFILE_CHOICES), + help=( + "The type of benchmark to run. " + f"Supported types {', '.join(STRATEGY_PROFILE_CHOICES)}. " + ), +) +@click.option( + "--rate", + default=None, + callback=parse_number_str, + help=( + "The rates to run the benchmark at. " + "Can be a single number or a comma-separated list of numbers. " + "For rate-type=sweep, this is the number of benchmarks it runs in the sweep. " + "For rate-type=concurrent, this is the number of concurrent requests. " + "For rate-type=async,constant,poisson, this is the rate requests per second. " + "For rate-type=synchronous,throughput, this must not be set." + ), +) +@click.option( + "--max-seconds", + type=float, + help=( + "The maximum number of seconds each benchmark can run for. " + "If None, will run until max_requests or the data is exhausted." + ), +) +@click.option( + "--max-requests", + type=int, + help=( + "The maximum number of requests each benchmark can run for. " + "If None, will run until max_seconds or the data is exhausted." + ), +) +@click.option( + "--warmup-percent", + type=float, + default=None, + help=( + "The percent of the benchmark (based on max-seconds, max-requets, " + "or lenth of dataset) to run as a warmup and not include in the final results. " + "Defaults to None." + ), +) +@click.option( + "--cooldown-percent", + type=float, + help=( + "The percent of the benchmark (based on max-seconds, max-requets, or lenth " + "of dataset) to run as a cooldown and not include in the final results. " + "Defaults to None." + ), +) +@click.option( + "--disable-progress", + is_flag=True, + help="Set this flag to disable progress updates to the console", +) +@click.option( + "--display-scheduler-stats", + is_flag=True, + help="Set this flag to display stats for the processes running the benchmarks", +) +@click.option( + "--disable-console-outputs", + is_flag=True, + help="Set this flag to disable console output", +) +@click.option( + "--output-path", + type=click.Path(), + default=Path.cwd() / "benchmarks.json", + help=( + "The path to save the output to. If it is a directory, " + "it will save benchmarks.json under it. " + "Otherwise, json, yaml, or csv files are supported for output types " + "which will be read from the extension for the file path." + ), +) +@click.option( + "--output-extras", + callback=parse_json, + help="A JSON string of extra data to save with the output benchmarks", +) +@click.option( + "--random-seed", + default=42, + type=int, + help="The random seed to use for benchmarking to ensure reproducibility.", +) +def benchmark( + target, + backend_type, + backend_args, + model, + processor, + processor_args, + data, + data_args, + data_sampler, + rate_type, + rate, + max_seconds, + max_requests, + warmup_percent, + cooldown_percent, + disable_progress, + display_scheduler_stats, + disable_console_outputs, + output_path, + output_extras, + random_seed, +): + asyncio.run( + benchmark_generative_text( + target=target, + backend_type=backend_type, + backend_args=backend_args, + model=model, + processor=processor, + processor_args=processor_args, + data=data, + data_args=data_args, + data_sampler=data_sampler, + rate_type=rate_type, + rate=rate, + max_seconds=max_seconds, + max_requests=max_requests, + warmup_percent=warmup_percent, + cooldown_percent=cooldown_percent, + show_progress=not disable_progress, + show_progress_scheduler_stats=display_scheduler_stats, + output_console=not disable_console_outputs, + output_path=output_path, + output_extras=output_extras, + random_seed=random_seed, + ) + ) + + +if __name__ == "__main__": + cli() diff --git a/src/guidellm/backend/backend.py b/src/guidellm/backend/backend.py index 115c0e11..ff80769a 100644 --- a/src/guidellm/backend/backend.py +++ b/src/guidellm/backend/backend.py @@ -115,8 +115,8 @@ async def validate(self): If not successful, raises the appropriate exception. """ logger.info("{} validating backend {}", self.__class__.__name__, self.type_) - self.check_setup() - models = self.available_models() + await self.check_setup() + models = await self.available_models() if not models: raise ValueError("No models available for the backend") @@ -126,7 +126,7 @@ async def validate(self): pass @abstractmethod - def check_setup(self): + async def check_setup(self): """ Check the setup for the backend. If unsuccessful, raises the appropriate exception. @@ -136,7 +136,17 @@ def check_setup(self): ... @abstractmethod - def available_models(self) -> List[str]: + async def prepare_multiprocessing(self): + """ + Prepare the backend for use in a multiprocessing environment. + This is useful for backends that have instance state that can not + be shared across processes and should be cleared out and re-initialized + for each new process. + """ + ... + + @abstractmethod + async def available_models(self) -> List[str]: """ Get the list of available models for the backend. diff --git a/src/guidellm/backend/openai.py b/src/guidellm/backend/openai.py index 46bf034f..3618465b 100644 --- a/src/guidellm/backend/openai.py +++ b/src/guidellm/backend/openai.py @@ -92,6 +92,7 @@ def __init__( if max_output_tokens is not None else settings.openai.max_output_tokens ) + self._async_client: Optional[httpx.Client] = None @property def target(self) -> str: @@ -125,7 +126,7 @@ def info(self) -> Dict[str, Any]: "chat_completions_path": CHAT_COMPLETIONS_PATH, } - def check_setup(self): + async def check_setup(self): """ Check if the backend is setup correctly and can be used for requests. Specifically, if a model is not provided, it grabs the first available model. @@ -134,7 +135,7 @@ def check_setup(self): :raises ValueError: If no models or the provided model is not available. """ - models = self.available_models() + models = await self.available_models() if not models: raise ValueError(f"No models available for target: {self.target}") @@ -146,24 +147,32 @@ def check_setup(self): "{models} for target: {self.target}" ) - def available_models(self) -> List[str]: + async def prepare_multiprocessing(self): + """ + Prepare the backend for use in a multiprocessing environment. + Clears out the sync and async clients to ensure they are re-initialized + for each process. + """ + if self._async_client is not None: + await self._async_client.aclose() + self._async_client = None + + async def available_models(self) -> List[str]: """ Get the available models for the target server using the OpenAI models endpoint: /v1/models """ target = f"{self.target}/v1/models" headers = self._headers() + response = await self._get_async_client().get(target, headers=headers) + response.raise_for_status() - with httpx.Client(http2=self.http2, timeout=self.timeout) as client: - response = client.get(target, headers=headers) - response.raise_for_status() - - models = [] + models = [] - for item in response.json()["data"]: - models.append(item["id"]) + for item in response.json()["data"]: + models.append(item["id"]) - return models + return models async def text_completions( # type: ignore[override] self, @@ -191,7 +200,6 @@ async def text_completions( # type: ignore[override] a StreamingTextResponse for each received iteration, and a ResponseSummary for the final response. """ - logger.debug("{} invocation with args: {}", self.__class__.__name__, locals()) headers = self._headers() payload = self._completions_payload( @@ -295,6 +303,20 @@ async def chat_completions( # type: ignore[override] ) raise ex + def _get_async_client(self) -> httpx.AsyncClient: + """ + Get the async HTTP client for making requests. + If the client has not been created yet, it will create one. + + :return: The async HTTP client. + """ + if self._async_client is None: + self._async_client = httpx.AsyncClient( + http2=self.http2, timeout=self.timeout + ) + + return self._async_client + def _headers(self) -> Dict[str, str]: headers = { "Content-Type": "application/json", @@ -429,69 +451,72 @@ async def _iterative_completions_request( payload, ) - async with httpx.AsyncClient(http2=self.http2, timeout=self.timeout) as client: - response_value = "" - response_prompt_count: Optional[int] = None - response_output_count: Optional[int] = None - iter_count = 0 - start_time = time.time() - iter_time = start_time - first_iter_time: Optional[float] = None - last_iter_time: Optional[float] = None - - yield StreamingTextResponse( - type_="start", - value="", - start_time=start_time, - iter_count=iter_count, - delta="", - time=start_time, - request_id=request_id, - ) + response_value = "" + response_prompt_count: Optional[int] = None + response_output_count: Optional[int] = None + iter_count = 0 + start_time = time.time() + iter_time = start_time + first_iter_time: Optional[float] = None + last_iter_time: Optional[float] = None + + yield StreamingTextResponse( + type_="start", + value="", + start_time=start_time, + first_iter_time=None, + iter_count=iter_count, + delta="", + time=start_time, + request_id=request_id, + ) - async with client.stream( - "POST", target, headers=headers, json=payload - ) as stream: - stream.raise_for_status() - - async for line in stream.aiter_lines(): - iter_time = time.time() - logger.debug( - "{} request: {} recieved iter response line: {}", - self.__class__.__name__, - request_id, - line, + # reset start time after yielding start response to ensure accurate timing + start_time = time.time() + + async with self._get_async_client().stream( + "POST", target, headers=headers, json=payload + ) as stream: + stream.raise_for_status() + + async for line in stream.aiter_lines(): + iter_time = time.time() + logger.debug( + "{} request: {} recieved iter response line: {}", + self.__class__.__name__, + request_id, + line, + ) + + if not line or not line.strip().startswith("data:"): + continue + + if line.strip() == "data: [DONE]": + break + + data = json.loads(line.strip()[len("data: ") :]) + if delta := self._extract_completions_delta_content(type_, data): + if first_iter_time is None: + first_iter_time = iter_time + last_iter_time = iter_time + + iter_count += 1 + response_value += delta + + yield StreamingTextResponse( + type_="iter", + value=response_value, + iter_count=iter_count, + start_time=start_time, + first_iter_time=first_iter_time, + delta=delta, + time=iter_time, + request_id=request_id, ) - if not line or not line.strip().startswith("data:"): - continue - - if line.strip() == "data: [DONE]": - break - - data = json.loads(line.strip()[len("data: ") :]) - if delta := self._extract_completions_delta_content(type_, data): - if first_iter_time is None: - first_iter_time = iter_time - last_iter_time = iter_time - - iter_count += 1 - response_value += delta - - yield StreamingTextResponse( - type_="iter", - value=response_value, - iter_count=iter_count, - start_time=start_time, - first_iter_time=first_iter_time, - delta=delta, - time=iter_time, - request_id=request_id, - ) - - if usage := self._extract_completions_usage(data): - response_prompt_count = usage["prompt"] - response_output_count = usage["output"] + if usage := self._extract_completions_usage(data): + response_prompt_count = usage["prompt"] + response_output_count = usage["output"] logger.info( "{} request: {} with headers: {} and payload: {} completed with: {}", diff --git a/src/guidellm/backend/response.py b/src/guidellm/backend/response.py index 4c2d70b9..ec7e8e7c 100644 --- a/src/guidellm/backend/response.py +++ b/src/guidellm/backend/response.py @@ -33,6 +33,7 @@ class StreamingTextResponse(BaseModel): type_: StreamingResponseType value: str start_time: float + first_iter_time: Optional[float] iter_count: int delta: str time: float @@ -86,8 +87,8 @@ class ResponseSummary(BaseModel): value: str request_args: RequestArgs iterations: int = 0 - start_time: Optional[float] - end_time: Optional[float] + start_time: float + end_time: float first_iter_time: Optional[float] last_iter_time: Optional[float] request_prompt_tokens: Optional[int] = None @@ -120,6 +121,10 @@ def output_tokens(self) -> Optional[int]: :return: The number of tokens in the output, if any. """ + if self.error is not None: + # error occurred, can't trust request tokens were all generated + return self.response_prompt_tokens + if settings.preferred_output_tokens_source == "request": return self.request_output_tokens or self.response_output_tokens diff --git a/src/guidellm/benchmark/__init__.py b/src/guidellm/benchmark/__init__.py index 1efbffb7..a9da9e80 100644 --- a/src/guidellm/benchmark/__init__.py +++ b/src/guidellm/benchmark/__init__.py @@ -1,6 +1,7 @@ from .aggregator import AGG, BenchmarkAggregator, GenerativeBenchmarkAggregator from .benchmark import BENCH, Benchmark, GenerativeBenchmark from .benchmarker import Benchmarker, BenchmarkerResult, GenerativeBenchmarker +from .entrypoints import benchmark_generative_text from .profile import ( AsyncProfile, ConcurrentProfile, @@ -30,4 +31,5 @@ "SynchronousProfile", "ThroughputProfile", "create_profile", + "benchmark_generative_text", ] diff --git a/src/guidellm/benchmark/aggregator.py b/src/guidellm/benchmark/aggregator.py index 19780f68..7a210df4 100644 --- a/src/guidellm/benchmark/aggregator.py +++ b/src/guidellm/benchmark/aggregator.py @@ -26,7 +26,7 @@ ) from guidellm.benchmark.profile import Profile from guidellm.config import settings -from guidellm.objects import RunningStats, Serializable +from guidellm.objects import RunningStats, Serializable, TimeRunningStats from guidellm.request import GenerationRequest from guidellm.scheduler import ( REQ, @@ -130,112 +130,175 @@ class BenchmarkAggregator(ABC, BaseModel, Generic[BENCH, REQ, RES]): "that were not within the warmup or cooldown periods." ), ) - - start_time: float = Field( + in_warmup: bool = Field( description=( - "The timestamp when the benchmark run started. Defaults to the current " - "time.time() on creation." + "A flag to indicate if the benchmark is currently in the warmup phase." ), - default_factory=time.time, - ) - created_requests: int = Field( - description="The number of requests created for this benchmark run.", - default=0, + default=False, + exclude=True, ) - queued_requests: int = Field( - description="The number of requests pending in queue for this benchmark run.", - default=0, + in_cooldown: bool = Field( + description=( + "A flag to indicate if the benchmark is currently in the cooldown phase." + ), + default=False, + exclude=True, ) - scheduled_requests: int = Field( + + scheduler_created_requests: RunningStats = Field( description=( - "The number of requests scheduled (actively running but waiting for the " - "desired start time) for this benchmark run." + "The running statistics for the number of requests created for this " + "benchmark run. This includes all requests created, regardless of " + "their status." ), - default=0, + default_factory=RunningStats, ) - processing_requests: int = Field( + scheduler_queued_requests: RunningStats = Field( description=( - "The number of requests actively being processed by the worker for this " - "benchmark run." + "The running statistics for the number of requests pending in queue " + "for this benchmark run. This includes requests that are waiting to " + "be scheduled." ), - default=0, + default_factory=RunningStats, ) - completed_requests: int = Field( + scheduler_scheduled_requests: RunningStats = Field( description=( - "The number of requests completed for this benchmark run. This includes " - "requests within the warmup and cooldown period, if any, along with the " - "final results." + "The running statistics for the number of requests scheduled (actively " + "running but waiting for the desired start time) for this benchmark run." ), - default=0, + default_factory=RunningStats, ) - successful_requests: int = Field( + scheduler_processing_requests: RunningStats = Field( description=( - "The number of requests that completed successfully without error. " - "This is a subset of the completed requests for any that did not error. " - "This includes requests within the warmup and cooldown period, if any, " - "along with the final results." + "The running statistics for the number of requests actively being " + "processed by the worker for this benchmark run." ), - default=0, + default_factory=RunningStats, ) - errored_requests: int = Field( + scheduler_completed_requests: RunningStats = Field( description=( - "The number of requests that errored during processing. This is a subset " - "of the completed requests for any that errored. This includes requests " - "within the warmup and cooldown period, if any, " - "along with the final results." + "The running statistics for the number of requests completed for this " + "benchmark run. This includes requests within the warmup and cooldown " + "period, if any, along with the final results." ), - default=0, + default_factory=RunningStats, ) - in_warmup: bool = Field( + successful_requests: RunningStats = Field( description=( - "A flag to indicate if the benchmark is currently in the warmup phase." + "The running statistics for the number of requests that completed " + "successfully without error. This is a subset of the completed requests " + "for any that did not error. This includes requests within the warmup " + "and cooldown period, if any, along with the final results." ), - default=False, - exclude=True, + default_factory=RunningStats, ) - in_cooldown: bool = Field( + errored_requests: RunningStats = Field( description=( - "A flag to indicate if the benchmark is currently in the cooldown phase." + "The running statistics for the number of requests that errored during " + "processing. This is a subset of the completed requests for any that " + "errored. This includes requests within the warmup and cooldown period, " + "if any, along with the final results." ), - default=False, - exclude=True, + default_factory=RunningStats, ) - queued_time: RunningStats = Field( + queued_time: TimeRunningStats = Field( description=( "The running statistics for the time spent in queue for all requests that " "completed within the benchmark run. This is the time from when the " - "request was created to when it was scheduled to be processed." + "request was created to when it was dequeued by the worker." ), - default_factory=RunningStats, + default_factory=TimeRunningStats, ) - scheduled_time: RunningStats = Field( + scheduled_time_delay: TimeRunningStats = Field( description=( - "The running statistics for the time spent scheduled for all requests that " - "completed within the benchmark run. This is the time from when the " - "request was scheduled to be processed to when it was actually started." + "The running statistics for the time spent from when a request was " + "dequeued by the worker to when it was actually scheduled by the worker" + "for all requests that completed within the benchmark run. " + "This should be as close to 0 as possible, any additional time is " + "overheads from the system or the worker." ), - default_factory=RunningStats, + default_factory=TimeRunningStats, ) - worker_time: RunningStats = Field( + scheduled_time_sleep: TimeRunningStats = Field( description=( - "The running statistics for the time spent processing for all requests that " + "The running statistics for the time for each request spent sleeping til " + "the desired start time was reached for all requests that completed within " + "the benchmark run. This is the time from when the request was scheduled " + "to when the desired start time was reached. " + ), + default_factory=TimeRunningStats, + ) + worker_start_delay: TimeRunningStats = Field( + description=( + "The running statistics for the time delay between when the request was " + "scheduled and when the worker actually started processing subtracting any " + "sleep time for all requests that completed within the benchmark run. " + "This should be as close to 0 as possible, any additional time is " + "overheads from the system or the worker." + ), + default_factory=TimeRunningStats, + ) + worker_time: TimeRunningStats = Field( + description=( + "The running statistics for the time spent processing all requests that " "completed within the benchmark run. This is the time from when the " "request was started to when it was completed." ), - default_factory=RunningStats, + default_factory=TimeRunningStats, ) - targeted_worker_start_delay: RunningStats = Field( + worker_start_time_targeted_delay: TimeRunningStats = Field( description=( "The running statistics for the delay between the targeted start time and " - "the actual start time for all requests that completed within the benchmark " - "run. This is the time from when the request was scheduled to be processed " - "to when it was actually started." + "the actual start time for requests that completed within the benchmark " + "run. This represents delays from the best case desired start time. " + "For async strategies, this represents delays from the ideal system. " + "For sync strategies, since those are doubled in queue, this should be " + "as close to the time for a request to be processed as possible." ), - default_factory=RunningStats, + default_factory=TimeRunningStats, + ) + request_start_time_delay: TimeRunningStats = Field( + description=( + "The running statistics for the delay between the actual request being " + "made and the time the worker started on the request for all requests " + "that completed within the benchmark run. This time should be as close to " + "0 as possible, any additional time is overhead from the system or " + "the worker." + ), + default_factory=TimeRunningStats, + ) + request_start_time_targeted_delay: TimeRunningStats = Field( + description=( + "The running statistics for the delay between the targeted start time and " + "the actual start time for all requests that completed within the " + "benchmark run. This represents delays from the best case desired start " + "time. For async strategies, this represents delays from the ideal system. " + "For sync strategies, since those are duplicated in queue, this should be " + "as close to the time for a request to be processed." + ), + default_factory=TimeRunningStats, + ) + request_time_delay: TimeRunningStats = Field( + description=( + "The running statistics for the delay in time between the total request " + "time and the worker time. This should be as close to 0 as possible, any " + "additional time is overhead from the system or the worker. " + ), + default_factory=TimeRunningStats, + ) + request_time: TimeRunningStats = Field( + description=( + "The running statistics for the time spent processing all requests that " + "completed within the benchmark run. This is the time from when the " + "request was created to when it was completed." + ), + default_factory=TimeRunningStats, ) - def add_result(self, result: SchedulerResult[REQ, RES]): + def add_result( + self, result: SchedulerResult[REQ, RES], is_error: bool = False + ) -> bool: """ Add a result to the aggregator. This will update the internal statistics and add the result to the list of results if it is not within the warmup or @@ -243,85 +306,87 @@ def add_result(self, result: SchedulerResult[REQ, RES]): :param result: The result to add to the aggregator. """ - self.add_base_result(result) - - @abstractmethod - def compile(self) -> BENCH: - """ - Compile the benchmark results and statistics into a Benchmark object. - This is required to be implemented by subclasses to finalize the benchmark - and return the compiled object. - """ - ... - - def add_base_result( - self, result: SchedulerResult[REQ, RES], is_error: bool = False - ): - """ - Helper function to update the base statistics for the aggregator and add the - result to the list of results if it is not within the warmup or cooldown period. - - :param result: The result to add to the aggregator. - :param is_error: A flag to indicate if the result was an error or not. - """ - self.created_requests = result.run_info.created_requests - self.queued_requests = result.run_info.queued_requests - self.scheduled_requests = result.run_info.scheduled_requests - self.processing_requests = result.run_info.processing_requests - self.completed_requests = result.run_info.completed_requests - - if result.type_ == "request_complete": - self._update_stats_from_result(result, is_error) - self._add_to_results_within_active_period(result) - - def _update_stats_from_result( - self, result: SchedulerResult[REQ, RES], is_error: bool - ): - if is_error: - self.errored_requests += 1 - else: + # Add base scheduler statistics to the aggregator + self.scheduler_created_requests += result.run_info.created_requests + self.scheduler_queued_requests += result.run_info.queued_requests + self.scheduler_scheduled_requests += result.run_info.scheduled_requests + self.scheduler_processing_requests += result.run_info.processing_requests + self.scheduler_completed_requests += result.run_info.completed_requests + + if result.preempted or result.type_ != "request_complete": + # If the result was preempted or not completed yet + # we do not want to add it to the results. + return False + + # add base result statistics given this was not preempted and it's completed + if not is_error: self.successful_requests += 1 + else: + self.errored_requests += 1 - self.queued_time.update( - result.request_info.scheduled_time - result.request_info.queued_time + self.queued_time += ( + result.request_info.dequeued_time - result.request_info.queued_time ) - self.scheduled_time.update( + self.scheduled_time_delay += ( + result.request_info.scheduled_time - result.request_info.dequeued_time + ) + sleep_time = max( + 0.0, + result.request_info.targeted_start_time + - result.request_info.scheduled_time, + ) + self.scheduled_time_sleep += sleep_time + time_to_worker_start = ( result.request_info.worker_start - result.request_info.scheduled_time ) - self.worker_time.update( + self.worker_start_delay += time_to_worker_start - sleep_time + self.worker_time += ( result.request_info.worker_end - result.request_info.worker_start ) - self.targeted_worker_start_delay.update( + self.worker_start_time_targeted_delay += ( result.request_info.worker_start - result.request_info.targeted_start_time ) - def _add_to_results_within_active_period(self, result: SchedulerResult[REQ, RES]): - start_time = result.request_info.worker_start - end_time = result.request_info.worker_end - completed_number = self.errored_requests + self.successful_requests + # Add result to the list of results provided we are not in warmup or cooldown + total_completed = self.successful_requests.total + self.errored_requests.total + global_start_time = self.scheduler_created_requests.start_time - if (self.warmup_number and completed_number <= self.warmup_number) or ( - self.warmup_duration and start_time <= self.warmup_duration + if (self.warmup_number and total_completed <= self.warmup_number) or ( + self.warmup_duration + and result.request_info.worker_start + <= (global_start_time + self.warmup_duration) ): # within warmup period self.in_warmup = True - return + return True if ( self.cooldown_number - and completed_number > self.max_number - self.cooldown_number + and total_completed > self.max_number - self.cooldown_number ) or ( self.cooldown_duration - and end_time >= self.max_duration - self.cooldown_duration + and result.request_info.worker_start + >= global_start_time + self.max_duration - self.cooldown_duration ): # within cooldown period self.in_cooldown = True - return + return True self.in_warmup = False self.in_cooldown = False self.results.append(result) + return True + + @abstractmethod + def compile(self) -> BENCH: + """ + Compile the benchmark results and statistics into a Benchmark object. + This is required to be implemented by subclasses to finalize the benchmark + and return the compiled object. + """ + ... + AGG = TypeVar("AGG", bound=BenchmarkAggregator[BENCH, REQ, RES]) @@ -335,38 +400,27 @@ class GenerativeBenchmarkAggregator( "avaiable that match the preferred source." ) ) - - targeted_request_delay: RunningStats = Field( + processor_args: Optional[Dict[str, Any]] = Field( description=( - "The running statistics for the delay between the targeted start time and " - "the actual start time for all requests that completed within the " - "benchmark run. This is the time from when the request was scheduled to " - "be processed to when it was actually started." + "Additional arguments to pass to the tokenizer if it requires " + "any specific configuration for loading or processing." ), - default_factory=RunningStats, ) - request_latency: RunningStats = Field( - description=( - "The running statistics for the time spent processing all requests that " - "completed within the benchmark run. This is the time from when the " - "request was created to when it was completed." - ), - default_factory=RunningStats, - ) - time_to_first_token: RunningStats = Field( + + time_to_first_token: TimeRunningStats = Field( description=( "The running statistics for the time from the start of the request to the " "first token being generated for all requests that completed within the " "benchmark run." ), - default_factory=RunningStats, + default_factory=TimeRunningStats, ) - inter_token_latency: RunningStats = Field( + inter_token_latency: TimeRunningStats = Field( description=( "The running statistics for the time between each token being generated " "for all requests that completed within the benchmark run." ), - default_factory=RunningStats, + default_factory=TimeRunningStats, ) prompt_tokens: RunningStats = Field( description=( @@ -390,7 +444,9 @@ class GenerativeBenchmarkAggregator( default_factory=RunningStats, ) - def add_result(self, result: SchedulerResult[GenerationRequest, ResponseSummary]): + def add_result( + self, result: SchedulerResult[GenerationRequest, ResponseSummary] + ) -> bool: """ Add a result to the aggregator. This will update the internal statistics and add the result to the list of results if it is not within the warmup or @@ -398,11 +454,45 @@ def add_result(self, result: SchedulerResult[GenerationRequest, ResponseSummary] :param result: The result to add to the aggregator. """ - is_error = result.type_ == "request_complete" and result.response.error - self.add_base_result(result, is_error=is_error) + is_error = result.type_ == "request_complete" and ( + result.preempted or result.response.error + ) + added = super().add_result(result, is_error=is_error) + + if not added: + return False + + self.request_start_time_delay += ( + result.response.start_time - result.request_info.worker_start + ) + self.request_start_time_targeted_delay += ( + result.response.start_time - result.request_info.targeted_start_time + ) + self.request_time_delay += ( + (result.response.start_time - result.request_info.worker_start) + + result.request_info.worker_end + - result.response.end_time + ) + self.request_time += result.response.end_time - result.request_info.worker_start + + self.time_to_first_token += ( + result.response.first_iter_time - result.request_info.worker_start + if result.response.first_iter_time + else 0.0 + ) + self.inter_token_latency.update( + (result.response.last_iter_time - result.response.first_iter_time) * 1000.0 + if result.response.last_iter_time and result.response.first_iter_time + else 0.0, + count=(result.response.output_tokens or 1) - 1, + ) + self.prompt_tokens += result.response.prompt_tokens or 0 + self.output_tokens += result.response.output_tokens or 0 + self.total_tokens += (result.response.prompt_tokens or 0) + ( + result.response.output_tokens or 0 + ) - if result.type_ == "request_complete": - self._update_generative_stats_from_result(result) + return True def compile(self) -> GenerativeBenchmark: """ @@ -428,65 +518,27 @@ def compile(self) -> GenerativeBenchmark: cooldown_duration=self.cooldown_duration, ), run_stats=BenchmarkRunStats( - start_time=self.start_time, + start_time=self.scheduler_created_requests.start_time, end_time=time.time(), - total=self.completed_requests, - total_completed=self.successful_requests, - total_errored=self.errored_requests, + total=self.successful_requests.total + self.errored_requests.total, + total_completed=self.successful_requests.total, + total_errored=self.errored_requests.total, queued_time_avg=self.queued_time.mean, - scheduled_time_avg=self.scheduled_time.mean, + scheduled_time_delay_avg=self.scheduled_time_delay.mean, + scheduled_time_sleep_avg=self.scheduled_time_sleep.mean, + worker_start_delay_avg=self.worker_start_delay.mean, worker_time_avg=self.worker_time.mean, - worker_delay_avg=self.targeted_worker_start_delay.mean, - resolve_delay_avg=self.targeted_request_delay.mean, + worker_start_time_targeted_delay_avg=self.worker_start_time_targeted_delay.mean, + request_start_time_delay_avg=self.request_start_time_delay.mean, + request_start_time_targeted_delay_avg=self.request_start_time_targeted_delay.mean, + request_time_delay_avg=self.request_time_delay.mean, + request_time_avg=self.request_time.mean, ), worker=self.worker_description, requests_loader=self.request_loader_description, extras=self.extras, ) - def _update_generative_stats_from_result( - self, result: SchedulerResult[GenerationRequest, ResponseSummary] - ): - if self.request_latency.count == 0: - self.request_latency.start_time = self.start_time - self.targeted_request_delay.start_time = self.start_time - self.time_to_first_token.start_time = self.start_time - self.inter_token_latency.start_time = self.start_time - self.prompt_tokens.start_time = self.start_time - self.output_tokens.start_time = self.start_time - self.total_tokens.start_time = self.start_time - - self.request_latency.update( - result.response.end_time - result.response.start_time - if result.response.end_time and result.response.start_time - else 0.0 - ) - self.targeted_request_delay.update( - result.response.start_time - result.request_info.targeted_start_time - if result.response.start_time - else 0.0 - ) - self.time_to_first_token.update( - (result.response.first_iter_time - result.response.start_time) * 1000.0 - if result.response.first_iter_time and result.response.start_time - else 0.0 - ) - if result.response.output_tokens > 1: - self.inter_token_latency.update( - (result.response.last_iter_time - result.response.first_iter_time) - * 1000.0, - count=result.response.output_tokens - 1, - ) - self.prompt_tokens.update( - result.response.prompt_tokens or 0, - ) - self.output_tokens.update( - result.response.output_tokens or 0, - ) - self.total_tokens.update( - (result.response.prompt_tokens or 0) + (result.response.output_tokens or 0), - ) - def _compile_results( self, ) -> Tuple[List[Union[GenerativeTextResponseStats, GenerativeTextErrorStats]]]: @@ -499,12 +551,14 @@ def _compile_results( requests_tokens=result.response.request_prompt_tokens, response_tokens=result.response.response_prompt_tokens, preferred_tokens_source=settings.preferred_prompt_tokens_source, + errored=result.response.error is not None, ) output_tokens = self._compile_tokens_count( value=result.response.value, requests_tokens=result.response.request_output_tokens, response_tokens=result.response.response_output_tokens, preferred_tokens_source=settings.preferred_output_tokens_source, + errored=result.response.error is not None, ) if result.response.error: @@ -547,8 +601,17 @@ def _compile_tokens_count( requests_tokens: Optional[int], response_tokens: Optional[int], preferred_tokens_source: Optional[Literal["request", "response"]], + errored: bool, ) -> int: - if preferred_tokens_source is None and (requests_tokens or response_tokens): + if errored: + if self.processor is None or preferred_tokens_source in ( + "response", + "request", + ): + # no processor or we are set to trust the response/request tokens + # set to response tokens since that is the most reliable source + return response_tokens or 0 + elif preferred_tokens_source is None and (requests_tokens or response_tokens): return ( response_tokens or requests_tokens ) # trust response first if no preference @@ -562,6 +625,7 @@ def _compile_tokens_count( self.processor = check_load_processor( self.processor, + processor_args=self.processor_args, error_msg="Processor/Tokenizer is required for calculating token counts.", ) # no tokens that matched the preferred source, diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index a7ba095b..e882050a 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -108,30 +108,81 @@ class BenchmarkRunStats(Serializable): queued_time_avg: float = Field( description=( - "The average time spent in the queue for requests in the benchmark run." + "The average time spent in the queue for each request in the benchmark " + "run until it was dequeued by a worker." ) ) - scheduled_time_avg: float = Field( + scheduled_time_delay_avg: float = Field( description=( - "The average time spent in the scheduled state for requests in the " - "benchmark run." + "The average time delay between when a request was dequeued and when it " + "was scheduled to be processed by a worker in the benchmark run. " + "This should be as close to 0 as possible, any additional time is " + "overheads from the system or the worker." + ) + ) + scheduled_time_sleep_avg: float = Field( + description=( + "The average time spent sleeping til the desired start time was reached " + "after being scheduled by the worker in the benchmark run." + ) + ) + worker_start_delay_avg: float = Field( + description=( + "The average time delay between when a request was scheduled and when " + "the worker started processing it in the benchmark run. " + "This should be as close to 0 as possible, any additional time is " + "overheads from the system or the worker." ) ) worker_time_avg: float = Field( description=( - "The average time spent running each request in the benchmark run." + "The average time taken by the worker to process each request in the " + "benchmark run. This includes the time to generate the response and " + "any additional processing time." ) ) - worker_delay_avg: float = Field( + worker_start_time_targeted_delay_avg: float = Field( description=( - "The average delay between when a request was targeted to start at " - "and when it was started by the worker in the benchmark run." + "The average time delay between when a request was targeted to start " + "and when the worker actually started processing it in the benchmark " + "run. For async strategies, this represents delays from the ideal " + "system. For sync strategies, since those are doubled in queue, " + "this should be as close to the time for a request to be processed " + "as possible. Any additional time is overhead from the system or " + "the worker." ) ) - resolve_delay_avg: float = Field( + request_start_time_delay_avg: float = Field( description=( - "The average delay between when a request was targeted to start at " - "and when it was resolved/requested by the worker in the benchmark run." + "The average time delay between the actual request being made " + "and the time the worker started on the request for all requests " + "that completed within the benchmark run. This time should be as close " + "to 0 as possible, any additional time is overhead from the system or " + "the worker." + ) + ) + request_start_time_targeted_delay_avg: float = Field( + description=( + "The average time delay between when the targeted start time and " + "the actual start time for each request in the benchmark run. " + "For async strategies, this represents delays from the ideal " + "system. For sync strategies, this should be as close to the " + "time for a request to be processed as possible. Any additional " + "time is overhead from the system or the worker." + ) + ) + request_time_delay_avg: float = Field( + description=( + "The average time delay between the total request time and the " + "worker time. This should be as close to 0 as possible, any additional " + "time is overhead from the system or the worker. " + ) + ) + request_time_avg: float = Field( + description=( + "The average time spent processing all requests in the benchmark run. " + "This is the time from when the actual request was started to when " + "it was completed." ) ) @@ -368,7 +419,7 @@ def time_per_output_token_ms(self) -> Optional[float]: if self.output_tokens is None or self.output_tokens == 0: return None - return super().time_per_output_token + return super().time_per_output_token_ms @computed_field @property @@ -599,6 +650,11 @@ def from_stats( populated and calculated """ start_time = min(req.start_time for req in completed) if completed else 0.0 + errored_with_outputs = [ + req + for req in errored + if req.output_tokens is not None and req.output_tokens > 1 + ] return GenerativeBenchmark( run_id=run_id, @@ -633,15 +689,17 @@ def from_stats( ), prompts_token_count=StatusDistributionSummary.from_values( completed_values=[req.prompt_tokens for req in completed], - errored_values=[req.prompt_tokens for req in errored], + errored_values=[req.prompt_tokens or 0 for req in errored], ), outputs_token_count=StatusDistributionSummary.from_values( completed_values=[req.output_tokens for req in completed], - errored_values=[req.output_tokens for req in errored], + errored_values=[req.output_tokens or 0 for req in errored], ), times_to_first_token_ms=StatusDistributionSummary.from_values( completed_values=[req.time_to_first_token_ms for req in completed], - errored_values=[req.time_to_first_token_ms for req in errored], + errored_values=[ + req.time_to_first_token_ms for req in errored_with_outputs + ], ), times_per_output_tokens_ms=StatusDistributionSummary.from_values( completed_values=( @@ -652,18 +710,12 @@ def from_stats( ] ), errored_values=( - [ - req.time_per_output_token_ms - for req in errored - if req.output_tokens > 0 - ] + [req.time_per_output_token_ms for req in errored_with_outputs] ), completed_weights=( [req.output_tokens for req in completed if req.output_tokens > 0] ), - errored_weights=( - [req.output_tokens for req in errored if req.output_tokens > 0] - ), + errored_weights=([req.output_tokens for req in errored_with_outputs]), ), inter_token_latencies_ms=StatusDistributionSummary.from_values( completed_values=( @@ -674,11 +726,7 @@ def from_stats( ] ), errored_values=( - [ - req.inter_token_latency_ms - for req in errored - if req.output_tokens > 1 - ] + [req.inter_token_latency_ms for req in errored_with_outputs] ), completed_weights=( [ @@ -688,7 +736,7 @@ def from_stats( ] ), errored_weights=( - [req.output_tokens - 1 for req in errored if req.output_tokens > 1] + [req.output_tokens - 1 for req in errored_with_outputs] ), ), outputs_tokens_per_second=StatusDistributionSummary.from_iterable_request_times( @@ -700,23 +748,19 @@ def from_stats( ] ), errored_requests=( - [ - (req.start_time, req.end_time) - for req in errored - if req.output_tokens > 0 - ] + [(req.start_time, req.end_time) for req in errored_with_outputs] ), completed_first_iter_times=( [req.first_token_time for req in completed if req.output_tokens > 0] ), errored_first_iter_times=( - [req.first_token_time for req in errored if req.output_tokens > 0] + [req.first_token_time for req in errored_with_outputs] ), completed_iter_counts=( [req.output_tokens for req in completed if req.output_tokens > 0] ), errored_iter_counts=( - [req.output_tokens for req in errored if req.output_tokens > 0] + [req.output_tokens for req in errored_with_outputs] ), ), tokens_per_second=StatusDistributionSummary.from_iterable_request_times( @@ -728,11 +772,7 @@ def from_stats( ] ), errored_requests=( - [ - (req.start_time, req.end_time) - for req in errored - if req.prompt_tokens + req.output_tokens > 0 - ] + [(req.start_time, req.end_time) for req in errored_with_outputs] ), completed_first_iter_times=( [ @@ -742,11 +782,7 @@ def from_stats( ] ), errored_first_iter_times=( - [ - req.first_token_time - for req in errored - if req.prompt_tokens + req.output_tokens > 0 - ] + [req.first_token_time for req in errored_with_outputs] ), completed_iter_counts=( [ @@ -758,8 +794,7 @@ def from_stats( errored_iter_counts=( [ req.prompt_tokens + req.output_tokens - for req in errored - if req.prompt_tokens + req.output_tokens > 0 + for req in errored_with_outputs ] ), completed_first_iter_counts=( @@ -770,7 +805,7 @@ def from_stats( ] ), errored_first_iter_counts=( - [req.prompt_tokens or 1 for req in errored if req.output_tokens > 0] + [req.prompt_tokens or 1 for req in errored_with_outputs] ), ), ) diff --git a/src/guidellm/benchmark/benchmarker.py b/src/guidellm/benchmark/benchmarker.py index 51636e82..9303c568 100644 --- a/src/guidellm/benchmark/benchmarker.py +++ b/src/guidellm/benchmark/benchmarker.py @@ -291,6 +291,7 @@ def __init__( request_loader_description: Optional[Serializable] = None, benchmark_save_extras: Optional[Dict[str, Any]] = None, processor: Optional[Union[str, Path, PreTrainedTokenizer]] = None, + processor_args: Optional[Dict[str, Any]] = None, ): super().__init__( worker=GenerativeRequestsWorker(backend), @@ -299,6 +300,7 @@ def __init__( benchmark_save_extras=benchmark_save_extras, ) self.processor = processor + self.processor_args = processor_args def create_benchmark_aggregator( self, @@ -315,6 +317,7 @@ def create_benchmark_aggregator( ) -> GenerativeBenchmarkAggregator: return GenerativeBenchmarkAggregator( processor=self.processor, + processor_args=self.processor_args, run_id=run_id, profile=profile, strategy_index=strategy_index, diff --git a/src/guidellm/benchmark/entrypoints.py b/src/guidellm/benchmark/entrypoints.py index fa5d0aa3..c321b01d 100644 --- a/src/guidellm/benchmark/entrypoints.py +++ b/src/guidellm/benchmark/entrypoints.py @@ -7,8 +7,12 @@ from guidellm.backend import Backend, BackendType from guidellm.benchmark.benchmark import GenerativeBenchmark from guidellm.benchmark.benchmarker import GenerativeBenchmarker +from guidellm.benchmark.output import ( + GenerativeBenchmarksConsole, + save_generative_benchmarks, +) from guidellm.benchmark.profile import ProfileType, create_profile -from guidellm.benchmark.progress import BenchmarkerProgressDisplay +from guidellm.benchmark.progress import GenerativeTextBenchmarkerProgressDisplay from guidellm.request import GenerativeRequestLoader from guidellm.scheduler import StrategyType @@ -38,19 +42,26 @@ async def benchmark_generative_text( warmup_percent: Optional[float], cooldown_percent: Optional[float], show_progress: bool, + show_progress_scheduler_stats: bool, + output_console: bool, output_path: Optional[Union[str, Path]], - output_type: Optional[str], output_extras: Optional[Dict[str, Any]], random_seed: int, ) -> List[GenerativeBenchmark]: + console = GenerativeBenchmarksConsole(enabled=show_progress) + console.print_line("Creating backend...") backend = Backend.create( backend_type, target=target, model=model, **(backend_args or {}) ) await backend.validate() + console.print_line( + f"Backend {backend_type} connected to {target} for model {backend.model}." + ) if processor is None: processor = backend.model + console.print_line("Creating request loader...") request_loader = GenerativeRequestLoader( data=data, data_args=data_args, @@ -64,16 +75,29 @@ async def benchmark_generative_text( ), random_seed=random_seed, ) - profile = create_profile(rate_type=rate_type, rate=rate) + unique_requests = request_loader.num_unique_items(raise_err=False) + console.print_line( + f"Created loader with {unique_requests} unique requests from {data}.\n\n" + if unique_requests > 0 + else f"Created loader with unknown number unique requests from {data}.\n\n" + ) + profile = create_profile(rate_type=rate_type, rate=rate) benchmarker = GenerativeBenchmarker( backend=backend, request_loader=request_loader, request_loader_description=request_loader.description, benchmark_save_extras=output_extras, processor=processor, + processor_args=processor_args, + ) + progress = ( + GenerativeTextBenchmarkerProgressDisplay( + display_scheduler_stats=show_progress_scheduler_stats + ) + if show_progress + else None ) - progress = BenchmarkerProgressDisplay() if show_progress else None benchmarks = [] async for result in benchmarker.run( @@ -89,4 +113,13 @@ async def benchmark_generative_text( if result.type_ == "benchmark_compiled": benchmarks.append(result.current_benchmark) + if output_console: + console.benchmarks = benchmarks + console.print_benchmarks_metadata() + console.print_benchmarks_info() + console.print_benchmarks_stats() + + if output_path: + save_generative_benchmarks(benchmarks=benchmarks, path=output_path) + return benchmarks diff --git a/src/guidellm/benchmark/output.py b/src/guidellm/benchmark/output.py new file mode 100644 index 00000000..7d5aea0f --- /dev/null +++ b/src/guidellm/benchmark/output.py @@ -0,0 +1,304 @@ +from collections import OrderedDict +from datetime import datetime +from pathlib import Path +from typing import Any, List, Optional + +from rich.console import Console +from rich.padding import Padding +from rich.table import Table +from rich.text import Text + +from guidellm.benchmark.benchmark import GenerativeBenchmark +from guidellm.benchmark.profile import ( + AsyncProfile, + ConcurrentProfile, + SweepProfile, + ThroughputProfile, +) +from guidellm.objects import Serializable +from guidellm.scheduler import strategy_display_str + +__all__ = [ + "GenerativeBenchmarksReport", + "save_generative_benchmarks", + "GenerativeBenchmarksConsole", +] + + +class GenerativeBenchmarksReport(Serializable): + benchmarks: List[GenerativeBenchmark] + + +def save_generative_benchmarks(benchmarks: List[GenerativeBenchmark], path: str): + path_inst = Path(path) + + if path_inst.is_dir(): + path_inst = path_inst / "generative_benchmarks.json" + + extension = path_inst.suffix.lower() + + if extension in [".json", ".yaml", ".yml"]: + report = GenerativeBenchmarksReport(benchmarks=benchmarks) + report.save_file(path_inst, type_="json" if extension == ".json" else "yaml") + else: + raise ValueError(f"Unsupported file extension: {extension} for {path_inst}. ") + + +class GenerativeBenchmarksConsole: + def __init__(self, enabled: bool = True): + self.enabled = enabled + self.benchmarks: Optional[List[GenerativeBenchmark]] = None + self.console = Console() + + @property + def benchmarks_profile_str(self) -> str: + profile = self.benchmarks[0].args.profile + profile_args = OrderedDict( + { + "type": profile.type_, + "strategies": profile.strategy_types, + } + ) + + if isinstance(profile, ConcurrentProfile): + profile_args["streams"] = profile.streams + elif isinstance(profile, ThroughputProfile): + profile_args["max_concurrency"] = profile.max_concurrency + elif isinstance(profile, AsyncProfile): + profile_args["max_concurrency"] = profile.max_concurrency + profile_args["rate"] = profile.rate + profile_args["initial_burst"] = profile.initial_burst + elif isinstance(profile, SweepProfile): + profile_args["sweep_size"] = profile.sweep_size + + return ", ".join(f"{key}={value}" for key, value in profile_args.items()) + + @property + def benchmarks_args_str(self) -> str: + args = self.benchmarks[0].args + args_dict = OrderedDict( + { + "max_number": args.max_number, + "max_duration": args.max_duration, + "warmup_number": args.warmup_number, + "warmup_duration": args.warmup_duration, + "cooldown_number": args.cooldown_number, + "cooldown_duration": args.cooldown_duration, + } + ) + + return ", ".join(f"{key}={value}" for key, value in args_dict.items()) + + @property + def benchmarks_worker_desc_str(self) -> str: + return str(self.benchmarks[0].worker) + + @property + def benchmarks_request_loader_desc_str(self) -> str: + return str(self.benchmarks[0].request_loader) + + @property + def benchmarks_extras_str(self) -> str: + extras = self.benchmarks[0].extras + + if not extras: + return "None" + + return ", ".join(f"{key}={value}" for key, value in extras.items()) + + def print_section_header(self, title: str, new_lines: int = 2): + if not self.enabled: + return + + text = Text() + + for _ in range(new_lines): + text.append("\n") + + text.append(f"{title}:", style="bold underline") + self.console.print(text) + + def print_labeled_line(self, label: str, value: str, indent: int = 4): + if not self.enabled: + return + + text = Text() + text.append(label, style="bold") + text.append(": ") + text.append(value, style="italic cyan") + self.console.print( + Padding.indent(text, indent), + ) + + def print_line(self, value: str, indent: int = 0): + if not self.enabled: + return + + text = Text(value, style=None) + self.console.print( + Padding.indent(text, indent), + ) + + def print_table(self, headers: List[str], rows: List[List[Any]], title: str): + if not self.enabled: + return + + self.print_section_header(title) + table = Table(*headers) + + for row in rows: + table.add_row(*[Text(item, style="cyan") for item in row]) + + self.console.print(table) + + def print_benchmarks_metadata(self): + if not self.enabled: + return + + if not self.benchmarks: + raise ValueError( + "No benchmarks to print metadata for. Please set benchmarks first." + ) + + start_time = self.benchmarks[0].run_stats.start_time + end_time = self.benchmarks[0].run_stats.end_time + duration = end_time - start_time + + self.print_section_header("Benchmarks Completed") + self.print_labeled_line("Run id", str(self.benchmarks[0].run_id)) + self.print_labeled_line( + "Duration", + f"{duration:.1f} seconds", + ) + self.print_labeled_line( + "Profile", + self.benchmarks_profile_str, + ) + self.print_labeled_line( + "Args", + self.benchmarks_args_str, + ) + self.print_labeled_line( + "Worker", + self.benchmarks_worker_desc_str, + ) + self.print_labeled_line( + "Request Loader", + self.benchmarks_request_loader_desc_str, + ) + self.print_labeled_line( + "Extras", + self.benchmarks_extras_str, + ) + + def print_benchmarks_info(self): + if not self.enabled: + return + + if not self.benchmarks: + raise ValueError( + "No benchmarks to print info for. Please set benchmarks first." + ) + + headers = [ + "Benchmark", + "Start Time", + "Duration (sec)", + "Requests / sec", + "Requests Concurrency", + "Requests Made \n(comp / err)", + "Prompt Tok / Req \n(comp / err)", + "Output Tok / Req \n(comp / err)", + "Prompt Tokens \n(comp / err)", + "Output Tokens \n(comp / err)", + ] + rows = [] + + for benchmark in self.benchmarks: + rows.append( + [ + strategy_display_str(benchmark.args.strategy), + f"{datetime.fromtimestamp(benchmark.start_time).strftime("%H:%M:%S")}", + f"{(benchmark.end_time - benchmark.start_time):.1f}", + f"{benchmark.requests_per_second.completed.mean:.2f}", + f"{benchmark.requests_concurrency.completed.mean:.2f}", + (f"{benchmark.completed_total:>5} / {benchmark.errored_total}"), + ( + f"{benchmark.prompts_token_count.completed.total_sum:.0f} / " + f"{benchmark.prompts_token_count.errored.total_sum:.0f}" + ), + ( + f"{benchmark.prompts_token_count.completed.mean:.0f} / " + f"{benchmark.prompts_token_count.errored.mean:.0f}" + ), + ( + f"{benchmark.outputs_token_count.completed.total_sum:.0f} / " + f"{benchmark.outputs_token_count.errored.total_sum:.0f}" + ), + ( + f"{benchmark.outputs_token_count.completed.mean:.0f} / " + f"{benchmark.outputs_token_count.errored.mean:.0f}" + ), + ] + ) + + self.print_table(headers=headers, rows=rows, title="Benchmarks Info") + + def print_benchmarks_stats(self): + if not self.enabled: + return + + if not self.benchmarks: + raise ValueError( + "No benchmarks to print stats for. Please set benchmarks first." + ) + + headers = [ + "Benchmark", + "Requests / sec", + "Requests Concurrency", + "Output Tok / sec", + "Total Tok / sec", + "Req Latency (ms)\n(mean / median / p99)", + "TTFT (ms)\n(mean / median / p99)", + "ITL (ms)\n(mean / median / p99)", + "TPOT (ms)\n(mean / median / p99)", + ] + rows = [] + + for benchmark in self.benchmarks: + rows.append( + [ + strategy_display_str(benchmark.args.strategy), + f"{benchmark.requests_per_second.completed.mean:.2f}", + f"{benchmark.requests_concurrency.completed.mean:.2f}", + f"{benchmark.outputs_tokens_per_second.total.mean:.1f}", + f"{benchmark.tokens_per_second.total.mean:.1f}", + ( + f"{benchmark.requests_latency.completed.mean:.2f} / " + f"{benchmark.requests_latency.completed.median:.2f} / " + f"{benchmark.requests_latency.completed.percentiles.p99:.2f}" + ), + ( + f"{benchmark.times_to_first_token_ms.completed.mean:.1f} / " + f"{benchmark.times_to_first_token_ms.completed.median:.1f} / " + f"{benchmark.times_to_first_token_ms.completed.percentiles.p99:.1f}" + ), + ( + f"{benchmark.inter_token_latencies_ms.completed.mean:.1f} / " + f"{benchmark.inter_token_latencies_ms.completed.median:.1f} / " + f"{benchmark.inter_token_latencies_ms.completed.percentiles.p99:.1f}" + ), + ( + f"{benchmark.times_per_output_tokens_ms.completed.mean:.1f} / " + f"{benchmark.times_per_output_tokens_ms.completed.median:.1f} / " + f"{benchmark.times_per_output_tokens_ms.completed.percentiles.p99:.1f}" + ), + ] + ) + + self.print_table( + headers=headers, + rows=rows, + title="Benchmarks Stats", + ) diff --git a/src/guidellm/benchmark/profile.py b/src/guidellm/benchmark/profile.py index 4f7cd70c..23b8dfab 100644 --- a/src/guidellm/benchmark/profile.py +++ b/src/guidellm/benchmark/profile.py @@ -1,9 +1,9 @@ -from abc import ABC, abstractmethod from typing import List, Literal, Optional, Sequence, Union import numpy as np from pydantic import Field, computed_field +from guidellm.config import settings from guidellm.objects import Serializable from guidellm.scheduler import ( AsyncConstantStrategy, @@ -29,7 +29,7 @@ ProfileType = Literal["synchronous", "concurrent", "throughput", "async", "sweep"] -class Profile(ABC, Serializable): +class Profile(Serializable): type_: ProfileType = Field( description="The type of benchmarking profile to use.", ) @@ -55,11 +55,11 @@ def completed_strategy(self, average_rate: float, average_concurrency: float): @computed_field @property - @abstractmethod - def strategy_types(self) -> List[StrategyType]: ... + def strategy_types(self) -> List[StrategyType]: + return [] - @abstractmethod - def next_strategy(self) -> Optional[SchedulingStrategy]: ... + def next_strategy(self) -> Optional[SchedulingStrategy]: + return None class SynchronousProfile(Profile): @@ -327,6 +327,9 @@ def from_standard_args( if "sweep_size" in kwargs: raise ValueError("Sweep size must not be provided, use rate instead.") + if not rate: + rate = settings.default_sweep_number + if not rate: raise ValueError( "Rate (sweep_size) must be provided for concurrent profile." diff --git a/src/guidellm/benchmark/progress.py b/src/guidellm/benchmark/progress.py index 1e2e3dd4..1b86a4cc 100644 --- a/src/guidellm/benchmark/progress.py +++ b/src/guidellm/benchmark/progress.py @@ -1,9 +1,8 @@ import math import time -from abc import ABC from dataclasses import dataclass from datetime import datetime -from typing import Dict, List, Optional, Union +from typing import Dict, Generic, List, Optional, TypeVar, Union from rich.console import Group from rich.live import Live @@ -24,19 +23,14 @@ from guidellm.scheduler import ( SchedulingStrategy, StrategyType, + strategy_display_str, ) -SCHEDULING_STRATEGY_DESCRIPTIONS: Dict[StrategyType, str] = { - "synchronous": "synchronous", - "concurrent": "concurrent@{RATE}", - "throughput": "throughput", - "constant": "constant@{RATE}", - "poisson": "poisson@{RATE}", -} - @dataclass class BenchmarkerTaskProgressState: + display_scheduler_stats: bool + task_id: TaskID strategy: Union[StrategyType, SchedulingStrategy] started: bool = False @@ -55,30 +49,14 @@ class BenchmarkerTaskProgressState: requests_completed: int = 0 requests_errored: int = 0 - output_tokens: float = 0 - prompt_tokens: float = 0 - output_tokens_rate: float = 0 - total_tokens_rate: float = 0 - tokens_ttft: float = 0 - tokens_itl: float = 0 + worker_overheads_time_ms: float = 0.0 + backend_overheads_time_ms: float = 0.0 + requests_sleep_time_ms: float = 0.0 + requests_targeted_start_time_delay_ms: float = 0.0 @property def description(self) -> str: - if self.strategy in StrategyType.__args__: - return SCHEDULING_STRATEGY_DESCRIPTIONS.get( - self.strategy, self.strategy - ).format(RATE="##" if self.strategy == "concurrent" else "#.##") - - rate = "" - - if hasattr(self.strategy, "streams"): - rate = f"{self.strategy.streams:>2}" - elif hasattr(self.strategy, "rate"): - rate = f"{self.strategy.rate:.2f}" - - return SCHEDULING_STRATEGY_DESCRIPTIONS.get( - self.strategy.type_, self.strategy.type_ - ).format(RATE=rate) + return strategy_display_str(self.strategy) @property def total(self) -> Optional[float]: @@ -109,13 +87,17 @@ def completed(self) -> int: @property def fields(self) -> Dict[str, str]: - return { + fields = { "start_time": self.formatted_start_time, "progress_status": self.formatted_progress_status, "requests_summary": self.formatted_requests_summary, - "tokens_summary": self.formatted_tokens_summary, } + if self.display_scheduler_stats: + fields["scheduler_stats"] = self.formatted_scheduler_stats + + return fields + @property def formatted_start_time(self) -> str: if self.start_time is None: @@ -147,31 +129,32 @@ def formatted_requests_summary(self) -> str: return ( "Req: " - f"({self.requests_rate:.2f} / sec, " - f"{self.requests_latency:.2f}s Lat, " + f"{self.requests_rate:>4.1f} req/sec, " + f"{self.requests_latency:2>.2f}s Lat, " f"{self.requests_processing:>3.1f} Conc, " - f"{self.requests_completed:>3} Comp, " - f"{self.requests_errored:>3} Err)" + f"{self.requests_completed:>4.0f} Comp, " + f"{self.requests_errored:>2.0f} Err" ) @property - def formatted_tokens_summary(self) -> str: + def formatted_scheduler_stats(self) -> str: if not self.started: return " " return ( - "Tok: " - f"({self.output_tokens_rate:.1f} gen/sec, " - f"{self.total_tokens_rate:.1f} tot/sec, " - f"{self.tokens_ttft:.1f}ms TTFT, " - f"{self.tokens_itl:.1f}ms ITL, " - f"{self.prompt_tokens:.0f} Prompt, " - f"{self.output_tokens:.0f} Gen)" + f"Sys: " + f"{self.worker_overheads_time_ms:>3.1f}ms Worker OH, " + f"{self.backend_overheads_time_ms:>3.1f}ms Backend OH, " + f"{self.requests_sleep_time_ms:>5.0f}ms Req Sleep, " + f"{self.requests_targeted_start_time_delay_ms:>5.0f}ms Start Delay" ) -class BenchmarkerProgressDisplay(ABC): - def __init__(self): +BTPS = TypeVar("BTPS", bound=BenchmarkerTaskProgressState) + + +class BenchmarkerProgressDisplay(Generic[BTPS]): + def __init__(self, display_scheduler_stats: bool): """ Progress display view: | Benchmarks -----------------------------------------------------------------| @@ -181,6 +164,7 @@ def __init__(self): | ----------------------------------------------------------------------------| SP Running... [BAR] (#/#) [ ELAPSED < ETA ] """ + self.display_scheduler_stats = display_scheduler_stats self.started = False self.benchmarker_tasks_progress = Progress(*self.create_task_progress_columns()) self.benchmarker_tasks_panel = Panel( @@ -308,7 +292,9 @@ def handle_update_scheduler_start( progress_state.strategy = result.current_strategy progress_state.started = True - progress_state.start_time = result.current_aggregator.start_time + progress_state.start_time = ( + result.current_aggregator.scheduler_created_requests.start_time + ) progress_state.max_number = result.current_aggregator.max_number progress_state.max_duration = result.current_aggregator.max_duration @@ -323,23 +309,33 @@ def handle_update_scheduler_update( progress_state.in_warmup = result.current_aggregator.in_warmup progress_state.in_cooldown = result.current_aggregator.in_cooldown - progress_state.requests_rate = result.current_aggregator.successful_requests / ( - time.time() - progress_state.start_time + progress_state.requests_rate = ( + result.current_aggregator.successful_requests.rate ) - progress_state.requests_latency = result.current_aggregator.request_latency.mean + progress_state.requests_latency = result.current_aggregator.request_time.mean progress_state.requests_processing = ( - result.current_aggregator.processing_requests + result.current_aggregator.scheduler_processing_requests.last ) progress_state.requests_completed = ( - result.current_aggregator.successful_requests + result.current_aggregator.successful_requests.total + ) + progress_state.requests_errored = ( + result.current_aggregator.errored_requests.total + ) + + progress_state.worker_overheads_time_ms = ( + result.current_aggregator.scheduled_time_delay.mean_ms + + result.current_aggregator.worker_start_delay.mean_ms + ) + progress_state.backend_overheads_time_ms = ( + result.current_aggregator.request_time_delay.mean_ms + ) + progress_state.requests_sleep_time_ms = ( + result.current_aggregator.scheduled_time_sleep.mean_ms + ) + progress_state.requests_targeted_start_time_delay_ms = ( + result.current_aggregator.request_start_time_targeted_delay.mean_ms ) - progress_state.requests_errored = result.current_aggregator.errored_requests - progress_state.output_tokens = result.current_aggregator.output_tokens.mean - progress_state.prompt_tokens = result.current_aggregator.prompt_tokens.mean - progress_state.output_tokens_rate = result.current_aggregator.output_tokens.rate - progress_state.total_tokens_rate = result.current_aggregator.total_tokens.rate - progress_state.tokens_ttft = result.current_aggregator.time_to_first_token.mean - progress_state.tokens_itl = result.current_aggregator.inter_token_latency.mean def handle_update_scheduler_complete( self, progress_state: BenchmarkerTaskProgressState, result: BenchmarkerResult @@ -376,24 +372,6 @@ def handle_update_benchmark_compiled( ) progress_state.requests_completed = result.current_benchmark.completed_total progress_state.requests_errored = result.current_benchmark.errored_total - progress_state.output_tokens = ( - result.current_benchmark.outputs_token_count.completed.mean - ) - progress_state.prompt_tokens = ( - result.current_benchmark.prompts_token_count.completed.mean - ) - progress_state.output_tokens_rate = ( - result.current_benchmark.outputs_tokens_per_second.completed.mean - ) - progress_state.total_tokens_rate = ( - result.current_benchmark.tokens_per_second.completed.mean - ) - progress_state.tokens_ttft = ( - result.current_benchmark.times_to_first_token_ms.completed.mean - ) - progress_state.tokens_itl = ( - result.current_benchmark.inter_token_latencies_ms.completed.mean - ) def handle_end(self, result: BenchmarkerResult): self.benchmarker_progress.update( @@ -410,18 +388,28 @@ def handle_end(self, result: BenchmarkerResult): self.progress_task = None def create_task_progress_columns(self) -> List[ProgressColumn]: - return [ + columns = [ TextColumn("[{task.fields[start_time]}]"), SpinnerColumn(), TaskProgressColumn(), TextColumn("{task.description}"), TextColumn("({task.fields[progress_status]})"), TextColumn(" "), - TextColumn("{task.fields[requests_summary]}"), - TextColumn(" "), - TextColumn("{task.fields[tokens_summary]}"), ] + if not self.display_scheduler_stats: + columns += [ + TextColumn("{task.fields[requests_summary]}\n"), + ] + else: + columns += [ + TextColumn( + "{task.fields[requests_summary]}\n{task.fields[scheduler_stats]}\n" + ), + ] + + return columns + def create_task_progress_state( self, task_id: TaskID, @@ -429,4 +417,105 @@ def create_task_progress_state( strategy_type: StrategyType, result: BenchmarkerResult, ) -> BenchmarkerTaskProgressState: - return BenchmarkerTaskProgressState(task_id=task_id, strategy=strategy_type) + return BenchmarkerTaskProgressState( + display_scheduler_stats=self.display_scheduler_stats, + task_id=task_id, + strategy=strategy_type, + ) + + +class GenerativeTextBenchmarkerTaskProgressState(BenchmarkerTaskProgressState): + output_tokens: float = 0 + prompt_tokens: float = 0 + output_tokens_rate: float = 0 + total_tokens_rate: float = 0 + tokens_ttft: float = 0 + tokens_itl: float = 0 + + @property + def fields(self) -> Dict[str, str]: + fields = super().fields + fields["tokens_summary"] = self.formatted_tokens_summary + return fields + + @property + def formatted_tokens_summary(self) -> str: + if not self.started: + return " " + + return ( + "Tok: " + f"{self.output_tokens_rate:4>.1f} gen/sec, " + f"{self.total_tokens_rate:>4.1f} tot/sec, " + f"{self.tokens_ttft:>3.1f}ms TTFT, " + f"{self.tokens_itl:>3.1f}ms ITL, " + f"{self.prompt_tokens:>4.0f} Prompt, " + f"{self.output_tokens:>4.0f} Gen" + ) + + +class GenerativeTextBenchmarkerProgressDisplay( + BenchmarkerProgressDisplay[GenerativeTextBenchmarkerTaskProgressState] +): + def handle_update_scheduler_update(self, progress_state, result): + super().handle_update_scheduler_update(progress_state, result) + progress_state.output_tokens = result.current_aggregator.output_tokens.mean + progress_state.prompt_tokens = result.current_aggregator.prompt_tokens.mean + progress_state.output_tokens_rate = result.current_aggregator.output_tokens.rate + progress_state.total_tokens_rate = result.current_aggregator.total_tokens.rate + progress_state.tokens_ttft = result.current_aggregator.time_to_first_token.mean + progress_state.tokens_itl = result.current_aggregator.inter_token_latency.mean + + def handle_update_benchmark_compiled(self, progress_state, result): + super().handle_update_benchmark_compiled(progress_state, result) + + progress_state.output_tokens = ( + result.current_benchmark.outputs_token_count.completed.mean + ) + progress_state.prompt_tokens = ( + result.current_benchmark.prompts_token_count.completed.mean + ) + progress_state.output_tokens_rate = ( + result.current_benchmark.outputs_tokens_per_second.completed.mean + ) + progress_state.total_tokens_rate = ( + result.current_benchmark.tokens_per_second.completed.mean + ) + progress_state.tokens_ttft = ( + result.current_benchmark.times_to_first_token_ms.completed.mean + ) + progress_state.tokens_itl = ( + result.current_benchmark.inter_token_latencies_ms.completed.mean + ) + + def create_task_progress_state( + self, + task_id: TaskID, + index: int, + strategy_type: StrategyType, + result: BenchmarkerResult, + ) -> GenerativeTextBenchmarkerTaskProgressState: + return GenerativeTextBenchmarkerTaskProgressState( + display_scheduler_stats=self.display_scheduler_stats, + task_id=task_id, + strategy=strategy_type, + ) + + def create_task_progress_columns(self) -> List[ProgressColumn]: + columns = super().create_task_progress_columns() + columns = columns[:-1] # remove the last display info column + + if not self.display_scheduler_stats: + columns += [ + TextColumn( + "{task.fields[requests_summary]}\n{task.fields[tokens_summary]}\n" + ), + ] + else: + columns += [ + TextColumn( + "{task.fields[requests_summary]}\n{task.fields[tokens_summary]}\n{task.fields[scheduler_stats]}\n" + ), + ] + + return columns diff --git a/src/guidellm/benchmark/test.py b/src/guidellm/benchmark/test.py index df88daa6..3e73dfdc 100644 --- a/src/guidellm/benchmark/test.py +++ b/src/guidellm/benchmark/test.py @@ -1,10 +1,13 @@ import asyncio import json +from guidellm.benchmark.benchmark import GenerativeBenchmark from guidellm.benchmark.entrypoints import benchmark_generative_text +from guidellm.benchmark.output import GenerativeBenchmarksConsole def run_benchmark_synthetic(): + # logging.basicConfig(level=logging.DEBUG) results = asyncio.run( benchmark_generative_text( target="http://192.168.4.13:8000", @@ -17,25 +20,33 @@ def run_benchmark_synthetic(): data_args=None, data_sampler=None, rate_type="sweep", - rate=5, + rate=10, max_seconds=None, max_requests=50, warmup_percent=None, cooldown_percent=None, show_progress=True, - output_path=None, - output_type=None, + show_progress_scheduler_stats=True, + output_console=True, + output_path="benchmarks.json", output_extras=None, random_seed=42, ) ) - dict_output = { - "benchmarks": [res.model_dump() for res in results], - } - with open("benchmarks.json", "w") as f: - json.dump(dict_output, f, indent=4) + +def print_benchmark(): + with open("benchmarks.json") as file: + data = json.load(file) + + benchmarks = [ + GenerativeBenchmark.model_validate_json(json.dumps(bench)) + for bench in data["benchmarks"] + ] + console = GenerativeBenchmarksConsole(benchmarks) + console.print() if __name__ == "__main__": run_benchmark_synthetic() + # print_benchmark() diff --git a/src/guidellm/config.py b/src/guidellm/config.py index d9cbf5e3..be71c544 100644 --- a/src/guidellm/config.py +++ b/src/guidellm/config.py @@ -141,7 +141,7 @@ class Settings(BaseSettings): env: Environment = Environment.PROD default_async_loop_sleep: float = 10e-5 logging: LoggingSettings = LoggingSettings() - num_sweep_profiles: int = 9 + default_sweep_number: int = 10 # HTTP settings request_timeout: int = 60 * 5 # 5 minutes diff --git a/src/guidellm/main.py b/src/guidellm/main.py deleted file mode 100644 index 56b2456c..00000000 --- a/src/guidellm/main.py +++ /dev/null @@ -1,346 +0,0 @@ -# import asyncio -# from typing import Any, Literal, Mapping, Optional, Union, get_args - -# import click -# from loguru import logger -# from transformers import AutoTokenizer # type: ignore[import-untyped] - -# from guidellm.backend import Backend, BackendType -# from guidellm.core import GuidanceReport, TextGenerationBenchmarkReport -# from guidellm.executor import Executor, ProfileGenerationMode -# from guidellm.request import ( -# EmulatedRequestGenerator, -# FileRequestGenerator, -# TransformersDatasetRequestGenerator, -# ) -# from guidellm.request.base import RequestGenerator -# from guidellm.utils import BenchmarkReportProgress, cli_params - -# __all__ = ["generate_benchmark_report"] - - -# @click.command() -# @click.option( -# "--target", -# type=str, -# required=True, -# help=( -# "The target path or url for the backend to evaluate. " -# "Ex: 'http://localhost:8000'" -# ), -# ) -# @click.option( -# "--backend", -# type=click.Choice(get_args(BackendType)), -# default="openai_http", -# help=( -# "The backend to use for benchmarking. " -# "The default is OpenAI Server enabling compatability with any server that " -# "follows the OpenAI spec including vLLM." -# ), -# ) -# @click.option( -# "--model", -# type=str, -# default=None, -# help=( -# "The Model to use for benchmarking. If not provided, it will use " -# "the first available model provided the backend supports listing models." -# ), -# ) -# @click.option( -# "--data", -# type=str, -# required=True, -# help=( -# "The data source to use for benchmarking. " -# "Depending on the data-type, it should be a " -# "path to a data file containing prompts to run (ex: data.txt), " -# "a HuggingFace dataset name (ex: 'neuralmagic/LLM_compression_calibration'), " -# "or a configuration for emulated data " -# "(ex: 'prompt_tokens=128,generated_tokens=128')." -# ), -# ) -# @click.option( -# "--data-type", -# type=click.Choice(["emulated", "file", "transformers"]), -# required=True, -# help=( -# "The type of data to use for benchmarking. " -# "Use 'emulated' for synthetic data, 'file' for a file, or 'transformers' " -# "for a HuggingFace dataset. Specify the data source with the --data flag." -# ), -# ) -# @click.option( -# "--tokenizer", -# type=str, -# default=None, -# help=( -# "The tokenizer to use for calculating the number of prompt tokens. " -# "This should match the tokenizer used by the model." -# "By default, it will use the --model flag to determine the tokenizer. " -# "If not provided and the model is not available, will raise an error. " -# "Ex: 'neuralmagic/Meta-Llama-3.1-8B-quantized.w8a8'" -# ), -# ) -# @click.option( -# "--rate-type", -# type=click.Choice(get_args(ProfileGenerationMode)), -# default="sweep", -# help=( -# "The type of request rate to use for benchmarking. " -# "Use sweep to run a full range from synchronous to throughput (default), " -# "synchronous for sending requests one after the other, " -# "throughput to send requests as fast as possible, " -# "constant for a fixed request rate, " -# "or poisson for a real-world variable request rate." -# ), -# ) -# @click.option( -# "--rate", -# type=float, -# default=None, -# help=( -# "The request rate to use for constant and poisson rate types. " -# "To run multiple, provide the flag multiple times. " -# ), -# multiple=True, -# ) -# @click.option( -# "--max-seconds", -# type=int, -# default=120, -# help=( -# "The maximum number of seconds for each benchmark run. " -# "Either max-seconds, max-requests, or both must be set. " -# "The default is 120 seconds. " -# "Note, this is the maximum time for each rate supplied, not the total time. " -# "This value should be large enough to allow for " -# "the server's performance to stabilize." -# ), -# ) -# @click.option( -# "--max-requests", -# type=cli_params.MAX_REQUESTS, -# default=None, -# help=( -# "The maximum number of requests for each benchmark run. " -# "Either max-seconds, max-requests, or both must be set. " -# "Note, this is the maximum number of requests for each rate supplied, " -# "not the total number of requests. " -# "This value should be large enough to allow for " -# "the server's performance to stabilize." -# ), -# ) -# @click.option( -# "--output-path", -# type=str, -# default=None, -# help=( -# "The output path to save the output report to for loading later. " -# "Ex: guidance_report.json. " -# "The default is None, meaning no output is saved and results are only " -# "printed to the console." -# ), -# ) -# @click.option( -# "--enable-continuous-refresh", -# is_flag=True, -# default=False, -# help=( -# "Enable continual refreshing of the output table in the CLI " -# "until the user exits. " -# ), -# ) -# def generate_benchmark_report_cli( -# target: str, -# backend: BackendType, -# model: Optional[str], -# data: Optional[str], -# data_type: Literal["emulated", "file", "transformers"], -# tokenizer: Optional[str], -# rate_type: ProfileGenerationMode, -# rate: Optional[float], -# max_seconds: Optional[int], -# max_requests: Union[Literal["dataset"], int, None], -# output_path: str, -# enable_continuous_refresh: bool, -# ): -# """ -# Generate a benchmark report for a specified backend and dataset. -# """ -# generate_benchmark_report( -# target=target, -# backend=backend, -# model=model, -# data=data, -# data_type=data_type, -# tokenizer=tokenizer, -# rate_type=rate_type, -# rate=rate, -# max_seconds=max_seconds, -# max_requests=max_requests, -# output_path=output_path, -# cont_refresh_table=enable_continuous_refresh, -# ) - - -# def generate_benchmark_report( -# target: str, -# data: Optional[str], -# data_type: Literal["emulated", "file", "transformers"], -# backend: BackendType = "openai_http", -# backend_kwargs: Optional[Mapping[str, Any]] = None, -# model: Optional[str] = None, -# tokenizer: Optional[str] = None, -# rate_type: ProfileGenerationMode = "sweep", -# rate: Optional[float] = None, -# max_seconds: Optional[int] = 120, -# max_requests: Union[Literal["dataset"], int, None] = None, -# output_path: Optional[str] = None, -# cont_refresh_table: bool = False, -# ) -> GuidanceReport: -# """ -# Generate a benchmark report for a specified backend and dataset. - -# :param target: The target URL or path for the backend to evaluate. -# :param backend: The backend type to use for benchmarking. -# :param model: The model to benchmark; -# defaults to the first available if not specified. -# :param data: The data source for benchmarking, -# which may be a path, dataset name, or config. -# :param data_type: The type of data to use, -# such as 'emulated', 'file', or 'transformers'. -# :param tokenizer: The tokenizer to use for token counting, -# defaulting to Llama 3.1 if not provided. -# :param rate_type: The rate type for requests during benchmarking. -# :param rate: The specific request rate for constant and poisson rate types. -# :param max_seconds: Maximum duration for each benchmark run in seconds. -# :param max_requests: Maximum number of requests per benchmark run. -# :param output_path: Path to save the output report file. -# :param cont_refresh_table: Continually refresh the table in the CLI -# until the user exits. -# :param backend_kwargs: Additional keyword arguments for the backend. -# """ -# logger.info( -# "Generating benchmark report with target: {}, backend: {}", target, backend -# ) - -# # Create backend -# backend_inst = Backend.create( -# type_=backend, -# target=target, -# model=model, -# **(backend_kwargs or {}), -# ) -# backend_inst.validate() - -# request_generator: RequestGenerator - -# # Create tokenizer and request generator -# tokenizer_inst = tokenizer -# if not tokenizer_inst: -# try: -# tokenizer_inst = AutoTokenizer.from_pretrained(backend_inst.model) -# except Exception as err: -# raise ValueError( -# "Could not load model's tokenizer, " -# "--tokenizer must be provided for request generation" -# ) from err - -# if data_type == "emulated": -# request_generator = EmulatedRequestGenerator( -# config=data, tokenizer=tokenizer_inst -# ) -# elif data_type == "file": -# request_generator = FileRequestGenerator(path=data, tokenizer=tokenizer_inst) -# elif data_type == "transformers": -# request_generator = TransformersDatasetRequestGenerator( -# dataset=data, tokenizer=tokenizer_inst -# ) -# else: -# raise ValueError(f"Unknown data type: {data_type}") - -# if data_type == "emulated" and max_requests == "dataset": -# raise ValueError("Cannot use 'dataset' for emulated data") - -# # Create executor -# executor = Executor( -# backend=backend_inst, -# request_generator=request_generator, -# mode=rate_type, -# rate=rate if rate_type in ("constant", "poisson") else None, -# max_number=( -# len(request_generator) if max_requests == "dataset" else max_requests -# ), -# max_duration=max_seconds, -# ) - -# # Run executor -# logger.debug( -# "Running executor with args: {}", -# { -# "backend": backend, -# "request_generator": request_generator, -# "mode": rate_type, -# "rate": rate, -# "max_number": max_requests, -# "max_duration": max_seconds, -# }, -# ) -# report = asyncio.run(_run_executor_for_result(executor)) - -# # Save and print report -# guidance_report = GuidanceReport() -# guidance_report.benchmarks.append(report) - -# if output_path: -# guidance_report.save_file(output_path) - -# guidance_report.print( -# save_path=output_path if output_path is not None else "stdout", -# continual_refresh=cont_refresh_table, -# ) - -# return guidance_report - - -# async def _run_executor_for_result(executor: Executor) -> TextGenerationBenchmarkReport: -# report = None -# progress = BenchmarkReportProgress() -# started = False - -# async for result in executor.run(): -# if not started: -# progress.start(result.generation_modes) # type: ignore # noqa: PGH003 -# started = True - -# if result.current_index is not None: -# description = f"{result.current_profile.load_gen_mode}" # type: ignore # noqa: PGH003 -# if result.current_profile.load_gen_mode in ("constant", "poisson"): # type: ignore # noqa: PGH003 -# description += f"@{result.current_profile.load_gen_rate:.2f} req/s" # type: ignore # noqa: PGH003 - -# progress.update_benchmark( -# index=result.current_index, -# description=description, -# completed=result.scheduler_result.completed, # type: ignore # noqa: PGH003 -# completed_count=result.scheduler_result.count_completed, # type: ignore # noqa: PGH003 -# completed_total=result.scheduler_result.count_total, # type: ignore # noqa: PGH003 -# start_time=result.scheduler_result.benchmark.start_time, # type: ignore # noqa: PGH003 -# req_per_sec=result.scheduler_result.benchmark.completed_request_rate, # type: ignore # noqa: PGH003 -# ) - -# if result.completed: -# report = result.report -# break - -# progress.finish() - -# if not report: -# raise ValueError("No report generated by executor") - -# return report - - -# if __name__ == "__main__": -# generate_benchmark_report_cli() diff --git a/src/guidellm/objects/__init__.py b/src/guidellm/objects/__init__.py index 724e5930..c2a75891 100644 --- a/src/guidellm/objects/__init__.py +++ b/src/guidellm/objects/__init__.py @@ -4,6 +4,7 @@ Percentiles, RunningStats, StatusDistributionSummary, + TimeRunningStats, ) __all__ = [ @@ -13,4 +14,5 @@ "Serializable", "SerializableFileType", "RunningStats", + "TimeRunningStats", ] diff --git a/src/guidellm/objects/serializable.py b/src/guidellm/objects/serializable.py index 23e6845a..7977df62 100644 --- a/src/guidellm/objects/serializable.py +++ b/src/guidellm/objects/serializable.py @@ -18,7 +18,7 @@ class Serializable(BaseModel): """ model_config = ConfigDict( - extra="ignore", + extra="allow", use_enum_values=True, validate_assignment=True, from_attributes=True, diff --git a/src/guidellm/objects/statistics.py b/src/guidellm/objects/statistics.py index 4b2d3465..c9f25b96 100644 --- a/src/guidellm/objects/statistics.py +++ b/src/guidellm/objects/statistics.py @@ -1,7 +1,7 @@ import math import time as timer from collections import defaultdict -from typing import List, Literal, Optional, Tuple +from typing import Any, List, Literal, Optional, Tuple import numpy as np from pydantic import Field, computed_field @@ -13,6 +13,7 @@ "DistributionSummary", "StatusDistributionSummary", "RunningStats", + "TimeRunningStats", ] @@ -83,6 +84,9 @@ class DistributionSummary(Serializable): count: int = Field( description="The number of values in the distribution.", ) + total_sum: float = Field( + description="The total sum of the values in the distribution.", + ) percentiles: Percentiles = Field( description="The percentiles of the distribution.", ) @@ -137,6 +141,7 @@ def from_distribution_function( minimum = values[0].item() if len(values) > 0 else 0 maximum = values[-1].item() if len(values) > 0 else 0 count = len(values) + total_sum = np.sum(values).item() return DistributionSummary( mean=mean, @@ -147,6 +152,7 @@ def from_distribution_function( min=minimum, max=maximum, count=count, + total_sum=total_sum, percentiles=( Percentiles( p001=cdf[np.argmax(cdf[:, 1] >= 0.001), 0].item(), # noqa: PLR2004 @@ -483,14 +489,18 @@ def from_iterable_request_times( class RunningStats(Serializable): + start_time: float = Field( + default_factory=timer.time, + ) count: int = Field( default=0, ) total: float = Field( default=0.0, ) - start_time: float = Field( - default=timer.time, + last: float = Field( + default=0.0, + description="The last value added to the running statistics.", ) @computed_field @@ -507,6 +517,26 @@ def rate(self) -> float: return 0.0 return self.total / (timer.time() - self.start_time) + def __add__(self, value: Any) -> float: + if not isinstance(value, (int, float)): + raise ValueError( + f"Value must be an int or float, got {type(value)} instead.", + ) + + self.update(value) + + return self.mean + + def __iadd__(self, value: Any) -> "RunningStats": + if not isinstance(value, (int, float)): + raise ValueError( + f"Value must be an int or float, got {type(value)} instead.", + ) + + self.update(value) + + return self + def update(self, value: float, count: int = 1) -> None: """ Update the running statistics with a new value. @@ -514,3 +544,26 @@ def update(self, value: float, count: int = 1) -> None: """ self.count += count self.total += value + self.last = value + + +class TimeRunningStats(RunningStats): + @computed_field + @property + def total_ms(self) -> float: + return self.total * 1000.0 + + @computed_field + @property + def last_ms(self) -> float: + return self.last * 1000.0 + + @computed_field + @property + def mean_ms(self) -> float: + return self.mean * 1000.0 + + @computed_field + @property + def rate_ms(self) -> float: + return self.rate * 1000.0 diff --git a/src/guidellm/objects/test.py b/src/guidellm/objects/test.py deleted file mode 100644 index 019488ea..00000000 --- a/src/guidellm/objects/test.py +++ /dev/null @@ -1,364 +0,0 @@ -from typing import List, Tuple - -from guidellm.objects.statistics import DistributionSummary - - -def generate_stats_outputs_tokens_per_seconds( - requests: List[Tuple[float, float]], - first_token_times: List[float], - output_token_counts: List[int], - epsilon: float = 1e-6, -): - distribution = DistributionSummary.from_iterable_request_times( - requests=requests, - first_iter_times=first_token_times, - iter_counts=output_token_counts, - epsilon=epsilon, - ) - print(distribution) - - -def generate_stats_inter_token_latencies( - latencies: List[float], - weights: List[float] = None, - epsilon: float = 1e-6, -): - distribution = DistributionSummary.from_values(latencies, weights) - print(distribution) - - -def generate_stats_requests_per_second( - requests: List[Tuple[float, float]], - epsilon: float = 1e-6, -) -> List[Tuple[float, float]]: - distribution = DistributionSummary.from_request_times( - requests, "rate", epsilon=epsilon - ) - print(distribution) - - -def generate_stats_concurrent_requests( - requests=List[Tuple[float, float]], - epsilon: float = 1e-6, -): - distribution = DistributionSummary.from_request_times( - requests, "concurrency", epsilon - ) - print(distribution) - - -# Example Usage - -request_times = [ - (1743163300.403223, 1743163301.9923513), - (1743163300.4043713, 1743163302.015711), - (1743163300.407332, 1743163302.0157585), - (1743163300.4105933, 1743163302.0158577), - (1743163300.4118826, 1743163302.015753), - (1743163302.0636709, 1743163303.6263561), - (1743163302.0829957, 1743163303.64966), - (1743163302.0883937, 1743163303.6496801), - (1743163302.0896976, 1743163303.6497076), - (1743163302.0892832, 1743163303.649676), - (1743163303.701261, 1743163305.26519), - (1743163303.7232263, 1743163305.2885003), - (1743163303.7202537, 1743163305.2885823), - (1743163303.7239172, 1743163305.2885487), - (1743163303.7244577, 1743163305.288734), - (1743163305.3386924, 1743163306.9014742), - (1743163305.3575644, 1743163306.9247215), - (1743163305.364691, 1743163306.9247289), - (1743163305.3667543, 1743163306.9247417), - (1743163305.3670223, 1743163306.9247096), - (1743163306.972128, 1743163308.5344012), - (1743163306.997972, 1743163308.5577517), - (1743163306.9919195, 1743163308.5577645), - (1743163306.998968, 1743163308.5578134), - (1743163306.9986732, 1743163308.5578017), - (1743163308.6057122, 1743163310.172029), - (1743163308.6297, 1743163310.1953459), - (1743163308.6132185, 1743163310.1953993), - (1743163308.630173, 1743163310.195401), - (1743163308.6345012, 1743163310.1953926), - (1743163310.2397547, 1743163311.801659), - (1743163310.2674718, 1743163311.824895), - (1743163310.2598615, 1743163311.8248944), - (1743163310.2720146, 1743163311.824908), - (1743163310.2664378, 1743163311.8248801), - (1743163311.8722754, 1743163313.4339283), - (1743163311.900499, 1743163313.457236), - (1743163311.895735, 1743163313.4572852), - (1743163311.8999357, 1743163313.457477), - (1743163311.8968449, 1743163313.4572716), - (1743163313.5064678, 1743163315.068746), - (1743163313.5301821, 1743163315.0921736), - (1743163313.5289028, 1743163315.092185), - (1743163313.5287688, 1743163315.0922654), - (1743163313.5301483, 1743163315.092442), - (1743163315.1421392, 1743163316.705413), - (1743163315.1619313, 1743163316.7287838), - (1743163315.1561904, 1743163316.7287745), - (1743163315.1638331, 1743163316.7288203), - (1743163315.1655514, 1743163316.7288005), - (1743163316.7788277, 1743163318.340335), - (1743163316.8018036, 1743163318.363607), - (1743163316.8013813, 1743163318.3635497), - (1743163316.7962375, 1743163318.3637505), - (1743163316.8082492, 1743163318.3635893), - (1743163318.4177032, 1743163319.9805896), - (1743163318.4415567, 1743163320.0040746), - (1743163318.438249, 1743163320.0040603), - (1743163318.4454482, 1743163320.0039873), - (1743163318.435946, 1743163320.0040472), - (1743163320.059263, 1743163321.6231477), - (1743163320.0803068, 1743163321.646229), - (1743163320.077479, 1743163321.6461928), - (1743163320.077346, 1743163321.646171), - (1743163320.0767386, 1743163321.6462295), - (1743163321.6983657, 1743163323.262142), - (1743163321.720098, 1743163323.2855532), - (1743163321.7210836, 1743163323.2855496), - (1743163321.7272742, 1743163323.285537), - (1743163321.720461, 1743163323.285713), - (1743163323.3382583, 1743163324.9130867), - (1743163323.3578243, 1743163324.9363666), - (1743163323.3580399, 1743163324.9365273), - (1743163323.3582692, 1743163324.9365368), - (1743163323.3700652, 1743163324.9365063), - (1743163324.9848232, 1743163326.548244), - (1743163325.0123115, 1743163326.5716817), - (1743163325.0087984, 1743163326.57175), - (1743163325.0095794, 1743163326.571734), - (1743163325.0168707, 1743163326.5717053), - (1743163326.61861, 1743163328.1839957), - (1743163326.633079, 1743163328.2074084), - (1743163326.6254075, 1743163328.207643), - (1743163326.642937, 1743163328.207623), - (1743163326.628424, 1743163328.2074928), - (1743163328.2542157, 1743163329.8191519), - (1743163328.2807877, 1743163329.842159), - (1743163328.2869048, 1743163329.8421898), - (1743163328.2784348, 1743163329.8422613), - (1743163328.3039563, 1743163329.8646753), - (1743163329.8803456, 1743163331.4419262), - (1743163329.8897858, 1743163331.4652398), - (1743163329.9109275, 1743163331.4652941), - (1743163329.8975961, 1743163331.4653), - (1743163329.8834872, 1743163331.4652634), - (1743163331.5126238, 1743163333.0786362), - (1743163331.5387557, 1743163333.1019342), - (1743163331.5308228, 1743163333.1019886), - (1743163331.5362875, 1743163333.1019595), - (1743163331.5372543, 1743163333.1019611), -] -first_token_times = [ - 1743163300.4774668, - 1743163300.5430598, - 1743163300.5430408, - 1743163300.5430439, - 1743163300.5430408, - 1743163302.1260648, - 1743163302.1790159, - 1743163302.1789834, - 1743163302.178902, - 1743163302.1788735, - 1743163303.7622797, - 1743163303.8150222, - 1743163303.8150568, - 1743163303.815119, - 1743163303.8151877, - 1743163305.400849, - 1743163305.4539392, - 1743163305.4539232, - 1743163305.4539037, - 1743163305.4538555, - 1743163307.0341141, - 1743163307.0872338, - 1743163307.0871842, - 1743163307.0872376, - 1743163307.0871847, - 1743163308.6717093, - 1743163308.7247245, - 1743163308.7246864, - 1743163308.7246675, - 1743163308.7246442, - 1743163310.3007226, - 1743163310.3539567, - 1743163310.3539433, - 1743163310.3539903, - 1743163310.353892, - 1743163311.9333777, - 1743163311.9865408, - 1743163311.9865537, - 1743163311.9865468, - 1743163311.9865608, - 1743163313.5681715, - 1743163313.6213248, - 1743163313.6213076, - 1743163313.6213694, - 1743163313.6212204, - 1743163315.2052338, - 1743163315.257879, - 1743163315.2578883, - 1743163315.2578585, - 1743163315.2578545, - 1743163316.839769, - 1743163316.8927107, - 1743163316.8927133, - 1743163316.8925958, - 1743163316.8925962, - 1743163318.4791622, - 1743163318.5325277, - 1743163318.53261, - 1743163318.5324767, - 1743163318.5324028, - 1743163320.1203272, - 1743163320.1740332, - 1743163320.1739714, - 1743163320.174067, - 1743163320.174065, - 1743163321.759952, - 1743163321.813341, - 1743163321.813346, - 1743163321.8132503, - 1743163321.8133996, - 1743163323.410784, - 1743163323.464367, - 1743163323.4643219, - 1743163323.4643698, - 1743163323.4643369, - 1743163325.0459666, - 1743163325.0997808, - 1743163325.0997987, - 1743163325.099642, - 1743163325.0997283, - 1743163326.681901, - 1743163326.7354028, - 1743163326.7354014, - 1743163326.735491, - 1743163326.7354689, - 1743163328.3170884, - 1743163328.3714006, - 1743163328.3715012, - 1743163328.3715947, - 1743163328.3944945, - 1743163329.9375503, - 1743163329.9934423, - 1743163329.9931898, - 1743163329.9932914, - 1743163329.9932675, - 1743163331.5760312, - 1743163331.629882, - 1743163331.6298609, - 1743163331.6300056, - 1743163331.6299996, -] -output_tokens = [ - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, - 64, -] - - -generate_stats_outputs_tokens_per_seconds( - request_times, - first_token_times, - output_tokens, -) diff --git a/src/guidellm/request/loader.py b/src/guidellm/request/loader.py index 768ab150..063b0b9e 100644 --- a/src/guidellm/request/loader.py +++ b/src/guidellm/request/loader.py @@ -113,17 +113,7 @@ def __iter__(self) -> Iterator[GenerationRequest]: def __len__(self) -> int: if self.iter_type == "finite": - try: - return len(self.dataset) - except Exception: - pass - - try: - dataset_size = self.dataset.info.dataset_size - if dataset_size is not None: - return dataset_size - except Exception: - pass + return self.num_unique_items() raise ValueError(f"Unable to determine length of dataset: {self.data}") @@ -136,6 +126,21 @@ def description(self) -> GenerativeRequestLoaderDescription: processor_args=self.processor_args, ) + def num_unique_items(self, raise_err: bool = True) -> int: + try: + return len(self.dataset) + except Exception: + pass + + dataset_size = self.dataset.info.dataset_size + if dataset_size is not None: + return dataset_size + + if raise_err: + raise ValueError("Unable to determine number of items in the dataset") + + return -1 + def _create_column_mappings( self, args_column_mappings: Dict[ColumnInputTypes, str], diff --git a/src/guidellm/scheduler/__init__.py b/src/guidellm/scheduler/__init__.py index e2addfc2..5b9c2cc8 100644 --- a/src/guidellm/scheduler/__init__.py +++ b/src/guidellm/scheduler/__init__.py @@ -8,6 +8,7 @@ StrategyType, SynchronousStrategy, ThroughputStrategy, + strategy_display_str, ) from .types import REQ, RES from .worker import ( @@ -29,6 +30,7 @@ "StrategyType", "SynchronousStrategy", "ThroughputStrategy", + "strategy_display_str", "REQ", "RES", "GenerativeRequestsWorker", diff --git a/src/guidellm/scheduler/result.py b/src/guidellm/scheduler/result.py index c86a655a..48ec64f6 100644 --- a/src/guidellm/scheduler/result.py +++ b/src/guidellm/scheduler/result.py @@ -71,6 +71,7 @@ class SchedulerRequestInfo(Serializable): targeted_start_time: float = -1 queued_time: float = -1 + dequeued_time: float = -1 scheduled_time: float = -1 worker_start: float = -1 worker_end: float = -1 @@ -110,6 +111,7 @@ class SchedulerResult(Serializable, Generic[REQ, RES]): "request_complete", ] request: REQ - response: RES + response: Optional[RES] request_info: Optional[SchedulerRequestInfo] run_info: SchedulerRunInfo + preempted: bool = False diff --git a/src/guidellm/scheduler/scheduler.py b/src/guidellm/scheduler/scheduler.py index 32efd091..e603a3be 100644 --- a/src/guidellm/scheduler/scheduler.py +++ b/src/guidellm/scheduler/scheduler.py @@ -161,7 +161,6 @@ async def run( # yield control to the event loop await asyncio.sleep(settings.default_async_loop_sleep) except Exception as err: - print(err) raise RuntimeError(f"Scheduler run failed: {err}") from err yield SchedulerResult( @@ -184,6 +183,7 @@ async def _start_processes( multiprocessing.Queue, multiprocessing.Queue, ]: + await self.worker.prepare_multiprocessing() requests_queue = manager.Queue( maxsize=scheduling_strategy.queued_requests_limit ) @@ -281,7 +281,9 @@ def _add_requests( if run_info.created_requests >= run_info.end_number: raise StopIteration - if (request_time := next(times_iter)) >= run_info.end_time: + if ( + request_time := next(times_iter) + ) >= run_info.end_time or time.time() >= run_info.end_time: raise StopIteration request = next(requests_iter) @@ -349,6 +351,7 @@ def _check_result_ready( response=process_response.response, request_info=process_response.info, run_info=run_info, + preempted=process_response.preempted, ) raise ValueError(f"Invalid process response type: {process_response}") diff --git a/src/guidellm/scheduler/strategy.py b/src/guidellm/scheduler/strategy.py index b7f1d0bc..d0eab157 100644 --- a/src/guidellm/scheduler/strategy.py +++ b/src/guidellm/scheduler/strategy.py @@ -2,11 +2,11 @@ import os import random import time -from abc import ABC, abstractmethod from typing import ( Generator, Literal, Optional, + Union, ) from pydantic import Field @@ -22,13 +22,14 @@ "ThroughputStrategy", "AsyncConstantStrategy", "AsyncPoissonStrategy", + "strategy_display_str", ] StrategyType = Literal["synchronous", "concurrent", "throughput", "constant", "poisson"] -class SchedulingStrategy(ABC, Serializable): +class SchedulingStrategy(Serializable): """ An abstract base class for scheduling strategies. This class defines the interface for scheduling requests and provides @@ -45,39 +46,6 @@ class SchedulingStrategy(ABC, Serializable): ) @property - def default_processes_limit(self) -> int: - """ - The default limit on the number of worker processes for the scheduling strategy. - - :return: The minimum between the number of CPU cores minus one - and the maximum number of worker processes allowed by settings. - """ - cpu_cores = os.cpu_count() or 1 - - return min(max(1, cpu_cores - 1), settings.max_worker_processes) - - @property - def default_queued_requests_limit(self) -> int: - """ - The default limit on the number of queued requests for the scheduling strategy. - - :return: The max concurrency value from settings, ensuring there are enough - requests even for the worst case scenario where the max concurrent requests - are pulled at once for processing. - """ - return settings.max_concurrency - - @property - def default_processing_requests_limit(self) -> int: - """ - The default limit on the number of active requests for the scheduling strategy. - - :return: The max concurrency value from settings. - """ - return settings.max_concurrency - - @property - @abstractmethod def processing_mode(self) -> Literal["sync", "async"]: """ The processing mode for the scheduling strategy, either 'sync' or 'async'. @@ -89,10 +57,9 @@ def processing_mode(self) -> Literal["sync", "async"]: :return: The processing mode for the scheduling strategy, either 'sync' or 'async'. """ - ... + return "async" @property - @abstractmethod def processes_limit(self) -> int: """ The limit on the number of worker processes for the scheduling strategy. @@ -101,10 +68,11 @@ def processes_limit(self) -> int: :return: The number of processes for the scheduling strategy. """ - ... + cpu_cores = os.cpu_count() or 1 + + return min(max(1, cpu_cores - 1), settings.max_worker_processes) @property - @abstractmethod def queued_requests_limit(self) -> Optional[int]: """ The maximum number of queued requests for the scheduling strategy. @@ -113,10 +81,9 @@ def queued_requests_limit(self) -> Optional[int]: :return: The maximum number of queued requests for the scheduling strategy. """ - ... + return settings.max_concurrency @property - @abstractmethod def processing_requests_limit(self) -> Optional[int]: """ The maximum number of processing requests for the scheduling strategy. @@ -125,9 +92,8 @@ def processing_requests_limit(self) -> Optional[int]: :return: The maximum number of processing requests for the scheduling strategy. """ - ... + return settings.max_concurrency - @abstractmethod def request_times(self) -> Generator[float, None, None]: """ A generator that yields timestamps for when requests should be sent. @@ -137,7 +103,6 @@ def request_times(self) -> Generator[float, None, None]: :return: A generator that yields timestamps for request scheduling or -1 for requests that should be sent immediately. """ - ... class SynchronousStrategy(SchedulingStrategy): @@ -333,18 +298,6 @@ def processing_mode(self) -> Literal["async"]: """ return "async" - @property - def processes_limit(self) -> int: - """ - The limit on the number of worker processes for the scheduling strategy. - It determines how many worker processes are created - for the scheduling strategy and must be implemented by subclasses. - - :return: The default processes limit since none is enforced for - asynchronous strategies. - """ - return self.default_processes_limit - @property def queued_requests_limit(self) -> int: """ @@ -370,7 +323,7 @@ def processing_requests_limit(self) -> int: If max_concurrency is None, then the default processing requests limit will be used. """ - return self.max_concurrency or self.default_processing_requests_limit + return self.max_concurrency or super().processing_requests_limit def request_times(self) -> Generator[float, None, None]: """ @@ -517,9 +470,23 @@ def request_times(self) -> Generator[float, None, None]: yield start_time # set the random seed for reproducibility - rand = random.Random(self.random_seed) + rand = random.Random(self.random_seed) # noqa: S311 while True: inter_arrival_time = rand.expovariate(self.rate) start_time += inter_arrival_time yield start_time + + +def strategy_display_str(strategy: Union[StrategyType, SchedulingStrategy]) -> str: + strategy_type = strategy if isinstance(strategy, str) else strategy.type_ + strategy_instance = strategy if isinstance(strategy, SchedulingStrategy) else None + + if strategy_type == "concurrent": + rate = f"@{strategy_instance.streams}" if strategy_instance else "@##" + elif strategy_type in ("constant", "poisson"): + rate = f"@{strategy_instance.rate:.2f}" if strategy_instance else "@#.##" + else: + rate = "" + + return f"{strategy_type}{rate}" diff --git a/src/guidellm/scheduler/worker.py b/src/guidellm/scheduler/worker.py index a633f33b..0102a6dd 100644 --- a/src/guidellm/scheduler/worker.py +++ b/src/guidellm/scheduler/worker.py @@ -26,7 +26,6 @@ ResponseSummary, StreamingTextResponse, ) -from guidellm.config import settings from guidellm.objects import Serializable from guidellm.request import GenerationRequest from guidellm.scheduler.result import SchedulerRequestInfo @@ -54,6 +53,7 @@ class WorkerProcessResult(Generic[REQ, RES]): request: REQ response: RES info: SchedulerRequestInfo + preempted: bool = False class RequestsWorker(ABC, Generic[REQ, RES]): @@ -77,6 +77,16 @@ def description(self) -> Serializable: """ ... + @abstractmethod + async def prepare_multiprocessing(self): + """ + An abstract method that must be implemented by subclasses. + This is useful for workers that have instance state that can not + be shared across processes and should be cleared out and re-initialized + for each new process. + """ + ... + @abstractmethod async def resolve( self, @@ -95,10 +105,23 @@ async def resolve( """ ... + async def get_request( + self, requests_queue: multiprocessing.Queue + ) -> Optional[WorkerProcessRequest[REQ]]: + return await asyncio.to_thread(requests_queue.get) + + async def send_result( + self, + results_queue: multiprocessing.Queue, + result: WorkerProcessResult[REQ, RES], + ): + await asyncio.to_thread(results_queue.put, result) + async def resolve_scheduler_request( self, request: Any, queued_time: float, + dequeued_time: float, start_time: float, timeout_time: float, results_queue: multiprocessing.Queue, @@ -107,9 +130,8 @@ async def resolve_scheduler_request( info = SchedulerRequestInfo( targeted_start_time=start_time, queued_time=queued_time, + dequeued_time=dequeued_time, scheduled_time=time.time(), - worker_start=-1, - worker_end=-1, process_id=process_id, ) result: WorkerProcessResult[REQ, RES] = WorkerProcessResult( @@ -118,7 +140,7 @@ async def resolve_scheduler_request( response=None, info=info, ) - results_queue.put(result) + asyncio.create_task(self.send_result(results_queue, result)) if (wait_time := start_time - time.time()) > 0: await asyncio.sleep(wait_time) @@ -130,18 +152,25 @@ async def resolve_scheduler_request( response=None, info=info, ) - results_queue.put(result) + asyncio.create_task(self.send_result(results_queue, result)) + time_til_timeout = timeout_time - time.time() - response = await self.resolve(request, timeout_time) + if time_til_timeout > 0: + response = await self.resolve(request, timeout_time) + preempted = False + info.worker_end = time.time() + else: + response = None + preempted = True - info.worker_end = time.time() result = WorkerProcessResult( type_="request_complete", request=request, response=response, info=info, + preempted=preempted, ) - results_queue.put(result) + asyncio.create_task(self.send_result(results_queue, result)) def process_loop_synchronous( self, @@ -150,22 +179,15 @@ def process_loop_synchronous( process_id: int, ): async def _process_runner(): - while True: - try: - process_request: Optional[WorkerProcessRequest[REQ]] = ( - requests_queue.get_nowait() - ) - except multiprocessing.queues.Empty: - # yield control to the event loop - await asyncio.sleep(settings.default_async_loop_sleep) - continue - - if process_request is None: # stop signal - break + while ( + process_request := await self.get_request(requests_queue) + ) is not None: + dequeued_time = time.time() await self.resolve_scheduler_request( request=process_request.request, queued_time=process_request.queued_time, + dequeued_time=dequeued_time, start_time=process_request.start_time, timeout_time=process_request.timeout_time, results_queue=results_queue, @@ -191,18 +213,10 @@ def process_loop_asynchronous( async def _process_runner(): pending = asyncio.Semaphore(max_concurrency) if max_concurrency else None - while True: - try: - process_request: Optional[WorkerProcessRequest[REQ]] = ( - requests_queue.get_nowait() - ) - except multiprocessing.queues.Empty: - # yield control to event loop - await asyncio.sleep(settings.default_async_loop_sleep) - continue - - if process_request is None: # stop signal - break + while ( + process_request := await self.get_request(requests_queue) + ) is not None: + dequeued_time = time.time() if pending: await pending.acquire() @@ -216,6 +230,7 @@ def _task_done(_: asyncio.Task): self.resolve_scheduler_request( request=process_request.request, queued_time=process_request.queued_time, + dequeued_time=dequeued_time, start_time=process_request.start_time, timeout_time=process_request.timeout_time, results_queue=results_queue, @@ -270,6 +285,43 @@ def description(self) -> Serializable: backend_info=self.backend.info, ) + async def prepare_multiprocessing(self): + """ + Prepare the worker for multiprocessing. + This is useful for workers that have instance state that can not + be shared across processes and should be cleared out and re-initialized + for each new process. + """ + await self.backend.prepare_multiprocessing() + + def process_loop_synchronous( + self, + requests_queue: multiprocessing.Queue, + results_queue: multiprocessing.Queue, + process_id: int, + ): + asyncio.run(self.backend.validate()) + super().process_loop_synchronous( + requests_queue=requests_queue, + results_queue=results_queue, + process_id=process_id, + ) + + def process_loop_asynchronous( + self, + requests_queue: multiprocessing.Queue, + results_queue: multiprocessing.Queue, + max_concurrency: Optional[int], + process_id: int, + ): + asyncio.run(self.backend.validate()) + super().process_loop_asynchronous( + requests_queue=requests_queue, + results_queue=results_queue, + max_concurrency=max_concurrency, + process_id=process_id, + ) + async def resolve( self, request: GenerationRequest, @@ -286,6 +338,7 @@ async def resolve( :return: A ResponseSummary object containing the response from the backend. If an error occurs, the ResponseSummary will contain the error message. """ + resolve_start_time = time.time() response = None error: Optional[str] = None @@ -314,12 +367,17 @@ async def _runner(): f"Received no ResponseSummary for request: {request} " f"and backend: {self.backend}, received: {response}" ) - except asyncio.TimeoutError as texc: - error = str(texc) + except asyncio.TimeoutError: + error = "TimeoutError: The request timed out before completing." except Exception as exc: # noqa: BLE001 error = str(exc) - return self._handle_response(request, response, error) + return self._handle_response( + request=request, + response=response, + error=error, + resolve_start_time=resolve_start_time, + ) def _create_request_func_kwargs( self, @@ -363,6 +421,7 @@ def _handle_response( request: GenerationRequest, response: Any, error: Optional[str], + resolve_start_time: float, ) -> ResponseSummary: if response is None or not isinstance( response, (ResponseSummary, StreamingTextResponse) @@ -382,8 +441,10 @@ def _handle_response( headers={}, payload={}, ), - start_time=None, - end_time=None, + start_time=resolve_start_time, + end_time=time.time(), + first_iter_time=None, + last_iter_time=None, request_id=request.request_id, error=error or "Unknown error", ) @@ -397,9 +458,11 @@ def _handle_response( payload={}, ), start_time=response.start_time, - end_time=None, + end_time=time.time(), + first_iter_time=response.first_iter_time, + last_iter_time=response.time if response.iter_count > 0 else None, request_prompt_tokens=request.stats.get("prompt_tokens", None), - request_output_tokens=None, + request_output_tokens=request.constraints.get("output_tokens", None), response_prompt_tokens=None, response_output_tokens=response.iter_count, request_id=request.request_id, From d4f8c1ab089cba1cbbe01ed48cf41299bea2a08d Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Wed, 9 Apr 2025 15:12:12 +0000 Subject: [PATCH 15/43] Fixes for bugs surfaced from testing and enhancements to features based on testing --- pyproject.toml | 2 +- src/guidellm/benchmark/aggregator.py | 159 +++++---- src/guidellm/benchmark/benchmark.py | 283 ++++++++-------- src/guidellm/benchmark/benchmarker.py | 85 ++--- src/guidellm/benchmark/output.py | 55 ++-- src/guidellm/benchmark/progress.py | 284 +++++++++++++---- src/guidellm/config.py | 19 -- src/guidellm/data/prideandprejudice.txt.gz | Bin 0 -> 241795 bytes src/guidellm/dataset/__init__.py | 2 +- src/guidellm/dataset/entrypoints.py | 2 +- .../dataset/{datasets.py => hf_datasets.py} | 0 src/guidellm/dataset/synthetic.py | 50 +-- src/guidellm/objects/statistics.py | 301 +++++++++++++++--- src/guidellm/scheduler/__init__.py | 8 +- src/guidellm/scheduler/result.py | 23 +- src/guidellm/scheduler/scheduler.py | 31 +- src/guidellm/scheduler/worker.py | 68 ++-- src/guidellm/utils/__init__.py | 2 + src/guidellm/utils/colors.py | 24 ++ src/guidellm/utils/random.py | 6 +- src/guidellm/utils/text.py | 16 + 21 files changed, 934 insertions(+), 486 deletions(-) create mode 100644 src/guidellm/data/prideandprejudice.txt.gz rename src/guidellm/dataset/{datasets.py => hf_datasets.py} (100%) create mode 100644 src/guidellm/utils/colors.py diff --git a/pyproject.toml b/pyproject.toml index 6145ec8c..4eb171f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ where = ["src"] include = ["*"] [tool.setuptools.package-data] -guidellm = ["*"] +"guidellm.data" = ["*.gz"] # ************************************************ diff --git a/src/guidellm/benchmark/aggregator.py b/src/guidellm/benchmark/aggregator.py index 7a210df4..10ff8d32 100644 --- a/src/guidellm/benchmark/aggregator.py +++ b/src/guidellm/benchmark/aggregator.py @@ -31,7 +31,7 @@ from guidellm.scheduler import ( REQ, RES, - SchedulerResult, + SchedulerRequestResult, SchedulingStrategy, ) from guidellm.utils import check_load_processor @@ -123,10 +123,10 @@ class BenchmarkAggregator(ABC, BaseModel, Generic[BENCH, REQ, RES]): ) ) - results: List[SchedulerResult[GenerationRequest, ResponseSummary]] = Field( + results: List[SchedulerRequestResult[GenerationRequest, ResponseSummary]] = Field( default_factory=list, description=( - "The list of results from the benchmark, both completed and errored, " + "The list of all results from the benchmark (complete, incomplete, error), " "that were not within the warmup or cooldown periods." ), ) @@ -183,6 +183,7 @@ class BenchmarkAggregator(ABC, BaseModel, Generic[BENCH, REQ, RES]): ), default_factory=RunningStats, ) + successful_requests: RunningStats = Field( description=( "The running statistics for the number of requests that completed " @@ -192,6 +193,15 @@ class BenchmarkAggregator(ABC, BaseModel, Generic[BENCH, REQ, RES]): ), default_factory=RunningStats, ) + incomplete_requests: RunningStats = Field( + description=( + "The running statistics for the number of requests that were incomplete " + "or preempted during processing. This includes requests " + "within the warmup and cooldown period, if any, along with the final " + "results." + ), + default_factory=RunningStats, + ) errored_requests: RunningStats = Field( description=( "The running statistics for the number of requests that errored during " @@ -297,7 +307,8 @@ class BenchmarkAggregator(ABC, BaseModel, Generic[BENCH, REQ, RES]): ) def add_result( - self, result: SchedulerResult[REQ, RES], is_error: bool = False + self, + result: SchedulerRequestResult[REQ, RES], ) -> bool: """ Add a result to the aggregator. This will update the internal statistics @@ -305,24 +316,38 @@ def add_result( cooldown period. :param result: The result to add to the aggregator. + :return: True if the result was added, False if it was added because it + did not fit within the warmup or cooldown period, was not requested, + or is not finished """ # Add base scheduler statistics to the aggregator - self.scheduler_created_requests += result.run_info.created_requests - self.scheduler_queued_requests += result.run_info.queued_requests - self.scheduler_scheduled_requests += result.run_info.scheduled_requests - self.scheduler_processing_requests += result.run_info.processing_requests - self.scheduler_completed_requests += result.run_info.completed_requests - - if result.preempted or result.type_ != "request_complete": - # If the result was preempted or not completed yet - # we do not want to add it to the results. + self.scheduler_created_requests += max(0, result.run_info.created_requests) + self.scheduler_queued_requests += max(0, result.run_info.queued_requests) + self.scheduler_scheduled_requests += max(0, result.run_info.scheduled_requests) + self.scheduler_processing_requests += max( + 0, result.run_info.processing_requests + ) + self.scheduler_completed_requests += max(0, result.run_info.completed_requests) + + if result.type_ != "request_complete" or ( + result.request_info.canceled and not result.request_info.requested + ): + # If the result is not completed yet, don't add to the results + # If the result was canceled and not started, ignore it return False # add base result statistics given this was not preempted and it's completed - if not is_error: + if result.request_info.completed: self.successful_requests += 1 - else: + elif result.request_info.canceled: + self.incomplete_requests += 1 + elif result.request_info.errored: self.errored_requests += 1 + else: + raise ValueError( + "Unexpected state: request_info must be either " + "completed, canceled, or errored." + ) self.queued_time += ( result.request_info.dequeued_time - result.request_info.queued_time @@ -348,7 +373,11 @@ def add_result( ) # Add result to the list of results provided we are not in warmup or cooldown - total_completed = self.successful_requests.total + self.errored_requests.total + total_completed = ( + self.successful_requests.total + + self.incomplete_requests.total + + self.errored_requests.total + ) global_start_time = self.scheduler_created_requests.start_time if (self.warmup_number and total_completed <= self.warmup_number) or ( @@ -445,7 +474,7 @@ class GenerativeBenchmarkAggregator( ) def add_result( - self, result: SchedulerResult[GenerationRequest, ResponseSummary] + self, result: SchedulerRequestResult[GenerationRequest, ResponseSummary] ) -> bool: """ Add a result to the aggregator. This will update the internal statistics @@ -454,12 +483,7 @@ def add_result( :param result: The result to add to the aggregator. """ - is_error = result.type_ == "request_complete" and ( - result.preempted or result.response.error - ) - added = super().add_result(result, is_error=is_error) - - if not added: + if not super().add_result(result): return False self.request_start_time_delay += ( @@ -473,10 +497,10 @@ def add_result( + result.request_info.worker_end - result.response.end_time ) - self.request_time += result.response.end_time - result.request_info.worker_start + self.request_time += result.response.end_time - result.response.start_time self.time_to_first_token += ( - result.response.first_iter_time - result.request_info.worker_start + (result.response.first_iter_time - result.response.start_time) * 1000.0 if result.response.first_iter_time else 0.0 ) @@ -500,11 +524,12 @@ def compile(self) -> GenerativeBenchmark: This is required to be implemented by subclasses to finalize the benchmark and return the compiled object. """ - completed, errored = self._compile_results() + successful, incomplete, errored = self._compile_results() return GenerativeBenchmark.from_stats( run_id=self.run_id, - completed=completed, + successful=successful, + incomplete=incomplete, errored=errored, args=BenchmarkArgs( profile=self.profile, @@ -520,8 +545,8 @@ def compile(self) -> GenerativeBenchmark: run_stats=BenchmarkRunStats( start_time=self.scheduler_created_requests.start_time, end_time=time.time(), - total=self.successful_requests.total + self.errored_requests.total, - total_completed=self.successful_requests.total, + total_successful=self.successful_requests.total, + total_incomplete=self.incomplete_requests.total, total_errored=self.errored_requests.total, queued_time_avg=self.queued_time.mean, scheduled_time_delay_avg=self.scheduled_time_delay.mean, @@ -541,9 +566,14 @@ def compile(self) -> GenerativeBenchmark: def _compile_results( self, - ) -> Tuple[List[Union[GenerativeTextResponseStats, GenerativeTextErrorStats]]]: - completed: List[GenerativeTextResponseStats] = [] - errored: List[GenerativeTextErrorStats] = [] + ) -> Tuple[ + List[GenerativeTextResponseStats], + List[GenerativeTextErrorStats], + List[GenerativeTextErrorStats], + ]: + successful: List[GenerativeTextResponseStats] = [] + incomplete: List[GenerativeTextErrorStats] = [] + error: List[GenerativeTextErrorStats] = [] for result in self.results: prompt_tokens = self._compile_tokens_count( @@ -551,22 +581,40 @@ def _compile_results( requests_tokens=result.response.request_prompt_tokens, response_tokens=result.response.response_prompt_tokens, preferred_tokens_source=settings.preferred_prompt_tokens_source, - errored=result.response.error is not None, + errored=result.request_info.errored, ) output_tokens = self._compile_tokens_count( value=result.response.value, requests_tokens=result.response.request_output_tokens, response_tokens=result.response.response_output_tokens, preferred_tokens_source=settings.preferred_output_tokens_source, - errored=result.response.error is not None, + errored=result.request_info.errored, ) - if result.response.error: - errored.append( + if result.request_info.canceled: + incomplete.append( + GenerativeTextErrorStats( + error=result.response.error, + request_id=result.request.request_id, + request_type=result.request.request_type, + scheduler_info=result.request_info, + prompt=str(result.request.content), + prompt_tokens=prompt_tokens, + output=result.response.value, + output_tokens=output_tokens, + start_time=result.response.start_time, + end_time=result.response.end_time, + first_token_time=result.response.first_iter_time, + last_token_time=result.response.last_iter_time, + ) + ) + elif result.request_info.errored: + error.append( GenerativeTextErrorStats( error=result.response.error, request_id=result.request.request_id, request_type=result.request.request_type, + scheduler_info=result.request_info, prompt=str(result.request.content), prompt_tokens=prompt_tokens, output=result.response.value, @@ -578,10 +626,11 @@ def _compile_results( ) ) else: - completed.append( + successful.append( GenerativeTextResponseStats( request_id=result.request.request_id, request_type=result.request.request_type, + scheduler_info=result.request_info, prompt=str(result.request.content), prompt_tokens=prompt_tokens, output=result.response.value, @@ -593,34 +642,28 @@ def _compile_results( ) ) - return completed, errored + return successful, incomplete, error def _compile_tokens_count( self, value: str, requests_tokens: Optional[int], response_tokens: Optional[int], - preferred_tokens_source: Optional[Literal["request", "response"]], + preferred_tokens_source: Optional[Literal["request", "response", "local"]], errored: bool, ) -> int: - if errored: - if self.processor is None or preferred_tokens_source in ( - "response", - "request", - ): - # no processor or we are set to trust the response/request tokens - # set to response tokens since that is the most reliable source - return response_tokens or 0 - elif preferred_tokens_source is None and (requests_tokens or response_tokens): - return ( - response_tokens or requests_tokens - ) # trust response first if no preference - elif preferred_tokens_source == "response" and response_tokens: - return response_tokens - elif preferred_tokens_source == "request" and requests_tokens: - return requests_tokens - elif self.processor is None: - # no processor available, fall back on unpreferred source or 0 + if not errored and preferred_tokens_source == "response" and response_tokens: + return response_tokens or 0 + + if not errored and preferred_tokens_source == "request" and requests_tokens: + return requests_tokens or 0 + + if preferred_tokens_source in {"response", "request"} and ( + self.processor is None or errored or response_tokens or requests_tokens + ): + # we had a preferred tokens source that isn't local and we either + # have the data to return something or we don't have the ability + # to calculate locally return response_tokens or requests_tokens or 0 self.processor = check_load_processor( @@ -628,6 +671,4 @@ def _compile_tokens_count( processor_args=self.processor_args, error_msg="Processor/Tokenizer is required for calculating token counts.", ) - # no tokens that matched the preferred source, - # calculate locally based on the value return len(self.processor.tokenize(value)) diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index e882050a..580e2a23 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -9,7 +9,7 @@ Serializable, StatusDistributionSummary, ) -from guidellm.scheduler import SchedulingStrategy +from guidellm.scheduler import SchedulerRequestInfo, SchedulingStrategy __all__ = [ "BENCH", @@ -87,17 +87,17 @@ class BenchmarkRunStats(Serializable): description="The end time of the benchmark run.", ) - total: int = Field( + total_successful: int = Field( description=( - "The total number of requests in the benchmark run, " + "The total number of successful requests in the benchmark run, " "including warmup and cooldown." ), ) - total_completed: int = Field( + total_incomplete: int = Field( description=( - "The total number of completed requests in the benchmark run, " + "The total number of incomplete requests in the benchmark run, " "including warmup and cooldown." - ), + ) ) total_errored: int = Field( description=( @@ -186,6 +186,15 @@ class BenchmarkRunStats(Serializable): ) ) + @computed_field + @property + def total(self) -> int: + """ + :return: The total number of requests in the benchmark run, including + warmup and cooldown. + """ + return self.total_successful + self.total_incomplete + self.total_errored + class Benchmark(Serializable): """ @@ -260,6 +269,11 @@ class GenerativeTextResponseStats(Serializable): request_type: Literal["text_completions", "chat_completions"] = Field( description="The type of request made to the generative backend." ) + scheduler_info: SchedulerRequestInfo = Field( + description=( + "The info about the request from the scheduler about how it was run." + ), + ) prompt: str = Field( description="The text prompt used for the generative request.", ) @@ -376,13 +390,6 @@ class GenerativeTextErrorStats(GenerativeTextResponseStats): "before the error occurred." ), ) - output_tokens: Optional[int] = Field( - default=None, - description=( - "The number of tokens in the generated output text, if any, " - "before the error occurred." - ), - ) first_token_time: Optional[float] = Field( default=None, description=( @@ -459,22 +466,38 @@ class GenerativeBenchmark(Benchmark): and end times for the benchmark, and the statistics for the requests and responses. """ - completed_total: int = Field( + successful_total: int = Field( description=( "The total number of completed requests in the benchmark, " "excluding warmup and cooldown." ) ) - completed_sampled_size: Optional[int] = Field( + successful_sampled_size: Optional[int] = Field( default=None, description=( "The number of completed requests that were randomly sampled for " "the benchmark. None if no sampling was applied." ), ) - completed_requests: List[GenerativeTextResponseStats] = Field( + successful_requests: List[GenerativeTextResponseStats] = Field( description="The list of completed requests.", ) + incomplete_total: int = Field( + description=( + "The total number of incomplete requests in the benchmark, " + "excluding warmup and cooldown." + ) + ) + incomplete_sampled_size: Optional[int] = Field( + default=None, + description=( + "The number of incomplete requests that were randomly sampled for " + "the benchmark. None if no sampling was applied." + ), + ) + incomplete_requests: List[GenerativeTextResponseStats] = Field( + description="The list of incomplete requests.", + ) errored_total: int = Field( description=( "The total number of errored requests in the benchmark, " @@ -582,12 +605,21 @@ def create_sampled( ) if ( - self.completed_sampled_size is not None - and sample_size > self.completed_sampled_size + self.successful_sampled_size is not None + and sample_size > self.successful_sampled_size ): raise ValueError( "The benchmark's completed response have already been sampled with " - f"size {self.completed_sampled_size} and cannot be resampled with " + f"size {self.successful_sampled_size} and cannot be resampled with " + f"a larger size, given: {sample_size}" + ) + if ( + self.incomplete_sampled_size is not None + and sample_size > self.incomplete_sampled_size + ): + raise ValueError( + "The benchmark's incomplete response have already been sampled with " + f"size {self.incomplete_sampled_size} and cannot be resampled with " f"a larger size, given: {sample_size}" ) if ( @@ -600,13 +632,18 @@ def create_sampled( f"a larger size, given: {error_sample_size}" ) - sample_size = min(sample_size, len(self.completed_requests)) + sample_size = min(sample_size, len(self.successful_requests)) + incomplete_sample_size = min(sample_size, len(self.incomplete_requests)) error_sample_size = min(error_sample_size, len(self.errored_requests)) sampled_instance = self.model_copy() - sampled_instance.completed_sampled_size = sample_size - sampled_instance.completed_requests = random.sample( - self.completed_requests, sample_size + sampled_instance.successful_sampled_size = sample_size + sampled_instance.successful_requests = random.sample( + self.successful_requests, sample_size + ) + sampled_instance.incomplete_sampled_size = incomplete_sample_size + sampled_instance.incomplete_requests = random.sample( + self.incomplete_requests, incomplete_sample_size ) sampled_instance.errored_sampled_size = error_sample_size sampled_instance.errored_requests = random.sample( @@ -618,7 +655,8 @@ def create_sampled( @staticmethod def from_stats( run_id: str, - completed: List[GenerativeTextResponseStats], + successful: List[GenerativeTextResponseStats], + incomplete: List[GenerativeTextResponseStats], errored: List[GenerativeTextErrorStats], args: BenchmarkArgs, run_stats: BenchmarkRunStats, @@ -649,12 +687,32 @@ def from_stats( :return: A GenerativeBenchmark instance with the given statistics and metadata populated and calculated """ - start_time = min(req.start_time for req in completed) if completed else 0.0 - errored_with_outputs = [ - req - for req in errored - if req.output_tokens is not None and req.output_tokens > 1 + total = successful + incomplete + errored + total_types = [ + *["successful"] * len(successful), + *["incomplete"] * len(incomplete), + *["error"] * len(errored), ] + start_time = min(req.start_time for req in total) + end_time = max(req.end_time for req in total) + total_with_prompt, total_types_with_prompt = zip( + *filter( + lambda val: bool(val[0].prompt_tokens), + zip(total, total_types), + ) + ) + total_with_output_first, total_types_with_output_first = zip( + *filter( + lambda val: bool(val[0].output_tokens), + zip(total, total_types), + ) + ) + total_with_output_multi, total_types_with_output_multi = zip( + *filter( + lambda val: bool(val[0].output_tokens > 1), + zip(total, total_types), + ) + ) return GenerativeBenchmark( run_id=run_id, @@ -663,149 +721,76 @@ def from_stats( worker=worker, request_loader=requests_loader, extras=extras or {}, - completed_total=len(completed), - completed_requests=completed, + successful_total=len(successful), + successful_requests=successful, + incomplete_total=len(incomplete), + incomplete_requests=incomplete, errored_total=len(errored), errored_requests=errored, start_time=start_time, - end_time=max(req.end_time for req in completed) if completed else 0.0, + end_time=end_time, requests_per_second=StatusDistributionSummary.from_request_times( - completed_requests=( - [(req.start_time, req.end_time) for req in completed] - ), - errored_requests=[(req.start_time, req.end_time) for req in errored], + request_types=total_types, + requests=[(req.start_time, req.end_time) for req in total], distribution_type="rate", ), requests_concurrency=StatusDistributionSummary.from_request_times( - completed_requests=( - [(req.start_time, req.end_time) for req in completed] - ), - errored_requests=[(req.start_time, req.end_time) for req in errored], + request_types=total_types, + requests=[(req.start_time, req.end_time) for req in total], distribution_type="concurrency", ), requests_latency=StatusDistributionSummary.from_values( - completed_values=[req.request_latency for req in completed], - errored_values=[req.request_latency for req in errored], + value_types=total_types, + values=[req.request_latency for req in total], ), prompts_token_count=StatusDistributionSummary.from_values( - completed_values=[req.prompt_tokens for req in completed], - errored_values=[req.prompt_tokens or 0 for req in errored], + value_types=list(total_types_with_prompt), + values=[req.prompt_tokens for req in total_with_prompt], ), outputs_token_count=StatusDistributionSummary.from_values( - completed_values=[req.output_tokens for req in completed], - errored_values=[req.output_tokens or 0 for req in errored], + value_types=list(total_types_with_output_first), + values=[req.output_tokens for req in total_with_output_first], ), times_to_first_token_ms=StatusDistributionSummary.from_values( - completed_values=[req.time_to_first_token_ms for req in completed], - errored_values=[ - req.time_to_first_token_ms for req in errored_with_outputs - ], + value_types=list(total_types_with_output_first), + values=[req.time_to_first_token_ms for req in total_with_output_first], ), times_per_output_tokens_ms=StatusDistributionSummary.from_values( - completed_values=( - [ - req.time_per_output_token_ms - for req in completed - if req.output_tokens > 0 - ] - ), - errored_values=( - [req.time_per_output_token_ms for req in errored_with_outputs] - ), - completed_weights=( - [req.output_tokens for req in completed if req.output_tokens > 0] - ), - errored_weights=([req.output_tokens for req in errored_with_outputs]), + value_types=list(total_types_with_output_first), + values=[ + req.time_per_output_token_ms for req in total_with_output_first + ], + weights=[req.output_tokens for req in total_with_output_first], ), inter_token_latencies_ms=StatusDistributionSummary.from_values( - completed_values=( - [ - req.inter_token_latency_ms - for req in completed - if req.output_tokens > 1 - ] - ), - errored_values=( - [req.inter_token_latency_ms for req in errored_with_outputs] - ), - completed_weights=( - [ - req.output_tokens - 1 - for req in completed - if req.output_tokens > 1 - ] - ), - errored_weights=( - [req.output_tokens - 1 for req in errored_with_outputs] - ), + value_types=list(total_types_with_output_multi), + values=[req.inter_token_latency_ms for req in total_with_output_multi], + weights=[req.output_tokens - 1 for req in total_with_output_multi], ), outputs_tokens_per_second=StatusDistributionSummary.from_iterable_request_times( - completed_requests=( - [ - (req.start_time, req.end_time) - for req in completed - if req.output_tokens > 0 - ] - ), - errored_requests=( - [(req.start_time, req.end_time) for req in errored_with_outputs] - ), - completed_first_iter_times=( - [req.first_token_time for req in completed if req.output_tokens > 0] - ), - errored_first_iter_times=( - [req.first_token_time for req in errored_with_outputs] - ), - completed_iter_counts=( - [req.output_tokens for req in completed if req.output_tokens > 0] - ), - errored_iter_counts=( - [req.output_tokens for req in errored_with_outputs] - ), + request_types=total_types_with_output_first, + requests=[ + (req.start_time, req.end_time) for req in total_with_output_first + ], + first_iter_times=[ + req.first_token_time for req in total_with_output_first + ], + iter_counts=[req.output_tokens for req in total_with_output_first], ), tokens_per_second=StatusDistributionSummary.from_iterable_request_times( - completed_requests=( - [ - (req.start_time, req.end_time) - for req in completed - if req.prompt_tokens + req.output_tokens > 0 - ] - ), - errored_requests=( - [(req.start_time, req.end_time) for req in errored_with_outputs] - ), - completed_first_iter_times=( - [ - req.first_token_time - for req in completed - if req.prompt_tokens + req.output_tokens > 0 - ] - ), - errored_first_iter_times=( - [req.first_token_time for req in errored_with_outputs] - ), - completed_iter_counts=( - [ - req.prompt_tokens + req.output_tokens - for req in completed - if req.prompt_tokens + req.output_tokens > 0 - ] - ), - errored_iter_counts=( - [ - req.prompt_tokens + req.output_tokens - for req in errored_with_outputs - ] - ), - completed_first_iter_counts=( - [ - req.prompt_tokens or 1 - for req in completed - if req.output_tokens > 0 - ] - ), - errored_first_iter_counts=( - [req.prompt_tokens or 1 for req in errored_with_outputs] - ), + request_types=total_types_with_output_first, + requests=[ + (req.start_time, req.end_time) for req in total_with_output_first + ], + first_iter_times=[ + req.first_token_time for req in total_with_output_first + ], + iter_counts=[ + req.prompt_tokens + req.output_tokens + for req in total_with_output_first + ], + first_iter_counts=[ + req.prompt_tokens for req in total_with_output_first + ], ), ) diff --git a/src/guidellm/benchmark/benchmarker.py b/src/guidellm/benchmark/benchmarker.py index 9303c568..d471e3f1 100644 --- a/src/guidellm/benchmark/benchmarker.py +++ b/src/guidellm/benchmark/benchmarker.py @@ -28,7 +28,7 @@ GenerativeRequestsWorker, RequestsWorker, Scheduler, - SchedulerResult, + SchedulerRequestResult, SchedulingStrategy, ) @@ -51,7 +51,7 @@ class BenchmarkerResult(Serializable, Generic[AGG, BENCH, REQ, RES]): current_strategy: Optional[SchedulingStrategy] = None current_aggregator: Optional[AGG] = None current_benchmark: Optional[BENCH] = None - current_result: Optional[SchedulerResult[REQ, RES]] = None + current_result: Optional[SchedulerRequestResult[REQ, RES]] = None class BenchmarkerStrategyLimits(Serializable): @@ -187,53 +187,56 @@ async def run( cooldown_duration=strategy_limits.cooldown_duration, ) - yield BenchmarkerResult( - type_="scheduler_start", - start_time=start_time, - end_number=end_number, - profile=profile, - current_index=current_index, - current_strategy=scheduling_strategy, - current_aggregator=aggregator, - current_benchmark=None, - current_result=None, - ) - async for result in self.scheduler.run( scheduling_strategy=scheduling_strategy, max_number=max_number_per_strategy, max_duration=max_duration_per_strategy, ): - aggregator.add_result(result) - - yield BenchmarkerResult( - type_="scheduler_update", - start_time=start_time, - end_number=end_number, - profile=profile, - current_index=current_index, - current_strategy=scheduling_strategy, - current_aggregator=aggregator, - current_benchmark=None, - current_result=result, - ) - - yield BenchmarkerResult( - type_="scheduler_complete", - start_time=start_time, - end_number=end_number, - profile=profile, - current_index=current_index, - current_strategy=scheduling_strategy, - current_aggregator=aggregator, - current_benchmark=None, - current_result=None, - ) + if result.type_ == "run_start": + yield BenchmarkerResult( + type_="scheduler_start", + start_time=start_time, + end_number=end_number, + profile=profile, + current_index=current_index, + current_strategy=scheduling_strategy, + current_aggregator=aggregator, + current_benchmark=None, + current_result=None, + ) + elif result.type_ == "run_complete": + yield BenchmarkerResult( + type_="scheduler_complete", + start_time=start_time, + end_number=end_number, + profile=profile, + current_index=current_index, + current_strategy=scheduling_strategy, + current_aggregator=aggregator, + current_benchmark=None, + current_result=None, + ) + elif isinstance(result, SchedulerRequestResult): + aggregator.add_result(result) + + yield BenchmarkerResult( + type_="scheduler_update", + start_time=start_time, + end_number=end_number, + profile=profile, + current_index=current_index, + current_strategy=scheduling_strategy, + current_aggregator=aggregator, + current_benchmark=None, + current_result=result, + ) + else: + raise ValueError(f"Unexpected result type: {type(result)}") benchmark: BENCH = aggregator.compile() profile.completed_strategy( - average_rate=benchmark.requests_per_second.completed.mean, - average_concurrency=benchmark.requests_concurrency.completed.mean, + average_rate=benchmark.requests_per_second.successful.mean, + average_concurrency=benchmark.requests_concurrency.successful.mean, ) yield BenchmarkerResult( diff --git a/src/guidellm/benchmark/output.py b/src/guidellm/benchmark/output.py index 7d5aea0f..aae0fe89 100644 --- a/src/guidellm/benchmark/output.py +++ b/src/guidellm/benchmark/output.py @@ -17,6 +17,7 @@ ) from guidellm.objects import Serializable from guidellm.scheduler import strategy_display_str +from guidellm.utils import Colors __all__ = [ "GenerativeBenchmarksReport", @@ -115,7 +116,7 @@ def print_section_header(self, title: str, new_lines: int = 2): for _ in range(new_lines): text.append("\n") - text.append(f"{title}:", style="bold underline") + text.append(f"{title}:", style=f"bold underline {Colors.INFO}") self.console.print(text) def print_labeled_line(self, label: str, value: str, indent: int = 4): @@ -123,9 +124,9 @@ def print_labeled_line(self, label: str, value: str, indent: int = 4): return text = Text() - text.append(label, style="bold") + text.append(label + ": ", style=f"bold {Colors.INFO}") text.append(": ") - text.append(value, style="italic cyan") + text.append(value, style="italic") self.console.print( Padding.indent(text, indent), ) @@ -134,7 +135,7 @@ def print_line(self, value: str, indent: int = 0): if not self.enabled: return - text = Text(value, style=None) + text = Text(value) self.console.print( Padding.indent(text, indent), ) @@ -144,10 +145,10 @@ def print_table(self, headers: List[str], rows: List[List[Any]], title: str): return self.print_section_header(title) - table = Table(*headers) + table = Table(*headers, header_style=f"bold {Colors.INFO}") for row in rows: - table.add_row(*[Text(item, style="cyan") for item in row]) + table.add_row(*[Text(item, style="italic") for item in row]) self.console.print(table) @@ -220,23 +221,23 @@ def print_benchmarks_info(self): strategy_display_str(benchmark.args.strategy), f"{datetime.fromtimestamp(benchmark.start_time).strftime("%H:%M:%S")}", f"{(benchmark.end_time - benchmark.start_time):.1f}", - f"{benchmark.requests_per_second.completed.mean:.2f}", - f"{benchmark.requests_concurrency.completed.mean:.2f}", - (f"{benchmark.completed_total:>5} / {benchmark.errored_total}"), + f"{benchmark.requests_per_second.successful.mean:.2f}", + f"{benchmark.requests_concurrency.successful.mean:.2f}", + (f"{benchmark.successful_total:>5} / {benchmark.errored_total}"), ( - f"{benchmark.prompts_token_count.completed.total_sum:.0f} / " + f"{benchmark.prompts_token_count.successful.total_sum:.0f} / " f"{benchmark.prompts_token_count.errored.total_sum:.0f}" ), ( - f"{benchmark.prompts_token_count.completed.mean:.0f} / " + f"{benchmark.prompts_token_count.successful.mean:.0f} / " f"{benchmark.prompts_token_count.errored.mean:.0f}" ), ( - f"{benchmark.outputs_token_count.completed.total_sum:.0f} / " + f"{benchmark.outputs_token_count.successful.total_sum:.0f} / " f"{benchmark.outputs_token_count.errored.total_sum:.0f}" ), ( - f"{benchmark.outputs_token_count.completed.mean:.0f} / " + f"{benchmark.outputs_token_count.successful.mean:.0f} / " f"{benchmark.outputs_token_count.errored.mean:.0f}" ), ] @@ -270,29 +271,29 @@ def print_benchmarks_stats(self): rows.append( [ strategy_display_str(benchmark.args.strategy), - f"{benchmark.requests_per_second.completed.mean:.2f}", - f"{benchmark.requests_concurrency.completed.mean:.2f}", + f"{benchmark.requests_per_second.successful.mean:.2f}", + f"{benchmark.requests_concurrency.successful.mean:.2f}", f"{benchmark.outputs_tokens_per_second.total.mean:.1f}", f"{benchmark.tokens_per_second.total.mean:.1f}", ( - f"{benchmark.requests_latency.completed.mean:.2f} / " - f"{benchmark.requests_latency.completed.median:.2f} / " - f"{benchmark.requests_latency.completed.percentiles.p99:.2f}" + f"{benchmark.requests_latency.successful.mean:.2f} / " + f"{benchmark.requests_latency.successful.median:.2f} / " + f"{benchmark.requests_latency.successful.percentiles.p99:.2f}" ), ( - f"{benchmark.times_to_first_token_ms.completed.mean:.1f} / " - f"{benchmark.times_to_first_token_ms.completed.median:.1f} / " - f"{benchmark.times_to_first_token_ms.completed.percentiles.p99:.1f}" + f"{benchmark.times_to_first_token_ms.successful.mean:.1f} / " + f"{benchmark.times_to_first_token_ms.successful.median:.1f} / " + f"{benchmark.times_to_first_token_ms.successful.percentiles.p99:.1f}" ), ( - f"{benchmark.inter_token_latencies_ms.completed.mean:.1f} / " - f"{benchmark.inter_token_latencies_ms.completed.median:.1f} / " - f"{benchmark.inter_token_latencies_ms.completed.percentiles.p99:.1f}" + f"{benchmark.inter_token_latencies_ms.successful.mean:.1f} / " + f"{benchmark.inter_token_latencies_ms.successful.median:.1f} / " + f"{benchmark.inter_token_latencies_ms.successful.percentiles.p99:.1f}" ), ( - f"{benchmark.times_per_output_tokens_ms.completed.mean:.1f} / " - f"{benchmark.times_per_output_tokens_ms.completed.median:.1f} / " - f"{benchmark.times_per_output_tokens_ms.completed.percentiles.p99:.1f}" + f"{benchmark.times_per_output_tokens_ms.successful.mean:.1f} / " + f"{benchmark.times_per_output_tokens_ms.successful.median:.1f} / " + f"{benchmark.times_per_output_tokens_ms.successful.percentiles.p99:.1f}" ), ] ) diff --git a/src/guidellm/benchmark/progress.py b/src/guidellm/benchmark/progress.py index 1b86a4cc..9fd1e956 100644 --- a/src/guidellm/benchmark/progress.py +++ b/src/guidellm/benchmark/progress.py @@ -25,6 +25,7 @@ StrategyType, strategy_display_str, ) +from guidellm.utils import Colors @dataclass @@ -46,7 +47,8 @@ class BenchmarkerTaskProgressState: requests_rate: float = 0 requests_latency: float = 0 requests_processing: int = 0 - requests_completed: int = 0 + requests_successful: int = 0 + requests_incomplete: int = 0 requests_errored: int = 0 worker_overheads_time_ms: float = 0.0 @@ -73,7 +75,7 @@ def completed(self) -> int: if self.max_number is None and self.max_duration is None: return 0 - number = self.requests_completed + self.requests_errored + number = self.requests_successful + self.requests_errored number_percent = ( number / float(self.max_number) * 1000 if self.max_number else -math.inf ) @@ -128,12 +130,55 @@ def formatted_requests_summary(self) -> str: return " " return ( - "Req: " - f"{self.requests_rate:>4.1f} req/sec, " - f"{self.requests_latency:2>.2f}s Lat, " - f"{self.requests_processing:>3.1f} Conc, " - f"{self.requests_completed:>4.0f} Comp, " - f"{self.requests_errored:>2.0f} Err" + f"[{Colors.INFO}]Req:[/{Colors.INFO}] " + + BenchmarkerTaskProgressState.format_progress_display( + value=self.requests_rate, + label="req/s", + total_characters=12, + digits_places=4, + decimal_places=1, + ) + + ", " + + BenchmarkerTaskProgressState.format_progress_display( + value=self.requests_latency, + label="Lat", + units="s", + total_characters=12, + digits_places=4, + decimal_places=2, + ) + + ", " + + BenchmarkerTaskProgressState.format_progress_display( + value=self.requests_processing, + label="Conc", + total_characters=12, + digits_places=4, + decimal_places=1, + ) + + ", " + + BenchmarkerTaskProgressState.format_progress_display( + value=self.requests_successful, + label="Comp", + total_characters=12, + digits_places=5, + decimal_places=0, + ) + + ", " + + BenchmarkerTaskProgressState.format_progress_display( + value=self.requests_incomplete, + label="Inc", + total_characters=12, + digits_places=5, + decimal_places=0, + ) + + ", " + + BenchmarkerTaskProgressState.format_progress_display( + value=self.requests_errored, + label="Err", + total_characters=12, + digits_places=5, + decimal_places=0, + ) ) @property @@ -142,11 +187,141 @@ def formatted_scheduler_stats(self) -> str: return " " return ( - f"Sys: " - f"{self.worker_overheads_time_ms:>3.1f}ms Worker OH, " - f"{self.backend_overheads_time_ms:>3.1f}ms Backend OH, " - f"{self.requests_sleep_time_ms:>5.0f}ms Req Sleep, " - f"{self.requests_targeted_start_time_delay_ms:>5.0f}ms Start Delay" + f"[{Colors.INFO}]Sys:[/{Colors.INFO}] " + + BenchmarkerTaskProgressState.format_progress_display( + value=self.worker_overheads_time_ms, + label="Work OH", + units="ms", + total_characters=18, + digits_places=3, + decimal_places=1, + ) + + ", " + + BenchmarkerTaskProgressState.format_progress_display( + value=self.backend_overheads_time_ms, + label="Back OH", + units="ms", + total_characters=18, + digits_places=3, + decimal_places=1, + ) + + ", " + + BenchmarkerTaskProgressState.format_progress_display( + value=self.requests_sleep_time_ms, + label="Req Sleep", + units="ms", + total_characters=18, + digits_places=5, + decimal_places=0, + ) + + ", " + + BenchmarkerTaskProgressState.format_progress_display( + value=self.requests_targeted_start_time_delay_ms, + label="Start Del", + units="ms", + total_characters=18, + digits_places=5, + decimal_places=0, + ) + ) + + @staticmethod + def format_progress_display( + value: float, + label: str, + units: str = "", + total_characters: Optional[int] = None, + digits_places: Optional[int] = None, + decimal_places: Optional[int] = None, + ) -> str: + if decimal_places is None and digits_places is None: + formatted_number = f"{value}:.0f" + elif digits_places is None: + formatted_number = f"{value:.{decimal_places}f}" + elif decimal_places is None: + formatted_number = f"{value:>{digits_places}f}" + else: + formatted_number = f"{value:>{digits_places}.{decimal_places}f}" + + result = f"{formatted_number}{units} [{Colors.INFO}]{label}[/{Colors.INFO}]" + total_characters += len(Colors.INFO) * 2 + 5 + + if total_characters is not None and len(result) < total_characters: + result = result.rjust(total_characters) + + return result + + +class GenerativeTextBenchmarkerTaskProgressState(BenchmarkerTaskProgressState): + output_tokens: float = 0 + prompt_tokens: float = 0 + output_tokens_rate: float = 0 + total_tokens_rate: float = 0 + tokens_ttft: float = 0 + tokens_itl: float = 0 + + @property + def fields(self) -> Dict[str, str]: + fields = super().fields + fields["tokens_summary"] = self.formatted_tokens_summary + return fields + + @property + def formatted_tokens_summary(self) -> str: + if not self.started: + return " " + + return ( + f"[{Colors.INFO}]Tok:[/{Colors.INFO}] " + + BenchmarkerTaskProgressState.format_progress_display( + value=self.output_tokens_rate, + label="gen/s", + total_characters=12, + digits_places=4, + decimal_places=1, + ) + + ", " + + BenchmarkerTaskProgressState.format_progress_display( + value=self.total_tokens_rate, + label="tot/s", + total_characters=12, + digits_places=4, + decimal_places=1, + ) + + ", " + + BenchmarkerTaskProgressState.format_progress_display( + value=self.tokens_ttft, + label="TTFT", + units="ms", + total_characters=12, + digits_places=3, + decimal_places=1, + ) + + ", " + + BenchmarkerTaskProgressState.format_progress_display( + value=self.tokens_itl, + label="ITL", + units="ms", + total_characters=12, + digits_places=3, + decimal_places=1, + ) + + ", " + + BenchmarkerTaskProgressState.format_progress_display( + value=self.prompt_tokens, + label="Prompt", + total_characters=12, + digits_places=4, + decimal_places=0, + ) + + ", " + + BenchmarkerTaskProgressState.format_progress_display( + value=self.output_tokens, + label="Gen", + total_characters=12, + digits_places=4, + decimal_places=0, + ) ) @@ -155,15 +330,6 @@ def formatted_scheduler_stats(self) -> str: class BenchmarkerProgressDisplay(Generic[BTPS]): def __init__(self, display_scheduler_stats: bool): - """ - Progress display view: - | Benchmarks -----------------------------------------------------------------| - | [T] N (S) Req: (#/sec, #sec, #proc, #com, #err) Tok: (#/sec, #TTFT, #ITL) | - | [T] % N (S) Req: (#/sec, #sec, #proc, #com, #err) Tok: (#/sec, #TTFT, #ITL) | - | ... | - | ----------------------------------------------------------------------------| - SP Running... [BAR] (#/#) [ ELAPSED < ETA ] - """ self.display_scheduler_stats = display_scheduler_stats self.started = False self.benchmarker_tasks_progress = Progress(*self.create_task_progress_columns()) @@ -174,10 +340,15 @@ def __init__(self, display_scheduler_stats: bool): expand=True, ) self.benchmarker_progress = Progress( - TextColumn("Generating..."), - BarColumn(bar_width=None), + TextColumn("Generating...", style=f"italic {Colors.PROGRESS}"), + BarColumn( + bar_width=None, + complete_style=Colors.PROGRESS, + finished_style=Colors.SUCCESS, + ), TextColumn( - "({task.fields[completed_benchmarks]}/{task.fields[total_benchmarks]})" + "({task.fields[completed_benchmarks]}/{task.fields[total_benchmarks]})", + style=Colors.PROGRESS, ), TextColumn("["), TimeElapsedColumn(), @@ -316,9 +487,12 @@ def handle_update_scheduler_update( progress_state.requests_processing = ( result.current_aggregator.scheduler_processing_requests.last ) - progress_state.requests_completed = ( + progress_state.requests_successful = ( result.current_aggregator.successful_requests.total ) + progress_state.requests_incomplete = ( + result.current_aggregator.incomplete_requests.total + ) progress_state.requests_errored = ( result.current_aggregator.errored_requests.total ) @@ -362,15 +536,15 @@ def handle_update_benchmark_compiled( progress_state.compiling = False progress_state.ended = True progress_state.requests_rate = ( - result.current_benchmark.requests_per_second.completed.mean + result.current_benchmark.requests_per_second.successful.mean ) progress_state.requests_latency = ( - result.current_benchmark.requests_latency.completed.mean + result.current_benchmark.requests_latency.successful.mean ) progress_state.requests_processing = ( - result.current_benchmark.requests_concurrency.completed.mean + result.current_benchmark.requests_concurrency.successful.mean ) - progress_state.requests_completed = result.current_benchmark.completed_total + progress_state.requests_successful = result.current_benchmark.successful_total progress_state.requests_errored = result.current_benchmark.errored_total def handle_end(self, result: BenchmarkerResult): @@ -390,8 +564,8 @@ def handle_end(self, result: BenchmarkerResult): def create_task_progress_columns(self) -> List[ProgressColumn]: columns = [ TextColumn("[{task.fields[start_time]}]"), - SpinnerColumn(), - TaskProgressColumn(), + SpinnerColumn(style=Colors.PROGRESS), + TaskProgressColumn(style=Colors.PROGRESS), TextColumn("{task.description}"), TextColumn("({task.fields[progress_status]})"), TextColumn(" "), @@ -424,36 +598,6 @@ def create_task_progress_state( ) -class GenerativeTextBenchmarkerTaskProgressState(BenchmarkerTaskProgressState): - output_tokens: float = 0 - prompt_tokens: float = 0 - output_tokens_rate: float = 0 - total_tokens_rate: float = 0 - tokens_ttft: float = 0 - tokens_itl: float = 0 - - @property - def fields(self) -> Dict[str, str]: - fields = super().fields - fields["tokens_summary"] = self.formatted_tokens_summary - return fields - - @property - def formatted_tokens_summary(self) -> str: - if not self.started: - return " " - - return ( - "Tok: " - f"{self.output_tokens_rate:4>.1f} gen/sec, " - f"{self.total_tokens_rate:>4.1f} tot/sec, " - f"{self.tokens_ttft:>3.1f}ms TTFT, " - f"{self.tokens_itl:>3.1f}ms ITL, " - f"{self.prompt_tokens:>4.0f} Prompt, " - f"{self.output_tokens:>4.0f} Gen" - ) - - class GenerativeTextBenchmarkerProgressDisplay( BenchmarkerProgressDisplay[GenerativeTextBenchmarkerTaskProgressState] ): @@ -470,22 +614,22 @@ def handle_update_benchmark_compiled(self, progress_state, result): super().handle_update_benchmark_compiled(progress_state, result) progress_state.output_tokens = ( - result.current_benchmark.outputs_token_count.completed.mean + result.current_benchmark.outputs_token_count.successful.mean ) progress_state.prompt_tokens = ( - result.current_benchmark.prompts_token_count.completed.mean + result.current_benchmark.prompts_token_count.successful.mean ) progress_state.output_tokens_rate = ( - result.current_benchmark.outputs_tokens_per_second.completed.mean + result.current_benchmark.outputs_tokens_per_second.successful.mean ) progress_state.total_tokens_rate = ( - result.current_benchmark.tokens_per_second.completed.mean + result.current_benchmark.tokens_per_second.successful.mean ) progress_state.tokens_ttft = ( - result.current_benchmark.times_to_first_token_ms.completed.mean + result.current_benchmark.times_to_first_token_ms.successful.mean ) progress_state.tokens_itl = ( - result.current_benchmark.inter_token_latencies_ms.completed.mean + result.current_benchmark.inter_token_latencies_ms.successful.mean ) def create_task_progress_state( @@ -508,13 +652,13 @@ def create_task_progress_columns(self) -> List[ProgressColumn]: if not self.display_scheduler_stats: columns += [ TextColumn( - "{task.fields[requests_summary]}\n{task.fields[tokens_summary]}\n" + "{task.fields[requests_summary]}\n{task.fields[tokens_summary]}", ), ] else: columns += [ TextColumn( - "{task.fields[requests_summary]}\n{task.fields[tokens_summary]}\n{task.fields[scheduler_stats]}\n" + "{task.fields[requests_summary]}\n{task.fields[tokens_summary]}\n{task.fields[scheduler_stats]}", ), ] diff --git a/src/guidellm/config.py b/src/guidellm/config.py index be71c544..16378856 100644 --- a/src/guidellm/config.py +++ b/src/guidellm/config.py @@ -74,24 +74,6 @@ class DatasetSettings(BaseModel): ) -class EmulatedDataSettings(BaseModel): - """ - Emulated data settings for the application to use - """ - - source: str = "https://www.gutenberg.org/files/1342/1342-0.txt" - filter_start: str = "It is a truth universally acknowledged, that a" - filter_end: str = "CHISWICK PRESS:--CHARLES WHITTINGHAM AND CO." - clean_text_args: Dict[str, bool] = Field( - default_factory=lambda: { - "fix_encoding": True, - "clean_whitespace": True, - "remove_empty_lines": True, - "force_new_line_punctuation": True, - } - ) - - class OpenAISettings(BaseModel): """ OpenAI settings for the application to connect to the API @@ -154,7 +136,6 @@ class Settings(BaseSettings): # Data settings dataset: DatasetSettings = DatasetSettings() - emulated_data: EmulatedDataSettings = EmulatedDataSettings() # Request/stats settings preferred_prompt_tokens_source: Optional[Literal["request", "response"]] = None diff --git a/src/guidellm/data/prideandprejudice.txt.gz b/src/guidellm/data/prideandprejudice.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..8c7a10727c239964fff2e203f07318b29108eb2c GIT binary patch literal 241795 zcmV(rK<>XEiwFoug!X3w|8R0?WMyG)WN>n2YIS63V`VOMcys`az1xl>Ns=Y_u3y1; zr#rB~KJ11+1hTTaJ7c%44sEk5_&49kX}c<+ww zV@f}b@2&mNrtg-;9)}+q>?hjIv2A<%iFq?lo42lgpYgT%x@uTPFhG&k$=G4vU z*iH5u?Xa|yeaSvDwB5^TH(sZcjRb?Xchj5Mf6U|Nz3uzWvHR8@Ff9Dwxf>2MMvp(t z*J&EB!vPOFHs`MI+Sy;-{?#6_ABVX+w3B_mnfaB;esDQn`*fMcOFJz$4tP9cj4j?` zv`>tzyCj*x#?_A2)v(ZLUr> z8NX_|TU!~MpY5g@`02EaHXNG{+f$pymIjY~#89e{oSQ>yKW*Qc-`hz8`2C2%@Ey5* zY!a|dHt(mAzqd56_FElaYqOv5BA762ufkQ@hzW_G$<2-f=UxE%(HwZ}zRe z^!(#^xp9wd{*V9b|NcKWyW8fU+HrbmH$U`!J3K!B+rOpsGY4+()D3Ul!XL0rJ=#Xm z;D0&UqMx->www{3ElkI@T7|i}&05TQCv-kl$$q-Qm+g}{Nwy>V1_O!ndYr~Hhd)O2&p={(x$`!E@rX}qEx!m}Ocbg*-d zgJ6H#;dn0jN1B(Zjlaeo`QtcE_7dIs+#b5dmhSdc9m#~{wD+rQk?lfzr*`wl&W6?Y z=VtcZ$WwXbHksS$%{GHsKKr0OV%H`&2&eICZ(@^$KeYBW`^M0ITg-N}ui6S+l7@y8 z&l+r2zQjNLjt26UKlpXVIM2<*9crgW7BqkU&!?^bRkg$%5Uf%AY4byEbMF`JNz*+} zW(wah-!^!>sx7oV=yCIltoE@R@Q*ZWFiG~6mu~9kPcaI!$2L7*?B(BWk#Vdx&GBf5 z0kd$!vw#h_4u>uM(#1#y=0O zo6vJTv)Tss!9zQDFT9fG)@~Gh1y(2z{P+C#FPlG^{ci35wwro*OVhvF06%TA(c9SN zbhF#()plEKw)AY5&$HbNW;^VM`q3two3t97HX06}wqQQ1Z{6H2wBpU2=Fv83!;P`G z`IyTq@zlT)?e`70GL{R!|A;w@himb87ru-S5B3rMj{VVI?vOAL-@xa&?cZ>Ry`Stb zZ0v$MH5V+By#p3y(SIyhzlyA&Vqc95*tv2XCpW{Dqd{r7fzHr-$o{tScs%=^fW#C*aTwWsac=DL_A z++COS)r>Chhl^cJIHfu=Hb`xAJFrtTPIi+wwve{Va28sas`n8G2{#CiOnRBjoIPAG zP{EQuHn`^MiZ-LbL-jsh`$O8bvOv{&-u%_v zqF-zyc4h(aYJY&w^M~uck##=U0#D|qwOr(vahuL|yzB>YPxIIId%M*ZZD2c@FYVAy zO|PlbC22o&^D}6tYiQpS<_Saps#(pmY|qzomV8RgnXtR1+w~gz^L2j4Ncgy9N7~$t zBOjnGnWyYub&#}q=KXZNTzj+6TN-wAOl+bL_;Aob2G}@~7ATZikK2bH0UF|A+4Hf3J=PjkMcD4xiJd^TI>>sBd6v;;=NE zUAMS#(l&)Tp*uZ)_Gz#k!}px!p5tq$riTN*vR(hJ8O-FHUlaeDFy@ET^=unizWg6< z=3G4T25p=bNuXL&Sc+-(OA`!>Ja&)8q?MwYFulWtQzwYJ^Pt*rNX zeP_0xaa(eO(&}PeJH}u3al5p}2~I1`t3nAb#}lle&)11=)wz}B2KH00yy)C zGjm&3omwcj|1Ru@)y z*7o|BM$Vc2pKX!LXp3RT^sT!6@P)Gt5#HRnxI>TOp*OZ=Ursi2^97#CadboEkJ(@m z5*E(Bh@j(WE|=|r{C$N=Jn41>=;mXQc!!5+qw@{RX%n!#+sAJ7q-F+Dtwm9Jv_G7- z)uE5rgr3ZNHQ#AknNAymg@{n33jR(ri9+qv?;e&1fOqpz27;9C^$6|=zm zh>nn4Z2s0ZELV9PHvHp#Wa@#!c)+`!yKl=?9su|B1|9{hXfxRrv2^>5&ha+c&cIgC zgzlTm)FJZVaOli`GykWL9Pz0*$HJ+u?fLJpTkc0azj7HF})DLx(3_dp_u3X12G@U>Bba3ZDr#0hYS8|1A%< zv$RcaPP@T&{Fxh6Z+IQtWwLnod}$|lab|z&p_^@2FF;2Q^l8$u8TPU{llg^59vs_J zJUM1h=8opVy_+7diRa+bZQ%e!(0sO)$5J82^;o0?Fex=aI(p88D6I|m;I~WcfqP) zHF|r)3_Kmjz8~N1*~ylE!0P>ZhwbY_G(%vkcgSAW#+JArZ~pzsqG2Ik_6}PFC-e1i zXyJx62eZ}ltYxNg+de$_4NM#K!PVg1f)l*Huz%6X#kBC6w^_^}%rh+A^XgG}%3Oh0 z@auVD>0Ax~$$EiTvM=xD`LTv8lgS7HPzh{Hb3Qla;s4Nge{ak)IBj)z!rovV8^SuR z2UCiov@PY?f_0c^B-;g;H;CuU7KG=-|uiRIV1`@LX*XBkC zWotI7f#1Bq1sGtfk{cfj*_7VM}+GPf2%PEyM>FgaunlT?e}?-9yf%c z+97pw_(9)ngW?P@820m=F_53f2iqp_+5sL7@YsqoV^6jVw*bBlsFp(Z|FKd;@ysfdX>Rq;SczC`TyzKvWZE#09kZD6t=Nm@; zwD}G)W}c<}_Ls6ZPUC>Fa=`m;+F#FfdTr-DZhj~W$o(F_)xfI483`aIIf{i7fFKo+6X$2Nz#djJEoIJ;o zXnyT#xBoFmw*9v68@YIOTBGGz|NWToRvCnm6EU6XvBgF%-XZf@%j5wd0iRw2PL5RK z^lGGE0KaK#n2WtQ2Ctoq-+bmOJUgBYyu$u=oBr5L76$&K;6@@H=Y}z_kOw3M9}BWZm?NByIQqz z+Swv5x4E@J0`{5@m-*x45TKyLOSV_cIish{9OApXC4Dq+XUb!NS&Flb99vaBa*LcFsM+~`#Rc{m^ zq<8GB$DBkV7Uo~Ks1*1Ti0C{fF1YzYTN!%U6mCxCO>77_L)eb>o|!qo8{9csgp@Hi_}DXo zE?E}EKY`)`O69i4X3a4RiB(2Zo&oX8q-J<%GWzx?B=5q;c1G zFY-Q`{l2|u9}1#*IC&&tXUaa5;YGNO8I;|olGOr~H6gAu-Bze!7;NAU`a5uSa+`u+rBLGB3i8&3jrB7vHrG*9b?i#vF^06&CpMrt)ej0lj zQMjo~Bv#AG+c=_PmncTjI3q!M-i>WWx@Fo%FfAi8>^M(B=*A16XUm+gUtSOnPZ0NfN`e+k<;yN4JI87C_w{ zfWU2?VNP>0E?s{Ut3dHU#jSzONq!-fuz)_DS#X_(95}nfpd^kN*1yT2!quG|P2JKL zzy}@i(Nx+4;S0fN$qaoRJnMG=uvr_;%+va9^D^Ul5JvRfnf#{W{Aumm z45L^6no>nOx;@J~ct>$uaUIX_l|ap~5hJOY4UVU3;!@E$S7`Gx0c$@vfkLs#a6RwD zr`5wfX2yxwk7rl`KQRS@%rAtQa$09vm^kSMyJ;F?T^jN>tkQ1MN^q*??pql!*H{y| zP1s4K-g$0aQ-&RA+wqa1RgMfa!?dItW8@}o4+>K{r`XI)e;xJPe0tL_;|ur7=M}`j zHAaTPe}1d8=8jl+sr`XYt$edFf*J8ZA5*$tMRFthaeR$Dp08RnAjk7LW~kXjRFTNP zX$QEK`WgHC8^M2)2wR{!Fo$q`5N7&Zan;C8d^cNevnhwdPH8s@J=%A0)sBswOdy~5 zvMmhW;Gr8H`sQ7D>`!JKktHH8C}HTn3G&GOYQbs|{^i$x-%R_P_VrFy3AHQOsOEtm0P6FPOv?a8wYB_;l*<44Y~P0IBUL9mVFO(}k6iw1ue z{k80<11GYKgU4MsgR-F@y)C?Se&0^J8}Adpjb<`X!hHC$*=&z9nJl~Rpk%)dF}vrR z59dmwxDM~2Xy1I?9PL!1f;nlA-yLn7S?+bR?QJOJ0Nf`k!cD!HmE3oircal4y!36P zt#KSa=k=xk?PLUJU)XdqYCmM`0Hdc%(F`xst?)5ul07MI_cO=k#vk;_P6WhaCO%bmLT{C5 zl*bH9&~2gp;a97F=*$_(hr)9azJe*|o#ZuxT1&GG+}z!C!U@Z701n+?6WYHP$D?3f zd}Yp=bBGyc*snx_)m{#soZ_|gQ_Wt{A(7S1p`~_%ofBRn+_0-ns#+jgIop=KXV$p* zObH=eI8-h*-8TYXOdL8#=*2eSWBLKqkXf(8UwZ3CVsQNTS&4lNY6WvBs<1(cC3mE$ z4CA_MaoFH{(igW^wV5T20?A&uO|ZR)$3apcFfD7J{E*HObpJ+%I+GE?dyRNKTKR83F6tR+}?p1VJ0JDYlnV6-k|F zh52L3f%~7uUW;lbR%alsOgdOOtaQmWA_C@UzZr?fRfRca&Z(Fo{N1l(2a=c~?;itt-+g#ps5Oj?+#L`wg*z%s*|6;z1)ajaNBj4|(b|m9yZ@(RvJQxrU2K7z zfPr>_+aO5jUx~Yhn@LCmob5wZbKrPcSZDU$a2Ci~M%<&*m*L9_8#%7H`;$~KU2Nq4;Vi1z7Zg0PQyGTUYk zQZ^}X0Nj;sree34d0})&6XS(?C$N8*0>J5x{GaB)iAN~|^Zd7L^3CXstSltE`nB5K zYbr-wKk$oChtc`BjS}2kr7P1%8&+N_dn30|*5y2MTbkoovA*COZp|bu`%|QG+na(n zEE2tWzvu#>=npv6wklf;$LUv#-7Z92KQlg1g4CDQZ70hG&zEKMd~Tng3h&$&d>E6c zr&!gCE|6R@k|7b{05ufG8Ryjc)q>hQomhL~O{@X^Dlt+Gey zjh(k#&Kp=RF>6dd%RQWy>(#Rr>TiKs5G+@z@a=Jxp{BR( z)v$5eGnZ4jO~gU)2<049KCRbGZSK8y_p9aRXYxn~6(9w#%0^mzpkv!F{0ZRB$~&6DdKC8}!@Ze; z>`Z+5!L`dA_0_O|z!5FR>GeLfxm}UowKabvT#TrewM0k;mYK;Yado;uji)umyuz38 z12)&udd{x_SSI!2*0LgC(vjceywPB>aD|8}Q(J+^*GwkH{x;yG4L&k7P;tm{I-L-r3TyYS!nqv-OBnB{){3%}s513F# zY!kzp1lv6Wm@pz;&`+InMq*M{zKSk97H2M>yUzs#hTuY3{7I+C{VPN<=3miGE2#&e zoYx_TEMlF@u7Svp(w{|UEBHI*?mmCNLe{Bk753-RntkovnY;QK?)Q=|D|m=w1%;NN zP4-o}pDyi-fsL|C|Xy@mZ8G*}v)| z&16=Y^LzK9?iioJ`z40`>yOOcxA+M8&E;R8e`kA|XZ-oW{$Gw*jycoVh@4Q7!CBC0 zj@=#UIS}+pM}lr+0Wg9}=WiZWhqzEbkrMe&;~}%rw{)V1F&F0%qx5UWPQyxqW4=tD zXT?oC>`Bm>}S415E$1GNswT)kG@YNNYY4wg|W#YvKP+qr;Ll$0Vr@I zmZ1>7e~j=gF+5WN5i+^i-QwG~J<@7RuIPj&3JX@|RY5G$Bg~<(zO0~ZoYNQ7Y{wG+ z)OrD^P<0#KbiI(GM@*@{^Dfhf*ggJ@nB6Vi(NlnkBlG>&VqLtNze^NNyZG12L=8sj4 z;a-@6a|IQLGXH&WLM!)~szx(?9{{Y6>_+rpzN&=l+149+2gAOg=qy?JfC0kfO_Q*e zt<2rRN><|3@|#r@GpdGDZ@Xb?w_CElkYd<(6%m`g ztSX=qRy>bhMU9K$6UZswi|-1{?9*@Gr#NOQ%X5eV*}XYp=Fp(1aqaQ|GtB)Q=Z%mN z{n=(v^u<~|8Sy%xa^6utGSlP~Z^lK`z3~z4amfO4Q9;e-Lci&b0^{jVFn=yy66V|# z9b#VbbZe-{qMf3c1QsDAZn~P6>jduymMfN2^&J@!l`p}_UYA;LJy?rCI_RSB!o|^%mybR z>wt)E!J=u@S8Z6v2kZ*}rOeYB15M!&f}_R}yv`wm@M5;g?xg8f>8&_hx~(wbZcb|; zEDH1p_>i65HY($$SIv7hm%-z&4uUhABy1WdmwDu>sM>-D8tByxU+OD-wGd!|`Q2QW zmCl9#a>?0w8ssry9DdqbCVm^x;f034|9pfZ)h1_g_;Wm8%z?%mCz09-x6S#+uS)Bc zadTQ_gVJo)y*Pi*-GUEjGHtHc5$;$wxlmbZZ&pOv4;$qOy`vp3I^T-?vE)_v3@555 z6S7xjqvRsQh*q%SA?KMiX1z2xlO7CXcTdlXWnC}GC7DKG)n?0WH<&Y^xm~4C9HGkVkK^7;sAbjmGo^G8V6=!H?o~k zfBxipNZW^pP{)Eh#n-VP{3QpJGf>YHv^DFX#>@D{;icsjf0U_Y^I`%w^|wkfVI zF<(&m$5YY0MyWvKC7MoGNvtmrD*Oe(MK|Sqe-sCkDd{IQFYk}}P})bf2&w2CYk!28 zhC|x!eaK4{9S0dCPk*%;*0;mDLZFBbOPC=;Zq7c@lz?eRSDw7P3LB=4cPWbjS4XZ+ z#tX6T)&YfZp;Jb(ys}hpTHJyjGp4*H^g&uu0!V#oq6%|(LtXXZvwPJg@o9FrrdWm0 zU$_!I$tUs-L~y7Khu%uwD|nVgRs;kkR;+8+QN#j&Z}$b!@f47~pR7=#gtW zxAuXqontO?y{Ov>8oQY7QsD}aCS%gb73W02)EwG7-cC0F&&FbBiP7Lpp1}{2K%vsk zcHa&bO&`+TdOGU}yp~4xE5eCdN8@sx%pOt4(Wy+^gl!s*XK2(MMgoTGLHc7lX%)SP zOPE-xNf(X0BRc@yJ`+($a0;ye5nyG{-tn?;WdF%f==PF0V*oyVw+qEp^41_;=CyZr zyY7geE}lUSg^`38r#>Bv@1#OfDWAaT(`u1Ft@9V8UlvncT*k>WUk4p|iV#v}WL+WP zW-+SKSwA7m>LxA4q(c?=o*OA|bNI$kFUrD#I~HqHk$P05gqd%t1@-=5>1Njsnh!Xkvs zK$$z$7v-+NV@A?T@44GQVQ@@M;H<blQTAmvWD}5Q~m(5k{(a7;Hu3G)(=RvY;u>;?|x6%x-pon5FH#bEoP< z>YH8S9P!H@c8&;XBW7YE(+Phb_GNZJR17?_HE>Rl&|l5MeI)a^Q|<4@I}_ZiFxZ=7T9vLR){}O86I}$J+BL|>>Yt0 zS~TE`Ed3xH_|@;sxbiE0HfD_L=2bwshG$)+GG9X*6eda`Hr*|kGK|H0s$W+SIkHPB z_@%Z}kS`GnKrcbz@|ar-z4 z%;?PaWyX`rvW{kOD{f^tgoI>Y8Cc-|%V*YYZO8g4=``n$4sxU6y@yKH1 zoQ7Z`6xsYjW=kuIFierxh|DE@AEl=?jwA2(tyMMBOWP-2136ddQ1&)*wK}L1>KlSa z0idgQ840X{`;=TGaT2HCB&#S!HD-#(CL_Xu8amCoYKU+(1Xj^x-sy4X_T7FWn^<7}O7$-sm^ zD3r@?f#nh(mcpQ;=X#^M9zet`G6Cmz=x1Tmxou*B)p$;I6*9GKndQieV4m4pzxXPB zl4>u6*Gl=Vr^-2Y*V*^p)eOnh{ax-cAkS1^$hCg1a;vion9`y8Ma2{i#?4yWo*&Fy z3vnt4hA}GeyVN14KJtp+CM$fn@ zY_#GNF^6RLx4)^yV!4j)mAX79Lf~>+r2H1q0g=^1*1Utd!e{~gq7o}73F$o97`CCQ zvGvjOK#Z^qwozUvS1VRi$$B)&i!tIt%|&=}Hu!gP87WP~l{izRt(jg~Dr-J#VwEfs zB~ih4-t0!8^FBM~XjAkeU(dc;tT%PH5O8pUR%Gua3%L*9lfuOwpR3J^py(22NgY8i%*k^nDYZ=MowWtr1dnbS72nO3+;zESL= zx(TModhuFBDtPN$?kLb;CX`MHGudJTxy!n2{?;lgmy8Kv%p z=4)qL=?Wj--XAG@Tj6X+LzJZwuFPfoQQBCtiz@RnwsBVhzgeV$&y_hX*`9R0wL?O=;SJQ+Z$6FQiLgLJ<(IR34l+Q&KTm zQB$Dy<9&LC%BvEJ{_1iPKv>&-0|Rr%ajB?Glru=Q4zw)Kb}FCw^SF$ZHq2?&c6xXS z+RxX(_d-(=Ee7!Ahlj78P$?=`A)@o}pajSX?7UkMa(j5lpH2EKX6(O@yKbn{OMh|Q zr2N93JBg~0!-ra^`Z1rw;y?u4jfXrC!uRz@B}}L69%cJbJ`p)E98|Z@ALKbJ&?U^9 z46%E7ZIuO6X#m}*ByhiG;VU6Y5B?dC3$B{;7vWvc%a@2;lB(pHrz+9$N=P<@N02uD zU)VO^C-QI8uDy+f_=!S&_My8@)=nUV$8~d7u_qDH7V1A@t)e&rB1)e;9jWsFNF{rv z*4CM1E|o=Mhws1C9l2vFPFlB6XNk3Uqw>w3)+vd^QUmeF+jHI=Byw{B1pjJkF4qZ1 zMOY3KXY=z`IreM;QZ@=a*M(P}6^axniY}s5heJ-yTt$UjP?S#Xm(5=wvc%u`LC>h%F^h6&eD3<>{SbU&{Jvu`~t*mTNO6gG@!;XhQwWd*TF zfJ@H+F3KE$uzp8pg1>WqtYi|SaT50)eZ9;`g@y+UCxg&Y0ZZ_R3YVGnh=5YU3+%f~ z$)f$>qGHZ2q?F^iv59ETUp8N7KYcoK9#?r*>q|E7Fqw}Zs;8Rwy23h25;fYD>S!3) zIdd;vyO*!MG8-Ro)$FPTS71{Paj-EJTB*V;E(M2GQH%=Z2v%e&ykEv5j$fmqz82fn zTNg&vDJbN8Q>WFdI4pVfLRWbNcd~5EIK%&V_B{n;d3tSjwN%ei#yM*iT*a65OvQoRc=WCE9ss9`OD5uilgD@ z)z?`6d}85Ium@0cY+21lQESPrad$qixo#_g>!0YxW2t`Y<@0hdC~&Gz5VRWqc2b8M ze6}3zT$v_SJ^YnAtgJL?MZnM|n8%dUvo$r#y+_4=+Y%*nE6P!FpiEj?+vJY4TnN|M zVi3n*jHkqBZ48Hc=DWd6yZCAKy8-61gac1lH< zRZHCEAk0ZA$ENFDOVoE8-}C07a`+=g+uoh>2;#PWml z!Mx9<^eOUXF2LZ8tY&Tirx(xJAtN*-&ywsTRz$xcEPB&$p(W;W`IYT=O|p!nU8|O4 z+4`U=sMA@`S9?X9k7Mc1No4KRNh#m!LG>0ayWl6>`d=Cf?55-R z?ZlL}=Y=K=xdC7V?*xzmY(|!?TNYi^ zCFbmsm*d{o_PguGv}Q%LR=q3U3E`f?L|RXBzEx(KAvn#RCl>M&DQ`J1Ijta%wvfVv z_KQ7Ta6cljT@+X3s0F7d`)u1l5K!%LA@E9pt~-`imBVooU&~}tOuIclRBAEd@``8o z2t>u$I_C&rT}zUOMUpeMZs4myN@*H*&8|}}Dl1HM!?C}v)ii5TO;Hg7FwO1fj5WN} z8}s*)z!PK$9PkYMDg0F7G^H%-Nm@-V{U1D%kZJTM>f5=n?uJv~?&G>Mn9CEDvfP)e zfY#7abUMemyf7iH(kK;^lf37OH-nR}m1Qs^x~ZBHuhDA1eN#{bqo3vI)sLZtSv+Y2;N8n(mJZS#Ws&U&4yocJXAW+@vN(Ip99jOJKpC>U0mkDQEMxpD>5 zWXR`gQR-{L9uVgEdSUE!*LVA>QyM^CEtN}bXF8NXgDg!cK`IZ^^!lI~d_Cs&P|Tt> zy%;c3Rtv?jlCY_I9OC7AVA9*26xszm za?~8fwgen>d5kuUZ4Jy6$mHiGReqY$$h5U!1V+h1{i&yFUP&Y1j4$U$UGhC!+cqz}R ztUeu<**mAtckP(GAW1pn>p(cNJ$Q2-z$e%YGLVO{a)1Px6D>UQ;W_vt%A$&=RkNo2 zq3Q$MTE?9XKjhD2NIqiN@3Cj zN*5^SzuR;}H8Nnydt49Cg>vqiY%uC)w$3oSf2z zGn_oeVT974iZs}`bb5r*2+yN=&EVOexzUuQs=~TJouIA!yt*vI= z*323Cx==L^m#gp&38fVC>_rVCzC=-jFwm2;=*tznr}bo>Nmze4URBFHC5Up8_q>ga zgZC}!+m5#yCd?J7QylMVpUj;!8*T4KZH?=eyZ6& zC4}yTORZmtP)ZZslyzSj&1IQtXjfnx#Gfm)E(bHsVZ1B^Zhl9eDY|4vnMOW6q51Mu zcV%t%^f7lgFHNyfccz2}!j|YD@}^rRM8N%$KeTK{Q#D-!(Iq3EnsPnS*-H zLvgTTs8RyjHO@~R8FT+EYM+`EYe%AiHl*viQAEj>Xz>%oMhS)J(`hkn!_ z9!RvpO5`u#enM}FDo(Gny-ckUPWEpnV=~`^LBN{w#}hQJgsRoFLv^mR=HWUV#~zMe>AKkW1$J0Fo?e?Q zi+J55=U+rS&(bzt;^eD;8nboI!4?S^k)hGp#pRA}f`wAzu;^NtZ627=Sb9e=T zeXx`K^>B+EQze)s0VaRe}yXQ%;&Mte0Q6!{=*wMq!yT$bol*G;9*{agEup7m2zwHW)-*>;8Jh(4ro&t`)O z=djF=5XHi6i(*phU(Rw8)2b|!otV{z*{*mqL=~;04rY|QHN#YiB<6ofvjp9my31rO z|B1eBq16}wS?R_0XV9iPFLBD9QSW&m_tjw;yeJA~h<=Za$B~F3+x?OYibFRdu<{-F z{$5^DVke@)K?n_!0V-gF5at}8J3-EMrQlzRRlfA2H z(PuA(X)G=|JNb6~pb&gle^I2x=r#^x0?fx6GbTvM(}E0nPs#hS$ONxsapy95Oydbk z@_XS}G-6e1$v=;YTPiNF zn`~u!Xq9Tb@4%rP%Z-HepL6L9-V09Cz9=Jaa(7nyp({(C)qbDJlg; zh*o)`RC|47dy@Tq&dn@A%6;R?dh)IQeAEQT+3^=v=LfTF?dRMBMIXqmJkDesjofBM z&pGoZY#xfZM--^}(6h|coqk7J1jP-4ugj~0pDG^A@JTD*TR<`C) zR5Vulx%_z4A0nH27}cdVNsJQV@3@dL=53)tl)PsT!=?o{c-@!*%NS&-Xkqh9ru!7T z16J2cjRT`yyAiwFnqqUlrH`-j1?r3&sgTmuyleKdt!s-$bs9Pc6$ZIeOaY50qp%@T5ZfvN z&$8su^Wm+=HR^<7OBo8 zmi{+Fj%Fphh_kpj$Xe3<6JR@wHyo}d6^BX(O|lS>XK$gBW%2k*DCj;}RH`TED=O28iDAhvZYmONBInw6kN1B*A86 z$B3J^al*E(YQFS&+~n#CB|c2o$X|pi+W-1zB+Zp@#zr^^dK`OK5T;ny3jnD9VAP$$ zwFt=DAWOs1?j$nPQEwPX3{J&;vA zsJb(BKD^Gvz!r-{+f_!qI+Pb7K=Z3~Gpi=fcAv)E2TD`N<9 zT&K3G2xbR0ffBCHs8dTW0TU-#p( zm$Kbg5sx~MU7Bd(L3b&_l{*H4wjJL{5sI92FA?z&8l-U`Q<_z@GshRvSpmg}zTIsf zhFU#pv4`BsJ*!_yT*8UD3j(%5?akVA)|Nqxd+1(<9CCUx^E|S}PO5t@mxeulV!!SC zYlLQEnHS%(q_?BY5nT+Xk^X|0H%Y;i4!`iO_n zF*F8+pFOQIH*a_;3^P6$DEu&v(Ng4S$%@sBov-PG^LEd4JVhrbu^a^O$6WG zw~&vE$|9tyy!hchJK#O&l6OqhR1&Ii?cvhS*F7WcTVgZ0Luw9!Kj;2)N(2Szis<6QoqH@FfRSaeOk4_pC^~)3;L-oUvO^L60$RE+i;#n)D!1yz z?7j|9Bjv`*Sq$^A&5OIz&vCoJaQi(cp0%X?O-RwM*}rac=pAiv&YqA9Zk0W(GQY7* zU)=`Tv~8m;j@R&Pn6Fqv4ZdQyqukG?FrMo$dJSa6^{T%L?#ZKB!*R~f+7bsVo6rV=&i_uK$;-`0P zR&_5^GaP)mJ#W$J!pcViNCWk&!IQ^o!4JuNiBk4d*XEI(X)eOT zta`ySDu*3+KK4*i-i@nJCND0(+?;lR(!4;9^Y^SurUQ6ENK?$FMO!oGTx!&GnWgiS zfiG_o>)Y_ly@sOaM9h=VodyMXHSA)dw(N7~+HZkAduWp)BeU)-x@vgItJiA{^2eDy zs)`P_k+T7)Wuf5gwf(D(fpMa%;;TwrJc~_J053q$zv!l`tDHQe8#!lmFuTW!>NR0S zSKVlAQHwNqI6}sNH?{nvR*-sKm+_72cTwk-soS%y zs}t1EpH`$En@KE(SFSAMqMq(h%;0-xE%wKaU7@Tz!~0T6Gx8p{Xyo=7CR>U=XzVQ! zMdz04)~1;bFEWLC^~GD}5Y`>olB)YV;L<*N&Rw}OJnWqC!Z5ayPBZoDc<_4r>d5KiT?|kO4Lfj4$JkGKea2UDO>VL?E)PMxpZP-eH+9-dZek){DWAqwwak; z8CRRUx>7jDPB+oih~i?+ktMqv$>u0W30li9As1Rz*o01K=_Ve0>fqQ=_q!61T;9iY z*Ff4qzAZUQ-Zo2Bi>`L*El{_v=(kX|msK5yR_fxs1-l7v-7BIe>75Y_OV={XMY-m4 zuJ1^>w$r_$29)32$h0UVf`Zv$YFud^8w`$~U1;V}I~|4`oPB5q>Y_h}xUOT12-TKl zGfaAYfrA@y{OVt z^r<4WHxR8pgx3^3gX)Hdj&A}oyUt==xa2071geu3t~ph_NKmZX_H`>SB=`u2YZkx- zI(;3MZB*`1{|WD?5+U=1E|P>8y#z?j>VEc4LIPn+b7cqb(|0<^xq9KMH1){vWe9X> zu6oDMpObIs4rH04neo=_?|R=j0&D}`7CKMptUK$AI+nC)k}yLk>H>uFo?QKhTkY&6 z*E~)nsU}go$*x4NO}mCx`Tkh3x_4^Mepx)KmPG@4sY#sg%2o+g1AEEiszczK4mS}7 zQ-Q`b2;C0k%a@NGaUByV=DSF3GYiG@%oc^OQcgUE!oVq-|lK`ews-|CYUE3TU{&uzPp`OrEvS(%FoV5k~ z{0Ae4^{&)4eanC_jdT#ryivH*puimYhbU`RI0f0g1q>YlqoF? zotkc*m$^M2nPl;}ghDjlc8aej{Y9jFVZ>b`AROj@q%T;C_*EW3bI%tCd32$i>bMQ*~wNM zMu%7h?uNOp+_kiJ*H$Ieq4~fYw|K|)YCkp?I&eSoP`e-C4JRw~rn;Ej)eIIc{NN&K zPP&JJSJE`gg6gsk&>Mk=2OW6Ly0zO!Ew#{}VdIrtQZtXBP#kVW98)p3otIF*`$eiXnrw|HnVvWG^x5K8jq^$7u0>2!`qJ#>pmzIpafcR9JZcD)$BGhrv*zQ(EV#Uquy^Rg+<`imqA*=OXX-QM-^ zyD{oYB`u{GFy9iv0F+C_sfNj@ib!08T3oRtB`93uxWk$t?P;yyDUi=n+z6XtQvXyb z&(Eh-nS>-sr;vEui+>hEGrXGNqTm0QwkgOQdzBd?9~UTtxbEIYoRT}i zXP`cy1Q-x(bu^F|`WJzm`U@UE{~NE@#o~XFzF^Y#RdNPWk=_&zIOJr@-}7PuX!N+w z94kt!aqK(AG@rSFW9NW+C@K{iXYE6Ny4$j!LoADlf!MoQCQ`$M*;0?Msvq7E)A+r6 z^eNw?0iIol{+wUgB<%NGh+op`}> zQD{i$0sB;@D;sY+7ZPAn5#tI+tn&S8UADJ_)qXc^1GT4PX{VRvA|&%& zG8R9q*`I3G`Fch2edc*4oXFO5juYutIcpZ>T>#pf!TT)_DO>n+dpj#wGQ$|W@$-*7~t821`IHUQoP*qoJp|)}ps+yK_7ELJ=EJ1ZodOQ_1 zmG3s%{cvVn3Ev~HQk3B(Rdl@}QK6P)Z1?<~nHMOqLy=Ep{zTf4)Rn)t4{p^+<^^{> zBV%V1bW4hc&CPVsvsPVvEUE;n(Xq>|YcEW-WKnL&EOZLbtK*c6L1d|AuO=W=aPC+N z^`wUHrGldFzi@6zEO;zVuq3x>=Qif7HuPV*p$*^Z)AvJ;?5w3x(&wxOU1xXBqEy%) z_#)oXwvYO4dq)=)VaZzZSg6vdopbM^Wy6Z1DtT85FIgS~Yg z?xFQ$DES`q%JzHYzIadjx=;p=ii+y~A{wQ;jW$I5yUJISt=y45jyc{3uiFpnqnZNV zWc$XuOZ+f9Dy0M*Zo@fwe`c@uurKFmpvcI*jB>GB`B0**RqCgdA<(sc*LrcA1q8h? z!5eaIE7D21(N0xUqWYnEKSzht!{^8WmEA5o=Q8u}E=>yikZU-b9=*dGV1_PWAPu;K ztEMn-mZ>X9SngfWPTfltcdwh>S4yO-$nzL<3TKG9T$dMDPRpl~lXDE2psrOb^`!26x%P8g$#hj%(aV)U#SY1KC}?Er z2EiP$G`6Yzh9Wr-a$N^RAdTv7kZl~NfJNWkk4AE&n^wXG+A1&9^akt!A<4keJ^flK zAAC&0JXTbmW)IoAHWG)fu2CX0&32uGLVgVHY|7Fyfla-)b!3xA;8AkKvEFH(DOb+0 ztA^?8H?*m)@c8cli0&@HQyzoU{WT{V^&`NEyo2i+^|64q>864oXRsb!FNa*#TZ8@Y zw)Z?qSmv!b6>>fL*XT9B2MZr52X(X)8QcGRVYz2#8}&ls2Pg%6ODZWW*weDE=u`K? z4#6Y~Uk`_#**~^2&Xp~(oxQbEr$9e4#(s|4lnPPKEsLKU4{^q&JM)|HMG|$m*?X6UPBCU zRh^qNc~TbiCS zGn6Yoi)U`7M)5fv<+1|vYav0 zT}~zxnQb|Ml=fouLxUbs&%%t7|D(rry=+T+izB&>^J(i{+!oaj7ilDk{?YM4Wz*g6 zjLNO-Nop=r#cq4bN@<0Z!6{&*2YbQo>>%P1h&m(-*)!0bcLeBc|D1Uc`yf<>LWL`m ztBzGF#|-J`lY&S`4V^>A-00Q`%^nekghxP_4I>BG*gsfuXWnnAggU;ZuPK-&mADh3p~>8~*L$wvR#$wTdlE&iY%9<&$n_ z<`{OXfNs=FW^RC2zxe`r)O}~FQchB53ERpP1`D6!Ua>kwUQr#48F1oFqY2)7D^HTR zA5T+rfoRRTQ{;e53$l( z4gDr?(3J+&2g@O%O(v0`?RaVqZzxa*xI~i8IwptQ81S4wqRNAbBbfb7?FkULrHY9& z_7T>GB)^{(on5G)T$b@X>!wl9^k`-X);?>NfI#U=V)Q>%#sZbMP>FhuK_|xa~hly z>PXCPBFt4BP?65`WOzuY&~E$C7rKJwWM7tWI?b%Qi{=O~{ilZ2{@s!}6V9o*Yn~kD z!#P0p9-xncd;FC}gDS@!G6A5eGB5zqW;G4pjK*?8fK=U07=IaDgG4yKiZIRn1rAas6A0 z1JPHE^U3yrd{=S})D_wmV)pk)f!RoxTi|6vO{Tm_ zAYuD}ADznOAleSLG@DL52W34o`WyDQNLI4g<`C*If*jA$L?NeRLXc<}v#d~IFn=zL z))n<6SL=ntA*SDoZl47%P|pVv*oexLm0XMRnZJ%V7j-Yt>V%$*7EkzFWFDIP%_S=y zJ)<+lvy#%gNhB_>b#f_QrAmj}rmAF-(h=~mcDdyXQGDQ#VE>@G8nWCB_ISY8+ExmL zb1GM@V2N9#_*QN^!$57i9@P0&lzbNG=45KEj_y6iL!~JgyOodc8coiYx0hAb=K1l1 zF-)tv^{UjZa*H$kOi~zH^g80lRI|A_3?$ffo^xKgM9or~byDV7H@|?v)98(BUDhqN z_&8fy_!Ukm7k1V-%wDV*{qEHrfhBO1zD8vTWGtYbEVOGgH7~pzJ?|0s;p`lS;+)A= zRi4>QYMU2TMU#ut8Q{K)gS%!8Da|j@z*iDV#cvE<>`*XZhNYG_OI0u9N-clKYn?kD zg5T%v+Y%COYS@C+s|pvt#|HJ3)mt1h8S=dCAY!b{yCzq=ETRt1{31k%=KxlpW1fh;pt;{dQ%>&~ow?Xk%0 z1$73AE#G0^EszQtQfWn};5)}lWX5EajIQ~5pJ7#mL*?lVZDU*Q3-!T55-+!#WW(1C zW_Qa)|>l-{;dvc2;F2moo@2G)ZtpyQ}Pd=1iI^B zO6R$u#9g<1f!Ls^CCrlWW#ReI>as63e6|@0LKaK8&JWN11>qzhjihpq;PgH$LP%Gh z4H8k~bUoLEAp8>VuM$s}0-kw^I3rj$Sx73k4UL+-iW|!WQYY`i4t0i+z+>D>lG|mcKPXKX4QQ2UM5W&ELM_{c2|v(Bw^7`LlI-`o79k9)(#W}S z;4oH7)D<20s~_#Jr4Pr!E%_~DBORpE_kDCAtctuo2nqgc<>cRDB7@=YUYCGo7w88f z_@n*o1uo`?zJM-%#$BO;jk5Kt-$6->WS02}#Nz!eXQwhkag{MB@aGZX$+NALj!elZWer{jWUO_I4F9%sVYI7bckh6s?;m^ELO8l z=4D`O93UzF7Hr=GIg1ZD=_EmrrGjWw=2(D(J#ZKc`f?(RpYXhj&Gzj+7g%;ghS=PU z@!<)r1+&<7sC;DyAQyEzzoaUOpc~(tyQHcKeg#xdx}?=>%PvgO+IYy#mp=5rdl?fQ zr+I;@T11cNAF7U8dDf-|slEp_0H1TXlNnS=A#Otlr)suUh2Vs&G|CX1I(xINEFPYs ztpGblxyW1xo%Go`8S4A6M4?L7n8_C(;PO4wnOUjVMTXAq>lRU?MKb}s4coNfFhH&> zi0Y}PsvWp9$L+DCslKBqepl6AsYiK4IC(<@Y$`;MUn)|?iY)6Zh)Wx_4d1D`kGtl( z9-I2gw)3R}$&){&_(W||kS<^)OWYU=nhKo7B4+RV3Z%B~_1n@T$ZCnaY+Sl;h=j}D zPInb?V`eZ>ZLvWiZ2LxT>p7-Hs`HvxT#n+2G%G z=%v%iU?uu;F;XTA!?ATZ7S$<~lSm@eofHZ&Rj(_~1Wp>wilZG_XNRqt4XcP#$Wx;e z7IvG{9Q&L5{j1XyU^ecn76OS#!3lwRDm7^8OipS)9I%nS&%BRvc0d?s_IorF>u~lhJ#{}+h=1;9 zR+=B0*=)xAw7QukhE-J4otFUOty^0cYWz+?*l(XBH5a*sw^g$Tkmy6L(R?OHubxF{ z1Y!vvH!g@+H2U&_#XYc@R0TYoThZW5!L5=78Bw0Z@`&Tdusib7}}TK1$LT96vRY? z%*}!F5wjW~l`ufx{B(D+> zp5W6(ZbMl-WQe)&qm8j@4@g33LZCG%=H%4>!vGQGgFJ=bx6|%6pSp@t_{)8K@|kl& zn$E5IZFL8vMu~_XGNlM%R-pxCL!E+2RZcWnVG%i&!q+5!{C^Nvxv|Id7L>2CCztxJ z!%nxjT5WEP#yo8~RxsdsKKZ6Q=<#?C?cUai_Tm};+_!;f0A!n5<(F4Xi3;p4kbDly zaYmd1$h*Z|8{9W1mJ&A%GUVkztqu`T@+9TqVJUrfXN|aOLmot!Dy5X$il{wqismiU zD$@l{EUh_vbHP52ez^}ULpCP6 zC2YP!gz~$v8KZ*}w*SIbL+&8qH!5x?Xt`g-V|MU~E;GXTPg!a6+%7riLfvonHqWe( zGkb+%25fBJYP5i`-h|gda$gZtJBLzXk)U=4ibl|W+T2TgM`P2@B4IfC>eckkTnp6k z-_?IL=jQLxRpO%?!+Q$o(~6gwrR&6u*i(o|Y-MiNdZ?L=$wjLU{||k?ero-#Pn)QR ztxPEk;8a)DHY*5VTUQZpH@+8gtEV1fTZmDx*=N*~lLcvh21}9QrU5_&sotr+kp@!h zra>t;HAo+X|7K!!REIX_#2X}bgP@J3mdl(kNT#;kk(bE;IpPzN#A2#Zp(Yrtn{pRSXsg2I# zKMNw4!$@`Vu!#deWy(`rRCPaTA@p}zjmqisEVUO|6Y9u>2ke1v&|LA|W>hbqdLnd)yI4&&e3f_JcfN6i^nDV&0m;5iz2UsG-z``uOf6ZCEW zKXLHK-0yykjjjv7azSC`|Ex%^fDNMS)fv^}?nyRl&hMH@Gc!NpmjWf%M%_5_hKM&nT`&=sT#RD~8ciaM5ty)5G3-m}Scwrw^om;b|9i*Mq0luod&qVPxIf8JuPBVMg!L6C2 z_Xf$y``C!6P2m;5H+MLWsS-THR(X*OQpIzm4k51#E(nf#3YM{#CUL|Uy^~n484PoQ z>p<^V(WbjH?K;vVU1xU4{S!M7d{=D-X+Vmmi`}g6Ed2aICwJXTE9zQz1RbgKbExTp zcm1L=DK`}i;QtSh777bq`wHGmPrV_Mt5gDoo-x7o8Rxs|7?@1+L|@kwghGCbP*p8Y zv1fNT6$hf7kK?Z$ceL}3bN%hP9YPjlrZ}WI`VzL2n9F^+QEuFHY5IjS;BPcno}XfN z)55zFdzHrK9}@~&tyrX|mCx1$ox1kOu=b#aRh}L6AQV3nZAz77VI|hI1P3q5ieRtE zOxwdE%==bibZTr(-^`BR(!Dj&aN4uI7W6{^>?)RLadoLH$@K=I?XU81f8v|A+j zLh!`*6dQ(;bG1nO#LoCpOq$YjJckFW(cqYgkAbKpLW*aaHF8x*W;CXl`OL|`FcCd^ zeZl@!u~+bjG#{&TB;dul&={icC<*ttucq&D*_Em)bxu(Z|6TU3Gr=4=n&o6K7FpEX zTOV$^Ck@Y`A+4>Fo0LejrP>0xt2AwAkS&tXgPbHSmsRdz0s^#Lho!zyq>JOC(T?sQ zB(A!biim-8DMtTF17Ou4Kn}UIdS>rnQjxqVL>-9_msYS(u%xDKJT#29=lgl?eck35)Zm5ZJy09i1f8t%^m4!cC!a< z>R#D_=v_sCkhbEs2##zcB^~k$D}!7P-d>Wv2^u(0Ik$N5;_y-eB(@0kM{(N(qmhJA z$7hHsrLHI^09=>llmMjdMF%*Xnk;+B%R}3F9K5Qo?uT7ypE4;?e&M>7W2WMWqgK@9 z;JU8FRDQbzwpfa@%g{J20SYQ{Rh69;@+sPo)paBfA~%K0I_Hm7GZRn2siz6NJMY$= z>kOQ%T2jV>8vs0!zKr`{`tnQp;azTRM1hMx-f|1`i~??p$b$PCeroJDn*Oe9J1i8Kdx|o_ zKsYyt=KPz1L&vIcI~lArL6De9a`jS)W*iM{T-EU7y_vPplWKit{XjHN{ixVB9=$ab z*L0;DG*>Cm0(Y%?ajF4l_*=P>QhmtPs(hRE98&3`8u}=E@3zMfiC@=N;_G!(M=4#s z<|}g9GYau=7_iM&?R~v98A?}pR)l7+2~W|L7nIz0iSALAI`ewUFiy>!VO)~=F6A7P zqlb(M>NH6vSmg|WI=e8RxA4@tccsVBn=0J?#6GqX*>~WJKOt(83%F1ETKK?<$wkSzBXEh?Kca=uH!<85yL5|5F)y?`Ui%M|wUU=*$My^-huV6_UI?I&IC#$j z0kl6r&lphWa5`+xah1+qCN_y`sVoD%c;0=&@37y5x>YJmMb+lJf<0L%RiqqeK6q;u z9l7a>P(Jj|QRNX3@v2#(tee*$j^vBgN`Iwk9*?l<)HJRvDN=~nwG&s16iX*#GcQ+* z5qOd1N#ZPryfhG=vx&Q| z(%Q2aE9>5Sg($`YGL*wA+?vT3Rl`Tf(_}-p;zuY2Ilj5Ak!bNa9Ervtz_<=?jX1U5 zUTMYA$URCkP)CWVpapDmgf3L8pzFX<Y6;~k*cwWPyba5^E z;jX1lUp9Sr3Zmyu?v_->>tzme2qA7H?U@9<7CLc8f#&E6>0F82Hd}SJj`|B@@pV-+ z1xlC8d7PY5mg~%uSF{zS0ydIZO#nRpBeFkUdq@or=>ocyvKu?Sb$bhCT#)Y6ows+2 zJhrK?=gVo;{)p4Pi?LmR5A=*Ke#;>{4MJ}GgUOY=#XzOl7@JTO9`&_QOzT4|g z_8R}(x!`n&0&t)yC%wg2%_}Twbu2bkW!`!Y{=#CJ!)zT|p5#>074m)Wyi$0v6 zM1gSu#NN;Gsoc^tm7TKWSZb zwa;CDRn524JyK|S`^~PShDX>ytIoJ!bP@^%BEHo2#0MCr&de0+fWek>tHjGVGg*G~ zth!sAO1sc~Nw>O$%XDqO*9SQ(yAC6d`CJM|aM@(F_(hlD*__%){3m87e5lc%>5jXW zJO27JG0SG^*AzEzAu*{>7GezY_dQrg@fYf79+vkOR5n63xvTS^sx}p_$gozMbojGX zrB(#qP;*4>C2+M>@#Rk$rRDO_-2iI{SEaiJ#6U|^xA4xY#%oR`_fkV8h2bZnW^L83 zJ(JVnm7(Eo;}=!#OF~wvJU0l96!(R2MKtx@Dvvt92KI1Fr0R?oSgOlz)i7zP8WoB6 z#porJj%b_9wWVT3ru}Zf5~o8IZ8=*u*snt=g24p_jr9~AM56(?ufW5D8^BZhr4Uh1 z`W`qrjUBde_=F??>;O6q2Mep2Z{S(Uhx=FyxpB~SIn9+t@QNW9IH-iR51X=1Wy^ac zkd+lnzT{<;TZx7X99iT`Ld_&&WoNAH!vzNgV)W$>Tg9PBcKa0aJh&dnTxhn3v1BF_ zsK!s`9^Rg7!2Qw38--vJ@RQSgb(i4R!@yi%M|1GBAI9q zkm}};(Rcg8OkfqCv-2%B$EVGoqP;oZS@2<*d)rpe_^oFZ^FM_u_YjB%iMqK9g@Yrj za&MK1HdQSq_J@+hk|e$7^W|91wY9@H&t{zTh6+*P*dD9i*+C9XI0`QOFHIb$BMfgH z$&Q3PJz=Pt`wS^a)oBzVE9EvM>Nm0q3JIl2UKFub4(a@d+cvnC(Rj{y@ooHFvO z<~CR3up4S#qUwNC-2hJWUKv@;U|>^C`$e5Lzeby?3w3_25=k55#C8ODrvMi*o^69<^n|SL-1Td`DUk9O*_Nru-t!Brmb%N&D0#NyDR_y(3U`yB#^t@42-ZEjx8 zIImPkUuc^CnsxbL>E}z^yl&k$D=hx1yO?fk7j|JB8R#No#eK^lF@z6QuPu3_Dr+{p z&e2{z{JZF~nML$$G%gz?PB^kn{rI9!+WH}e^LA8CKactt!K263gOQWF4pHQeJf#av z>0~jfk~5`yKX6J+mf}Ui2Kpj?EUHr-T&xu-iw-cS)`{ir%NxQGA;jG&iJ(|e| zms6Z_GB1PequO0V7E7IxYP3BisvrB+U%8OFpKecsfxdeTRlbI%G)9eY89dQnYE;!8`4 zFseDFq6a^5rTnSfyre(xlGs7#tv88Ra8#}1sXm)~FE+dBdRdf0?}q3$>ngCmtDf*= z=MsDW05_#6_pO{yc58ySooUHA?}h86I1h5wAqf0awmE5uvj6Mhz$kBl3g7#>fX&nN zRRn2kUZn|3Q6;akbr1G!j+)QWb6t4@lCJ?XrMf`3n}e4C8>1F`oHK8;LKhi2P)zCv zw+7LrbOU&>beVo%b!Lb3JL%$83u&}z>U%{`I)k;sztqpd^3lDCR7&0TfhIImkjDfX zj`RYuJR8pPQwT%L+>n;Qg;W=!?VW;oDorJptXl5Z<_{q-Tl&vXzSv6?ccG1f#Bj;p zhIm!SF4ewIyD7+9%!l;zSX8c<&GZgsDLRxzG4!qtv{*Ya_MCh99~00);wt8z0n?$C z3j4iWLg<<$(3zWIC2SVT4c_4-KRPQ!K#+ac!cx0B>*6W$={1{_N<}i1rkygs6uD-0 zBn4wEobt4u9LAh#p+f_LKRfJ`Jk=2R?-MAhT~p09W0SNGj~rpNwF z0}u6^XKx>k{qB@EN%GBEjXVVJn*(#HkDKqX#)G^fP3Q~ANemLyvN)45r(q@F-VJXS zTk+5qX6qqmd*^<9@5TIuZ7n?wR!SE4)K}N~kU&I;s#kgvu9`iL8!vOrVNsYhk#8Pr zAPq;&E}dKwvL;{5lDVXe43hyuLYDo>H73}yhz8xwanD1oXN62=-6`&{bx!TsD`^u0rpxT`iupV9Bh_yl=13td%OqSHLM;&cQsqz6JVH zG&hiNWMg(FRL)mjc+Np`-?}Pg&Gn<2*{B?KB3)4S>;kyhIc-40=f&ErZf%k`owns} z5&)KmyhdaJ=Y0b2GsVoK#dQ4v?}L1|FnN#qRYVr8gm%@oK$38jUu>v*!O1R)`0gJ! zo4xr(==fuIfI0ftHqTs&RK9J}>(I^7KZ7qN4n2DtpbJ{AH!YQ;RRL}8v@>RAk7N0I z9mT9awRhq>g$?jNWuov|$ObrwH6f`^OE8+|H$pgs2X$2!%kW&kJ{KT~S4xg4urc_U z4~=)5gx;K=Hh=I3fVqbpFrx_Dk)HrcY{mm8_O3P?VBV*@c82T;P2?O`!?vP39~+U& z*as2mXggC485GNSm%{(c1U$D@h3Z=R>_Zjs;~h%iF*!KCA;`T^KX1JFYE^`(R4NgTO%?nU zpDTCFqKfzx%~$EuitrOOHeXN{n*}M1TRV<&w8qMJ_3CW%7qn9zFIN)y7(k378Fo&Jt8l!<9J zEBCx4yt*fRvo6Gn&y%t~Gh{on)~P&K$q&7k5SgSkWRFCmnKw~l%`$$JYtE_($FaHf zJ*(7h&SeJ}2Ce5Gj8kaa(zVn*m;w8o3(3u?*A3t*GRVCijhEVM=yNGdVHKYF#Uh9! z71g{*hLUf^tuM`AR+>`5bfkv z^?i47FYmyTq*8t^H_)oO>#e1fxqiuc+qwew;{=K@kYeywS7r34>sh?oRp#oq`f=j6 zJ9INzif~cRlpI4Zl%TF#h&``Kx~wfk$-n06pq zbw_n%32q-TgXCS$#Wr1tN6~Y$-KE-A_glwz1$WW{&a!4SKrAUyEn-d>pCxG4{k^D( zdd2nV;enPmP*rFvt*ThPl3HGB=}J^ zwkbR%x1n>ctt;b_{kqNK9K@Rhm}J;4nb+UG+$NOaQ@OS_w;=^3gaei3QS`5jn@fwN zzPTQl>?ifkEW*BnVkgduHZqT+Ze!ZRAb;MnTbZulW4ROHq$1s>*r63Os~XmZeZp^e zXUNu-RC#mz4gF5ec2{h@D}_7v6NQj!3$K0!b`y*}L8em_#Vy|Fx#RIgnO}vzSSS zjyBP&(!|cltJ8ES|>P)Qgc^Td+d!c{hd~-aCMi8yCfEl@%w70W_8=m{{JQH zZIT;FvNXZHo&vG!76EyH;36b4tE<9iDyvdDVmhT(6eD3IEggVyV1P4Ft3F0AdNJFX zYt&8pB%|NYd#VO{%&!W}TEsXl!_-%kw+C&p98dQgBb&s`@j@QWqnl$n}ZFLm&% zF;(?IeL3W&rhvXA3!k{z16t=PBlu?atw;HKzZyqj%lwbrJ~t-hl#FB!3G?0h*T%0~ zbf;R6GEwQk!Jj3kcJp6=`g)xy54=_L-Co9&zY^X@PUUr*9nWv8l2FCmQPKNb#II@j zo(^M!5BL3vx@U^Btq>G&Zv)M@6&NqSP`-ZZY`+k0#H6WHZBc)^{t0S}t_mvPCV5Tl zukyX(;OV>iMp2j9AxGU^&E);2$URbPzmHqdrU&mLZkYoLM+9l*+++lG#T~O|HA`{e z1hI;#1vnUqDZnr5wm9ZN?2HKu=-1no^S_HPCkf>rV^Ng~+@w>R`5Nh|kh`Xl%;%H+Q|Gm$z6UI}_Xagg@)d$rCa( z>W|?k#!H!>aOYxJkBGl}i*a3DzMsA?<(3HCXu8zQ5h#`p4s~}Ia%32SU}Q|t?d?Xw zP87qIu7j-|EjM zrPk(UcXy(U+}-&ZA5kD)*m8Qb9;ALK3r{@UB;_FXPCk8XJRolqrC9ol>s$rj)JQOD z;+7i}G8<{k`^752{ChtrHw1i!ZtEtaFkUdyZBQs+eJ+b<8WUv2(hGoy za8GpPZjJoO=$Xb5VJaS;h=c}ivz6`rM6?C}acgnirbKbrNYZ9%21c7Cw(stT1m^w`x4p;`%5DsZB*m#y#r{Cb^}*B2ba>MoA5t2HIb zH3`oB^1A-oBq~EQYHjKh)zb77ZWv2E6N#W2!F;w?nuNV<0@Rw>yWpBQIFihY5p!smY9Dn(5eInc*%eV=W5FE6Y+kW4BqF=ylAVa zvh%e1GxKIhZJ_7CPRe4*Qka7`r@yXJo%K`i~xb-|%+_ZGzzg zcz{`u21&viIo88mV`Q*|!ba)-qX;h#8|zOqJ;b$COZa+A)8-&5>H4j|)*xLUImU-2 z{IohJC-qum(40QPhOy!5?ghBjTWF#=b(d-cV;u#Kdc5z0U6n9nQ%;}X#^=Dy(?Q~s zQQ7IRzZPBV#w(($@=D1eV`SRwi8>L&#+@&FD_FS|o`6*DIt$}+8`$b8BwE=q&e6nc zy~;oJya-Pc0y)?x_N*haEjDb?z)SaJ;OTIJ+i1lgRBVxYljTiAB49{Q@^euXh*SGT zZHhL59vd{V_1m-0Uhk$G1F~)Q#%;$`!lANr2#k{zs5qu+_?LVdrqe6xv7Tx9`bUjH zr4Gwb1Ix=KS=Ixj0(Q2Vg`J%C8(wJ-0KqU~dFzIhw;Pm`T4@1VXp1b7-FNJ=g7FzAupl7PfF*}u2x`!eRQcGg?!5@TbHL9^XR3t1GzW`O_J+~2 zi!kUr16ceZXS%0bl$6o2Q|=aNdm`<~GcyQ1t{#PL2aO!Ms;F>z)SB{>Q{DPaBPN`; zXd{2mZ(iA_D0TRcLQ@N&L5e(8GPdiIiyb{q66(%B1-P30;zEt!v#jWtm!&l>jZ?YG z>~CcE43fm{)yTt?4g~WY9e25Tj0r8SCl&1+76GM5bSAt^xP!jbC_jOo4qw^r`b3iJ zx65uF`)5Ph%S!dv=NV29!{l9li_Ptd7XxghBi=>4#j;x%iK)7aIrQ|p|lkMDhocfZ3P6Mt*8Jmt${Igbu2vrC6Wso^-r;*-8a<75XH$m^XI#Ro!%^B?VetHxM}w-&*<#0Q*y;jtCD=75h>Y-2Buu~efy0%W z;~~o(Tes}~Mh!;W%4u;tL(~x0F&O$e#Db)65^5<_p&#%H{1Xt?qba{Fj zPcm2LP+_MT1VzS~SF<0?nn2L);HAk#=nZnasbMSi(@Bbkbb+Sm%c6QM94%PcHra#v z%I5tpIW_ghx{MD+Qxp%NU6*1JbT>R-JT3})iwGUKjxEH>JFE0?e@@Fd>;1y?-r^QG zC&99&bG+LaDB9G9gaj*!HAN<0$(cgbS4+Z*MJ?>Z`N^ReRj1p-Ct(>q^X-ORzYO23 z1Jhe$d6zLeBVY#8odd3}XH@QRd~dUNLCT34*^ePeeX7ZnENb=j@>FCTLKFM1HItp* z7aO8I6DDo*4tQTs{kVD0Z2o$&a2)qbbf9@K@AS#oOk~=I@xp1%&VzKA#LCsF))lAx z?%04Ug}6bo<_<1$zQkYcXJ$h=Eal3lhe6KT7e&9Tq?)-zN6Q(DBXRD9%CZ? zQ-@0W=t0|{gfw$@=4t1UuINPB^J6sDAJ=cHsq7o6?{g306G5YoPuat>ABrU20gnS1g7Ju4$5Kx}#XU^eK#Ph=QoT_PnX>&bzOyyCR>WtjZFGC*+SPhX$aj1#&W(hIgK(|Ty6Ajc>Fr2 zEM%#t;uxA7f9iW%(!vSVM?s}A>+2GA!xvDrM3ecjE$m=v$Qo}c9vLbE$N51+<(1Zz zvfbe{COD8Y+WGPx?XsMllmIGs!cSUdg@C0d^KEp$06{>$zr4@Wo`yFVGAD_PJZ5Tr+_{>bjr{}&-KMi1}dSa2J1iJdwE8Y6O(%!NRH?b+C z6`g>nx^E<&00&3+H0er3cchNlikw17Xdo0X?|WaJ?01>nPEbfTdIC&gebs2Nh}qV& zW9kpDxwr{hAz?{oW>bh|v;Fllz}`=B1`LsB@Vz| zBo?f_0u)$mmfJ+MeZSa+atR#EFKbz~}autF;^#JP?sr&Ycu zl7c2j5RDY7c($LJ5JUg{JiafiO;YDSCi7Op5sYXhODi&~TZr|I0P^8}_(P-36cVjj zXuXN}tvdJ#iHI1{v`ppl)EK6jTs=+?xQB3;BTK1Q6>((hb)ODonjIt@hJ)HK+cNAg zc7od&FJF@jPVxReU*+f#7(Ycey$2y_xmkzBG^-j`-X~pd@A1k;->*xnz9b9Sl)gQ3 zM3P*O*(y=?^B)zL}E*Eyrlrdz76ztqd;t^3*nTMPbXX`0Zy>*MIt zAGRiRD^0g8)VU30ci)%kRDeBwjp#e8pN3amw;;5~YiTkGPRG5Vg3*1BjE&=vt?!OY z)sbj}Y8BOvnYYa^ssdeR*jz3cr657mh(w|vOdh{A?4J(hKiR#U6R+Ce;RhwtAR(|^=X#e zcN0R*8%2gH$Uaeox#08kPV6zz0?46xl-xLN#E}||$30yQ>Z!zyQQwI@He>_mL6N18 zdnbtXt$USj84@yxSR*)sOo%&UIKv*?8C=+`0 zz@i=N_B#&K(a=+{^|oG|@%OjkLSq3#D!Ov7)<$L{HdY-%4VR-iDe#~YH;+>9m;(aq zQF9ZHz=LkS&HIaGro+?gRT|4`U|$C_hMA*pOq)strdoLt$iTQQ?-C_xsaaPmmLJ-@ zJOnLeT7JBnh5mH1&}fk&%FYjB?8 zOREuSnh2cdp#08c0&}855cPyuikC#%(Ag!NJWt!Z&?t1$eA=vOqF^* z57vKSzHv$`cxJ)VQO*0DJo2qe5#WPtKNOWfest$^rDiJWjCJy`?Wb|lGT2{B3LrzC zCEm14n?Gp>)>@+3Qx2|g%yL*pQ95%n5rEPEX>rKdw9KB+AV10P& z4_)?WD+E&m-*W+VJxYEYQ>6oXl`Yipb!!Li;luB#iL4)+1K;^Au3`-U}+VJ8kKo@43Rget;Q z2EUh3)k|=NVw(!f;|?8*OCy4Q#<0=^uB*Xx4-J!2qhoA3ViaMQD8QDGw}t>&QImwC z52uVLJ#B}u*0>i4-J7DPBhJP+jtj`V3@y>P(RZCW^6W1hbq&$ z?N&&;UwV*;_ULyUx7Jg@?RTe|uQTNXmB%v`WsP6b(4E%0FVpMAqa7ROY`e>pe@)sP zc;cvmYK?~@I?bg6py`yZrcA7Dm=r!cjjh9k z`2~myuCT-tN>g&*+)QaBj5R;xjD&l!j0L0JrE|Dd(;>q0`v9q+&LVu^H+3?QTDg@t zv`!|gshT0yXMf9HeY&PBb0Lp&(hsWF`&3tD{H}p^h(c@N%c_58Px-EcwoVhn!H(IS zR91aXNc%~wg9cl6*7&h&+94CF@#ONJm0Toj!^iM-j1kANRu_AlKxUF>pYsfCn9-hhO#G6U8llq`tp8{-^(PWcet0a_)ZCDG0^EW4M3sM|I1 z7KU0V;C##&|4iU_I@SW=h`1bb59mqy457Zuvy($7kJ&@Q;U%V>{#g7XJg^%y2OW5_ z9Zm>=7JAjQF@+qCKZUH)W_R^-G^3CNu#m}A|1=s>cMAVsd!kA*l^g8E4AIWvpkDv? zLFM#+{*I>&;iB|KymfN%6E99vZ6`WIN6<*c3$0B@igXC$FfBtZZznN_kbiWd5A`>c zD6Vwm!lHNGhtE`;W>`nqi9V@yxc>OIy#-?FZc7A#(OODfX*Y#JoLG-yR~&~9WL>&# zbKgrNM00sLV8WE$CU=)j(}<1GVGlJzMD#B1$r3&v#qKQRc>)u?)q;&d+GBmwGp{Uk z{X(iGvhtpA>Y-H=gNyp^r6#M#z}ktQGy+@>y|ZBwU*6br7EZSxnx{GE5^=RGr7;Co zhBR$j%2JTS+#j92sZZ1)bU7fBX1xY#S9OoBK~2-rv%oP)hHY|(=nj8~MgD@g9HrJv zFINExg8G%uc$go)&nCa`CF1Ra9z{tF^Eu)fsW|E^3)u3D(Tj&>3^d^7j?ZsD-Eyie z%bem~X%-`_LPHpR#~TID_cvr+N4dlAhu6z!Wqm#ZZ=j>q)_W~bZF@?mqQb%0Lon!zcs3KaybQ@gTQAStau-3??e#NJU73zxV2 z7UHp=hy5X#SGFFKxCV*@6S=434r0aIf_cpsVYQr;@5Yy_W@(JYKP!ud;=d)hfE8(C z3IrmLMXXthEZ<14(Nvd(y92jYG&#>{blUaH@EmWK*H1jZE_P`srX}Vl5~uUScj$>0 zn6+ul@jC;icK!0-3ucj;WuaZAsd6N!<*79X>P^iRdOVDeIgD0kn;W|8+zT^L@(i_K z2MyHPFYIIbDjuS2f}b%p+v-cNk!WXGn8uG3xtVR{@&V-{;S-)*Lc`_v?(R5TrK$Oy zKcV}6Fv92VuD*#vC-_nJt3#~GlvYr3VA zMzzW%lmWojm^Qv1@+_$gt&ZX&t`MFf|7QgwKk%6Ba{OpS>sOkZ{A0~?r%|2#z{sH*PhiQ!fdVFjF5?f8pnrO-q|Pzg;f z?^y+-*{)xRvX{g7eXztsu^$;xjj$%*J)EXKp&@bRpq@oa{+hLslRk zM58`t8U1gw6>=+z|XP7|$!uC|OqM zT-B6zFV!X>#g zk&1}|)17Y)r@S!7-Q`*F02glv)UU*Ltgfn0vVp`2Q@zE8^57kHHX{AsCCBNL8Ivd9 zj7~v2szDD!W`5vdTH0wV+(_vKBCsGRjv5L0$dHHfheA)4oxM>UCj9L&Y-U#=B zDc)QojeU~54-tuy(BInriN7ZA4(lu@oyCYcH?^=fm#VYU|m0khF zsh{ePYj*Ib(^W3_3Z;^RM@I}fvSd?bD~rmK&4`trru#!V;VTOyd9`r?g%r=*e)3Gi%dG(iheTfC zHxlduLwIP|J7EWf+x<9g@8|3~OSU=qz#Z4VgchsM*SlagdM-}thpVwc z+payfyxXqvAQGiQ_0PWO@RveFkkrCxL9Ls&3cg?+m1ttuB4m~+Hz6gQflGFym&j)O zPCxmj)=W@QGfshm!Y8_TH{)-kvc{bT;^)(47p*1xDuqswrsI~Vwieu~C*Mco@6T)we+Ob07G@82?~ylhjh!JJg?*mQ+1MMYj`p1+y;IsuxK51? zSW2^O_ieIfdtN+%X2o?=~q(@{p_X;WHS`J3tKdt=mxa)zB94P-_7CWS&5aQ}J zSk!B0pyc$-&z&%n?kF*Z@0iMU%m3@q(h-p@W)Q}}Oz=mfP?Gt;ab7`>sqVwa5S|C0 zP1xl9XRZF2Gp^E$&Q8QEXNkjDA$28!@;PR;cpD&K+3T}aDxDZ`B$ zC$Ew-Atpccg*|6kxjWVeSQ)4DQaJ!x7hp8549l+5-pmTVgLUfSU0} zq+a@tHtpzfi_T=pYMzLTm4l4pcc+GuX!JK5oKX>WM+p&cw$ zm`J?pFCj0m&n`mSXD)9N}MCjL#Z+1!*36yQ^iF}QduYE!&*p=kfY1QFNALK~hPEp68A zP%Nh6)ze_k?%SniO@%!GpDP+)$RpvSE-|mLMOHo>r-k}1|GUQQ>1hCC`x;80D6&7F zhB*9tHkIo?-EaQP_d4mvY~&~-lOd|S^8N`cO9|aHv=F3L>vqLr<|W+3P7kYpj|8tI z1maclW^!jWjOL3!bJvqSjnSJi-nJFd^TZoP4hONjDdym6h4FZa z(bYs}Qk)r>g8Xqmg^4kR#AtLGGl2+AY$5aLm&=jZms_)5XEfj)b012=aB^MXF^X(h zWsy}VWr?5L(a_#xscgc1NXstCpRPe>aUSF1xL1!(VFBR}zgU`ax-r|77nqbnPM=JI z8e0YB>eszZmwx`kPx!&Y+88)ymn#*gRkW$TpL!4bLJl^@nS;q4`OpE@+0w~@a zfx3Sikpkq0^%z?Oj`4uA(nsK45D!eAyE`uNKi4n$fy3nb`Yz93uRruSVa!dRrnBtb z(y8iouJAR@N8jhUWGEq;6}kgjxUgFRIj<6!2P|Ukdl%)ken+|8`lP21u2n z8yQ{BoDA0YYPkY(HjZ6p*cwotN(EDMA{2Ph2@93Jvd!Cc|eo%~NRwU}Gb4Wa@(5s2%?%aiC&c%eedT?hYIA zNrP^x?ABX-uB)Fq1?OAGAdLnt8bUpsp;17Xv0Op8NAv9Mgv0 zvHM=*AK62N{d9Ju7NE89qW_G6$dQ!)*-C1*_84etFEW@=5TG}O{uZPRSSEMa+nOr~ z4zMyNScPf>g)=%cqI(qZWT*^v`P3wdmdJgfb4%F!0-F|ciFS}V3`Ym3X8y5+@yJe0 zWw|f55z)3o?9@MjE3C66J)Y&>#c^SJF3?h#e#Z4)ju;v-I8gPz1_}56(jqXLe*FEc zm+Sj3qR}+eF9^TNy;l&h$bhzTke%KkuAZX-9Cc|Y;+)h zOn9#BCc~&OW&nN1lY%4Tb-a2^wtWBCMhHP5}-IGs>EljlwrMAsToeoV7 z^{vx*2zM7y;S$x8mB34_;?eu7oAZ4yt8Alc5gtPlo|>QVo6E7@S*|a0OG^@ z%2l3nZAqlfWh{JOqg}|q})@B8FHvZl`&ICXvybCbmKhe$P%E$ zV+o-r{dCrGOg<*qRI3#@CdP{*yFFBbkX_-&z6Z9im)Q71j>72zdsi~-*juRibIK&A z3@nooyw8&FbDP$GR2ECccTOMPPK;ZuQc%HNvz796;r}V39l`(|cl>E-NUG@^R(y7> zmYmM0Y}OivsH~XKb3ImaN%yO~f(uEvfs17OPGebdZRB}BAKuNe;Q$RjOyQ=Aman?k zGV?{tesdrWK@Q?YE+}bETX!T3DqPh7uG>cD%qZ*qceI{eDQT6XXuOW1Kvef=5@yi zJ1qRUq`e@I;w8`?xJg#SQ}!6UBlG3BA!a?CV~pVF?pbzs0E^7qFO6gw=!!rWH4_le zo?Z0lW1wLO0`QUOLcwZ6X$3qnh=+b`DRTLx^qC|3u)Zuqo9DOryFHwv`=u=o6E=jN z7B9h5$SctK$I8|#+o3PIO+$bQYrM1>L3SgNwL69u&uOqnp21W^GjTA1$(Qqy{#z(x z#_{XYIzUpE%hDuEs!ULp`I&NTnM3K(otjGTC}f;U-%wqZ6s*_19$`GIG%k9Alo;9X z?{KfEgo#O4UH11O3?aN_0uj@s!L?t=;UdT&D#vkEu4+seIpV$Kb|Wd{mtBtRO+|_& z`{n50<~nuq1G+kiWpukzb*>WA`@R>YG(%fi$yrkVisf7!>gb1>VtJ@>nsPuquKD`0 z_e z6R!3OpWLZC5n}{(nR7QUZ&{~*AcHI$oqK+Fh!U|iM0rtZLI2HU z0s0{KML|~&gzN|X=Q+=fy{BzURWS3YJdU&Ooy1U((Y!$7jD2&!lXw76DMDTE&hwi} zogS6Z4AzVCr#C6u6fS#SWgY*sChF1_X2ct@R7}=XuN5(63*%6sbPW8raNv2Sl3P zwK~>DOMJ6SDkdL>-Ei0@L;^fpIGAV&xEi{k7W4fDEXruV(j93BPW$B<+I(17+&H96 zAwR6^@ga)Qt{K9e$M9bw$wwrbRE`XVEPph9W6@n==~oSM1iStDylGCYiCyF)vrHrJ zFSd_bV;>?bSb{Ao9{`=u0&Lc~yxnWbXjuz1QouPemrTU(0QpGjQbNaAC$v#YFqBra zO=dL6?FEor0-oMqxDCso;oVzkPvCn~!prkF4c=ZrD3_)xqPRY zHyaArCw55e`dk2cucI~|uki5e^#=~8E;f{Bn2}P2)$HYI!nVV-f#MmMVT??@Tol+U8aWu~de}lbCEi zgF@`c%oC_h%9H7orUolvZ&I_dkHr@GDbgek`Re+w-|1<$y@efV0!^%dp$Y7ZXimk) ziLteXuUq@u7jGxyN=a(8rpKU?l$Q!8dDUdm@pAYRY*v00gd)DR4SM%PJAG&T2;MXA*?D}d!_0>0*O zv%8_pFS00kp+~V=>#Ngz)VYuQI1H3b)^t^$lWF4g#Cm0fUn!8 za$d0gEoM5dvfZvz*GK^6$<3+T#CzP&^g2kSD`QP4>ubFXvf>)CE0S0;p}>}!cOODU zJ~TC$yQ^AI8`nkanFs2br1*Cru9qA_2ew`P2Q4f4Q}uI>0?{sF(gqA`qCQGaHfxRf z_3ln1(3%}QwYnL^+;MppttQ%c%t)1rV~{kS$Y8M) z!dz?)uPAk?8rLgi`7Bl$Lp4)WAX5dD~WWnf7&-P#bKBpXlLP z^Gh=d5#CtoBLqCYjgPlVlQ|Ti@I-ro_Hrm~4G5cJTsem7p~>sI|E!UvVmVv64^;B| z|KU5tu?*dW0I{@sTnvGFjbK>tR%5*>N--R;yl8gT(sqd*S*eYp9k4;={y|}$Zxa~d zGUF0&ZfUQa2(H|~yU3!bX5^^F=H7Y?s5r2tnEE6Hu3~{sWpi&!+)ZI(`YuNz%y{%v zK8SA@z>4sb>R}_3w@Hu^EgT%WI0n1)^&0kPhqOF)JVk1?Zxi>!A4+N(0JytC3l#z| z74&_N)$&K(=QlwXgJA&`kD6*+-fP64PF6=qHgbIy^%$9}C@4%MA+_0IH(2U&12YM~ zl+e%NZ)PzfLRq2{X$8SVN(3uE76nJ`D_3GZKT`xf8XpuJN!?lvdkf+8v2aU~N>wU82`N!!-ZNV$6 zCg{><=!z5d>4?a1!`GdRk7ABXxD>JDA~*f!7%Vdm}=wpr%hgw;shFjy(s9m#Gjyr#2(LI7+g8w48+ zG(;eO*R^+vhg(vQXBmHgxH0sZg86*3JQ63_OPXn%=%N$_pBykGe}ZP}Mg700iPF&i z@t0$!OnbLAm+ARyjTcCI*kEY(@!T5kC)ypjlXwU<@!{Mg8rh+NmHV^0CPJ(%9a*kr zL%a+~NjIfd#hzTHNpTklu&E&O-J&wk8Xh}U=A4MUce>G-_k zkUDjp&h>z?<>k~+O205S>rjz|25>9YIR94PGwn5KXU+Gw*rO^gQwU}fPZ(m?d-l(G&~UhYVxh-ve!@>Xp2bg#$QIeW z^S9>iiP?qj#4xQ3)uz@g!B3=+t7xxr!Q(2X0k4Vvenn}8(Y5xxMb%bfI#c)X_ArMMl3L2mOIo#wGJ*AKW%Kh?GR{P3ji3B9%XygNtsR;K%kt1lRW$^bbIH6PdG)r} zrnAeoZEAvWx$XF5yZI#SLy0EJqP~r)T!r$m;!-=&)2qD{VY*kPZdRb(BK~$NuXccz zPsq32F}2LN>?h)|o+NfT*J^s*qNJxrpBe10|6lkLPp0(#?V!5m3gBjv)n8v5Sk--?V&=uEe9ev zlH8ltF6w^xP=>eh*iGIzhP&W;;KiZFkTbo=RMUi?>GN?4mn`&?)ZtlRcK;bpP~yTvDo^0Yry!&dJ3=4 zB8>k!p3qLmN%9c>b$Ayem?HAp_Vphn_{<|wR^9~evNHJAkDAZufh?_^wAC-A!wbc+ zrO#RV&VC%K8(TyTZ4SfXFl$%btY2iNdWU{K|R1>&?PEuDmY^PWIF`oR{~wAgwaJ zqW|?OsLo!L-)7Q+$yf-?^b;e=0}7L&EL?uADi$pITzC56J&9QWcaZx~_sss<>1aOH z?e_w9Wi7_(EeO5Jt&)Tn<~uY8XizQIJrecG*8DOzU|BwI)i& zplUsc1M(tV;Hm3P({SNhY%aOoge7vk&vUwnDe0H9L60%j%$0*vlG3`j{TV?crpg+u z7URubJqiGxKC|#PEFB#`-qu(iA4j=3>g+Ye9n8el{(M+#Xd+V(+}MqJ52!?eqpylY z!3?>UK)dEgbtNaZ1qEE8Dn51u3z=1xDJBgLl@h_UwWuEsbvO(wM|ot@>{~tvUy=QY zUIvM7WR4JeUs9n9wZ2m=dJ95|5!Cdu77^?QAuA=G260y2bJDa5_Yr#Nb#bL1$hNyh zDQLw-NZ{q5)^;~A=|KN<#LyUV2iXHmh}}k#v)dy(!{%V{u<9hkt&^ z`nA8*RQ&~1FXGvU8imgfod5^tML9kx=-~niD`!fOTCCZ`>Ghf?jqQ8Uh@_tPe4bDCOb!rIsMW?OpmY@SMlNEO4@}i?pzq$FKF-&J$qtQW%-7Mqy zH9VlPxd6RyGgiEgMn^Z=%WhPIxL$DYnK0y08i?Kw6JiSncZav9jJyaPD7=oou`E9J;I{i-Jj+0GVy83U9q~*H9#ZYwf%z>GuG93)a1ZZy zv)w74NnM%CD17Kq8j;ozntR`~Ag@0z82ZIK2@i?7Go}c%YbtP7n07wt21!cah`lu| zp)^KY9(j?KqN6FrT;?s(K}A7?ut zYuVA@M1!wy^M?rC;_AKQnqs0c&@hBOY)joXvoE%)>hk7E+h(Pf-pX4XiCt(bKqqjb zI}RaCc6!o`sobDa2P3_XE8-VFh!nRFD42d9kMlw)T|lo z5g!v?fIJrQ^z=yyFnN<@%NiXcu@g0N48@sHP%DTW_R? zBt#~}E8as7-f}=+Q2jnjhc)k|S#xK39T_rYNI}d${^-`dO5oyd8Hf&@p`@e@njEBj2O{BH!rkuJ zJHARHLZ+29GkvT_@iE+Pp`IbQWA8`(F&yo8@9>M{fYg=d%GQg*XlbHPwb#&z%dMz( zKxO_k>%;}kmM_PL6Y@Frs@!vf6vzqB?=TxActb(DRscru9PE61zF7@#XU)k2ePmrF zLQFbhD{g?2PURiyE5bcWVeH(5YsrI(kVmmV9M|pedyK^hVEboN)sMlD85@~A=ok9c61We=+DmnwmH|aQ5}=5666LNYVX|y1m$aZKFuCZYIkkj zosrg&w+Q!q|3V=G3~uxVahqPZh5GC8&faI5B6L8oIpbEuR}p z`yp$oMhC+?0^{sRtwo7T4eHV+&$hFx2^@eqV|kz`{>grn+P>;&LAl|_hEm-(32v01 zi$q_PSq|iVm45aZjR=|VbYcT-q^%cdo~U08b+@RU>Ffnzu#4}CCR$?|bs8Mj*K0M$ zF6}>lnxOU~|EH6*q0r|-8~8wt`|$4VMelrkaYLjPL)a@Txp7Np1qHmCt0Ls+MQDe& znK0lT>^ZduP*8{Kkwif8PpQU91yO`Z7(W2V!R~t=-}}wK9(%~uv_|eY2ELx=%X|5B z_!P-H%Wt{%F^zTkKsD~LIxJ2%I7sdFLM3Nr8(?8_mu!ixl~%z6ia$8piN<7|vmHil z`s!D2lZZ|M@mQ>n*#%)|x#61ZmBdS)pJss`r*tt7H}WFolqm3lw@M>WOH$VoSY-UG zL?^mLFNODXlZUY>Fb-qyqr0$D@|%dD|FD*kDUt>2h_==Xk3*Gs^0;1PmWLhQoV1ks ziJsH6o8lCVRI!W)Nsr1u;Zs$&?#wCPGTrw#(6)2iU9J)k?&IcnB;Vn3F6I#v`r%v; zX6MzODmL+^cyH{+>D4$1h2Z=0x0U1E+1*69jHmNYLD}x9+H-)l9$ajf?!WrI?&oG} zr_?eAG4<^OS<#DciLc_30q|2H32)D}>3I|`wCK{`_9B{8gaTx*BdPfM@dU@0niFlW zYC2y=y0GTe87Kk9V@*i$_dL>OHqrH^n3jfpC}yJbhxB=t_eq#Np3gRbOIaFUq)Fpb z)79l7N$#C~Ed`E*i~CHRBHfkMe_pQKdM5Pjvf>hS%d zsfo~-7&8N)bTH^nQz&B5#$HIs$`%%{?3QMsK>?OhZP<}2P!R|OWRz2q`Bq{eHCrdf zUyj1N0d80Kttgz)^Gi#Y@xnV=yxLWk&&LKOd3j^&c10ii7|vpNMSVdHamsD;Fe?dC zkI8pIX9L?GUGT0cKu-Zf;Q(Ue3nG3h&M_#Ryo zN**DKt4-?`s!{=iQd_$M(uN!ApdZMJ>PL#POB{%wM#hN$_22*Ja=yIRm4MV!BiX!+ zC=rA^`_)1V^8BzD0I6{K;ojQ3$?WVR^>JMhfQ16L)sQApM0M&V?1Rk2T$PsjphrpN zZo1fS+J!}u-Zs5NH71QcrA)T0F4Hx$E=$5}L`N&Gozz=wV0S@#a~?qqG;?%N_A@Ql zTAyphC{KI3mKp@Uj^`!dkM7G_@bt}!Nz_3PXKz(*0tG0iT-cSx#1q#+YSPu4j1eu@ zI<(SB+qh8fg~ej%i1&E2ba2q%T@hGhF9fI<)NUG;UVvQAFt%zDU5+317?9Q8H%nJn zTdNAgu>_i=F=p1;(*@p?yBKw~RU|6jyJoOp!x1UH-O#G`3iTQGly7Cpp`^swe3&|) z3X=bAFh#G81*|?JHPF{XI9<@8NXggJa7i|S2>+)WD3L3#NL@tMf5p)C<%Uninj9>C z^_Fk)zR;a%DtF>W|8mRvO{DS-Yx)*Mwq4VMMHuxD6^2|qBsNS|DUtgmH%Wm?+jE1X zO=;$BDqk67liHq81Xec(WI=^!BJC;Exw1r`*2 zt|5Z2AoLs@vGi=hy#*1r>0R8)Wi(;O=GB8glQGZx4eu4CF%)KrDX9 ztuo1vWRi)8Dr<;)uyJ(t8x8ahrA$L*Vvu95U>m%gyMbbB76~BvJ#FMSX{a>$^O!E& z;@r0-^O2|kR}I8Y>fM=d?7SBFSnH{|s1mTgy}zkT!E80bqq@+Ez0hdwgCufFFeQ;<@u$LVAC()xAa8;;#oN7+i zvj7H6XT$CVl7;ZONq}Y4IAC~F`=y_RMjcujW&S~p+Q?4$tpP@;|0pPnwvB#_qvEpx zDJg8R%BbxR$et*%8llBb2^7$-yeq~Dmg;ELiI{LA3oW9csOmdViiLzPenY~cYBa$?9T zFWu?LIjWi_5~bXBK{+%rn#$wlC_!<7FmL3oqR5LQpH**aZl>IU3j^xmlKx!Q)857G zft)$5YP;}JGTfd68phq4!EU=nBQX!$_oj@VYJU=o*>2WDtLYR&0V+N&v^c*$dit|w z&;#l{?Ct=h+<{N|@^Zs0+S1=dn-y48RLP@ue2%wl4cfg|=i&XYkjNWmUI&WSsUJot zm2W=H-(TwbAtkzMq^CyK_C~CJ{ehzFKV9`{1egi=V*$v78pYda^|y7*dl97`$$;{l zzpSjiiq5Ia%oC!EN=)rhh~tm?@@d0hQvA^)qxmU>UV;zH?{%P2v}p`^f%F5>mtB?XTP*U zfX%h|3oj<0_U||Djn*Ogj>af@u9U+)a@qy2JSH4<(>|h~On@$xf zr2W7>SXHc0&nwx6T$MRn@9|w8fo)oy)AcLzi$Dkg%K^ z95EcF=0R!@_Eswt2YDi~D{W;wZivg?Z0E!xANT;5?0}+(vMS-{HY*`0sKvj+4l&xw zN|zLs@a64_#I=)uOwsrzWJI!BSCA&6{PEi+#H9wP`!q^N#wI~Lbe&c+lA?jpIbK8~ zk;;uqiO+AIQ)9-6AzJ94}E&P^bMUs;!5XR}DV(R4qsGOtet z4Z$lV#5CxWkvjza|3ldS__J%oE^VsRbi$-)t75ZmviWpPip=fYqYZl11m7`d3U|nI z2r)d$Btou1R91s(=CJHMfRl-nEs`aIQ z47xL=SG{4cYw58ONB1@^%qs=eK`bTPb#_lf6YpjdXIp2!U&X1Fg;5Jg`G)gKhFQe1 zd%64Z-2dPL7p-k5B15}9G>AlnF>()B%aBmQmi>qE^mJwRw%vRw&E392t-N-}CU! z7&X5Xi;;l#?aOb~?}*LyCN4SL9T>L7`!PeAkYIP!8u)8*^L{I)NjuJdx7)OHs$W=l z{Pou3p+mcaWQk@;Op9%(@Nd?bAs!YA;qjlh{p=Ke={btS_~qp84>n~{o}X4I%9!e5 zJ}8+=eg~~Cn+***T84s@D`KJ;>OY@GDc|txQeOAwFsFvd&mNUhzK0#hkF-hagt&U} z%y2pYoZa~0q>2{0wd^p~kJb%UzX=ErG~RaQfP?FV_h>&L>O{&6(l9G8Y(i0Bh<*u@ z$#4zl18oH3lu6kk7oZ1wh+CkWJ>cYx=Z7HfuE_^ zlnT{A*p~Fw7HX1b&Z3lnY+aoRbQ{txBm&%~DFCe3zq1hoT1Bz@8*+TOWWI4ric)q| zo!C+2p(|dq{R5{}Du&$+*^P)HR;8B!IzZ~(KB_N0Ek2~*AZUqde(Ii6GH*X|T4g}O zizN?2%r=#w>QlZ*k*ir^6*7A`^QY-Z-kB~qXlR+TrB<~hk2H*xFa_$C*FW9bGIa@w zPpTV^M}AS_yY@=***g}{`vG8(+Wz2=jeLSnU>9YH`llDFP}}R$Vd$JqZ^{=|kKuG0 zvQNC5%Zhy#3iE7KoGjrkfx32T|76_Jki11Iw6`N)** z&UEt23uR=B6qHHE`ZiCy=zH-qf)V&S-@^td1UHeY6Nk6AEP#5A>}sYbcfHxDhk8t6 z4>y(8H}4iX&T=?~Z|67jENLJh*8%Q4NO+-;kc;`gnIIA%NiVIM&WMRlnE>p`nj^i2 z#W0CI^?6V+p3s~-FOvL;g$^ICX}Kp&IBcv}sf3-$$F`JP_hhmzX7BC5ByCWaX``N2 z^p)(pK`t2AF)sg_+J@@{#8fncd_o6R-Thsx=$QlDuDZKx%VNqxMDhRBL~U!1amC)ltZDe|-{P4j8H zIStlq8%(6CS;#ww#(_?d^^-~6uQqC@THAQ&iYC3L+vHI_|MOcD#mNp-3Yn{iZy06F zh6pzsjD(`vfBM~|_?$4boyO+NH zx6NT9AO=ANOP3N~Z1NKAIivJA)KN_SgvZxIi8#KPIS^uped_7fb(u|C;I=n-e*1TW>%SIez z+z#?1T10!s>tg$PI$Q2xZyAHO>@1002PfwZfi)<&LqTE61wEx2(KvirQ|shX58OLSCL$dcPUFb!h9 z$Zp7f_&}q_Z6^B~+%$D)$Yq*fQunth)MWbAf^!zV76*H3^$hIvvm`{;u z-Ix6YbaRn_tSjew+@pK(IKMo$rqAPf_WwK2a{(`&41cDXrYc9R^D)y< zGbqbst4|@lV*e=jjZ((is=|ly2J#J8WE29O!R$o;O1DJb=4S-U9}rZ$E>WGnUut#y zj0IHk*SUNBn*R1_H(tN%5vLlbU2-Vofp-44ppw^`rrt3){9g)wITF?J++~s5Rpz}L zU#|Fy@pqEu#6-WzGj&7_g-ZjpkOkXrRfF)${e;e zH{J)*mFgd)yK7!)^e&tMYWXS6Ukm@Ua;elAW4_PU6wS*!i-Di<)F&y?c5|irGKe+c z%SPoD{~~3ibtsTFCi0R;9`Ed6d>;f9MphE=Vl?{i<`PuHspesOPDx2-MdmCR6@MyJ zvX!hJ!qUvM+I*p$pTZKt!54Vl#^i$fv)^oX*y!!xDzusZG@$pBCw57wIxMjZT7%40r)-7491bIWX)v9msX)uR= zVpbuIX;<3ZW?a$YAA0*y^#hdV?=87>sOgm^-W@j3>b&)+zE4X)N5v_)Lp3F%gVrn6 ztrWNFa#3l(uGCAhUVm^mNg1k_C%u6-EUi$xFPc0$9m)UycBxTXVsfy(U6fS)+!HFv zM+8c?e%MQDtn}=cM&8(}q5im=A4?PVrwHaCFheTnuiHIoC^&(>^W4A|MWbe9Y__8e zkrrJKFn~I$QsxS9JE;nnjVAX`3B%|e2GB2;eYYDjkEg~^Q~F;S75>q*>G8{>qXNN% z^&6PVNGRufZrc>Rs-a+fS8l^?(~BPu3dyON%q9J8E>1X(PLpDW!t{CGEiQOdSU`6U znq_81B4N{EX4>~_PE~bf=7{d+UND39c??YUrWB-)_g2pi#+6FboY12WvjvOhkth=$ z6l7_Irv{C$5PSep_@6%$b2Ivj19g|fq>RdF>Z_H*fj3x{waDG?ROPoBS}^0o*0ey=w{u$TOFKh{q_@QRP# zN)DX>A{bJEYmq%hDBf;+1IYd1o+&c|;PCF3L2v_@TV11JWo^&6kKkqJ0QqXWN7J!V%3?7G{$ zOhL5ywZMT~1iSCmoE*dA&kRpg|EdAwhfNGMTJ6H9Bcal8?d{W@kPRpZzOTBph z@FpQ@M~KSU|ATtUHlnCE0O6kO7t}>~usAc-&okZ05c$;l;+*8xKhl8#Rb%m(E(3WQ3Ctn-63+$UrB^&ZFpI7RC9Ir0& zwjy;d#I@Bs5EjJd^37JPR>&e1*9D7VL4uIZbKR@|C>|z%h%3q$$lKO}%l}Sl!JBDG zKbXrd#7?_ATtGU7v-u zwfg@r`%5S^t)uWSl6#QfT+Iuy&s{h<+h61v`S^6=#@b)w_ZCYMX#9)n=5dpMsKt~m z`LIiP0*2SP9H#58#22#IpN1qFh+5cOlw_7*^jT!qP~u@Qy1W;+-gN|n&R`uH%{}~N z&E4v6y&2m;UICVYqpl93^2>NTx7|4PT;6ir^BmKX$eD0zZ%2Sdvk#XSgT6#}ukF1O zW*Am}@zgbDVd1tBN9)GA-~5bs#?)!sV34uu)Fk^kUrsTg&^-zxzhM#`NKiYKW%cdd5wrTa8=4?Re z{W?)Jd^~-+Q)ycDMx#g1>ym>nq{$$qxjo-Y2y=QS)Gl=Tjg?fdzwfFl`uVZrpta%u zuN6KZ&`1;|Ag8ip=^4M-zARL0KxKDwN%b7&Tys-A4kHX;`;-`#@M8} zTNFOhEN4|C!dHz%Z>*Rp=Ub1rt@LQQ3VZQWm|#m)3bB5XBtnOi@wwB$vb87CO>fp* z5p9&sqrGd5{(2dx{$_+P(E}vb2cfsa`;d1M{Ea29N4 zg?N0AVBdZc?LEg$u$)8VTNwAZboMH1xO)%W*K|*&*PTpBq6I0nyokskpW$?pl>7FV zb9Sv5-`begHrkotAw`S3dQx`%TEa>XK7?$gb ztReh5M{9L8hr=g+h8p@<8)1Wyd7P8|F_gwGQ5<8PKzNvWhHpd5s+MpfOjQ+;HNO<2 zJfi`$o&^t0c#-_^8cz2z)L=4iojcCk%7C*}H>$>8NC2bOOJVaB!^ z-}9DtQCYJCS}(8JZ=Sasfu?yO4yVa52ky?L$(y@2y78;0I1|+v9=e`c^AzgOnYjAB zD4aIp*S_`?s(OzX4GVxxL&Ke#vXN%ef!|=;fH}n)m)r@h)L-}yu`1e4pR7n?&A6Q? z{=}M4F!|t3?Db8}#bxl6TC*%BQ`6AW1!B5z%V zXiC}9AABh}d%~o!IBn`Q?}IlU?vMB=U^>U9Ge_dX5*ONy>H=?rw)5%JunmTEOfb+a zf~+4Nyks8u-^6IZQg5P{p7lzCb82ldOtJy4`R(2}LTRm`;Erhn0QYpmS$MIb`8oCa)i?74AGfwsJkN)&50Z49TuR% zx6}mZuV>T~z*7wCKkQ3=380bSZ>pMR*D*v5bTCG9NOJV!5&vp%7UfcS1gA_7U-oW- z`{|5&$UP<4>Y-Q+xArp`gYTDy;L$SGFC9)_1%M9+GD5)j`1W|nQ*^?)ETrf`^c>Y( z^y;~%BMnct6B;~qn$vY-@h@=rz`tT+rh#pPAm?-rQs=JM5?xHG7QDYf5U$?9C86bk zc_oeS6DjDh(q!ZeEJ|L1rb=_ZdEVFD>vj}1BnNy*aFK2qhowlid%&JiA5u{71lw+9 zh@IY1Tnp6e0-Uj1w~CR8pc95r_VU_%#)CruQfK}5p&c(Bq! z`$IeY8A4pxuA}AeeNUVxevp89w;V(z3#X^los_+e=3KDZ^A$+@*q0mC++){R)D8&k z&28$ae6Zc=956b$P7^qu33s#6;Gf`9VzK1rz7#8+AyDCuhdzYV0HFr4%3P?jfa`Xd zbns;&dk!56qBfZ@e1~(Jba!;tmcE<`vis`4e1wY9%3@+#>$+fBLmbqLh*Pj8r(000 z_qB|9y|99399e<>3Ok{n`hI!dV|9x0BWFjUkGuW%>nZ%2Y zghwhrd{u5BkE^QG@@santY1Y$;$*pqZ?H*Bt%#*sn6#BCDgF85pq zcyLp#h92Uj_<@&W;7X+tOzUf#4hXa#xtci(0jvsq-uW#Vzi6Gj50#*U0Jhbo@qOu4WP| znTh6zZOqF-63T`0K@UuR>hVVEyKH+jj58!(Y1mL`Z`XAen{YbY{kuTF7lI1?&2>DV z29!u#$v=(QU(2@3%j&sRb*BVl{<`!~1cKrq_H)^%B~!2-clLs0bLn4?Z7$KxKIlDO zPt~fvd^JV0-j%dP#16z?X?8OHyPNLsV)DoPUAx4W3YB`)jyhACDYXRIhTZ~@=@CqN zBdAgE7;+Ym-MiiPnzq~_1W00@!G+;?!_asvXtA>@>N;N&;E8{oIUG-Qmwqo1j@yH*JW>&7W#c z_i`;7J?+wV!KNvg6h_TSV5G$6Pp}G>9a7mEBjkqhxf1S(iqZuL&?SL>Y|op#~!C8(bW&hcc8IhP?J%|;=8^Kh%x{Z>JW<)`cDTNIpP1Z5b6%H< z1@b~)*r|}YgV8p^5%aCt+Dsi$G8vYB%S_|CaxEIZ+$Ozd@UEtmQp1Y|VT~FA4Jfz$ z0~6hcjSZcLl`mh~x8)^fBk1{S(MZl%F@+6|yJVeJe+68c%;R5oY1KK{)c}A7jL1c^ zfNPyJ^I2|#$7?T|qc9kbW~!6mp`;>(n3+x{CK-A=+syA@dY$|Rn21#=#?U-TQe=c| z0go{*GCanScTbGk)E&EamRAFhg%x=vsBt1nRsZ3B^K)3<&?iB?6vr_Ci~!Kp7dd+@Sgn4@NnZF7Xa=L5>_rt)Mpo?c5DRLa31KR7?cox*UUoJV|k5)r49;8?@AG z4xJv!H#gCa;0a?Q@+=GCo22(vteZJgVbihuppPyUBQ1Y6Gly?)?GVA)SuWw)qR1`w zRFT?*8)USo$UQ%+!El_5wbxs{->YS{&@7oz<_k68?oKv5hnwird!I|T2^hsuv_P@=7`@%1?bGG9dGjQ*4NV7 znpEODbNI zzYcLg{7C}QYw{El_36JH(*4~oiN`Y90mO}!5KRGV{#jDSC?XcmOS=dhc_wc=&NFuE zcr>Mj`Yq)X0qjfCd172DlMBg?2BR*tZ&`{z59wNFvni}I$<4}P-0S|Nkpakj+})k$ zrG)IKV>8}iU!MnSRtFUCBYdvwj)>~hmy?+`{>2YLA?6n7VnuPD@dGXC2~FJV63s+d z12|Gr*|y)sd_)aRhL@sCp=m{LZebYx?QAFa4?!~EU;@Q;IIs-Yv2$^eTF@sD6hBW3 zp&6502HfN=6t@^;vVJ4s{p(vdfp`nT;##`{JUQ7*t-nbQj6 z3&jSs*}^P<{Xia=$NDIb8t06A)l2;NCPE%HSp6f}TP|ul5TN8b8($r!;+ww?EY3n? z2S<&VavxoG9*6ROUWQ%0E07(lf1C>bNoBH~oQ#YqjAp438dJ=0Nck-mf`vqy-D*BI z8E)-07qG_D1S`E@&$c{2C82!)!g3*o94{x~JD8cm4IE*y&&X%B(my6O^uN`*`1#iJ zoIj<*t&;QG3}NEzvA9M~l1`IEO90ZjZ2R^->sln3Z-Nn*ljrrGq3ri7ch_=oO-})bZIGnHT`&vWCsw3bW)SbwMmh5LQQ1Vcyo{|v& z$erv-X9*W}mhC^{3A?^paR<=gN>?~1f?bwQz(`x-BM0IsNrSdFI;*y!fuT)`ZrA^F z96y$RF2%x(*tUn(7^<1fcy|IkfT4P&7mMda1OAEE%kNhz3ihhW9pPJ=P6&I2xbrwD zFkG`OL;XP0Ox!X!OPC#K7Ui0v>W1A!3E0GUm3I?P(ziK!zj#$w^Rc*x)H72FTJrM$ z<7wh3@mI{<^iRYk%LMA^C;lATD zo?6$am(Y8OI;u%gA0_LeJX7=RoI}=&^76F17-S_D4Q6Ic@^a|jh4}VBDX#y~U?1Pj zR>en=)%;}1t>I+;Qwioc4*}Tq$D8*iPiXY34|UVw{l@;tzDJ2ROWxP6(nzoW9FQVL ztS&<~m*Zzfi(Rw9#Y6X+11H7t(2E6bYL1#FtAuF1hsuj}_PKtntW@aibKj*!FNkVWqgc@z=*HIwS2_y~WJMu=D+%$8|`||TSMW5=!KN*U# zZTBFbpcAKre1_!W607m+n-V|S6OFrE680if$fTPDtyQFL_>0X~ysMkz4Wu_suF{WN z_k`e?$4mKiAdJFB*d?{o+)~O>nZz5Ymby*^4xBxCb#rvfO-4B16R zk zV>e*62Rv_QEZq+*a5#2J`c&WYFmv(g_F2|)sj5s263s#?P;50E9PGYQW6UW=QN!9* zH~QmxJ#|6f_c@Rl7dEJ0$tnJgL?4xp!d%xXgbcgpZ-;8^xnAO=^T_mp_Z!NiCsBSGpoQf5}V z1>C_OyG1;ortEHD%Wsk}&;Vc!>%%s6DEe_^NUju&dPd zA*$do{CjgD+9A7SzM*JUk{=K7h6)2poX~`+9I6SHyXC zVG)WRh|+RgQo{hNVX=KRF*_p`hhYYyq}`=`H=|Kt&wuVE0u!!dO*F^0jIL6rSjJ~k z9_y>vTG^_@UqMIf^vJv~(2}hI&B~(_mxsETaqnueu@JRiAjCr;%C%935T}R_NO24{ zAHa!8F*7*bTD?P_D|&|&YMpx{gaPj{yEDepj2oz=BlNOI>@7BM9B z(FN{8pcQASJk|w=!J-v*f^iVhUIGK^O4e)fblI!ErWgc<=HWHssQXi|tjXXcBpn;G zn)$|DL^da1sdE9@Z>|)*d1_v1)ODDY`LT5M0)c>HQ}Q|@+$2Q3jmw7JxB9~YpQ?AF zbXB(N^dWbRfbcb_L7vIlhha~jfMRT!n!jI+<20HHx3e)(4eON47bZRk&X;`-`Q*rA z0D99QQ`bv9j2WUn6n9ufs90`w{`myBD0A8W6u9r;v2}c>)AJi9g%(;0p;-AW*vDN` zd7$ojBEFiE{kVDjkN>+~^fh1MHOk)Y>FJZC z?9mFAwZ@pfDwSZvGDzzA3cGo=2G8hg;nO?@PgO7IOrgm0A;UBTukCTW`JNfg0tn&b zztzCj5GAxFI>YOcW~QeBMMg(q@9?Mxb#Q=MdM3wCYQ7&w&nOR!{1M)qAh=`l(9Fn1 zJf&8nPhXIKg+e6$4(na=I&mJJ6muFGysU`3F&d1K4;Pxb3|(Klfq~krdaTwv9bSyE z`Zl^8z8s@umBDknIRc}FfJO)E>DCMTU+NCL4fb?Js@ZaIL* z;9QhCl2M$nIH*TVgxQwEzSgYaNZZKzN?)V7Bh}|^NSF;Ao?(+6$FwRVi_aH%-}0yX zP}c`%YMy?%N6&!|J6r?|eUM+L*lVruu2%s=$`aixdmf{CAg4^A0k|T9fhp zo{}^a|K$^E)z@b)-InO!2{;x;n}Wq0UsC|{VkwKC&l(W83-IfNa-zo9`bt2q5+KliX(@?v60fPia8_{?AuaL^P4qP&E;a@nJPl3qgkH)HZp zQ%CjJyD_*drnpdvzci@ZIq!XMC?^|-PV6$L;Y|@C$U1c`AG9$EsLe--@0ogoExK9R z`EC3rPF7yNPa_+1TBBlm@pO7d#<-!m=;ja{!>;U%zkiauT;%X6e3UA6_0Rm+xZw`w z0Znoqf(2<{RXlP4!MCMhf)tBeo4x`3j0}y%X?UZ}e|p^;B4uKFr&8b^>VMuEFU1i( zD@d_@k>ubNT&x%(Ln`Eoo0mN~Z&0-4rrfWp!UxH$gXeJ=%`LRa8gXTKd-rAK`$i#_ zjk&kGhzWc_Ip=;F>;8LY4&khC6bwyoLX?Kq5|%k3YivqHL9Flc{#0Y&PL!1v?1hQW z@djUrOCa@8pX<*;7|7{Hr%_kWY4JJLIAXuQEN-fHQ>e~*8d=B~s_@S>zjSv9ag@(% z)qoSRIC<=p!WbZ7bs_;X4Oi)y6D8PebL*w{eD?dzf2}1QOqw)e70jg*z^*9?{5qT} zx^dGEZYZ&}8>9wnlYfRE>XWSL(+d$0vLbF(&4u?pHMOs?H9 znzi33S{LekQ}vvDa4H_muRwsH&cloe!L_c-w^7Nl&Z=5(^@CWHV5kwgcA@FfdKCgz zY)dfXLvAz-hCyaW=QdMDR|ab-#wPF#SqOAfep&%DQO?pHL*+vK?{{}I=wv@Yrrk*_ zzFL#_)99A)LPuozYcyOW{%wg$rh6GgVV+LvE9#LJ3nWi3ifvEhdSrVN#gxv<17SR+ zUX9V<%xmv&$`7ajhxiAx($r@&Fm>tVaqz3WUa^W?6Exvbo$$FD#4vqn;E zn@=5QeDFgwsqHX*77+3SMbgQ-RE@i3f%i?0fL238<`Bh(^Vz*>X51FMZ(8@B2yhEh z!~i#8>j7p2j#(%78IE~9F%Raq@v3Z^_gj4u(d(nMBudTu@obLD0#q4*3l0)KHHite2FhxyW0(Yf`O?jDS##y(=-!HrR z7rQUvZ4Q;DH7#+)yP1y9Xh;bh*u)FUZ;3pO#OIWM}ektcW0g$f;dWI|{{R1#( zWe+BX|AD)vtVlXKO0t3i{5O`8K3P;gbpN?l;%6{}ukGDz63wZWlS&93kuVImq2_r; zY1&DUcRd96?Cgm_eHm%tC#^nL=T8d!?Uu${<*P%*L`(`hzhAfxmLL<(pLIy-Q1iWU zh1#nfJHhxoah^Ixm$3CV34{Z;K^7CV6Mk*=Hc?aCz6){;CBv>wK)nh>2!w|7(BOf; z9X&A?PBpC;zlkB+C3_<4J>U@9fgu~Co<qy@{g zBaid^c9B2gVsDI07w4WAv5i9;?NFIOHHIy}?4L%Yj?t(12^B;ZG}#+y%1yML3lwj( z`@bsp$I9FDv}AOQ6_aG6#FPVQoI*|DS!Y1sVM0NA>vx@j_|u*9^}WU z3w6zEyevV_n(V|ki5ej|(n~F?#vk?9=fEL?(0@|vAYY5Jp3(`Lah@YnnLJ7BH!Nv1 zUFw};0Q*^1T6n4Je+dbz`xcJK?`L#t`gY+koD!%-F7az6ooV?jaDi63?Z8*bRVn(Vu>sn9Y4dd;SN>V|N$zFwxiwW=j4v`zlBHAn0$#r(5J z0(e*On7#IcR9VkPr zK$sWPo!*8{B(MuB>$B?slF%NRUV21PI-;K_mxjX~#@G>mEQzt|t@fA2mhFOOg>F#m zam2k-Kk2{xZbwA*2h4AK2$VBoLfx>WD0kMV-dd2#{~@!`j`ck^fh}vg=DgIxO8|y~ zebG&&n)$6Tw2n?DBB6qL^vs(EK0Q^UZ1qg7Fayr zSe`QuC|eH>|kenPK1aTg&irQop99kJ2f(Rq#2mt=(^pA>f*uBDurX zANj*AvRcEN+wWgynh2m{xS0IFt1!TadmqjnNY%#F)yYXVQ z*Pfx&-c|Sr{)#stK0{WPBfOuwxoQ5?VAdp6>hnI&RNN28goLf|+pUvqB)0N$%+ZAZ#@(vs z-VV59r6w;WO{hDj}Nw+xqa#~H?&y(zv+Iw87q!zaVqTW$Ll2mGD zO}L&TPBZE0H96|9{;Y%ArdXY6v2>EPBc+lw{(jj9=%%k5|1MUxkQf4lg9+6+c0^TZbCdHXOSjd?U*P_l zt9;TLp_T}G4`h~vlxm95xTje|cj-rYgMe9Ka$%p!6q=XA#Jfmb*unLI-fYoaI@X&D zVps4}!UMW7GjnqWR%M1q+JS{C3VlumLxp5wL=3BT$V#n=wIuT*So_9oBsxYBMM9iH~r zACx5UO>=;c0x0`ttyrHl{N;bZQ)cbrC6nB!6>_p&T#JA`G z@N>erAdDV(ttl^NewUsgHCWd#I8qVxRy9V`@i&?SQJ-tMgmtMLC#*Q|T%!(#ANYQ^ zw#rQ#PJ@6{z3LDd(F==tX}kcbU@Z{fpWuN8OAR25Pj%UWE474^>G?PMdA4VHXxRPX zJAJ^=wz@h3B^sOslIU-3R#KL9#MGx!2=#-$Ux1D(hz9zom&$z`8@P&Ef1Q!p|I66B zEH{#*=Yo5G3e-+ALvl5%xkOWmfbGi_SYf<%yvP5?n6fMlUp=N-;Q$826_ zUgSK<_;>k#_W-kI#!P1FN|Hca+<*Ts--U1dCBTzCc5xNs^Yk>yfQKlzW#i%ixK#H* zle-KX>P}Y8B!XaLJyDS+>6qKB=FsR*FyRLrl6qcW5_g@9c@I0|F~85J{Vtk&RLtG8 z^Mlc6Fh(fYBxj|STlJHxbN=bdJ3_hX8*cmDz+C-NlJ+n46FKFH?4*REJG&GM>J@PU z&@2;{&qBAF1eiFyAEKt?sMl#wO*$^@^Gi-)JiT1qd8el7fy?Zha;u&S@Fn*_iNBnl zsBu9py1}M*csQbHlF}@uPZPZI6NlwToSArqb#K&jK--b8x3y12V(gob;wcC9qdu=D z7cm35?AqPnY|$u?Gm>JILU_hzg2R-CP!K_*MKq;=-^TGt33P$h6WG4Ei*3)@z7-i9 zPDWe@lXnSAaEic|fVWH>X+%lz9`6sHzwNeZ5$H&@TL>OjlE5ZE%5BSEZR2_WzxbrF z4g3X=`Co&AL?L0GhvTn6xpcPdor`F1<<4Qxb?i^Y0=1Jwdn;|cZ*d%eESzTjaS?RC zpgNJvQ6jjf>aGq9khQViL%!gKk)4A{pP|K7BfE;q0}{k1wcij@YoiMc5h<-D*klM8dJ0(M(W_t+`lacuJ_NNc*OMIoJT*sK_cJ>JX?gid zoNj|hHt+X!E5pc(|1WXHaAUvem>;ff^Y!oDc+y4+>s##E?;{^D@G8YbO8&~_%)C9` zVYb1K(uF?wlYtY=}j)*Vu7j(Z1*Gj(Uh1=+UrV8U(4BFtU-G zUaH}M)KaCgtQ}y<72}Hw&dVE8!+NSq`G?P)lRIk`DJd?N5N__$dgLH5?R9J{RAKw| zqnd512LhJa0c!YTS{HB%wD||_bGEPkk%bl}?lp5f2BN*2Tw|;jo?Wt_WdGj=LxZgo z-kV+=%^go6bKs$DMf-_!(7->>JF4OHffDP#pSvF&C6vp<+b;yS`kgN z7kfZrBL`?FF!hy|p;>F^2*WF8rU$eY*2Jli*%%Px7``qE*M*f$u+~IBVeY!9F}i>L z)NwE}Kpdm9Zm*OYr_qONU0w`gpTnlM+Y$C6d-nLv$}U|f?ufD|ibygUZ~G*j9Y71+ zyBTjrGT%sQtp(z-I=x!-gu}HO%ODgHL$)6C*rnuz6)WKCj4^POG8?At64!gwjv9ez z;nIP-o;CM{qhjPcSd;P4+Q4(;1xWKq#6!+^B~US|j`Oex|Z;l0&H#ul8&{lxVWDKI)?+!|&N5eNC4QJ|Lv}3%y>V zq*@H;Vr_e#{__Ihi^i8nwf8l6jCz05`0Vr-w;bK??@8$3i0TzT zu+ey!7M~1Eo)_S5UAo}f6dR9dSJf@^H0q2Fc=)JooIJ9(=`8;CDD%> zg9eFZepkAFbUoC`PokBm;x#nHTR0GC!) z=@vaexpXp5t_c$WF)p*Cv$+1}*$9EIBfM$MH#2fNTxWKWy9whX=<_mu(G~vN+)?S< zD_=7d!^|piPJJ@u0YMOG@Gsvc2)0fgnkXdxCI!OdN{8lge_xQ^*;GVhy%d6SbXDUir+^mTX-FoVX; zZ}oJDE{F%QODErl{JF+Yo)OdQtSAq)L+E6vl^|ygX=QsZ-4G9TuG=D=SSQcc_KB)`fH&IjuR8EL)lUm1@gk_wQ2cqo z`t1Z5Tc1LfYVI;Eq(RRTBp;@QLJp$jPCC=>Wwi31dvT=q#;+biM_v*k-iN`uuW| z>~c?f7oGxl^!K5;Moov%!7Se9#4tfVKRbB)*qx_tE z^p%^UkulYeh}uEidztK3PB0fQO$YTY1ykw|s&Ur6|~5i-H%7#l#hBh4D68 zgk1uZR^TnNf=^Lv?z8ARu~A>r7{|!v!dxLdUxS$ zCf$5hGot9=uW7(B*+GDt&8^QFk=MYM`I02?vNW27%kqHwMz{Y*1q9hKYhE zx{QpfyZfi!T~wW5eY%oMb?J`3+LvjkIs0*D>2bIDGac$;JJT;++Xx4e<)Z6FyV3i+ z`O@0Ct5!Jhx%O1b3Z+M7qWYC?7J{%sgWANhJ%FY*!Ho^z$bCL^Q+!zgd4;7`tS}(PUGn^!R4Sy$uCVtgN28mpzUkpUU!#sqTQoo z!4YF;c@P5>3KlDQ3n`o=JQ;1{Nsa^TutZuX6PHdojplAQVh+<;trnbt0TFu|bF9$s z11H?y#!&(4ibSaS zU7O@S${4;YmDH78FUrd7IMP4iW%oQ zrn^mIq;ILR`|6&m9o%lT#XESPEMwL->#{CDlanx)(E>$F=W8JaaS*Oj5yV{TDbA?d2 zitTXgy=I^)(pSQhsW*G%ZzBGGOcmEt=e77vK3&&AHy&@B)3ig=DY_MbHA?+ zXrmB%yPPc?5fbSM)$aZv+=cu8t!n1T#t%)^`0^&i1XeD>uGwFjT|JL>sA(8a|1n}; zhqSSot)k>l;|EpQpjKOjX&YguaXZld38e~(n%uXZll!&;9&x(zcx7__Bxk5_$F;|z zV`u4*Bd<*da&^My&@n*UiJDSoK~a^^nv10>CRO^&tUULI)5`_R>hk7U18odv3H|)6 zLw9F^5KD@m(2S3(Ov{#p#j8IhQ8Xj$o3Sh+^rs_r7Iz(U?beg8*N810SyUtk#{bYL z!Vt3w=`d4E?q&y1dA^*ubrs1z2jt6gbA}xTI&ny=tJB%o8Iwp5ZHTc2M8;RYZCoSb z^f_KuI7wuVo+St8)48#Yi_F~i4Sk#}tc3qhPJW8(-Xzko!3T?5E&(4H=)~q(7ED5k z&0{Mh6Fu%$zQC9Hn2@~}gG>)uhr01VsMk6D&v`qRcqUu<@wz&}R=(UnT~5n64n5P{ z8rxzvr9kd^H9)p#6f5@boJ)gY(i*IocuK&ZcWj@N9No+*@HEE=Ou!+8ALd`*Hr6vOVy z?VYOg#TG$CjgaVTexuMsWWzH8Q^8TbO?yE$4<+Q(np%B(j3N&_l9>8nK5=J*h#_@vxu@-IX1__osn-(`T(>e6a|zJupk|HPUqSMA zlfUf(r}Gk2JGbDdB(GDaeanaI202rwnlsb)g)C zs-;%3hU`tC-^%(8p3ZEVLQnfnKcuZj9)mTW18p)sU-s$i=89lUDXchV8MN)nL=Z3- zD$DuS4;cY|O6wzsa8F_Gp84>AirlEQL5(p)O*aN+ys|iAgMm??qAgn<691Y;Nl$nl$E!=NL z`o3YrfaZ~gj!w%u74X8NtHZ}6E^qDS_&pOqI&NVwf5FuLSkI)p&7YrxSQNO{{NOd% zDvNFvfUFWMoyb za4fW(Nu3R-qA`|=x5WD0nG;?Z{c5*?-WFUK@ltv@Pk?rR7bTUTptn7kUuS}hj zI1-{p*2--*qpI^c$*P%qmfys8KX-Cbq1P9cx92fYUhqsX!X%V9tUE0gib<>;3&Wpw zWD2Nvw_S<(7h_p_xrH(09(U}~hz;6wddGA?Bpnw}B-~FI2)1j{o!!Q;$YgV(7MV`_ zCC}Y0Lwr%3qSwpwu;`{*kT)~HfLXCI1@s9H6r~t(m?UrnYmf|+#1ixFn}~F7#a^u( zhuoLYHp##v!fCi3$SDMHq2}?|F0J%I0bERJ6^NPmd)0qy=yS3(>zSgg2MxYKm=B4< z$W3dz0)sWlmOz}x)A;i%xAqSkDxhZH{;rKRHh-G_{`YG#6@@m&k=TjYs*{4TvVrxg ziFu6uVaBGQH7E1*kmqZj$13rQJQhkC7sTN)l^FH zQiTvF)X0s}CM8Y*0vp~@V@Rtl@_Tz4agsig27}ZC9+|b2ny+C#I<`>mf=y!L3YMh9 zb<4@AwT%m_RV~Kjl~kE}T(FaDZf{*Zi;;IH&jg(VQbQV}?Q-k3E~Mo_X6qtI6P;r8 zPrb2+x^f;YE-4NeGO&{FH%GS386i88W25>6z>(h>I>>LF?Uat1Kz^!*t08z72abmF z3JO$MmNE)Qvao1A(VFydZTcM9&ej*QM@D5*bPbJkoISe0Glx4 zPxU+l=WltPPTjN|0riF8zZ!x~Hyt?X+UgOt`SotvH8%p%Z$30mB2nTzPs~RFWeo9V z@b$AP@ob1X^uyUI|7>A#nblm}BklMOoD{R_kg-smCHxV>wCRF07H^#=KFRq_g#Y1` z*qUNlPyuQhOT2s94e3vpb=&WG=X9n?L3VW&=GIpN5!i|umg|jp*FO;DtF!PQ)e$^< zN;SaKk=tjeah9dBRW>y^zc;34-;m058VG1;tgO0K#gPh{g0x3c1x{f3l-q%|;y(;loq3==LN5pS z8P(;C=JyLw$0A_BwJa>ua4K+_-EIDek+^|_y6{YH7AgTgVP5%|IW{6>ygW&pM?Ua^HZupcnK57DIGqG1yjMH%PT zh_It%#{^qzr)WBO*P#4z*-exrlAr60zF<^Pl9Lb0hwTk%gFY$M`c|vBr%B}t1Ku`d zDT=IyxjC3!9di1+&|&uLN&%F5k_sEnL5b!OdmxJxrx;L5kIN4}I* zb`^LqLQ*1?(|GnB`Mi<{jdY$6K(%(HMc}iTI^3A_Vs)Gqutuc?4S!n?X0?s@;uJM& zm^0$@A8J_bU<6FD7#%q+u5o-Zq4*!SsrgU)b~Y!k#d;9Y?olJi#b3k=0cGSt7Ruv2 zcaHa^G0YO=v_qnezi)2o1%!^q2wuhOyuTJf%8}nKLc)Fe#Kp7L9Z%0mIo~K+2D9c- zI;;W4<`Zy0NnJ-MQq2rvAGS9mMR~S2lOIl;wtQ}rETbk@g-|F0ONnM4@MtviG}fcj zy+?W1cPl8B9ys9xp%hgrW?{i&uAxA~ho{gsO;+AX2aSDiJ&& zJ9ddL(;#~jMXpxWR$_Uoc2}wzk+NJh^%d@1n?M7+>ZDq5wZN+*z54hW389(gs77gznLmE@$cT^Tf{&b!uB+^>yLe);l%- z*B#)19rT^xy54y@;Ks~Cuz6K4-VnYDZ)%+2&$y!ODdt{=)5S~&C5VHpWnPB81@x7I z-XP9W$DIoUEf7ur(CMxR3hjX>Xs;_FDg8vPAj%Wk*i%k3<|$ighkKaj}HOsZ4-*64(nG{ePW!Wo@G@zoo zNwHkhml|FFO_sN(l^^PD?rRltrrR}{IwUn&P&s`tQ(B#`sM^Km#?nVgO`sraxpUCHLP{=+!8a`uPOA_fc9K6XNM3>c;_%?m+NxOQ>!Vml@B^|5Y}@9zY3Er7o_WV1%OIp&RcVh&emUu zqT-Wma~pGiv20Qa;}Kd$&$N_r3e5n%n?ruLudssF$Y0bx%){jl?0mQ6eW5JKwyp9O zd^fd}l)k(B87hITQ!Z<+7b{mX6$Q}3LQt5z&>VHfkr6=tEL(~T1uPjR8W_~oJ6f8l zAHnhv^N75}iLm7wv;a#}eOYVgfwD)n?3n&12jC=4jUndrm!;VGx$R>O1!QfEAP_x^;imcG%64iLd)cspzHdy9VUpOcBl4FJ*lOS@@h|7f?ZYc8Wm4EJ!It zdK*v)w*LOH$ccx8eFn;OKcs0&dKEs`gX%Kw&$O$tHCj`{_)-`GdjRE1;jw)lUxrgfbj291ANXLxM zPc`D;N4?E;nb{^(t6mdhMqGx=z|13AN61i7+JL0Dqd=kda6edRQ55g_M6crUIaHlL zfBN+0+%)^!^NYzZyUH~Up_Dw4EX@zeog!io=g#CW7d9igdQps+<-M1%ots3iJ-E|k znsI9XNJF zR4|PxQ6#PBw*qis?YsdSl;`n)LX%0t8Cym;dR7vpPqbI5K|rVY0M-98Xt}R>Sv@p(Pain$PT&ymB)8{TyQRyT#Vy ztAaaNVy$QodFw8l^~BOh3#Az?mb)<}>Cn?u6SL7F)y+)-Mmo_(6yQk0plU*IM-ncc zo>(NL(z?pHH@*|4;z9t|la(=USsO+7W>%K7A@@4XW-9Ucw zEFojLYk#fF^pZuZMY2{MjaI#LTW210o9d&1_Xrodb3HZ&X1dIA31=TqKW_d>&G*JL zHbt7^Y<^>LDzSMS@3?*f1lah(9yJOcHrlH|7h;BX>FvHHig>BPg$KQjU{B`{;_R(+ z6yj_@w1cjBEv1<0@ctFV%kolJ^8jxeE zd3A({fDDy184$J=?wCXfa!aN|Y>4B_ka_<|_S+Agz;H1Q6R#6d(&8gMo-vF8hW8zD zzyetwJw{#5H3sj;LT1gmvz{8G)68@-l_;R2HP1&o8?_|^V5{)Z^L%O$-&#gIeFcT! zP2E(@_kv+f7DGL~?Eo(syV4^;5>NR_MTkk=A#sQeH3#YM*s%1|u^=6dV*1#5z|>8o zu0#?;0BX?re~cD=_gd2C%7d}c?-?hrsb3JyS6=upabt%{+>tUcFf zvmaOboS$ziRm_D-Se7=4zf~>oMo$7}$>|yGua~ifc2o3hpGmq8!D$-+-t@I88wMJ; zM<0T#O%h$X)h&|}G8%5_hP@lg+wpsMb|12bp1DY(?Du4R}PL2ouj!Wh| zb=SK3IU8{a9?<7@0Cx|C=j@|w#1ST@)O2naZVUvkeN`R?cRJTj8TLt>qLBk1K~^uC zn^IBDt<*zT&4^^$HiIO%Rwj-)B#cy_Eg#gst5yweq;3SQSf7ymTPM)Ue^KDCv1w8t zW(g`i9AsWkqfy+M%cOK+-PLA?;*Bm-=ckYZJsGap%ntts#loL68yP z!ObjYRdm6-7|PZF@!GoO^|Cy&HPj)z;VDm9?o$5j@0A0tgS zRO?#5py7C86{_5bFdy|bzXtA}IHh%KnX`N36!W_-5RbOlnrM_kCv!$tqACA_&m??4 z?U|%_$dG%$NCB@PKZc|n3q<7|ZrIV5SOmeiI+&oo3^}az12a&HI&vs=iiu`aTMT2> zVSlKVfz+gqJ(cx?$4`wTd<~D+^1;vj?$)C)z1eJfy0V@5DD$5-I`zCNYuzpqX1Q(t z?2g(HlQ3zZrIFO1pV!U}nC=L}uUDzD+SuONCLi`{s1ouQ9a`I{oixY!@B#XwxDtxc z@KS*$^!#_t`1}#-#oz`kmcS$2I!QYjY~p;=E#wIo6aKr+FQd6&*{TpNK<6#?Rsu*< z(jPIX;pM#L0Is*K)}oPNk7L+_ta0ns;XWMZW7y`7#XDFl2ILf6 z#%PhXZC{b5Vw{(CsL9-Q-nfgUrUL*bLI7MzIS%XgKJ~8P3Lp&q#|BpY z0G>x^F8_GLqG(P}>FcSx`#%vy0Zk!L+O9nxT4UL_V6r;yeq39AvTc*EEkE*Z^H+5g z7_?kJQKf=Yty3dm>7-4LNNEQqt>t-J>}<^|LU3LNnMo8)>!f*ueaW=3Tla3^T@@{k zs!k~mPsl1$pJoTL`J;H-glb)LL{*rlz^8>QG&^cWW6zC(^=;lPZm#$f9(MD?Eq|uN7+owpgZkN(x0;AC6!Skan1DcaUmod1-Q@`3YTjBaQRZc0 z?{m^4IR*CO6+c8|TWB>z^5(ConJ0{gHyZQKbXV=&bDDH{|WhlyYj~l1fR65#xX&Y z9UNb4g4P(eK7Kr%^582&vVrIg)vmvlcufbviV`d);1SZ4FiO!balf8mrablL%2q`b zetqAagzAe`?O2#PizNX$V$dC@IjtI zLnC20)lWF?L zLc3;$C8KnFwDdD1rC89z8IAnS%gHq4i&dPaBMGDQWhAiurzu>3Q!%tzUJ6n&oSGw# zUvS#t;d5j@9xcy~%9Y?lV;G#uy3Ni>Tm9ObpTaxm8;l3V!*z_UT{v>uGvTjaCOTNumN(ksFr-&r6NlmX))QrUK zc~Ov0#JQ-}3^4F{-AIhh#9370LX9&&X{g~hb^PP_ZneDDLp5pe+81->w&y7f`>j~T zuqS_VW&S}NrPQ6FT-9SKOw_@4X<}KqlF6uBnyhO`h#y~y85t`tiU5f*`3}LN&O6dp z9A5UJi-h*iX#g4VhS4eEuA8107gGb6(@)?Y$E6H!$z`6{+hu2F$KEvm)gW|`{-s|y z-WxLqbVvOtq?6m?-^M0eg17KC)9vx}E#`%}C}N+671G;q>ZUozb2 zZqo%;Tfm5l6vSI4J@qfrZD47oC?$D{4o`Q(1yT&ve%#1q(Xyy1+Pl?)>UR2%`ScYQ z8g}kP=P7C9(avL}`#r&OT)fO8ILFe9(nfOX=P7z&ze(_{aR{3lDxVv=WkbvlDSnqf zZOe6Sasj;aka?XTd8!wh1VD0m*yh8|n--bf$XwKyKdu1sSTuF%)pF_K%Y(BQPjjL# z_H`WJa6S6IWu`vB=_m}~mu{LGsNoP=nk;p-6Uq%#meQeEqD#X+!8PA{nK;jm;x?0Q z72nqtcj!P)28mb05|G=weuN!;27J3RRy>@V^Igsp(t%J(x!e5Pd;=~|+dB(L3rV>C zAPWlIZT>ROZ)AncK0d|tsPBf41=|VGfKW9%FDX|548DY`I|pIw1o3-tY22mw3$Dn( z5%PB z^;wjT)anV0VvXcoI3r8TWYUob_WZ?2%!}rh7vwO!gS(z3ff~-+f+x1PUOE~k#jBd?U$*m zQiMrW6^lf3Z<Z4(PE5|W96qSUL_rAEhXZK@nI^%SfP+E9)ezNKU! zAsj4z%r_)u2GVDUQi2<^?altIOwXIjKDv(mJWI9(3T^y#K>>Y?q#@CFRPYw;<9gL6 zz|{$fFHr?EK)T!2=Y$MJvV3S`NGYRPlAx9>6D*rRKFMiWd@L8mdvE54fOp&ht^v$% zD)N_3k)JL3p}y}u8K$y^=u;OUM#G{-3h{rFvO)j$KHv1Q{UCn0shGQIQLCGEbk=X; z_b`|!s%nTHm*8(hO`I)b(xyJ^>5UK-uD1|B9GdQQQQO44 zxm!blFKu=zIYO$=HDz-b{a*?v%${wWc%|qx{>7uH=xMV{yEx!y*<#rVi8Rm~j1^jr z$T)>vvG|fBxjA&QL_javp+IbR>83XAZj0Nu=Lpegz4bXL7X-+fxL6+d7N**bk_*gU zHq~p>QNqa3Z^Qn=bJVoK>EKZPADvFmHual&bQ8NzlG3sWNfU54sNBf$!)j40CR`Td zYKC`@0()Rud_t@x(bCzJXlI4t9|V{s0WN)HV^fZ04LoyCDWK88V|z27rpC7fE@?K3 z6%L8qf8~bRU7oNz$W4BxQ3u`Cj0p_slu(vq=N0e#ifxplCRj_Wy~6t=8L#90O1tI# zC`FYv%_?oqSg3Omf`sH_pvF4 zfChF+Uc#;Xu@ZF+yWWY8*0%q z?i?eCcp~mQk%~~^_I@x5gzlpy<1q8r& zQ+-O?0WB{tUAFwOHBat;Z0dHK#C+fa1WL80*Uup$5I8fD>g&f5c&D_I(p{+u?A(8* zy&-y0SbaqIF$9hynIO;M8E>VQs z*UN!h|Aha7RP!FAlUqO?#kYEsLQ!uB$9tg;XWf;Z_AOo95XL(bKfO3!i`_(_ZZk^t zg{l;u<4@$iQ$oIUSp4<8*mhXd7+BufB{9^rSYMu~0hv$$uCa9dj z8SEGgrk>IKSje9Bz_%5)% z(SC96TkKu4yC!@4`n?QQ^_)V@z8`llkg5r$go0UgdAld_P*;*=zzDRR`wpw?n7zdeqNh9M2_%=I<>-BNSY}|Cnxm*Jx&>I3@|M`u4xy$c--C=A`v^_&JC`@N2oDBUq#(AB+N&S_Kc=P!lQv$dNG zLi4uOTH?_>)yHZ?KSX#(`7)~>0yCD<(S$)ju9E(+0n~=#>cUR5bS`*eY z^p~}{f|3Lh1y<+nRm}}6J|kdBAV#+Ur9Ygftd;_zA8g|$)kGp9)e0Hcgk>SVp&_8k zds95SE-jX`#l*GUjPZ~>Nm+{4lF61j_+DjGl=$zGAb8h@)TQWUuKnul#&DX$KNB~W zKu-0{^Rtj8IdR=3`5$>yUEKe2nwPrv>Ur_=`7{c0qLEt-#B?{HH@0Lc6>P{?wu@<} z=KPsz1OMy4|5uJC+$O_L%WNeTzVtj*snri!>q6QOL}cRmNuzs|$UuA1dIkw}mc494 zsQ(boTGqvU12=EC71aH2vn+&*;_}km>Y9!{OMQz#zq`wd)W?UJRibmgZ*kBXN!`=4 zb;*(n_qJm|KQauweQGS{ayh2k#8WK;YNY_lK%>(Pk{3wJh06dq`Ts=b^vkS=Uywq4 zLTo;EHrhSpE}QzjfH>L+te2weS}1dSS<+ubZcjopO2PmdJYh^t#{b8YG0~L8m~^bLTOqkVY-q%D> ze}L`5VUl(k#+l#3&*Ic3@<`#{hf{A{7pfB4XrUlx3ky^#k9DH>QBANJ zb(Sb!4R-5TrZF#XH$nH#fZWB@da03(+!RaU*8tpfdnGZ#dTE6| z$j()Li&@O1*`*e5=X-$@A+(v%B`Te5tWwi(H3clxlj$t1Shd{<^PuG{6v4P)sJ-Mt z_1eT&1U4cKz*V{qxlOza_sM+0wDN(#-UvZ?BIt|$ioux~NYJzfqb)Mw#g0+aBKn>AT(o6mv8ZS{TawgVBe(rVGS6Z~nB`@+J z53yv+e9VIO=`_D<8kZ;67XYi?(5#hY^pjK-xYHRa{_tAE^ZB6Ddc82Ye;-kb1GUF2 z7a+N5Y*ivoQ_XXo0K=&8B!%SM*cP1`F29y|sgO&)9HcHM38|GrO-xB{Vsfm<=R$R{ zq@L7+kWmI(rH1$}y4nvcl0QaSVrZPGMo10RiL$z5R^&BD_0CL$^;yUoHYN4pf`<2j zF!^AVJgXo6pm_bzguZqnH{0cmOE`Ca{-tPZ%NAcK0ZiN%&p$SEjA8DaifjXp~mZ_IayeFApgrapI)ZnIQV*ibi*Hs zZ;I`Q<%gp@WCg(8?uCnC?T&@+m4PPH{!d< z<=@it)#zQM*6!3%XvCp@++FrB5s1UgZ6L4k zcJv z0Xf+yPy1%?2{nx|f8x4lf3;hbv)s(%j`r-Uzem0&%3b3$g;sqWyYAI}NlEjmH%4pO zP3+xMgE2Sx+Saul&P9CP4aHmV&Fq?6yY_jtS7kwe;IB(Z=O4IyAh@!>{y-&unED<@ z<33+513+(o5SoOO&rX*p^yN)#a0FO0c=TsA6%xRJM$71+%x8t*(Bs9@^ijrKW@?}F zuk#`u9!XsF0wZpiIGtmb^*p|-S8%sJPwUQJ8lfek#zf&=>fa>TP>oRF=$zl0yhDk4 zoDH^nWqugHRL-Jnua-H0jxB*4vs5`*05w6ssjx|b2Os=ToV1gAr>vyjH|xck2q5{E zyPR50=OTOy}+&n?}+7pNy0Xj(TOeLz$Ff{`_&We2)SD)?w^A= zPGXmsN6Q35hqFAPkgAGrNfI4Ke1mu^43m5hlf3D53L8MJA@hwe0lmxIaJ;eaONH?Q z?F&JCCUlRCU!XueB9HCpW}F!UW~V(Jj+^@cwHKp=cnYKWimcO?LTzCr-rLJ-+YV`H z;1e?*QkL2ni8pwtujN7K`=W+~@yB&$V=fvp*R_%ZRyAU2`58YFICZ;b!{x>)V zdsfqtni1g>_@yvw0K+srlPGyI5*ooyY$V7qbVshbsKEr@`WYD9$5WGH^|T8fuQr? z?+moJn4i}>vq$V@kz%zx1MgDgM#%uV;P^oD#SdZG2 zia5|a8FwLYDat79eL}niQ83d^;_qFw6NOdp?sG;{_rxV^#0w&(Y%pSA5qK?yGKpsw z+6q`gcmdgrN7dRhTGz+fjs$s6-dsv)lsYij(#LLN(P9pE)ZLtt8@+J(b+lV4Z3yZb zX3m`Bx(ya81xQ-DK_m~q0_g`f?SK5B-KYjuI-@XDXD2c{#Hr#n)j&>=@_{J|{aqi8 z+fvvd{670mAbd*SX*VtGv`n^HAX(ElQqS==0I%u&^ljL?kX$8{P4rD2vuCE)w4f4A zxb*LEt5$3f-14pb1Ol|BPyuv6-JQ3r4fl**4ubAs+OuG=Y$qRC<0foN>ZZCK`>4r; z1wCuzlQdm)tT!Q>{OBk4rBlqRC*MRFlunCp+I2-P%c_Q$=0nb2_C`8UspA}wI;qY# zsY#K3E}k)o*3>Up>5*+>0Md6t+`i(HetIR)x8tNdLOF?8nH@ZQ(jkF`$Kqb_M5K1T zoL@IK(fwf5)y#~0=1v<{kbBizvs})qq9gzYdiptO|1pfWmZ*jmFE1#<6+k2Kv6hPh zou7{6KQo9f771D*Yr~jI#DWJTOn=!d<1mrOth)Zy>8w~v9NTg051o$B`-?_^9YA@R zGtCCW1f5CG4+z+S5UYlYc__5IjZh;F+tc)T>y4Q6Nlv71*>Li*-HDBSE6U*-8^B*h zU}m*TpV?g$aXAWpb^9;blz^y zmTc>r0J%0p z9M5TGNWTrGmuS)~b4vXchR5{WTg`PMDswI0T7l{xCt1^;%i4rKadHh&%LBIAEGBlh z^Ijmr;+%fnJpeTz7J&!8@t^N2BqT58$sNJeHbC&O3i4uTs1~9jyJ00CQ~ZpOgset1 z^Zhr_RUy>3gSv{6-VtHz+xt4nY83+*PK$4x&3hMe=Hg)4uB_`N%If}{8aHRw*eb&7 zsQ^Sr5peZp8ro}!@;K;*Hm<&pPC?!2xn(U&(stJqbiLD8^TB!*+M6VfM4D9ncC#w6uZ(cYpXg%vp^IQ#9}|E)p(Oh0L?(p!gPWilaXsg% z>pR8EG*U?_GBU7pzH2`5T!a~ohZsb9v3yKaLOGfut4 zyrpZ6#uUti#ARb>EhA>rmnCFNF|LX0!OV4*tD9zT6P+M+F|F{;omaT4Ha~wJ`qpsbqrYDRNbx*SC(Ql1>~>(yoe$$9csX$IaHWX|JT9$TiSawJ>|j`SLdH zrog@@e+HNS98x|I@z-oZ1~?cnM7TJeX+V&N3I)UU`k$I&c9@(p*H(O_IXpq1XwehN@j5>V=8tEa{mG=ghBlrCUX8OpoaCXDw9C=O}pr2~Y$lUF3-NMAjW z9N0f0ahiJ>(WhbmM3HL4tC7}a0q$&t{qu835@S=BB3IR&r`rsgA3Z^P`sO=rI=_y_ z7f#=FPpkL&5|ETA-&0#N#TtDnH6}giLpksqlJieje({M<0;o_SSPc9t}8B!&?GkK zBiN2W2cHBffRIP=zqkk<41}(y#RGa5tbk7_Rf;j`@fxr0Pd7iqlnH6I*5iEjRyx_z zp~N27?V8K!#?2~q-pTB|{)7){!q_u@A*BEmh2tof-#eOBKRZn{qFrNw>DYBba0C_M zSHxW=lLq+|fUeK`^lP!&RFDDkaG6iq$t6__0@Zr50$5C-bKHRuG#d{{{XW~No_JIi z6VvJIG6#^IgT5U3XVc3-qSW8gaV=t8NZPVBn}mq1}t=p4@=2}~=! zzfM#V`u&a%3R~Zg`)tgh_p|BPyeY9q zM#DuN>mCOVqa5~5UDOB~k&wBXg&;|B{eX_~p_R1i`oI7Cmk7R#fAHJpvxfGsw z&KHR!ATG|wuC33$bc_ma=Gv^0zYV*wFAd8F81eXPt$JkErxvHpT82Z^1Y>)_#0X@npo_obZ+3p`CC*u z(bYC4wZO|{dcM+HJ|G9OChm>$6iDYc_YQ1IlFnw9gwx_&nU4j_X!wRch4GvWww)Yn zNqeUlR&^cj8jNS*(e7`ALsYuBKhe3LMd^bP4~vE?U9hw@dJiG8R)NoA*@r7d{~--6LEfG)J-#4`345|YMlydpU9>vs@mp6p2(u~44#p*X z2zveTf6a;a>9>8TL;G@tKwh_Xi?+ZNWDg{5!zC>t(`>f)3OQV3`meD=!N}o+A;}=4 zojiv-YulKy)UZc*-U_AGSPcjQY-5C}ya9Iqjt7&(HQScS_7A$IP5Wpi@y|gyx`SDp zfW;!I{>Q7nzzH^l>6Q|F2Ug}ean@+KFTQ-nj&z8*rkEE_8l)MlKp)3j2N>e8wHeN` zwjwjSe;U6}C|Zwd&^(y?l$y0{cGEJfJ3FXgniD+AECsXGVoJA$FNq<}y4t(CSc|H0 zren4c8YeWk5MS9Up4M1ATV?udy0M$7cN7H4IswA5lq z>r?C_KPb>q^{d~8YoI-vH)u+`6XCxqQEezNoTA`trvqTm8g|Z|2R(ry8lw*(EId5= z*=V!VdfNNBOU$^Kmd}pJ?5Y>^>vmRyQNGiahTOK&<^qlK>FbBQmxn~rAyLnUr#;VJ zc5gcjWr{yX8$KPH?432p6l6Nco;wBsnHFP4hEdiSAI*Md@W13|qGYlwGJ;L65^@gX z*&tcPz(N}>5VWMvaB@gl1Nzkyb=b`oZ6vWYWrR+g#e~|SfZl_s(ahGA|MZ3may%p5 z@|&R&(S%iA?V0I~_VjIk?YS0Gp+axFv^A#Bq%3PYJ<*=yays2=r{mYNRlbN$t?r0q z0R%wLVT1C7=3LB9=Qtui)YO8pEUcNkdnm)w5O&}DlvOb!YYe32k2N`U)vSZ?7k5LBX5HsmwpOF?BCrm`6%eg6y0% ze@L7VF`3tuctpoU)uO_y0VrZP29MFw5qNzIx-n|`y$_zm3(kYMks9He0aB?qBt4gK zLaHk=RLR3Tn^jG+mODY0MTVBtps*h3%uiLjDSUD2jpj@W20NNGGfz&_@J!utII4T3 zVBF4Qu0J}7HeMOQ^|fN$;X#%UJ|?Xk2o^ES_%5zP8wrKUW3)s|B;(!mZkfZZezAcT zk;s9T75EV0W37*dqwhO99nK3(a-_8f&(miVoJFGpwa((QZCfj^OJX0~?Bm4tVB)d? zE#;%t;mSyv!uRTYA-ADmr}}EWuFtFSUfx)kO{&@UxbJX_p=yL$>%BJzb^~LTCFk;A zqYhWEyEMdcw+0r_=jA@Se31IeL?Rdwq!r_Bga)+5uHEjms;`z0Gl6fi@e1Z>AgN1B zBt1E5bs0kbonRXT?PD#~Y`LJ(C%Xi(_+}()oW`a&E zTD^|?+Tl=rc|3zzeox9Or3Ix4a157S%K~VE)3={)enZpylJwBN^lZkl9x-85-|mls_%$AZs*GU8yBzkZoSz zE;n1Y>(g9HkwLKP`s6%rm9P(-gpmPYf%*G%&b&pwhxWyEP7r759F(0y+1gMA@(oZX zu}R^Bs=~h{(Rv2&YQG~*!_rxI{ZTCX`t$fvewnE+H^S=$cxl;~yiG)V{IL1mG2ibS z%)cna>lCWi#x=M;nc%7%mwWYsxTD#D_@n-wou%(K)FQD=ZNsLN&Q7aK9#R@5zE{O| zn+2Im;B0L^msr+MXoSP5?#7>N609+49(I&coFS-vo7um7C9qq{YPW=QXx-=13-3%g zaj(*l{kyh@S-f;mCit zi~S(mBdOA>PZ_25wjK;z7GhG>YJES{$C;_!zaX{x(|p?f(7#s=x09-XentR8kiw>4 zd>p19Q#%nM^T3@=1E^j%U#ll6-85^e0f|M0m--0C<=RJqZ9>YFTwe~4)=xJE=oCbw0A($6C;3>0 z-Yypj@S9qa+`!h)3BiP#x~ebn#F__4%;InmZ1NC2m)NA2ls(jYQn21SHk3_xooA4b#o#QLGIP!PPQYnXwHNNHu{<;gL&U-5oIMn z-z^pIrxESEFflJJUsEm@nl@2?HP8@Kif!C!-CFdJkMyzJ9&#NX{2N5kk7u4hu}^-g z=?c1q`g|$s+^vl--v?<00u8Y3TFKv_8~z?@kDzj{VEfb%TVYjj%NjXjkA2YpQ=&$I zn>)}sCKdM&!-nB)f+#HJ+h2~I7+u}`PaZ_uZdv7Xy;m{k=x&nOlE5f3RNr?dcyWda zReK4#eGi;z;7ZC9PDd2d(<%Qv%-f@0i;;1VjdW%w>)FKtWQxwOn!g1FjR>e*3Xdmn zZ;&0sxpsK-pEuz!buJ(BWjW8OKu(q>*>J?#H_Xf2 zWQmj ztApL1b`uL_-aF|CrWGvsPxy7Z+BKoTcY=Qu#7dKd#^Aah^y~Fo&3qeg0Cji0^|aF+)I=nI$PWr<1+U4fQ7 zFDbCkv~>xK`9RZ&r-?(y9$fmV!c&)-{$POwiv)Fwt!ZM@rVst*{+x9a#X|Po8MJD3 z@*DjFpI3nrZ2-wULhcI?SbFoFLr(AX)+53+F=o^KQSvtO$iaQf8HdXtJGkTV8yllN z)Ky9Ax^JUqNO`JH`{e*eK)Jtq7>+4UoKoow2c09Mx|$qUpb~$ZMjcYH*eg^y(~RI8 zI_GuicAI2{6bTJT2m8}>IlOWHX$G4+Z>RALY+)_H)^0Y9fz4)$iShc-Y!C;ZJvp1ZRmWd2_oGmup5%W2NtotcW)nBFqta%0yVKgF zh2xA;e(!>&b589ojrt)GL+=UvVrM@D<4_&xa5;+**}L&1FY~rF!?weXAmtG{suCd% z=AtL?@%)k1P~e8H=@+pzC^~r}}f-Yax zo!b!a*`=syZGBq+b#*eg+jy02%`|hJfNzE)_LwvfgN|)qbA-IxBb|f#c%8Ok_51dk z=JXAyXj23Y^V>vaZ>hmX=@yvYn11JRyJ(gxusn^I;)CjiX{c(xa>$-dhcImJxU2+> zoJDC4>yEh}_5}IwxcS$*p$E#~oCP}Llw}Ps;>)lJ>7qq?o^JgKngmWJw(lX?#pMT} z18a@m^tz{~n|u;QKq8e9_K(_Y>J6V>E*fL1KfmNF>w>QtIk8uZZ!!wX-yr0y9h>cN zni8%z3_?%+%sv=`wlF`JDVe?HvW3RkyVX=*1l{An4F?v5ahjuCDc}wBT11U0X= zFvL3TOP(xWP{xHwp!MA;#kklAJR5qLQj_%IJ_`hFuN@QyYF)|!8}IYb%Wl-A*XwGx z(qB>__snOyukgea^R(k)*WS`U?CkiskbfRx0Hlqun`B1ks%Ckre*|TGSWN7f@pYcT z#ro%Ot7OZLi8I~QK7BfdVQZ=Y488o``O~Mzi+-atjnbvl3J|qaV(%*%IH6bm+c9{m zQX~v|ABej2ci0z18+%&07qftLL@cuJgh9zwHLNG6QZW)xTH9y z&~lg4i7v6fu(wVqo~GwZylu?MF2`tPW>3%F07U&pe@ddIQX__H%yMoH&o$VchNo;5 z;)t{YT*|2)>ZaHHK)mN2>fLP0L^~`v5E@^ggw=C6Q3E19gp%nZ;5K5pI)xAQDLtgU z1`%elyP)m7s4{4n`K@SY(B$WQe?_y$t5Ru+*^M&jU}!|FX8DMWqegnN@rQj1okW`5 zJCSw`>M?`_;r5#Lmu=hjK zX;W*m3#nOO0G4Fh2asfLAY*`Pf+@iSW;E2$Mk9>tjx%KzNj3{cqhLe!kDnA7uVGgOV>`x-4 z0klT)w27oG?16$Xgp@1@1VSoq*Tf(&5@sat%7KD$r$W@6|FwEdGnKKWP|AjkVe4)4 zcTF8junJmYk))2M;PZXfj4hy(cPHxb(Qb{`wittnk}e0$sE9o~A#%*bb^V9D+JhB>ud-_Pzm4T#sm<>1{YS31PYXHm2--Jd$HR64^ZR<$zjZ zOX2w=>|hPu>0G^Skp9~&TsuzA`)}djjGsCPl>kaM+w)E~lTWa(TW+|*U;-4M{|J>(rmTy0)V$#H}s$HYCg0LpK1EhuX`2o)6c+ zH`fjOC;3#fA%a}MTonn*<&Q0nv5|V=x6ld3t(f+sE@;Nye#>)$08XWk-j{;Dk~vECu5oO z0C>Td-Vf#=W=Ur?Fr53J6H{1!=f@^Ac>C8G>ooJokNQ(uhqL$Oq^SWK-_otfXGj`~ za;~wm6zX&iwf702YoEN@1d`p$t9leeIUYbgEP>FXFlmoe2(2)8l_akZrWfm}A>x59 zc6M?BnVGip^Q|?$LX$p?*RTvOGj=D}tRx&9zwo^}V8^(7wEapc7g4CUPOh!r?ld+o#S-GtP+=K9nPgZq9=u;Zwa%dKiJ71OFxpca1|vrM?ZnEwDc?i0Q;jBIU6m3^#RFubC zw++XFCTzOYbdnqbU0kDYaMvj)TM}d-Gx+K0@;0#v*=1*;Q@|E!U#W@0$cCZj!(odU z-R(j&YrRlulI*YTofB&{EOuj@4#SfrsV#K0v(o^|d0x)iXd@-4C9&_`LFB}V12Y1U zG})8;-s&*D1Er<@RL_I$bSiI`l~?Jp#-M{kQrefOC{Hn@&N4&lQ0yy1{E%jo`+;Sy z-nh;li%7*F4w zQfbKajT$CVg*s_1A#$!_zq8)vd*O0e4EScy#BN-B!U)44Ml|M-|gm@@oJE%v?0 z{|@I&As>NJM{UY3EyrL9pRJmn#j3#2ABYttbYVIvk+AGoUO#G4bS+gQZzpB1rS(&E zDzDIOTjp8&)`1JH*=-IgxEbtuApz7-jTyAu&Ae;62GQm|Qov@{grSx_d`?_(Nhqx~ zU)(b4B;}gwV0Z&{D5zPnV0b%CeaP;lq3R3%Y1n=I+pgN=3^X-6^7}g4SfkmI4IXY< z>T2d}!*np@f5X zSM&-RPp>rr31FB1Nt~MTEtezd;_R*coNGtx26J`~6LFxMf^OGLwrcyc*&zd`Teuoc z?)I|5B4Me^TUhZL%j=~8PZgf4;VM$%J9^nsy6nYv(wMF@hv7(7f26iVkwl?a#G_?T za5V5)@%?3YU7FLKY)8Yqv;2=I79?y8iaLk)5c$>RZ*SgxX^f4)yrm3;e)nxenPF-w zULELO=m~aYyUki!Bj>omkdy7(@HK*2YKqjo)9Ihn^zKY1A2pw(qkG;<4D}tLyldQ>2NJ8 zE5R64kakq-upm}tjG4mTSDf1uo9AjXb)HW_R^*w1N5gX$zG?HW`$+gP4d(~Gd?5Ff zy%rfJX9@i2Q>bP%capQb8DEV0QKkOr6AXWYdL9hEdb#T8Bo)%*WR^WH+th26cSBRe z*yecs)wMe#TwhCV#?wjKROIp62?-FbjU6eHiBQ~0?rIk*hU__@ek{=`-b*0{a@OB%{fpwCP8`c7X z6+6n$e2amO4ZDhsb!!>s_Z-jj0%_3#Ur3bdy2;LFUfz2=i~ z6K+TTU8l?O=sJl@jJ4pRDb#_AvfP@c3DM^TZ1x~0vgKi#h*aPlsyV30QDPL zxt~T8O5Z&mmqTW{CF1RS+DPb8{Y7Dj{ATI-H zc7~L$C1l+8EOXHCG6eP`T*jMm+J|LTSwE#WV6c&f2B|+@``zZ>$9h^m20Gd9L3UQI z`4oiMJnOmFuV@BuRX}S*eOC=Nrn4pCRdxm1Hmw*022C34qv;%saG*2jHiJ*pk)EY! z<;(99A0-JNjNF>d(QJ_;`A2;T!T(c#A*uM_e&;r*9=Ly$6(v=#AIIbP+^g)=d-g6H zDXlGP0|p53%QI>1rLkEZqAR>2 z-jv4Y6GalKpiE=tM@l6d3FH-mutKJLp@dnkBog8 zCb_GwJ>r11$0(zYf8uJ%$kKY)eD>*Gbb@5~%SA=k9C{8YGjJ;Vh9G|1%OP;)f8Pus zZq*0Yo|_ZdaRG>wX3~l#qzWZ`12qBDt7vG8Zf|3+@yDgL1Rv@d@z4&5yEY*>OiL$f z#QiNhT`Pna{!<^#Eo;eQOC?sSt}+$3rse}At9QSX4$gFVyPRX1)`W~{*ws7U-p5o2Z3@K-)(tN>dQ&Vi6#W#45O1^} zrF>Gd5g6|kzoo3;Zu2uJwg+LH_qNKeOKYV#XzC@O@lg%Sv{eJVYb742W4e(gSH5$T^|vXztbbipIfb%nqfdp_p)xo197D^X<0|pw z#pl?Vrkh){bT%=Sd~fN45(w(__AvOnIo-0=+~bIRl;pF<2>xsx@}av}PqAcl*|hai z4hwb!Lu-e65YcjA=P;6n`|b-QptRX&K))bo4S3oKGvd2kzw7mUr= zee=`nJcsM5rYg5~q4!05+rFUMiHsn|PqYr_tEB~o)7X}npDfMR<*6bqaoTtA+;6Jx zjC}ef=<^(BYcixe49d23T26#@0x79-L znaeorxtC}fzky7J_J_aggL#e>wCYp4b|JtjevPlPwG!&ru@x`Wfq{?Jk>obFmUa=? zHRY83yhD06FcR$^8Ht2>IvuB>#=(PK&eitjh$zG{K7E?y-$tqOc$w;;#(({m=uUYL zSzbN`)><-@Ra)fU4W6_yfR2X31L29|IiR9GrkCm_zS;sWoLMCr7HABuLuYU3+uih- z+dI*si&6{g_1;FTWQPOR1)Rdosa(iFA;hm{Ax}u5v;i`$>`8eyh(&d5l4u~U9&R&D zZ{Ns7JAZJ_B;R%=rZKQ1^Bbt(cDI4obw9#lYY?Oae(}`x;P;sTyRbL1X zA?~nxz|{lvSK6EW)Zy$lH>wno+izZ@S7kO)TwrJCEAb#<9ChC4(Jq0!Zxb=`ve>L9Cg}zr$cInEXO5S=LfbH6*kK;E2XD zgG!^K3vc7}#o=H8`0d9w-Z&Zsa`GFGQHiwJo*m!W@>805$wG56dl^x2QaV+J!)*3m z>Y^?}%Ho50ui-cf$h*|GnX4K{U<6=^Idjdx*|WFg3G@}-+egtw@TX|;&XDUz`sXUP zE}h`;h}3TmZ{{Pa)hzKl1?x=sCkD;GPETKJK*!ba3u`XL=@lDjAz1_71H83#iZC4h zZQ475C6)v1)!xoqRsU8J(VU-Z_cx^iHeDp%z%DYT3ah%aXzsw;GsoA|p>&_Gec$Qw zT|`_|I@u76y|Kbyb}0#gJ$1*BVX1TB8CStu$uS>hO}(162d+p3*>QC}9CkfwLkf(g zam(p09=l2D>Fi>?IEj{HLNf_Ui>kvarsSppEo=OOkw~m0F)K((T6VD&N>14AH12L1 z#)*qAA^G)7Z|TP}-6Nj4^8kQwUkwztkq;p4dkJ+m8H0oe&gE!KXzxlC-WO{n&|%LW zMX@{@r+^qhEt6xDuIm^%QPpl!H1&j$3Id z(5$CfaR-&Ees-aM=tZdewM|%Te-u5Jqpl{{W?q7I1FYbapOyblh9yL&et<&dDQ+DP zX(!W)l7j@-I2U@RoIffrJ6~LMZa!YS2hS#VdyEGA@5TdCB|jLnp*ynf?*z4w+>A&l zu;dcjGN@^3KqyaAd6|tw23)tjL6g2xN)%pE-{Tes0ZybSJiDLqo_I9>5-7T-+;P2oy z`rD-DZKwm41Vuehzs3&#|6ys!Lxg6)1T&Y^R0u_|HD!t*0dt?5@<4*_Hihie|TR3Uw*{`BvUs{IBgw$v!f)9(3BIqVWx`NLNN~HU&{D zcM$++YY>vTl7D!u#nAbH^swXc%Lwpa2F%~>u0C|qJLwlrSY!SFZaGl09}+@p<+XyT zjYFELBRs(Axs?WR9Vn?dgXly>@d%oIkj#^<66TrQCgv)83Q{~?!NS{h)~oezLKYz) zh`_6bDrB!ZC{ct$Yp}q z=`l9_mTmGqMt2xkj(c#9Yf&@vS@TkiLY7-5cz=m+Z>#(WH6DIzmU|TLst@*&EU*M<<|!WB-IZVBQ0lnAdUR)ZDdVDW7S!i$Rebl^vZ>rXOBEJw zuWkd_kwj2Mg2J`|8q}jD35hCREc>!N)t`mK@jGtyd8kpQCC=w?$yU!j(YwwWMWW{A zVPLXHfc68|f4#{ns!=`$fD}Jgi zewUE=XX2EY7wXH!m{bmmJ?&EmqK&|sVg^@|jxy!%sV>2kp%4Qbgd)%HWuunhB(w3< zEnxW-4C#LJzRWBOQz2)1ZZJdl=~&eLu=z81*}SD6HowI^D{Vx6g9yGw1yDp-ldi=D zZ+LW1XGxy~Mjg>0Qd)r)e(IEnh?sl!LVk^_I@PpT4EBXbH45U~-7BXYeId zF18y(3`;I5Z1BPEQ<6lAi574$FVGfYr><76>KXtuHU&xLVEE&#d*?2~A6Ro=TN@ky3RQ#9{}&>Wk2jvPASFrDW)shzHFrTWBuMGinG;05|oV< zE#s{G>Al<2G~7#H+UXcXBZDQl|5h3R9@m|KlbMZYis-z}bzXw0;SOq24M|Ll{^=5C zj_gPAklAj6T1|}lkti}F?-+82J$eo|J}QhovI*Lq}k3_)kM0egqtjtd}p zgkkMN+`^POt?VIvDY;@na}-DDStB^hTWYw23{UDvEKh&f{H_h7>n1Vz1MfHwov!&s z(IRj)Xi~i2FKc%fpgd|icxAX2<&cm%)D(-KIGcZAh#rZ(%##k?$D?Ul_u{fifluE@ zG?avXLtQrYyFZTYZj9M0)RLx}HU_|mzke4&+4nsHxK@u=5qK}l1KTV}y)}4D;jE^15@t4x&4+ps(1EStL!_$? z6}~mrZ6LUF8%UZ?7G?97(;&Tw&4cd~&+wt#f(3BjKi=|ik;V+xN@wY$K}}MI?&Fob zoKzlB<)YxHY`TjR-jDM``)9@&YCOKUPb*5n-R7V9;3W3IA?>I+5k=H+8DYq+c?U!q zmGb}YoK?dk!K0(y3Gs}Pe4T9eudo{u-w>fIm*yN=5b zu2bEgjE+Wn0=g&UVKGB7^lIvYs0TT*R6q2|ffKE?*4(!0KvcP@nMs7PT@7l5Jd$aJ z&F5P093@>KB4%j86He`%@}2;IxbTrIxqWNwDi}jh5cR`qtj!(jldB#({?4=D4PJ&aa3k1B5r-dxv=?eslTDSCW z2Zva~D|1G76RqVZFWUN`v5z@MT&a2%kJCC3y--{T2X!ewS#V6$3AB1>7Q;U5O%_8y zJxSF$mm3;3B)O}F*VJM*qj?AYz?=B%n4}guVtE>b(^KVRL0!2OiIizvq8Euu zs*h~;BhZzE1b2OhWmc*hH0p@Bk?c3Q-IapEVHAP9Uck0^s{fh{3yfrzPPJkM7%~H< zeu1QWnu*AL_zF+zDn~1|Ca1_>)xanzk#+3bzySbIH1l~Ihz>p`Gt9$Ac_>kdi%9=N zqGtzM0MHmBt-vz9p3XmZ-&)t{ws<*jwa*^dT7L*Z2F`aZ!}u?!`ahmtFQ@v?Kc8#R z43(oXH!5dWrd@1$*-txlH=SIJijPZYe*kqdps-GMoh0Oe-R;z6U`Y85H_-0fvNr5; zo=#2C8}(hh#-HbG&nN7}z|nI+;*I0aWfI}V84un?X_qY}`@!>m`SBMVWg+rP^UvJYnHTF%lQkU&^Uf-mkXTMK6FS5I^f<0>j zjSJyI3>K2OCu1iA*0|gp+rNU8C>skwYa2CNhV>o2?xsx8#g+!s<#{v9gwhE`2e5Ba zVNF*%9uJ$@d-efL>ur*u($Nc5sh%DT&!lw3P0_!my^sgNX{_;o!su<+3?EwPbBEGt zZfPQc3Mr36M{4EMHSlKr5i!Lqof5;%;3tfx)a0n!oiC&F$mVKXdkUk(^Q*J4Lm=$l zkylzsA*6LS9}Lw)E}Wcth{O^{-P8X0R`$wx0xrPT8|9F0l{`eM*P{)6dr4b!Uk_;* z+UniLxW6`Y>(aDs-KUHbZDB)`tx!&U`UXtsb`0kxzU#K`xQMpnu1kB-F;?HMdv1oz z#Kj?uHN&L44+8<%whm}I?PbI)4TeswN81>GT-NFdovX^W11ZOTV4PeCZ`R0eaNniW zr@{Q_ui+x!89Ke&|Cq+l0vm^YZJ-EjUc_!goH>ZNoY2a$*1y@!teaSwZ3ryBP|}Pd zUixyIhsXsPz0*q#aMHx9OKsH@DP#)vLuO;Xf)PTia0?AFe+O+qhE>cM0xRc+7b92N z)z$JQ8d?(Iz;uKL8wp3zps41Qu5aW0?O{+-mC(DptN-1>JIPI4SGyv*B@`aGwu4;}*m zO>&2pwDO!~6VTEdISIjA`ls>qc$MrA`*s$RejJHi-6gnnt?7l?z?yiizS<$}$&Kpj z9}at5e%GGQ;DnRL`#4CiwCT%)+WIq6XWf1F3bj+O<2jt8`RD+8O8uYb`l_ui=qXe~ z+esiNJ1}WG8&?@vNOPBD?<(H=>g8MDzTVbf$h~933@gn*fq+1j_1(Z2s%?oCwmMNWy3=K8dDotV?5b`3q^oksIju=3x^`{xR z)Jg2=4aZ`>y%US52v(|Huu-MtIq1{}M z%HM%y+qF!v6@~Do)xKUU6s5gdMnf4TP2v1)#$-db1(+$P!!9AghS;r3e6}@J^9*Q| z1muxtxSYV(3}0h1=reKNHgSwh12zT|z>HPP1J6dTeX%RC;utSq{ptZeE z^JTX!y;;ZOe7#-2YN!vFV3e%O@J`m1YkGoLmAm${y7d}@YJe+s?zcB~DUDc3rkVpT zJLsP+*Zn%fQ3e%KGn;Owj&rZ@aK9`e6ZqK@9{D5nk$U^K%@Gji<^Y((cn{tBdiK=y8knB+XDl!95ev zzo8X89amVoO#`%&_o^F~UksPoO;1?|S$gR1~4{`r;@19X)F$ z03LAxXfCp%aA&O}$0o#30fg%l$r*#el+j3R$Ww3)&58eER6>H2ulggZTS94cmQL{yqL=B0t-UEiIT+CfXN z?JGWLWT-(A;tgiOX(f4!d7#VZ%|2 z?*`V_t?m9en#@@bJ2(gC<({tL0Q0+ZAS&9{?JuZDl!AzZ*VyD)^aomqCQLt?YeZ&2 z5HM65-S~+eNa>ErfSRP*A&;YaX(3(Gs4wL}nV3c^OTKQ3xUSqOcy%&JUnjA}3Wn%^ zyEivHVl2!{%kz=_F)hds-peEVI^{5So(?c8rry3A{+;SANsH0VseAu|zRLoDr(pjlz1t|@l$Wjlg=J)_4gr%z;FEU}F}95p z5V~`(7U?=E=w*aeth0wkU6;jlkm&QU^3TS8(k>kZx8Yu9dvm}ByNcMN>#bI59=2Zs zhv!?xW;T}$Bw1wOQbZBZ{8K<)7i~1GVQTK7H^GwjT^b}5p|2lFH30T^u-f7Y`&*D5 zp}78W^Si(ellz$0M0j_v@++NPT`HxhUoR&B=D)1Ee(fi{e1k_tP1t?5w+Cx<-b&2v zkJq%8-`;#S;Ev?ueK{7dB>zG^S}3Nnw0QFuPc6Eg7Ja`-YsZ;2qHAaLp}J)@zYRl5 z#u_rWBzM!4fgetw2=5-u5}1QV%hptMX|kD4k%g(Vx#Z&fp4N{??-J^zR@Y(Li}cUS zIs@C^-&ypu4mL1C3Uk!LRE)@J5yPZt)g?mC6itu3dEd8&wUO4C{M3QWB_av%ONXuF z&9e;w{MtBv)v3wV2+6M|jfsiy(is*X0n%8USsDv)&Q%c@C>M3QBL!N+s^6$>RTRgz z{h477w^n)BHO7NmoWJ7&o1Oym#?kTm74MRMpV(Z9xW!e$$F*v>kd22=Zcbv$k=m&!Ib32+mlD7wg124^Y9%D zCrOb#pHm4@1wKk5K%U&$ZS6Xfej zmHKG--OlddO$0L}pr?@$juunRNI=!+tdoW*AMxkgex>#l)-+CG@9}bMUL9<*?Q2$y zoyjaY2+Hc#LQ;3^MJEmjh>s9#DUf=U&TQXvz>+GKrx7A4UtQfh{`RUwm$_lnJ-CO5=+UPwwBQ z@(LW9y*;UR*DW%S{oM{vw~6 zCwZ~MX4%ZIg=*bpXUcAmH)o}LQy_VDqNOtJr6NChqRU7snOI)~&qdoQM!WPin0{1CQzilXO*y>4IswNV?%ngp`boP>h6;v~&Q* zf&pZrzNC9M>-mlTvC;FmSJgmfk4$YU!$|`6P`!E|=W#X;#;sN=-l>$)620XFyiSj2 zQ>Y?r^J@M#PVttFph_Ab%L96MYlr+gp1eXE69Jv8_hCAho>0ITu64TY>P9&^^iIEK zqoy^kK&F_7r*%)xYoYAvm8Q6~3{Q;~n0^?Yp~TW$48&CHsIYWtgVdCL6pFUA~~aIao4V=;QP$+ zwfrbnaE>d(9K!Ho(>!GBU+W&{ETM^oj}kH*P#?2v*=h zqEMdKrq8qbeFrVE=cGQOrm5d-&w<0$-3I&5MvD^%wS-wRgZqkK?%#0}H9n2{GW48l z$}}BmI~ZiCl?G7f@R&Aw?`#*F`hu*)T`LFIHqz__;#N{EX2nx>ncgm3lTC=pP|w#y zVhNb>GYj+Oh0+#0>Wn%>$1>k4n@xL3^DMo+xN)3+y5L0StJ0dR8x2>oQ`vrvl~4RK)&2;7;q04DWcm{{?SNVS4tk7Dz(cDAgqw9=COX*EJu~!(mfvl<|Z*;>y z@}vNzqPMr*W<||M9g`f5Iu$L}@-QVy26?#OqR8weG}&9CKN~~LswOA(bP{#xgLhIw z+Oc9qFvodfV=RJ!b>th)R7@FZK9QO+V>&=RNnyNE_-d&7V zt!Nx7Go#weeNP35CoZbPee?)#Gg0JKKiiGXpGqI*z6dyoMBkZ%D0XEpXB)V+K!`*N&=DT{ z*S*)Z@Wx9B?5+r8a4-XO^Y|?dne#wi4s~Y-{Mlj z?=AR$YDD+y`?0Rw&o3*F&K>~~hnG)}=?L6`X)WuF5@sk%l&?fBF7bR+LlK)pD>I1L1?%;t?c@K?{7D&A00ZZ4KC!KbcY|oJo6`#l4&JaLVSuaeVWufnG z5D)Jl-JzG7Fg4&QX&|46VTQ!`2}R|i4pExfROW;HxUk7M&ljB5;rMzA zWmf_Rqs=Tyup7;Q`-RZz8{$7B3s0VxS>8iKiU8ptu#VPbAG_tU+mWVMU$d1XY0PwF z<+q61dLdn{bK@wDTg2UXh}1giL)T=3zO*K+AdMb?$UroPrwUtL-;rDd3aNdemFw6D zCqZMTIE&P5kL$vGn+Mx}Ta5{gk2WJrLnV&7j_69U6C5Uz4V7Y0SW08O$i@NKE zo1!4JWjS%F42iAtn%bD^FPHL57-zD&=_P;8H&Dljr99RpN+LQ~S!w2cir8uG%J-Hm z-AsCm$BZUK%YPN^P{b6^?oBEgcFm~)XU~b%ms8AFN^_vOwjbYgk>>k$2$Z4W1JE>D zM2r)k8)73xf*XEVs8MXfMgQi>1c<}6GZHXl!TDZ&X2Tw(4yeH!0%|4OC8U3P2jW~ zKQ%LO=IuYk-5Q036GwWQU| z^$M`pKi2@NZq2`3>#Jr7TjW5OObIMpX>HFA{?_~7&7CfRf@Oo8E-MF-t%r|i!d(G1 z3jwP7$>-_mYKQH&=`nn=;MP+^HBLGMwYZBGx6@Waek7*Jd$%LK6r#27ni)~d+$2>A z+(L-jF464odI`P4L6R!Uj=n5Z0!J3)k%4hFy7&DLhC}P~ZM*ldn@)P%rTyFEjK_lz zUvsnzOm^^ve;gl9{10q;rAL}6#QcnJsvZ3O=z}R~sB1u)hK=1|#@wLa=cnR+#|t|+ zI=i}YMH)7-{@vQ`8mElSYy6~-tv&I48g^=@n2qI7F6p0!rUu}k+B<8>tw z3b=A{WpRTc*aQzAWb(7hme>Bpc*yw9F9;2P%l! z3wSY#!6WHIN9jPT0Wf(BMapGfnzdV%kX^SNo+g()mD0D3Ng(fs5|or020iCd(@X?t zwELC({1uBfWQVuhu`;(s!s>|mI*zX!7fD}JO2RM`r$kJ(ckp&P(p0Ra^pr+vdN_?t zO!Cf1bsmzsopxkXFRZbb!(1N`9K8ZcT#p469rh4LA#w7I>eKPW@?)%V+f`T1Ef8uS z13mHUbmEv2@Zyl)a`Q9vT=cBSAblH+sxC{c=3#RTm0(*LORv?ow9>5!?hw{|ND)CX zP&oLFSR%*FvCLrNfl6UGnOmlJphm4~U{Ss5=A=1HnlAaFz096pAQ>ADW5iL@{*pGk zktiW6mYJ`^ke2$$_U`b^&iTc74~0bn0h@Z-Y@Yk3$)<&PR7~PS#W;1gO@r-uLyaa7 zLxQisUyEM3b1s2JtC}g&ntE(4m+9Q?mMknKwV|V8ttj~r8e5KhGny^LxZC<#BffRn zr4CH~g#@MIFJ7q>xcbP4sK`8&Fo#147J|^jq-iU+-fkK%+W>c^STbZ0{yJY}%Bk`; ze%uQmJLDVqi^aK`#?<@S{GsmtXQ#|HCto6~o$94@ULmmOv+jEP*Bsw|`4WvG0;L{j zXcg>Y)?TbW*I6o()1mZd+qH%`Ipa~y{8tE3FS!zCgn`I*R&O$TgR%%7l)|2U!cFnk zv^@=lCO`$)#oMMiY`Op&?ooqO6n7SQD3wnrV$IRsAi#23EMWGxvsi&+ zCS z(%u}Fs6SIttHZ*V&Q$z?xfqF;oXo^=cpAOfEi%q`!`6!@`N3_86QX9H=Y8WSzi8Ol zTXSL{H$2qCd}rb}k*m&1L@VkZhG;JH+#EvftvgcOZw%GNkT$|`w$h;qeGZ&x*Dp^*MAmY-F-lPx zJP$`nSZtVOmjpa<0CdZ@moN!i%?1bU*e`W=y=S=;K`zwd6w+&`zdeA)^o5{}`oDJY zjYM&i5g;a#Kl9*_*b1S%A<$Y=%w*rWGh!xPAWKN7T*Lkf99Y@jMvAvsQb(s5i=AQF zbSk=TG)c9S@)tey~(;l`m@N;Qsgg`>1 zRH!P6DVTyDf24W+LrobQ$T+=RC5JdHO$b;Jf1Q7x1=IzyulWVU61G7U zNGVc-aABEU?-{KNJga&R@RjcBmo3bVr&eRz*QlXvHVMJTvwXn-Zh_W~3|1H8xo7Y3L2=X_ z3B_Xlzcm>5LQ#tqgZ4uY zpX&%%f`CL)rH&}iZhpXoOi387jB1?IL3@VD(2?T#vXnqzCDk;fzFG&aMW-0h`eQ*>+1_IS5>^V;g1;PtxIrWzN}FB_g9 zXuN<5-DF|&3tF8awj0XJ`q;mLPH+;tg`?QhXcd;7Dum&!J}8(-kI&QbTTK<0b0;n= zy&c5WWB@8_tv5_^>WIq`v9M=UB=y)ETmNo_)7 zlL(^lkydWK(w~zK_%0F{9c%}PepA;nEzl+>InVWU+I3_mlF3Wzo*u%VCuAr+21)qs zU*?*=HxsWHSrtv^maj^p9i0eI4*9ZBlHMP#_sv;oX|LA)(sLME@xkS+_i-hBM$o_6 zOM7Q9H;R6#^Fk%l2j*TrTm_^M3bR0f^xIm@FT7_&C#)q1@RKx1IE|0zFiBe(z^EI| z>G51Tpg$iUkK_h_Cf`kd3uhzz)*AZZJ@88G!d=mGyI=SzDA9tM}A%nuJn)E!%X2HDYcRd6N`#{L> zak5T+Yohuv`0|B(Q(4N7D>WlBFA-KBS#7OdyV;VR&Y|cfzp}ecz2JNAyN9vuX1#Ph zg7|5DrnRtKUhdj@ZUktDQ*&E(^X9(Z@qKpY#puVMVB}0}M0P2s{0K7rB6;TEwJTeLotB0d;(AaT9u}j$y@voF)7;Urp zvCxNB3yEVvYq}WHbH2Pz^5>K@r+Z`I(H9k{cyUTxLlV0=z66&yH3bF#c^yYIABYoL ztc}U;c1K`7m{@{@aAqV89fSbBpm`AQ-I&_?Ow8@DE!3vWd^7((5rKETBm2DtKN}+-gEsMiYnV zAu2t*n_4tdG}x4LdFqxGl=C#ao-Zd2HXdad6`7xV7(Xe9g!{)&r0o&XaTtq$S$jy8 z<6GaR=lWV|g@)$c+GdHJV-`2{jRDh5>p`sN!$^1B)BN~DRqBD}?$dGkMmn>=FI=U# zby$wEP|-M+fX(=&WfStHkA+Js0odS%WfyV=pJl zmDi`JMQbTb2tD2-zq(h7^RC|C82a(YvE~{1D=GJ3TEQ5q1$Zc2Fx&P7l%SqCLEEnH z;;)y(gGx8?ps>oZCq&~^eFKL3N!@Q;56=3-Ygpb&^f@eYZI3|F&#kh>cc;XGqQK!+ z)aMegY_Fke+TF`0>z>jHs&dDtr;FsR@Z9TeFZHHWXj+Yu)5LknWo;yNSi$A{F+5Y0 zEuH<|*3o8U4wy5YfwJRi_Z{=*T9bC#&jGp#tQ;d7xyn+er!th=&4M5}nw98Y1THRG zJFc`$Uws%*tV@9&wNBvc)zi_-wE-U8pOe{vpgz*4;SwGu$$VYvzbOoOhlWnMoJ7Yb zWzH}4M;p)-QdVtm++vJ9gA6}jM~_(QZE<4ickyonjCs>;LBAcoi9Lk(Aqq{~lyDqa zm(||L)Zi!7h4v%*Wq={0s6pX|GXr>5n7p>=SIBSPg$m!TtWi2!#R(%4lvRhO~W921N8Jy#NiEbH23o z6iRKc*cD*)ldI{4p=3Vh*cw5rt%8{ibEeBu^k6lml;P70PgL>I?TFQ+u(E;32V$2h zju>dJvdGqo`H&?8Z&bDDP}Vy8XeM|7`n;b{P5-bP+(IL_Q1B>GpEfw$57#s}MAwBG z@9Eqn-|E975)M3CblBj(J7SyKeZkMuP^`-aY*b+rtm&!5G`64K(R%p3kHdS(Mu|5Z zdKY%6Ko7bPg68mHT^PJb@}4@$%S)#(8u=KITwJRMXbm8JLCnzOhX!Jkr#Ji=qUDi$m0E%8_&^xlP*SM;K3;BmFgEs{>2{kDXG zZh#r${-r*nXF_E#WQZsDb6Bc@6Y?u^)kcWNeCUzA32F1Ud-S14c~r14A8P6LA|%sv zj3lgM)&t44Z!)V09l`Xd>EBr$EMx^K^r=z5-t~q{N)ymrev!Xvr6zp$@p+g%@M8j`f8l^rASqCy{x%fs_Q36WxG0Gn>2|W4#-LySO7gh!oL#l_d#h_ zQV^|wyp_5^o<~oF>+kK^9FG2+!x#+Trng{}l+@W|*KOA}U5%^2v7CNnOi&I&y7Mtb zl?}e;hUSKdQ|?w`B!Cvdqdxe#h(&nB9ua<0SJd9~xt{-$dtelf+2bl^|7|=yPL^0J z^E;mG!dPzIoSpkx2b;_^Lb#MJ;ei>^>zxz#6Bmt^3NsLf0=QrBKaGW6W-oUf^|4K6R8 zm~-}iyiin9keIqz30bn*lsc{+)~sgrP`^P8!>DTM$8GjSVp)$CD>ZNSm&2`CRXZf?osKCj5jU?P^&=9$~cFaij+s%9F3_~EZ}V9MYnZ|bNa3IZkJI`;p^*dBqb}g>>sQhjQnlG$5ypLj$-1nR3GLcG~ZhuHkX25|z2d zNq#KP*rQD z#zF*JVzM?cUdb7KlbITz3f*wv2+A|41S?ctFOUMrS*rxjK~M~C7TLW_h|N~a9KM#w zkUh2JY@iIb#ftCFfpGhuuyN54cYT*W0w`iHORaBr%z==pv|+H$U6vez;0W_}fe*Ge zfaQD6osqD~`Gh!lcc9i#mY@-LJGyfn z1n!5fRQQu)uX7&yebymDPS70HHV$KX1Uy{J8jm@?i5Z$Lh0qZ1czhy{>=KVfVUv0p z?Sk&ae(ss~#0;N1cfJO6Bai4j0xL* zM{wySc5Z}2I?)fnm-jPIs4jZM7-!zwKt*r0EO?Lz5$M;Dv2l*1?1~qhmC*vtu7ojK$tb3 zF0bb>eeXhhYbe(DHWC1uAI!g#YJ^f|Z}VCYa{h|d(Pj`ED=mnvAWAJ7vSuoy8s6<1 zHa431!B!9KKXvVa*L)sTZF1=tJLnoA*sRAHR7qGFyhnebSxIEOQeE7dW<#8vLy8sA zQ2<|29MsM9D3eWQM0m<~oi0ZnftOc9FgZO4FF`7vI_bBh$$q}| z>RZh~?gl=k|0qPZ6|><1smKd~(@fLrgFo9WNU#IsBR-QcOY(s@wQ}fhkWtGE|GA+qK%K~SvNhR?MW!R!Bx6Pfqik3Cp ze_SxgnOU`O-Y9sWNA&(V02&=$YK6t;*SzL596!B1AB!wN#$Ap~75@gg?Ci3kQ#E7h4+sGwU7#jBQb$Ye} z*)tWdeRomoIIP)}O^*gS%+2W*)8+uz|I|*wpYjN5GjJjAH9oIr?&FU(ZvgnJzGVpn zo&B}^x{_a~X$!bDWQASAkkYBFkoxi9~M^w(NtgvbyP4-<98At^y`4FQWmn+=MG*%UAlv(aDh zKDp8a@5ecqDDY1uiGd&v_ej4JsP}TcLbMHUVj;@L5@&4c+-?3n_~N0{9QeAM^N?w7 z6UIoBi&ts~K8;~S{db0CDq-j@lHyOt z9hR~v0-FNonEq~IScCtSfYj4@=)3lOWvFh`>?c@eG5$sCN#!_RCExqa>vX@HEfHCzjn88T18-tQ=s}*A2|CA>#^l=wkIidir5Q!p9yj`i1>Js7miJ?^(q$oQ zS!A2bOqav!iyg&n*%PRosrR4T+WMuw;}Nl*BXKIz31hVy;sDAzDt+VNYych$hq`A| zI&qL`Sn+7dUia#zt(w*5g$e5PbjQ@5g59&V7FSnEI0uzBF`jxr?l_;h=?v9^H~{8} zCKS+p(Yv?*8PbH~N~n5$5ZK_otGF!_{R0pVHrVhgW2WFCL-@HOe*?oLQDMBL8}tWU*X6RERb{zEJDj{RS+U6NSRZ!EmN=%@rwaEupY?Gc1m74?5~u z+6n{LF|7l!z0v2_07H#m9EshSDl{_lDL0(-pq%1c`F;H-)_M0!eNS%*&#^y(@pe*$ zN96clRzK`H)9OWJTfgnL{}ou`&H&k|ue9Suv--A@HkOXXvUG1}2nnQNNv~wSZxCOQ zDfKZLALMBg{Ga>B{2tLDX7GL3&Gz0Y!wefL1~-v`jPc!z>Nj>Q#3rVE4swZB)u28? z#Q-v3EPQ^Qq0h+MN1aDT0g(HCn0*TW+O>idsF`T4rZalTJ1GV;WCg9sJ?iOHtl~Mh zCAn&Q4@>96i0iD!c?%}4*h}jMCa1P}FE+>qmmm`9WgAu%kWGN5mb(E63pE-Zsn(W# zMa@I;4D5W46=e-iWPA;8<2wQz5 zz{b|RPOVm|Qtj_D0(vq<{*N+i=y!pYek03w!**e#6Z@OR*3;Sb#%g-RiM4zMfOyFfe`LFZwHX z0l+2Gjw4#nstK0rkC-O;)f9&08yh9n6IDMsxmJxU3Um>Md~K|FSwb^}u<3>-aW?pb z1oULq;0e8-(+nG{Nrh=7Kbo<`==Qa~k(d5JPQNUCJ^EM0>B*W+eBS(eh6-+dAvj2p zfhwb;3yJP!MM1gwj$FzAUtc^amDtM(ffv=c|YiP`2r%u(jP1LwmCj#CTc(t)xnK&RGI zm(oWdYfY!&@v?{M#oJU9;ZEG-z%vPr=Y8@T$N@7JsYXoWVu?`;O^DO=_lH zFzWvpxR`*-`FSU-s9$~^nj!3hN+SF*uM9%FGrs0X)=+GA#9O;bPHsRiE`uJ=Yrf59`+IH$k<#s?9J|>aHB6CRm z;0i!0Dd^}&m$c+|@4h8Aa&)%+Kns_x)_EZ+-^#>Tu^<5N)cJFhBK2cC*KTfJT<-8X z@8{c$3cc3ilKJwMbTcatbm@ooPU|HYzty2Vv-{ArgftR*QmTnCmr_%fmOkcuHd3Mo zC{ymyHu+ibkjP?}NZUJP0!f#fdZxeBBkL^23Fu!(YDnqfcscB!zDz0TXAs6oJEs>{ z>Wk67J`g&)9B_vVrd37Qadypq8dzr;l=UeLsCm5f#S<;!D4qU$zP_6Q~{`O%y zMmc@3TssRgM2-%}XH8JtHqY6$`kb%`_>#lJJ}QwEakpmY3P8GqLS5U1?X2=9jtQwwJ^tTo&{#^jTf>5S|oSX9^b6)AW2{Q7mB;#7ugc@RaPC|JrSLYJfJR zh}8eh7*y~1XJOcL3pu}|^~atF*& zp1wl#O0m}Lf$a^OBdnJ?U6e>u>!xz2F7SzKw4Kne8D~(=(h4S+WlZ5QO)DRbRKky1 zxG@{q{pW4&T#&~1^rDkz@w`YyjS*Rp!V%xnmJxzGXqAJ%?-&&_+O26 zUc^qNH|U2%4Vym%XxQZ? zTH^=R6e0_<{yF5+KH?u%! z@rkF`($O?#A5}LiF2;a^Evj!y2}2A-Oqep6noe9y%fbT-+hJ%BVB!*q=1vp6#j;q0 z6;13!a#%;HNUBvI+e;4ZnIY6RMcVgx)MJjQjsPm$tCO)BciqaM3sR9#iJULBXw~4o zO%5es{dTJz+?rdv!l5lp)V2+vu1IMds&`iQ7r&;6Srr z_%pLProK+rgm#winQ{SmXk<}K^pD4Q!8y$j5{-iICw}NuBt0{gfoIBXNtExf->bE9?o9Gi#IF7kDVYyI6H(^mQul8-5%Oh zk-CyzHLpCaQUlg~F)Fd*!~c5OJu^F@R2O2$Il7c-mBXp#4R8#N+8WoxFkz9{@k9tB z!8$x8L;W}vtO6?>7I1QC-AISklL&i4_c|0`H>o*_(N{;9(n`^r5eV|EWgAHqDLvro zxn;M6xr&3Cg-QUc=lVQyJck3BOwT}&XQGg-__RY52%B@_kY z9tiyrg5$lck}biOD11JVE=yjC6_i6tk|1sy8%8o$7aBI*N$M-%dDv;4jTULXn(uoE z3Cyq`g)M?4ML+xemPV!5`n6>LdfZ(ehqL~2I5&)M*K9_+J)U8YA4N*v`9>dxiRix? zRlePt5=L%T%R`(7#~9h;BXSj3E|l86K?32+_@C79$WzJ(>^tSpnlRNUH39Z^&jUH_ z^KlKgwEy(C%^&q5PPC=uj57%LeN5VJVp>YeF?-(N@>6#mqwf(_)Ous-3P|ARRV}T| zeX2DGWsr}Ja-)$CgHFlr6T;0W{bGqunOj4;T#QhCumu4F`bI4kMf5O{6a|SI@XF=* z$bKIm9)uc}s7{kX%7Kh&qOJ@ia>4Oe2)Mv)s&ZIfPyMtP z$IjElbMr9p&E!@6IUMEtE-JqM;jh8Od$5m3JU?s&o9@M@{K?O*KGHW!rIHKSKabyo z#K0le@!Ed3)8%Ua_|xW3^SdQ5;6-9;WZn13Xrkq^WZ&xY8V+hDbE^K+zQ?zR_Ju*Y zR;KBmF!@BKwF(0ex$Fli&+$$bj;DACw+_CB8-nh|f@QK39}n4OAq6I^@IvXeq_{X7 zd+xkytnqc`9rKCuzmpiN3;MSC{o`GWM|5*L$MJd3FSF+!;FF;w1$gwopX+K+lWQ**qU18=bCj~q#s4%BZ{~(^8`tm*>GivoN)?yXOb2?i`S=N zw)+ptGuJUIE@M-K~WMQOkMYEzK6DC zCzZ?V!OyPNhIKYf}lk%#SSXus4Q!v!iG`A>KqK0K^q7>eerS%0CE3~Nz6u!NMbheAyrX}=;*0@)zm`Tzb3v#KlmW<7egpSagw zyN2E$&xL5tsFgb+Uet_$2(o4NdKesE>Z+mCbBH?&Mfcnkcz+k(9+T7Cm;Owb%ggHu z=Gik3BKy!4o=RLZDx=l zTZYvx-*U4-3Es&jKwL;E$}W_F&Zp^VKSsu4UZzG;=<;;c>s?fu!RZWdLP_e;ZeeSY zQ)vAd`Nx=nHyS)?HME9(zMQ)i7}rv!U+R5+(T4^&&&Or!M&fQf6Kw7WfJsxt+a>zSp-mx8%=XZ|Jl0@J zFK9`!g#GihOhOX6`K9l2autSQwK#7U+mxxxCnyN(4dKza1zwB)a~e$sn=Bpj6f)En zq|iy)Qp(4^=F+?Q9337B>~!rGraXPipX#gbv2?xGJ`{R>#C*g*W6)D&`lMziMo1>N z3)(~Eou+KjO1`16I;=aACHgF$s7cnjX6SB&krte_#?p%( zUx3ih5nnNXAyp0M;nAw1d^zgCQfN5{)OGcwoYG1)%!bjSJU1-*>j!k?=MUyT9>(uR zc|O4jr>sQs2+%z-&CULvn8oXvAChjhBjr?5gnLi1t<^l{6V1)Vkh4?+VwZo*t+Ae< zu%F4lqe4HUgjk7933f^n#yd!aHGwP|OR$O=!gP$K5KybJN!2K>&6eMdx`HlCx09iu!3WFm4%G`hrjTB> zP5TtiCGXyQ9h5HIyU2VrEi86z7q?k7Hr2c|2GqPT+`i`;jWw)ve<;9?VZBIU9qUiX zPSTp`RG%Rg9ZMQ?*;|DoI%o$ycMbH|jc+rvAvjZro3fZf^=4?HKpz%8o1ViGQ+Pc( zE;P{^Ha0|IC72u-Bho4x8^*Qi&7kvBIfU|27&P{0x@)x3BO~+Zwd?7D_lp6Pl=8Go zdbBAyV@`+$y4itY0(bw%Q_VL;P;V#{033XR3KX$1Itt;;Dser%n}rKIO-xd4O`cHe zfWLUGY|AS@bS&TpU`sdzE+0AY5WZDNCaiZlHmUWPD+7VaPDCwaNH@IeiTmNFO2nH9 z00Fh&1MYaut-!14OFc_Ua33>sCpUBdG zP6&e}Du(wa*J>Fp^>mqL?STO=5^dxaB=Bm7ya505QBbU&R%po4wm7>?vLV=;3>2?m zKqTkT!pY~APj!+qvTZ)BukqDxRpYI`m2|Cg`bnBE9#_oPDLId?SsG=IIJ8hi+1j zwNZrZ=6A%sBC(4afQ2C1vN<4hV-elMQrITK7wph@ba2SMxC*+CR3YWug!+zZATY7W zKTTQE_`7U5w0R${WKLX~s$L>F+AdFZ?#>bG6k4Yb(VTR67SD+ohzD}PBzlH7w5Hs2 zsyu9n0ZkU5d{Yd|i>6zDnco|7teZZsl(_~bJk6ae8y!xcwdlwCQeMMItHb`PSwHCU zce{fdN%C1m`6EyDMN3Ig-i|B}WLPiD6L`EpQzOC~Qs{Pjz$ABYyJ^FIL;^uF=!Tr9 zaKG`RWa?CZ*%LZ9o)TF^n4aT)Fh{8wP>SeIPsBDy($4m1r*pjR6n1oV%y=RYAnBw0 zgnU|?pM75)Sh#-@Ne(I7IoPJNf$>4I^K`i%Ofh)B!YBtK+-tMD8$e1MM+N%vTIV@?U1#twIbw0zp(OFtefa@N2zKd!B(8gN3t%2;BO9%1X7#nTf^GC-xafn&+`15k5}W9*+(kn|NOq+ zU-$^?EBLvT4gL?_NH_6(9mvst`oFAChT|Obu)t3_Q|K9u1U80txA@tVqf=7OShj7F zvgwsDi^27w2^HR5vT~OT4=_}_FY5LNqE@pQ+%}q zpXl7fAd_Ia-E<~-tk2XC+QuPgo{bug9XGhCpBuPHZD6E#xUnTfSA2-YK55xoNQQJq zqC!$HooApbsI8owl@e=;6zEGTLJ}nrooVQ;m>So}5;X{Q1}?Ivjshg}JI>n#L{%%5 znuT>iv(Wiq*a|+v^O3DBDEYrr7~BNtmD;E-URn$)yJU~dOS1~0+p!-B$$~CgTc!RmBs9BiO=hjWQ(e5Sg zpszF77^s;5o)`)L(X(Y6yOcix)aJh9dw63AxN%#XeEnFK4pLNirs{wlvLSdNcJPVY z+|N4{ZB5j>^AD;|8jN0)sA$}-^c^rVr4I{HK|ws5&MS>YTVL{>BYR&~4drsHF{P*n zO+|`fqvyANg+vDLoxX-Qb`gQu$`(hp45fDH22mB z1&E8;Bf-O^K@#j&ze)CnaTMEZI*7<~Jk(|zGa>O*wHd+VG4 zS{3qBeoBq3yY4l4%x*;A+bA8rkh22c4xZ;scn;$&MvhM>>UriKvcP++nK5DgYnVVY zZ7vPWF`e?jGv%h<_VKZCw2_;9d4RQOv`SBB(73UqB=yxMWbK{y?)7*>cb~32N{d(k zf5XlyFTtut6PJ;269MVAVTeIb1YTGx){siOBs&+({|-cGPxAL0^D7KxWS|7 z*3oloqu^=j9-=M$v{lgUE!LCS-j1WbCYYS>InOZqEy=`n<%U~q6vH-nQWorNlB`Z^ z!Al;vPZtEPDwKq($DRv5Fx@`o!Pgl7$-Bun$nCPqj0H&m$|ilV)Cvxjd?=gPeW0$J zFAw+_63xZT=g}x36=&XPETGliAHGg}8 z`AC8bMJ41bkbZYf$gp#n=QZ~3SSSKuZ1oNlvBsJ#AQhYEr@l4)6dnz7I*Gh0KfS1wBt8S3haTY?s41=$Oq*>^u?$Gq-EAdBwO-iQyg@u?Iy4I&%V)NE|`TT+^ zWOxh623myaW)ntI!O#<`Cb;)aKqa-wBmGFca_fLMq-*`i5*&^cp{Nh*G)R^nnujDT zhQ>*9GGbuS2-?LNpH|^N)oh`0s=IM`^uGdBAVFrmIkcV{A)}>Fx(7bHWwtsbxu}vW z7$Ke9aC(B!1&xf4r)J$3B4E%N)4}tRVAryH#RB!NC93(!9{p(VmJ(8F$n&9pTpFQ~ zvlN*{?whMSOOd*cWOh*#EmqMtP%~3=rgf)aB}M3dvsn#Gw}sm2b1mU!w^5V0=>Wb= zdP+*dUFJpspRy3{tZCPWBQ212Pp@6>skH+>y0&4tO9hIH&MTmkijW^e{vK zHO;%pAVWwj72rNQ2Jn#@+Lzm<)ZxD<3t~fuXoTpnZijDNVUg>(2l={P6XS0@h8uE2 zR#K-BpCZEC@&YfJr3nmHG6kjo*2msKKaN@p{-m`c)6K#tk7JHWhxaj>j`{N4h{V!m zkE=ldAVO&mWs0&b&EHWA6q6PM%E$fE1*-Z%q>ZC2%vD)884_I2XQ4N)x?sCHw^RqL z`GeBk&&V^w{Cna#LX~;x?(Qzo?n}1$wM-IgXi2zGsBCRB(ixFQx zpzCy!iGkQgqTAABKOe>4VCDp)!WQueT{=>Ne>ivcf_h+<6jts5eHZ&*v$gTh;T3(8 z^fg1^7;^($EGf&e6TdmN-?JA*NVX6xF0w_gI zw_%PEYP^|WHM#gNHCqcYhpQGZO*st4I0Ehr4J{n) z^$tum(zsj3RktJ~$(XsJQBRep=@DLGg3T|X*8j_q)(k@7IDI$6iqgDRWf(x>F>42j zr7XCbPC$NbC*am|aPCOhqcs129p}^YG#m$w?^g377Hq>70oPsTp63rY{ z01mLe#mDPiLtcvp?wHP{YMhoB2h9N!(KMIjnga7R%!atw(i}q))MJ~71cuyrd8+(A zf_<*(uWny0Wi&@o)OCp89x+Z1p!zOtvv7Do1b-9eUVSXV=(;`XBiyl!i7#7%g&hj` z`Gvn4lq@#rS+cq34iQg+SzwntgQ4%zO9o0xaL}N-Ve7%m$_#X7rgM-BHQwe)?dOk$ zC4aWyHo@Y>Rv_lQC4A_SdrbQ!DR9<~PRWdtKJU8y$1&}}*cj^qS$I{6^S`|@2h;IJ zpnG5{*)+NwC9b`Cz=!T&4!6P4V<@5)eu6gk}nrcbB;vyd9mh@ntI^+6I0> z*{d?)uPW#|p`Lf;dZ?ZmfJ)ZfhxskXG1!t;+KBWLx#Z&Yc972+ znQHw|;|Y%-!u?6)!zQTLx9$K648cCUL>eAs&69J@(i|<-_}Mi0DROK>_3McXjhw?2 zDsSWzG|>o_Dx4Ns_L4!mQ|o)U#_XzhkdeBCPl<9!`h9A4Bg{aQSMV6_?@rHPbB{QD zuhoU)749oe5h$RroG%Zcys4kaX7It?Bn25Tt{{))kk*TKo;7B|mDGnHLN0&4ri8DV zv?qq4zOQ!RtiMU_`g! z`4*M$YgIc7h`0CtfzrQ6?KWzZefDhJ6OzaC<+H%DU`E+O%x+exi^OXl}usQdz zl!OW0+ol!c0Mhc&sb=8)1Zd!0WVo!@Jv27M=UaF;Asy0(|#X+YlZ$kSl z3V?uC>;W6qq9&cX$iJsl%!`3VP^&>#sxI(XmQ|iH{FjAy>Pr>Pv=}q%!hg*eCN4;A1EozXS95G+6Tj3MycI zKbbK)p9EWY9keJd_cy!kcK1K|h&*Ci8(rKf4n{pN(K5D}MT3jWP^P$rIktS-lO5}l zGTot4pDLh%G2tlLf!Ji!x^X2?Yd)ip>R#^_q7_Q?;0C{sHSsf~qj3PprcOC3h8>58 z{k0e&Rd2`e&vUC!eyi2;*IugvC#Bm?DJmX{;e3zEY=38_{>j>HZ}q#*Ym_nHaGusv zLy*v6iii(TMq9Kd@V9;DSk!L@>H1y>I&=MwM^t}qk)q?uIYO&~D+UQMI0_;vue>%K zCJ})??4)10i{7bbW&;!R&jdJI0-8H5-#+)BL#g9wBwo`eN0ZIyx-Vi%_HvbgpuDG# z9#p0hNt5Ayp&;U0b93Xluw@78@7{HZe6+vq&gbZn!uco_blWNOn0I-5xTgG1sFKu! znCJ|(jUuag>AYIR;EM}+Ns3&z1Dc+UC6dk-8QmMs_!xH9Y7&`AcDsu3VKUXF5gyW; znyc)uF_E5^F-pT@M7q!)#<8w_S{@<2=|=w9;#2Fs)*P0EQUQci$Sz@ODbDCpR8=VY z;k#b4s_m9fJq_z`?CcOk`#VXX(%P*Fi)qXy6(4A&^op_)UJD1L_c>-&yP2{7=20)T zzyw|LkprO9@H*{i0VdB4K^4B*ecT>al4^n?QMhY6DJK-chg*U4s6sZFBwDNih zmmyvbd|A77LNKm+1e|hxPgxKP%=r2uS-U@6@8i)Hl1pD>+R~;Q)VSyKkj9Qw*v7w& zkC&!9Z}aVwdZYgy*+3GzDGR}n-YG(ZD8jP^1KIPrX`#M?leizl zk8KDubga#y^Rt_$9M55P9>V2PjdE4_NwoV@Xdi~uxZRgEL;4J~7~EV*ui_vo@9QaN z>rnSaYY(!L!D3hq(uMo(R{bLIMy6zcbU>EN@=6OF@1x{6EAS0nZ>p`T&{`nA7JLhH zO0xa5B+VL8L{&I!ix96f}^ zL{m46(McA1W5)tUCJ>tNCHB}pF@7&jmzt@vqUc%*~^E$fw zAm#$XOI@aZ4-(cV(0Fb1;MXXth{N|!Bw~P<^mc0LGj3rzHta}u4zbDhLF~PkWGI3e2cJCzhB+=>@u>O) zTQk~B8+vcSZBH-8$Nm#Rz(Rzqlvy}TmN&P2sgG1#c)&6i`PGrvbfay_k-QWyYAU@~ zbcDWAv+k9S0cTz=AkdewzH&H}KoTKU3c?L4pUifJL7L7c8}^d6H_J+|ihM-7q4w@* zf)v>`0Ub^brH1{}m-JPlF=59-2@or}hI}PW-sjw;iTzaKzU>t6u8I1t3Vv(&Z`W+F zbdJwdlxy3Ma8R62fhXt$kT5y{-+8hX5S7*EkYNvz6EVmbrBEJRWXaU|@lwS%?RfoC z%COPs9q3I$jcAi|x{UWYs*r8FZTdQkjH%LeA z3F@ss8UT5QMyyF~iIy53mU&NI@1}lhq4aEhD>on^)dj!7P3FPWFFpa0PHdGpviR0} zT>kx9NCSllAw(ame+Fg^-3tfzmnGX4*`Zd9Z-bOTyeXti1RS2PMY18}g7$*cag1Z& zY1Z0q$Dw|x)c{-qusFaYbiTtAii4BNQc=cq7U278Qcy_!hw zVC^l5w-`U~w zP#-rVA5)lm+!4{q_=Zjy;O9DB=6@ZwleugC0+r1JK^E84tF) zmx@dRhN(Eh4aLmw+&`D?N~zFI5%UYnnqe7Fz|9m^-5CVL*Cfy(2#44X3wb}lS{b1hng@rly6#`H3rq_ z0E)8I$TNFxYkKT3jGVK{6pS4tr_l_T0T)|{Nk z=2u`A1#*ANf~y1ILvX0gX49XnHK3!AGKxp5)v$l>Pb1$`BVfO@u6k&;r1DgV2CIxI zT|FH~>5#!$GeVufH2D#|%n&%oXaX&lmG(@hg+YBhC9fr?DUl1i zH4|WQTm02hO5%mxW3(qPCa>qBRlNw*4646PQMAljw}sc7*m3U-jm}ONV5@2YD_Ou8 zbF~&@U#B0~>a(sxTT-s)Fdrr=yx(K@1|35zw#!MwX(10-yQ=I!8ymx#ZtZ288U zIPXz^n1AMR39*UO`CD|nC;(fx!_)anBjWxOb}4V#{hU@ExJ^*szHR!x&}`; zd~xRwMIBg|9QBh?H?!rnN;U%3^RT~2f!5mk=D@U6Fj~|`j3feLP=hrkj=SB0VlIF` zB`8k6aVmC7@(X_daeN+ACzZ5` znn)ZM1=EJ-z8sTmzilh`0TY!m)hD9gJ#?*T=x@|dL#JrGzx!M!R^hlmC+&}+85Xh0 zf>+8|R6`Ut^muBUpuO=l%eZ>K59&?X3B?6;f4Ycc?=Bqhz$4Q zOvoDaX5%-k8UbB8@qlLHdhOkwyxa7D z-*-nfaTv~R=lz|oXJ&m`cgYFmia0SXwPR_$rz5+*#YHdo7H^qq*?Hevws@?I2i&O5lO^EPKfH+%G zTPc2}_~fA%f8Q1`&^903>PY8Yww?MoBIK&k02{PgC}3oY^%}OgN!X)g)?zuYF!E`F zrBa=TSE|6A$~It_e%Z@hEeoAEx2X`eO%LuRw0FnsA(XarJK>&s^wT&2O^O8vgj+V) z$anOIL01ooJvw!mhTK*^Bm}u@xq0ANJ!Hsr09( z2tsxLTcOK`7&M=n@)+mqB;W_w0KXB!QPq9X;4^;4UerccU&Eb2k;SdtD2e`}6vkhTeb$X9#0 z7znUR6)e;}w$jlRiObS9R;*d6#sue_GsTj6Q(N&(#;`5#;rTqaq}&q@$RX(V-fe=t z>~LOd3K#0=PiN0wHvb>|n#I$R7dhm_a0xcK;HzjRlA~e^vxro>*5_VhW83OAwF4%p!Vq!D>MZLLed7WWk0=kdV9vni8!^T6`WPG zZ~~LkBc37MeQUHkg}}JbMCw4)qrB6rJkLP#zn*l>@~h2-s}7K4>N$ql@9K_LsgnH8 zddg3SfnwjQ%$BfF>U|O$;51vRMR%WJ$b`M6N`u2u15i%o$eteT)l*X6S*<>=Pvic0XAE~+*!HLbZ(R} zb8*-~hb5ZO4jsRG5Ju`cj!&vIE|=$LfxLRz zL7kZFm`nG=4O<0Z0>stToh-fs0a8xX6p56HvV(D)zVE(+$@*)wEK*I+X?N* zh_L{u!EP~I=LSS;vnZT0y&9EL0M%XdteQd}Xw61kwroyAf`2+U+!C!>*hRt{@?d#L ziXYaBU}w`pkh7$X-i9OfNsYlVEF|;0gLB3ZTAoE_`mX<^3++<30bVRFuZLn%NITwo zJ=2fJFp6{cv3v^7{F5p6#?kszTcq?qrX&qcn|d_;j8nhsC|Xq`Qx9d%a82`q)Vyf0 z9#KpTxC6I^G$G1wJ`DT8YNksEJ6g6k5?-AK2UHDEDMP9(IJgu~Xjft|>tpXhc^t!R zvljC=f(>gRqTj)3I6UlQ9NvV`%HVuUdIq;yWf>g{B$zALvSQn>Qeb&}az?n;Q8cjDTS7k`nhKB^%aBfQO zO?C5SXf)SFWZWszfqIBtI5MSAnnW&2H=Ivv2imLs`*?OOr5B{qi7=4qq%6aM3frZY zp?l0B0P!V2jM#zY}2F5z?hm} zk%~swrLF|f8PIucYo_TZmT&iyM832xGqKRns%1Z z1UrU35tN;xn)*5cllP^*)e8@Ibn>}A&@d^1^J2&ReN5$Fq3DOYQR#xf9H39HIS&ZS zT;S`@(gl;6$Rbe6mf*ipmfv3KS_Ij?d${%_7#D?XJcbR3oSXhmg)KMGF<@JF<==W( zn>_7_f#^`iMkmduM^Cm7mJ1nI4A<|OP7$?~s#B(N*<6<_VJ7_9DQkEX_M0sm%D7~f zPM(UN@I6Td$f~90v+jaO6k6?^W;*=1MWHnZwb#s= z6f{k#I1izY5_CB2$azb}lrv{oDK`_(rsG2A6_u!O)fBb+dRIvhi@>M>Hz^!wb!E5a zX#L8&n38|_Qg^gx3@QAexDKwPK;-5a4~HlkB||^#dCJ&;Ko^;a&rau2UG?=GwSFrL zV~>0FE6HC8-c_?U*lq&<4Ou|!JxhkH8$wEATVa0Jhzmc|sbXW9tv61)i{`Ym=(p>S zAOSo9cSsaGxM{gpx`*e_Eua!sFW5*SoUmW{TYi=xo{^4+^|qTrEOD@IPSfvH>q zpOhZPhA>P#OpfJ{c<^Q0Glqbto+5GWeo0J=8ED$?I_6>=gVEtHy+~~@x6nG~ms6+; z?w`KwYNCJuJwU?0iPCXbG)f#-<7|47`WXIH16MWcp3Q!~xi!G7Xs|*j5ZF?TjZU*>kdHOX!zhPOFx< z7_8JNpmkGs(5R@I>QYkHme=I8*k7LgGLn&rEV2=e>jsZKbJ31*%sz zu;0d$PioQh7u&cDwu&N+j^FT$Wq{I-x1^FOxKD;mX^SzN>2}|@jbBx>P_9)V+d=hn zG&G(bf9$Uv;)092&B4LDtO@(uXqXHh!@`w}fktwqI=R(WJCO56lZ)2si#hA=L|>-6 z&2Mw#RMliL>`D4&3)cY(TkSSNaM?+W@$QY5sne$$5zuu?Gr0k zZ+gLf^v4O8RjebPMIX-8@3u_a^s-4@gk}&u;=mgUQc!|2Zh@Zl6Vbi7#gH~2v_94& zixL-T5JHR&mk|CDZh?Odju2}5ZJ>X5a~zOG2VfSUU24?EvVTWRA}}7+ zy9whIdu@)Ehp+{4KJu{2tCdAfSC7<{Oe5UYf&EJ;R|zEeZ4+C@UT=m34f$CC7sI;x zw)w~U{EC)=1f){^LL+RI(BBj%gX$1Q4$lfgZO)c(j+}}p>`2$YIWzItSBZWZQ-KU6 zgC;n5A)#@gEW}Ov%&yIoAqHwq2oN8^RfC(*?i%k?G;s99s5D2haYaFtqAEu(FUoC` zo7!GN<9!uAHjM(0sF;b}2In{}G0m`B`HcU{G#^(lmNRz_b0z^x9%c zQ7LpPJYkK!*0Q;yXjp^8`%FFf6%U^1V_8CPR_M!XhmoJ6|YyHR{fv%AC zt+G)qPDx4Xp-6iONCY45re{vPiGW6RFr0F37T53(?caV1aORCtkiOm;N(9GrAB1E= z?Dtpsn<>|c#efWobiL8;!%Z_JvU);74(u=^v>1Q32{mTFL`x+k8!2c3O@BHp8;jxQ zuHSS}vFEQ*Xsx1mv%OY=1Tyz1F>YP8YvWr2%wfqL<}z*41Vz0BnKY5jwTLJF=G!eo zv_}>?wwT10L;N;b1m?xey*8p`p5IEH>4}wn5DP{KyIzh-0f)W^Qy+u#rgO>0T8;LX zV>(mu9j)|Wc<>1zDR+pg@tI5sRfMgo-BYb$L7D2a_Qu)ij7A%fG^qR~_ww)SRpRaq zd(B8Vf6ir8-vFTXmwm6rTjpa_hx9!SF{C?3>AmsFkbX zX1jmdlCy^%^<`;NPD{j3=% zGEyO&<=Z4Xmk@agpT*W|O4z`P4^3ng%)q;x<{ycMj&PJ`zqrZ+i=wn3VWMg)y^eBq z%joJxt&I3RI#Z)=&KcN{SyUci34a8q4**T=sI9cQn73ja5AXR~*R#lxr(sHmBySO- zT|epXXp~s9i72{1jZziQb-7yO@A~(o%>6y4&NgQlY{qVbYhUxQB|59Lg#*;=`6xd4 z=9eeQ4W4Nkrt~U%$9=2wLt;{FAK?-WVogo4s>g1x<)^ezd_9M8yK1V1E?Upu$F%B# z+9eYC1stlwHO)t`u@zRiw+-SeROb2CM;)9rMxXRHkOCdwEHpIlCR#UiOU{ijkS%#r z3iRulCxCn7BkkwhPzxl>nkXe@%;RId1zTX#DxjXcjEJ#>a2Le*n@OAm$&&ObHP4x% z$*6WXhr|juMxd;Rpzo2DqS^-cro!1W0Nc%qcblcMTEp*hE7oAvbI~p=1&IX@iL#M0 zU)ha^Kn;JfO)3-7`6h$hvw|mcUoG`H2!T`&a3icbANuH?>B6~aUO$D>d<^m7=?U+ z2c%E7GJo1%#GU`GBl$BnT9&CPXdTp`UkXJ!SI1FLY7N(zZ9Xm)V1^`p4Wv?$?zdS8 zb#d2!w>f!N`Gm?3n5op4rJ5%#tJ+M#A>crz@`zZ#%$%>CVM?5lbyw~&j02XT+`o+X z&D9n8X03L}kVfbv$bV3y?cM#(_^obXlpF)9+Zs|20;e`<6Q<($1+%AXK6zo^gIYb7 zJA{3k6t)eY5uh z(NO;p?YW<5qp|`Ulq8ZWjXDtDc+dp8!4<;2s6!3kZT@1)iqvw4M#S4(PMDL&t3n1d zs4lG!CuNtiWl^J7JyoXx;xVYDIlJR+PB{|qQV$RQvK>=1oE_jqMOGIq5e7e{>^zaT zm9JUxLY~HW_`iq3j8nYg{@cOcgOjydb+>X0cm#&vjedBUNPcb3S*ai_vRI4PZPf)B z1KSi3-V>%xnyu*Jq&y2B+%hU=Z2J(d##hi7LrMa|RO2gE)h@@5iV>rkVAM5IbC;_5 zV30^FPLLH-NY?RGM*R8-p-g%sCjL_qTxC0iYzL8ieY+f8vc>s)M48+#&S`){BmrK6@zFH%xBaM$p)imt z^q1jmY#i43a7y1~Ln+>&dsy`{se95n1}{tSRiEc=;InCK1Ez*VD#mUO%92d>pISe! zA;rn2Xjo1KMY!c2ObP~-DJw0d&mE=(c2TVZns?ChOvfY24hZ(L3t zIzc~7Pa6L=)S$vNBtb-rteX0pk8sz$PYrp^X14;Nc+4*t#SRCTM7=Gma5S+%^V|>U zPbApb_goKp0(BezGs&30*@4JTMJSqy4U{7$dcnfRe-VKgN{2Wst=JPPv<6Vj#va}P%ELe z%Xu#VoqC7Nq-X><^ahB;xbN$LvFy#grKhk1~)oRN166T>c62k6GRnoc~nPRfrFIiWRj zeO`9%HMuBmJO+W3p7&wjP-1lN$!4+$Q>#n0yb0!(ZR%|Qph2gTe{eEvkV3*R8;N@8 zeCC%X$P;_|VU*CH82CacOGF4yVdoOxWkmFbJ!}u_^^&kW$kRnt*Xn!4x!r73VGDby z7M5=*od$`{ipnwC!)+Fj8j$g}_UO zCIyx+L$OxlW>E_mSC{_n4}%2vwx8dmSewz=pxd)x3xNvtVZkS`84kmLj?JbAoWRcD z8g1RQ+?L$d%2CL8D}}}7DNK=ug=_hG*c^^ttiE{fPfdcj@05M3B~d{;1F|usPE%( zp9OZd-@B&S96jr~@856r6aZr9r5h1lm-m4^Mry;0P_VniK}r8Ia&aHWcn~a388kT> z9Mbxv>8&=q0GxA}hlq7n!{9le0BYU+6&?xLP3E<2Z)C=ZJ8hoVb5XxIT;ioVPNkR7 zA}YM0((%sQS3Pqi$C-STF{wnpphUdZtEO#IEAxCmd`#u17wT5NYlmZ zLRl<4vmcJnmsGCZWA>`a`T{;^_J&K$tgs^pcUdA!k>&w< z8B4rXTta;?hTZTgA`JNiV%sxWWnLcdQZ+(^ZndL*ffd+`l;41qvo;h>kN7rno!Q(# zPP?mYijhu2#KP<-gR<~oeRTJXH~0LNzUozH@o(aXlk7FET*)HJnh!3-9}mS}wVGF8 z3G?G>ZdJOX)(nL;)BRC(^f|Gnbww^?8-$=l`>~W}H`k#bkJ?I@Vp&gaiiz1uaFB`9 z0@#yeg%9&?G|`D2hoMEfx&F?xbu9uUiIrt-&o{*FT-M+~{zvfP#w$S>e8FN`nR7@F8_9BoX zTq4}@Z8<|urbbDHM2v({P-ibGeN5AyM_!Aqo%Jdq4YX*vRllFj3IJz5Bmy^fqzrAz^qjqr^?b4o!1b!U9*&|{EZoa?~g>4h{4EWjO za184I8uUmwHVUusy*p=3h*0r2@QjD%8_~67qCLO29#B6|IDk;R|MEJGk7M&4q^a`L z^tW+GU<^gN5DFVwr%_!4WU7M~m$5=0=nPB|_&JGR-OK`n~YA3Bw=y88gU*sm}{Bm?S zo|bH-(PcSV0z<>_tcT#@Rn`mn2~NtQHY5_d;m4Y)3Be2D?d_#`pXbv(L?5C!+4_r- z)}O||df@tp)Jdk6pjF=8lJAg}=0aL8#zo|c8nnC!#=p9xl#}r^vrtg+X$<8h^yQYi ziwsQZ_$W~g96AD?WH%0V!Fa21hawbgH>-aVo~dI|uYQ{Sx^Lz8m}v_rPUjl>v`N~a zJv&C-`?_}S5u)(V-waUX1W^Mwy3?sExv0V+aznpUH-OE&!sjMplqVJaU2Vgt@v7L^ z!3HeT>YwuN@GISEzFnKD<-jMU0o1z_8h{>_hp?!tX-0AUNoy&ghfb}5;p>dSrWaozQ))%8!*w5FmAUA?Y>gjIGv{=YU!k%PQc7*<{q zG56BB#HJ~g+&Z2ACF;T8DmH>t8W@t3^a(lkFp?SaxI!Z&pw$MVJZ}FoV_r8=Y3*3N zqDW>r3X4LchMZ*6(vb@GL=<`$rvtjsjuO890XnXou9uf7bXMsJgq@`&*3xY$uw$dtmB5Aj{XNtDNlQs(?y`MB{dmi&M4E9HlLc#o$n8&FzGtFNj`HsM>{nQ zerbKnPyvni)uH}iNf$E|d8U$uFj+$n$20Uwqu1*(ix5fF>)&>SR0kgldiX$L1nG`D z-pQq=4*Ok0I@MZ#g0V>CCg~AGs^Ott?F*nTv)=BXDQHN!x4X^1S~)WHh2BHyvNJvf z)#Mu*xFYqDjD`-yAQ>zRUW4zn;`bAgWVD z;T0&ABnUDIpU39Yml_XA3JDN`Ec`TFIH zw3n#(9RJ8gE?HB2$IH0it&Q3G8Zg~y*BE-`rghB=vvKIW`gY@MxQ1C#@Ji5z5$7b| z#;h2RC&<~IifEKymFl>9Z^`!6n+hi;57Ic1;47=k`tb(@*lLzHJf3D#Yy~`p6UJh7^cIB< z<4*Scv$(+O4!4`(Y6l1)0@NgiET zm^%5lY2eF;B%q}n;w~pPQ4hc;g0Dz4SFevZmgTXgsC(%*oZHl(L0b;&;2y&L*NOsO z&a8Qtu{=zgxxs(h`ffKDOf0Wjce>#3-mk&3hZ(m62M{s(<{1AtYkS%co^Ikbrga*t z^Ctnk10ZsE9ryN4yByPR+rEAIqP!w0Q0d)%bc;X`*=AZ!7#==jH?5T9SjR?5$+_sUzQh+7 zo!@S9Cr+TE-Tr(M$%rGNT5+BHcs;-T%3!oa>pmtIm$PW2v-CG&Eb1Q4CN1;&7D+He ztk!*8d(SJ836_4)6x5TAhgWZKKe&?3Jet~5y_2i>qD00clDv`_AcLo{;?2aT+Z22% z)KGeofKpcE(%ELVff(&Y z`y+*)o`T$or1OyltQde}&u9)~m_v^|flzL?aK^tHPN(@t+CmJxz@@%&JNtGoBEUz}3u@$LMsZ zg{#Ir#l=RD8snIf>rzL-@x2qOLxkaQ$~MaJMBM@k9-m2mB*?ZvuNp#axI6~$${*(w zX^{sbC53uY!F72}pl!T;$~kH-rk!)RGDR#>x94YQKiYkl7v|nr6V{4F7M(9kC7ma# zVm&ob$o59{J6-Qe=Ygu>#jCqPb%(&9W1i)oLMyZs^;1{;d^te70e$>H-G^HkrZvpH zlnH2arkwMO(rCu%Q=TG3L3nE#@A+Pj1$D|38$+@QPZ2%nJ(XQvP#}RBQ2L0IsvbW2 zwSI1uP|S3olOA)65>30eX?zzP!OWiyahEXlV+*0d6Bby#*+t&k=e7K4WxQai^t4x3 z3H?)v)f2z0O&(2s$@N%LCNTvuK4?(=(lDru|2SAe-t-1NpPpL1^G0G)i@-N>8w!0g zmuWPemgD=>XptDAZq1k%wVII}hs#ePjLK2u(>2EIctX<=NU?EuGum5bvtP<^Yybx( zaY;&&{LAow(V7Hp+rM%*v#=7}ro_JXtecH8-jA6W=>^O`zZ@kbU;tswRZK!CZ4kJ& z=I+j4q<-}_Q%qXD-6+p3H)Igt@kn2T zS@bsny3qM-9FubgR~ou9v2@@Dffi0F^M|p=q%PvGefoj-Z=W*~7l3fj1iUd98zK_G z__7AZr`fX>EVf5|Pz)cQj`tp8{9H*-BnWzG^w2cX8>72m+twZKA?>To#ji~ZaL*x&tsdExiQ?cg;xtY}a z=HTpab$xL>9I~Sg#a>dL#!tP|e_{EYp3sfj+_Y;u{CDbcdwec%8%y$DHMz3&E@Yfw4Dv`!LQ`nhrc4_d8=mW8C3P! z_H~0TcHV5(UY%}vC}=%n=;O@~n}}ULoS$Q6R8#513fJiK)&AGRiQb?c__2q4M4aoy zo!HdN_C+UBgYheynCOm5ru$t=QOb&gwAEbH&ZGKRH4spyTZcJ%+|0g8H)tiHYdCms zbF85KGw@zen-XC%^x}O+n}hw%3+P}Gt65n}qTJlM#%7#ym!77E#h5JN2M@exuoQfF zfNJ)zb(YcXyqAI31v+{mXWUjPf?j{|GNNcjdX}NjS#JxRwraa--xMdo(oK zaD&1LDEXgO?U@p~#GC^EJ{ioso1;u`NDshFa{cHQLRFCn@w`@#E*~Hn8t;gOdE2x5 z!9$P^zC44@e-W4z9}wd~xD+<42R7F&M%!d@0vZXsGc+b)<_;T~ZFi{7Jh?iWm=hs$ z!WcK3ueG%S>k_O-ir7JPj;k*SPtGK<2!{0?j?%}pk~3ZNwn02Asts$rOsAnR8KyL{ zu#vyfjLu&mDwSuwKSnsp78Evjn}55UmIxdo&^q+?(U!AQ5e%lL8?{4oP>x_tRJ~?W zvdyT+TfLmeAE_u1kvLw_fC3RP?4%1MnZ>?oOS5{dOJFLv=D+hLjSY5j zyq3f$i_@ozN_)D?;Xz6BRVbc95>2=T;xr)D_s)avaEJD^x0cAwaaZ?1UyVyGZhif-^h4~YUpp1>Nm_-#I zI!1)qh*B3(k>Xmu=3et51T@sO+~Ek@N!2lVd0*9bXf2WN);%_A68KFTmN($0cK`RH z&y$k2K83Z;qs6y>L#`5Y51ylI=FcXaDDWWLzvPX7-XM;DM* zqt|6%;YkY8Ve?6gJGWx5xaG5WuO09e;Vkob{a(l=44rUU5Ydup+SFgV0jOPLC<>T9 zG|_;V8?iGO?#SHZaAF8U{b80zYTR)iJG=>9H22hb!BHZ>z0bPdqGZbBk`MGpdrxBU zArioROcNC5zFSL5JRt8n&Yk&>^0ki4`!>dOhbyMO!$Us>ru9OG&i(XY0l+|0Z=+4Y z_ASs@Jzicy{UjjN^Q@!=($j^KMpD<^b!C^17KZ;v=H4m8X(pxejOk>g+R~tYibmg` zGP~(O%%MIL>*#gD$ys2jxQ6mC=$Z7m&$-Sl@ z9td*Pz{NRtd}OWg^Xnu>mil)l{~T6#gfRFQ>VNX)o2JiIO^?Wp?=+v96xWkPAm~uo>YlxR@tn*kFQ<6= zxHz==>q?&pG%iE9Z`|g_-XU&ab!woB#_~Hcs#yLKJt5esRfdDL^#<7zCM{GFOE$_ zzFdw%YUn$KkJqt94M*Ww82%POpSEqd--%(AZnGMJ*GOiZwoNCKqBSe+CimLcAstqT z5)QTS69u#CD|}1ecg zN)seY!4^$-TCR-Vzk-K4F5i`(BaYZ~HP5^Q%BVLWsh)>Cz=`sZlUw*h&@F*&VT;}E z5@hsBZQFtt)pECKR!1QdZJ0Lmigt}5O#EGy7%GFWHB6=L?!-pXbO=k!_JBFIJD)BT zSnvN{$O0N6kZ8>`4r6xrg1N1;pU+m5it+XGY_~rBYm~eZ zvUvDY4>8*a@n1?PJhubWdX&-JOyyZ+kV}+71{NhrNKXu<@_24S)J>NNBdE2|0#l*G zXnPfHbj>-@u97oxOy3mS@ES|XkqH%B7*zM2Fq z0EB72jM-VtFE7Ul83ga?d>XDvxXn(4#{6u|r`ZVOlC>OGCdH!S_>WXK{ru&N3;4rW zE6Yi-bTQsWM5vMs&3ga4=V%0@IhTMisSogHAwEUJ4Rf1r(9%xh zw`guqNX%-`Wi5)6bHsSU=7LHdcuagY>IXbmuJ*dBCW`PO^tVnvu~ESx;edBQ`W5Kd zCW)T>()mczOWqf9Xdrr=M8B{dw;_}I7)0in>XEHWza);wvh{^MaTk#`>WXa)hqj^K zi#Cg^#kVX?w^PA0xM zyKn&XEDPsrFdioLa|rgVWNuopV$KQ_N$G0?tSZSs_F6_+Gct5LNr$Js4ykPk$)9a& z6I-N|zLjiL8Z#S=%j~s%H@j)lAj3zfkiP#fVehisNRnjC}97x(WU|SX%`A8QQoA8O^`yBwZdLD zGHcoFHi4ytb$F|ldIHw)uq-w2}_!~P5kJ~QIpwT*-{npt(*WayC6Dbaj0c8BIuQ7XyU zxb>Js-jC7V>kxlhMqm(yM6 zsPjG>j0P5d4?<})D5B@jUYipePz#ORBLs+qXCSYvE{y|2^UX1b>l7d7nU+m2S}7Eh zw;IMWSFT|&e#doPFE8q?J@`;k|Ah$cy35kObJ)%A=at4ZZ|WqX1F%?2aLk~h!Dx@h z%kAP!vBfjr?>FJ{sM2rQ!0>lAsMBK8Va*W$Y!+HfF+5W-847sT$SxqaB;Vlf$B>r|cAJnPv72O!hYDMWiNd`?3 zkgkkKxXDcm0{_qi;{!+fSEg?`Wk!~1OU`4KQQ|M*{9!RcWK8mk^OKb~LR&Rr>bPGy zoLdutt*}##?xPKzv`Fg>{o9CIv%TlgLBYqr&N~u)@V~(Tmh9@VG{c3Tj84o9R~ z(xlhA>c{%hqXoN-W1sz8YB$tviXyA4aF{M|h&jKfy~W$*M9FD4wT0N7O<>!W-mn^| zAn@KIhEGCK{3?Ka99KFtv5L3-T9&%-H_r)#!RWD({=gsR&?e!q>+m5{8MXwF>Py?b zZ|+z$VV_Q}A!|OIs7{M_1xv;Dx>tQz+&#IWO1`&Uhe$+nKH3h!s!iP{s$i-c=@yVw z_P;Ei+H`>HE2Wpj%vF9lN|>H|ZH!W5!V=@AV}he7uZ8I-0|vUQiCGILwsVjVoxv+1(+hWl3EY#X)LatZ6|FRqZ55# zh(}L07&;}_;4tn+D+yrZq&+d~Bi>KHFoZp6wnhUqs86rgx}9`*q;pEj)H4g=%BOa; zANa`{06ulHcyJ zUsaDgbUxCdD#Idclrh?<3B1r7lhkvQy0D^MJf99VlQp{m`Ri0x_x0_T zl%0@M6huf<@S!@?+*1sofdHS#1B#s!+{V($k=%3Z;zB-9O?N8Lk~X02hK3iSk4AyP zdhGGONVk8RRsxV>OEM*%Pt5{B+KwkzsnSg9mXg}jL1Msz+}qaaW19C$Ns1_Y7wrH~ zx`>e4=sRN^H3N;Q`KWZAtYXfRhH!_v5_CIjQKvPJ8&`57U8vqQ6xCd=1F5O0UouAGpY6r8aYQGGGL!OOCs+9}S{8I zwYfuddpfwL8Xz%knVs$iij2@w?I1`Da%)%;*p1IE+1um3m;ZG?ajcp5v!qpmA&PPw zl~}ee&g=;y_OZiU%fi#v!lf_g65OU6_xi)NAMGB;_qVa87(ekhP-UJgfbrHT-2j81 zqz00O&A{H3^pqJ0-Cizi3uC6LdW>6iPV`G)38evI(!#}!_m8JPaRuNQwhzc~Sin?? zW}6(x1cbt;z#=Pq8rk2GTa<$PLqu#IUnIk7dB(G`<8dDBU~|34HYR9|D``vcs1XW_duv3PXV3nXNicEbBSCqvy8SoI`erQzi zH+$!b$-%;|#v0BU1VMGyOHwEG8ug$g<-3^rO2-8INeh$9`V|vbcJ_CP7U%r>hdu!p zQt`DTpkPAic^f94f5=ms5M&Hwv06Uo90xlM#~UemZ%J)9_OB*;OnvU1h9j>hBa+yW zYjUhP3P^iSi)dKiLlAAgsqBTVkT=)kFB<1qc)G5W;L z)nH~kC0C*Q4pLj0(;CI%u6?8{bpBi4qxHMC4Om?x%;|hBH(7k#nkQeBg=_xnS*|s` zp!$?v)fqX2pwJ<5;MkmrPKY!Ce>C1UE{hbR=`=$fII4UlO=;4ys9_Nuvm-1^sneE1 zOzPRx1R)4Ty~JG_u!f)cLv$+4pyQY*v&SL@V}o#CY7{n~L}I`a3EwhPnAj(J!;Zy6 zX5-$J`(z=mcopBmZ~4~RdD|-xvW-yBQGRzlk9O{*n#HZ`p307@7T8C^-@AB4eYXa}SbL;x0 z5?9;Ku34|9_0gR?Wb2|$UBW@V!|4SiH1X_+T8Dvf9#8$4I>cFn_fRA%DZ7BvChAeC z=zma+Or_dtGSfXCItp9LD~3wUC-=&6RvkEJGiPen%}3G}e&Xh8`%uP!Kw~v17E&Mz zoXoP{K-8`g@LBBVA|#(gww6tE^#h!AX>9Q#`tKOh;1JgEZ zB<<*!oIjb2xuoOBNgbONWkLREJqHw>P;I+ZmI`DiXD)u)tbxPt>q(T`^5h{a^4r+>h)vH2Zj#h{NT7$!ie3Z?lcI6Bh0)?>NlIXI z56<3Mwh)mNbffmC6tZFHdno414?E~!?4?7nn0wD;b9>W|3$5gy3*`x3oY{kn=F=#I z0yUj|f#C>oS0R5>_su^pJL+e<=e!DDhF1|qu<#aNEg#y5UF8Q!ksC6QX#RTMR=7@G z0D}bFgJ5FBrU!+dZO4$zAQAoH<&lX?SOjih};! zd7M%%C7z+Q%M%gVG*^D($SuU!){c)Q?3ENbLK0v&t^j&k7CgSYxlSX?(S2@&j=IrD zXny|$tA2i#Kv^K@XGcNdYQ)}*5&vS@elX(rbUvOG|z&%4@h{J=`sG8=@ zbk9wpt0(pA#jCdod@Wih9t@TorVw0RK(qZbl`P-gy>($r?NGpAbap?hkLkWqXc>0a zFw#`#7a@Hrzph2iIznFORDr`74VF11S!K)kH~US?0}ZeY6-5(F=*|=95!)1c7Dg5` z2<|sp+D{Fk&o%$0@Za+Mwq|eqj%l&=0x;H&92auZ>H(&@3bbEdH4)15!9`%4birVV zAn#a_x<;|vm?*wu@U6mkOKJvX&!vV3aT21*n4hhj8Y9XYQpxGa?N2#VvEB_U2gF;s zxXJvpuCU7qXxLPqh8$j^ovjgdZV9O-V#kvDolBEXp3Um%%)KCf_sgaYE@jQwWd&uW z(9Y|7a4IPcrA~~cb*`K57cZcAP9Us=QkW2B>$DBhn$Wp1wnEm($jQ+D8PxvqSFR9> zGL0V78N@Gg?J9gP*Yg%FeumCkBa=LF#*?UDf$3i917TdPw`jIMT0<$tlFq#XO;7!6 zgkPR2!bgg7uDD5S*)L3_=9ti2$A*7!KfF>fKh0qBS9n$wtiM<-{ zshRW{5a1eFaqA5#u~TbzHA&*4d8td0iaZ22z=y*0(9E@L6-)0^ydt}-2vS%&=`jyO z*)i_&a$OVzz;~DT{q)SVx`|!AV04Z22 zVvD+PE=kv$AgaF`1*tnX&u#&Bv+&~jW=uSgqY!-E;vs0-dHhABvyIG1jaF5~6cyzC zM!LfJGI28E6?eAmZjSRbWzw*}w$@favRA5PD?(x~qt|&Kolov{D|Rn)BSme(_mb_5 zl|)^Kh@=ZhPHF0?KOE|d+!`s?vf08ReAifJA&smrj%tni{8>CU!~B;gXFZetehVqY zdI^>~W~`VSJ$G0TlWTXeefI?VmYc4myOpbNhF zBj$9>Og5Cnbk1r@ZoojK?=^a2bZMncZ>PKZ|D>{Tz>1rOZSx86M+N~#OTZ(6PAb1a z>D-xNmvf9(L8lt9PcXu^Sf$`Js@Nut%5q!%2!@zp$|a!YmP!<~sQ~Uww0F+&eV80K zHv$rd#tC>;A7?aV350YOx<;;<-_leLk{q7(nWPWfLsw%x4Si(OrorH};!$s<+vee$ zyPJH{oe8E9v~wrp_s)!PY598HvL91Tn?@OAhcbMVeGC-R68@#c8Cwn*KiX+HYm(7%Zx&mnfit4^*wa{VLDt*839)H;tO&e^*+zY#=;cv$VC0={ar zD5R^f?mT-09)JUwrNfldQzLR%pU0FbQ7(eTEfN2wdCTQHh?WJ8;IMqLBp%8`U0XPG zM;1^t;-xkvH8!k!m?{%o>&^}B9-BnYCS#qTte;HEEw+F17{_4Ykh9f0bnWnxCwGP%Cr1!y;I2NzWjMOUgL_JQ-pa6nvwjZsA(dwkM*E^1is+%{Jj+c7|#dsKNznlWG_d* zFHzJtB(H|4ZJWN?V@;@ikvl`_D6Bj*hDXx;EI@Ni7IMTI`m4jzckfz|n<#`%&%x(> z1P!mflr;Qx?iuadd|N$Z^J$h?Tw_xmXA04lMhnQ*c6XNuSgk&tqxD?07j5eFYFE^i z`uNr+Lh81Ys6#DdS;E9#!$`_yZ%;OdG*n6Z4)%qit#*4;TViQK5+;;9B0-Bj}pY>I~! zX_j+LMkSK<7xMkmy$Qw6bo{=04@L-Nrq@K~`6Ruc-vZDd%({r*!3g4k9CW141t9G^ zO(Ls1|2JZLO^*Q}XzlJuaAq(7Z%8mPoBu*==~`HeXWjoKo>gKYpQ%3|R8VGC<0sDY z{b;#DkVFKwJ3z4tj)1L`gBpE)4mCadDF7W2g&{N}#DET8pq0>lh=%6mz0Eq#P-_{M zcVo$A991+-2!$;Hk|cPXX6la2{O-EIoJfd8@{8Y_$M1pX^;u{O0_HU#mL6aaJNepT zFRT-Yp!}d-BrUF>h<}n~R$~KnCUZ+a`HGV-92$npIUE<>2*Li>sIy-sqaHq9F(H!z zB>GREfOAlapFlT+LOFV*d6d@Xcx{El#31psa6pMmlEWG8EEeB*yXiSCQ>jQ0i?Yb>NQE6 zAP(D9 zP8przJR1RCA_|7j&>Aye(rXvpho;vWLQsc#iO7<@f(UmKKPr9nuBP4WT%kGe0UBD=6*TQ zgP4712_BTtx4C}V{O2ffJ7E%r_UrTU3ytW%t*nc-J(Hny(tzadpGFEjvTq5$ZFV6( zENCp69*I1mAPX6qnU zK{$mA5V|21RKK=sUGW4H#?C#k&F&;cPyj(dzQ5I#^pjD_=e12tb!I~Mo4-$icWbV5 zs^Q2p>z%6qyOE-SYSX#K4UYebf68M}8J`D@qKMIDcxuys5YO^JvE@zQ2wErjqLVk! zO0m1V0x%Oki4a^@$`21|ZNeUX4omCK-T{8uh}K3`Z$^e-5u2sM6T?QW*A@!W)Vz3x z?e1p-neqIk&-pQE_-hXmt}=2biOa;PKW2EPJVA&Xk9p+?-0D##uO!B`~%o7 zb~^`{r32!?g5qWb`V)(r1m;Bza>|J?X$_$RuN)25xmoA8IrmmD4dfJ3))w1H`-|YB z(g0eRa*(Jc%`!jwxF8Qq+-C@g4`y^gz_LqR-&#GYTK%%3Of{METef#yeHS9lYTD$= z-5nQCx^9c;Z47kXMvM_9vv%}bez1o8nPX_vk6N)Zj&_n(o)LPli$_sh1yk_d6M~@5 zPbl&Q{s>NJd--J4XjSh)lujsFKlf-j)Prf4=c-Y*)i>NoFqsPbk99Ymq$B5pA~?xy zovg;*(e`!Eq~tEqlR|Si{Lr|dpy9NQFKwDrw9Ap=Vrhztt}DWn5T`9v%T5#H3U5dO z&bot*Wl|5Nl-s@OSfvit65D$>1`^0(&6&h66tOW`Sc4~btD46Y8e3W@y|0NL`R7c_ z#d;^N=A=p;NTP)w4b$l(iS2AZ(dII`O?5Xe;T#=LmX{%usMaF;4!yGdKfqVmo2y$6 z^aCg-Tz0!r=k+p|6lj57<9T|$g!|?XCSz{|`rxjg77dLOpoiYLI<+C$R?qJi+UYJf z?wUzvOI>2Brc)EuSLj;fCaApI4>FiF`+8IG^;q4Sp>7g&(tum7U4nma{z9CTY3R{M) zP59G+MQ-3_681Wt2~%bSqQp$Gctor`?H%>^ebvi(NL85$$k=N)tciz z8(TW=9Awuok4W5jdAII`59R{$QudIP)d^A}ss9`%K5>p*xa;TLtKVO!gddtfJksc= zvBpFtTXjtae@tWlw>f+!_jYtbGiZ+l?D*VTBW-Vlab)2{PzJbR<)?}~dT4M51(Ja0 zn%`Y8;kK7$=NSak%1xS9 z=CQUTayD9Cb1;{&E=y$KPs6jzT8lRJQh-~E6)a1b$?{Y2Whmdbj%TYfc;&|qi=~S~ zWV|(efRv7!!gwSP+E*w`01!PkJKB{SW;=l~7rwdkD9>9w^~JFdN3g*&hH^*ZE0;v4 z^DssgirHIBvAgz2l030l?OJl2Mg*PX+^4`dje?r395}3P?yjd2k1$m_UyaDWVXU$A z|Gjyht9-D<^j93%<>m+WcBd6vSX^gYa_Oy?$c{WfEG`M1+{HQj>haUU_c4ZcpcOAo z$lYQSUe&2^_diTB2*aDsJz{6e{swS>>(}?7zYmV=aekV2R~ln44go?(x`mW})|6-8 z&B4y9v8>NiZ5ykXC?b*3HsP!F-Ch4^KIZ3ke|R9^I5fp^Gs_}>K03LuF+YD1QDqi) z#I@28iTMsQt%$r(fScS1jtCMl<*MJt zHtJ7-0uUP=oeW27jT_w zRd!`Vg;lT)>wD3C;|y>hFV|UWa>_*OYT`F`X}Z_*U*ry-Z3$gW1gmBTkC!L` z9hGU=&NV~CdTxQ5o7wnrGe$+56EVk9NAJo%#?MI zzU6-NYa>EaE=FtEclyYu|F{Wqwh-N3!Z#}hiF*)Eu^~vE=BX@=3?a6;l=TMG5fT8m ze%Als&{9PKo|VQD1Hew7Xq)cl(Grz9`V8=V8cy^`xqF)b*hZIWwaIFZSm4{u&u$k; zs>#-L^Q}Sv*oc<;3MP%I8MF>vUFi)A3LB|Z5 zD*dLju_~2MY*vb@FNi@V|_pU1*HK6txL!mn2Ia?~SmA+s(QK{$M<=z08{<0q^@x3W)9lOUK9TyZ z$Halck3y;3^?hEJMq>;{oOS??$5*g4w^F^^|1rOHWFD=edUCP_)URWGPvNOL(N6Vp zI;FrF|F4vlU^C>T;?q80wWGxK;=G}3I75P;*WpS(?`R$WX+*m?&-SOW#15#p|I-LW z%t&)3Y&8!RU%vP5l-f1_nBhi!6_?swywr0)E}tsfRgq@HIIu%9G^cUnaJqswikhlb zsR?JX1IOk0`!!la0CR^ILOF=;&z1B(jk~@oy%A*l^n+#-4RSNlgG*(qD*B;gKDvTCOmC9|-=%%~Qb`R`B)SbB-8H&*l_ zeafyK@XHPDuFV3_suv{23I|yImbCYYae03_4U@&MG&mw)RP}sk#J+T(!azs1>B87) z7Fi;fLrNtjrq|1hOgOZzw6u7FKmxK%CiK#%`rcidd-In->UZMK90v*ba@%Q99%tuC zP=}9acV1sEgd5W>ZqzChFJ`0>PAbA^@DQF1;r~IEPv%meM@+O}$!cuf_NOz(y4)_d zW&0TaXlC4^!vEr~A+dg8aMamhwaJg%1LU>arDb`lwlnpKx}fc%LpTf3y}%0Y zrr)xYc#;iA)Z1js2DvQc5B5!2=40CTt#B%cq$gVG$F_td*no0HX#prcA=Z~PS;`ZH z=fJ;TcheAb*JNc1*X2?|14| z)MKDbr;{>-tqIgBk2vrLe}`AfuYSEuJN3PDUZHFf?k}hLxVq$F2a=9n6hERK02oZ- zn04fUXuJe+gwIXhJ5dTc6B$v%Cd&^fRG+pa1#OL>Bj^H;vUFSLXb&h3%Kr>(nc1#E zCwSRUr*rDZKZF`W{S^PUlusq2J!qIxpKUDzUqQ-Qf7sKwDhHuAibanh{8O4YMPv^)sI)A0L8K>23)C>_p&mnDU(q}Y;prga^s(1h zr7F^BjvYt&Y|j!kQ>fI3Ha{We(MnEPW_IQUxyc#^Qv(bKm>mbZtN4i6n)XT=H>FTb zb$lU8@-Qtqhl6;eirsj6#o*@mlck!kX*Yg#px)Kg<8(PhY?i7!;`0{+K(dd#N>a39 zcEKRn)$88R@8MvwR@O`0d{Z#88*57(Qwomt8F{{<|J{nb+{79Vg znz~Lcf}J&nQ0rWf@1bMlFmb|>udvcio@`gDDJu#o@QEA%^K|iQ?uqsFT2@uTR+?7~ zcCxoa;YO)R=T_si*(FaP+XlROYfxQo}ixDxs;ry!n-)Co~r zyN1~zH%zkU)T7-R<&wi@itFy8++`Qy`;8EoLIJKp~{SngqpU5 z%K-YHNqPt^xm>Nyx#tLgi|ow;tF+H74Wu*p6XqHhv1el1;vv=jp@flZLz6slA5U0E>R%Zp`xY-W~jqecbAy zH9fbSx4n&2)2Sr|&$U&j8|c@Xa+ap%lwh+o7fg*3a_SE5LS0hT(`|$j&W%~Xb83<# ztp-f-u-5t$<_n?CfJ1un?l+?&F2Ylc4z7y;hsiSs8=O8;(hdfdlwd@;R9CPvJt)2n>=>tXj=lNhX6`&?HDx>%Cyfie8 zzZ^cBV`bPN**&#pOlw@S*jjt{guVJJ-I%qo-nO*Dz5#_p-41q6tuk zQtP&DN%&+4n$jw+L;M5F8ls!_bR|&>Uxw94;~F^)V|?C<_`L@=ZD%pfA71T#*S~Ky zVQhD)`c(Fk%S{g&|}rEX6!f?%T~V3voL9+AO`UpZv?ALazWxxoc)9Z z2?HQn9EBwMZOmRfA?&Rab1m9v)75np*Sn;1o8L!0yWkr1I^N8#^ky7kF;a3oA-ktF zkxFYKZTq@=P-FQtN>WyOzNVFRFewI^-yu~Jl!MN@C!$_0X+KsZ%V=xm8I6{1H~bbr z?8rL)J*{1!=kwT%=)o$rRdxAIFQ?VvKb|Yih!3|~MZa9w9P6NOePnEiB$Vj9z z#`Hm+&vdHbm*g>&b~Ybl`ucJzAkg0Eo;C+p-=pQC944w`ozu3kshPuY`tCKejCFrn zAEr~HU^-qeW7#wlB_0l#6;@qTcuh(d8B)mp|nHp zm`23pVX0@R{skVQ`w_d|UI8S6u4XQ3x*F>0ZbW2bNE@+imZrc!5FyZ8on>xxPlSP% zBA)WKgFM7em-i07pK4Gk+E+IY7D=JT2&35Y4ylIw&0pcSnXmmzJe&>}$`N2{ElVXQ z&QN2^qSqfN#+FDyS9Sw~$d#M!_(9<)D$b)D-Qb#pV&WW}!(5YyrtNAF7V7~MG^FHR zx1Y^s!Jk}uGp)y@(axMuvw_BG{=$2Ghfq4u$V~^!1pbL^z|YZLTec|@Lf2j_YyB!k zeh@orjcL@4X_68vdE-?sF5j6&XCXBAO#s^i6@DJ z1EJqDRrKF7zYbqdch8np2huYKJY`w@SZ-`_PF~{Z%&PAuFUEnA9oM9sbC+bpu{5OM z5>6IrQ1Nf@+&Omo5i*A6+ET~xjRKxUPB>e4Lz;F`B=9?IEot>x=F9j6<^5Dv+@Uc~ zbeL%w5|EjWyQDv-+^vWwMeU>H=&Uc?`MdwW0LuW>xW}DQlH3OG}q(<%p_@%e+gX0uRg~3 zn@;b$)=G3H+zgXO`t5mVpY!*=A)0=b((RAlO#uIr<|6||sMkZS zdgXTd!ljEf-S`&RG`P>K%(%%;qj1*<77-)ZIp+zp&Fmp2Pn=gChD^2VYsVgA&)?QR zpJVV3*Hgr>L0o$+q;VtgPj0<$AwsazNwfjC3}cPH0j|33R?3%y_X`XF896(EhCpvP z)p?;d-VID~oa>>_Ra&I=AT50X$B8+t9Y_d4Y*x+ChdyVzlUR)-2+#GFx2&@*D3^(q zk}Ys>q9u(&MeZ9s=;+kvgcfspZi7N=?M&TY68vIV`cGGcItz~l(}ur%ag+s;eihbN}c6qSW2yYS2ycv!NaQ@){2tS~%BKI^bJ^P(u0 z+G$Hs(@m1>ISRrQ*i~@bMRqOiuy`=6WlKM6NY-QjR!@x4j_!Xu*FS1iKd|JuT4=Ox ze^52$gkhUW)(&zZNjPf&nCTecE9#CA$1QcbILBDS%#>{P`O6RdugmoO9TNM{h*D-s zd^?-~gPsju_d(A0<-O*c=L=&l;YtZCKA}#usq0RKbp{)jNWZFYk3ZP2fEL zU4A*><4vdi(v8)g@OA=gp`N*4gTBCj zLE`UrDt6AP9($%GJ;yAh7r|s)2Xx4e)ukcX#z${~gmPadV>a5CeHs3~0}i zlwsSlum>+8y$((B^ED<9F`WG7AP`NLr_0q~LQ)l+Yt%))-`kS%9q8nh5&O5zE7P&C z`9(vMX1Zv;cxVk9KOhvI#69J+|1~TfTGX+L>(1A30r9M(=4i24OR`*!E&MEHf)B0!#a&&SGv)D+F?pRp4A$$Hqga@~ zrYrl3YmmfKL?UL3jWL0j#cQJgypKEj)qm3e)V?SJdnYrs6R_q*len9p9G6(vf@Z8> zG|IDnV5t1+oGe=3Ut0p-TT2DVI;R>v<5aow-=q0!*bYmGn&S@#h*~m2|A~Y0F!cGR z#wFweAjk1O(4GLx<+4Ph;nWH;J%TTEl!+|L2Rd~znc6Ki1wD*mle}XudblPrr+%4E{w^#R?iL=Xu^X=L%%}U(}2A07HHKTF3{pSU#`geqrIF?Sog8 z70G|rw~wbBTX4<3dPJi2*jw8hYl~{ow-&))(D z6q+k|LHKMUaQF4ZK8dhRk@>g2s|b}R9OlEGZ50wtppQ7y3;J~m*i0&_r?w*4Le!_- z@1>Egg_@7#e~~}JZ+e}*^^Q<)G&!pyp9=mKr+;0B?yq3!aDxT=b~kIhh1c}e#1W*V zds;^0)6utoXR}cM#ixSVK^bD!mNwj;wPF=ub!h*v1X!0Ru3RwO1O{s*DmOAY4>T{` zl-1KK$I%-4z}4JG+5m)RG)!>wF>3rZa3=MRc)oDGs`C!@V>&31l}XY0S(bZ@=EDXW2`#pE*cWmq0QU1X1qatDNj`YL}he^G-VcJ2-3dZ3UF zVJ;C@^dPp$cRYVB*{g?lwE);Rb{!ZEu-5!DW`d9+g@9lEtMFCD>-a+Ky6-ENsns?> z3qdCiEugxA_7Bz*n7sr-?hmnlcxjvxoSFHZ@zXCYOIZgK{7~uk&<$|YPS_R{Aq|>e z@;AROw@zN__3@tcUR6)IHo?*(T7AWNG_v8`JxANVc1UwTxy66OiP*L{g{B(6RkQQ< zN&LgddMG}oJ))T(qVBN2sujdVTy*>c8T}bcg6yZ?YjAsHrT6H6_G&xUYV{gThI+Es za)2LGZ~p5~un}QpaW>s1vv(#P zG~kIw$T>h634R3ZX*oC7#uE_%rza&bl2$!FgZwKWmi?ESkF}Ie{UyY)4skK=k&09O z>K)-|-3jL;_C%_~gPeHmW9`;#IH7Diky9!VtY_BKsXWbpj07^DzCR7GOMu_l{BKut zIja0%ZaBq|n6;NQ)M{GS@YXGvHZYu?Wg2TYZAX+2ZTkLv8V*8mb!|<4WQZmxdVz^) zWsXD5aG8m7gD486s14@}mt!ZI9a!b3da)p?esN4g!v(X>3(1AhogQNpCT|~{2*-wnPrAFs_l4$ytGJMXzb^H>_~${Pwu62KydlH}$V?4$D$|LHAAcUEXgs8NT+eTXcdjrv}9K*p-Di;pTkER&T{IL1Io~}6c@1CyXcKu2IdPI>& z+|OeS7Qg)8AuPtKcKL4PIZD;v@NAE-X9G15+r+No4@Pz2rf7-RmGf74F$_y*-o#l_9lK>IEx(i}x5Wr1&xFM*A1a#t#H24%$)KZ25Gl z7^7cUP<$3glQ)t!K4U)X4aRKm0Eu(quh{>1i{Zl;5d;5aJRUv!?tKb3e?8Sq#^HC> zWtdOR#Tnwt1C5B(5-TyGC+JsZG8UjDUaW`b1RPcY+lZL& zT9Tu`xFG0`qIr^i;ceTvDye3QVqA{WV)-=OBaCOWd-Hqt?TEw_KnvWNu^Sa}#6Ais zYH;~?S?#%!gSz^(7@W}#?|Nb9B@BTkD<7_?VU$7Qs3!+_OMmnc>lf3Ho`$xc3@d2) zlZeE{a)4bEiES_$b40_mgE}4~>F6s15TcW6fuZ|*m}F~01n>wBobS{$?nOot1Y>R^ z4m7lL>zhEM)6OpAa5-Nen(?c3Gi!7*Uqb>rUN7ng>^f0#Kz0{_X|)%O9?qvHsA#k(D=i>d{lA| zCZuU_m*QBSek_(J4jwk!_`e7U$8T|(VM8)@9z2zPreZpxdGPhJvsUB43xzjimI&}m z1QBGw9e47fe}9a`yZHi?zj2$}@Tz>fu~Vgde{cF6s%7P+CL3JJDb=hl%rdww@=4|q z{0y?QFn34ew$y^(w6|WHEfo?*L*@JyR-{7q2&zD`47LBK`rU;gMkU%d7 zx?y;a_JH^;^cNN3f)#0E+#^JYnVM+bIqYThmKYD-knF_ z=n|kDEjW+ zPpQ_w&)HSjJlB+-R^`yKxWo?7EmioC_1MVto?18dGeii z5)U$jz~BdDmTAvL{$daYU(})nY>f`;+BDVx7V>`|PSi^}8=>0afi2fh+ilPUpP_!H zXO^2PXuNz>2s~%}gk zbvy1>PS#qdS~opRkI&xSecNr1itkHh|@@Htk0&f!rsa zSJ6z3E!AHw=cqR{>~QRI9II&JX1)}w8~Xl9LXn-kZYPqrbb32aRJCAI;ped%t{tuI zno|BXS6?nu-ji9}OJI^Iyy`}TAQPT?1gq3DWVVsulp{W3Zr}&bu_#>1h(SSvun}5V2NuX&p7nLUePl zHU0?m9UY+05!VPPncp+ur#fsvL$nHKv`}6z0!&z>)hD_&lUD(r;Ip?F0Bcg(;c=90 z=L?T9O^I<2Q6ornynBXl?+IQ%?Ivv~cs%NnP;sC%Hm_X7C`Gb!VZ=zo8^f5mV`)$KJ zhb)ERrW`^*kBM;pk*fU7j20juIJ69tiSo3GCJZTLIeTSV$w1Wlp;jp&foDsPXefoB zHLZD~Hx}D4l$X&Fs$c({vfj;`aU~!PJqj$%A3QM2%6y>Hwom(yOi12`)X+9g(};$h zGZZ5=!!>AO*T78lWlCh0ttAx19IaHC#OfW6h zE5R1=l*rM@PN|VByrzRY*hG$ph^0xC-TJ5c242UHIW_B(l($q@!JghLT{qoLh#ECa zDkKpu(&?cF?Z(bPo!voGEo~rNXG1;IAgWOA?kS~uc@|B6nhdW?S-)C8G^>u#OPqKR zn9LDWf!Oig7+YJR6KtTexU|ZVYWT#Nw#qA%*|VNW7_FE`O(WQ18=n)Vc-(&DA8JXh z4Ecapv`@CE$LcOfR#v6G{zM)^yXj-vU4}gxpNixSUi*O`KPOFrhB*!KKd&Usw!Oic zwBsZCQ!oyfo)%p9<k=sj0A&g8R?g|uHFNX#A)pS=Z!(i# zq4AhK1DS-$CeLme9~{Syf%nu#z4fTCaa|8UZh##bs74#WD~d7Nafx`pys{7ott-N6 z{Vy~Jkddo(4wkp+g+{#^e+Y%g6~}1-rE-VU+49cPISU4ch_FEjx39in#7V{gadiN#>XX#~S|0zcCeLz~l< zjoPsLVnmL6gHUYK4Zo;)z_Bb{Uum8}7U=JeMG%Q`aM#1$_^{PM^dI<+dubms&yEkzZnWD0j_$;)HkfHYYaKE;Q=wS-d=MKi6)C+nFj zt==})ZU&2`>;te0-pJ5+t@W9at?AYFrGZAmkh0ZPZ~P_ z6Ab#Kbbk{(Q!SZ}=AOjqGVU(T#@2GTDT;i6pTu9-L0&WkcmFCb zzTqJ(#%~cJC2YZRox1GH(KlvCUGj@wQ!wy-agXhD9>d{NtnrKIq`P^AO+x+kjace) zy22tI)}GxP5fiG|{K{tvUJfC~Tbz%$!kYr?#aY4?z5<-e?^N=*~RJsm%=n!!%LW&N@ z{2=;YN2_uYDMt`wz7=T$9q06` zj22PC%!9E0?WUV(jF91%cOH>8G-(qF`^7 z&UfT};7bdrM>h`dd-Z?2P-DhT!Eli%D;J6Sn&+q!8QQQfX*lnaW?J+F$Zt}D@wFWt zl3rNV-t6&`t`>4d3e|6B1`d}RNKVr~$DNeSi~O3VrDK&%L~GQ@@{9nNmWL#Vg zWR=&eV;#n?>=?%z&y^X<$lY^-ANNdGYiHo-4e2H#YTY~c zb*4F?zZJ=T{dew-E_Z@rHCoN!U3Kf+B1}dPhH67SkKU)9y`dxXx1v-SKK^=+!F51! z5tO4zWPIi(auxl)a##Tno{}PrzX`)0~TL3CuT(yLjyYdHua^yKjh28xyCAQx5Rh;7%(%oOz9?rH`f4Z)f zZ{yZlK{y=^Fma-yWC+o+E{kU0DKlS-Uo*zU4%lm;T657M8razKBFoD>)&YM z&>cAFkwU4-og2s)Xvc+UGU5sURn2l6C665qQ+N z*>d_0bi?VOLrn!jV_h^=KXOb%#}4#d&v>^c`JFtqKE@jU;jv`(i2rK$RTdwY>nQv?WJ z6>_;VI~>`Tp=JlP5LI9MywOdm$GWH=EiVQJ5DaFQl=%-)^shhZFNSII81vGoy^4$b zS4UE(8R@TfxJD`%7{@5WGs9ZzX64n0O?R}9y6g#uP@pZLvLwjE&o{hVOXJ#fL(D6{ljvZRQ(0}ZAa;z4 zUOab4*Js%Z95`OT`uRyZ=bOeKB;!r6Z3BIkP;+tbdzUkw%Kj8uPkrs=4vFF@jXtY3 zCSPX|MClH-X!{C~v``21-3GjIr^UacR`plZptj}Gvrb!4{|wX4f<78eFo?adXtgP`;$A1U5N^`ofpB%*?enTuH9A&#Jp!u zhp~HS53Mfmj4*I|zK-@_Bxp9!iY&}P&js#L-{J@Sy}WQp)IARYZ1dK;Hc+;OemV-KVws!uGdH!M&lQoGYU zlNK?&56@|r{EyDuM8iEe{jl!c>vu#WU7;H3YG$M^6X&QbMlfk9tfo%xvadVesUuL0 zO4X?j6Uuw}yd;Tp5WMYwqcz>!hmuM_L@qIf3B{pCtLixo$Ag*DD5X5V@m-%sNk3@R zj~xM7H5b&Y-yPS7Roq ze4??8?8L*#D7J@nb0TZ$+Q6vTXU%KT>c-X}iY2wr5rPVjv9BEoq*v4o<5|i-_hO;I zI=wvHx~zSMMKhswu5juTx>)B5N__RJ0YS(%Yh6hlvyzq*b@{x`H-zz@u?EE7eX+D4 z*GiRId)nFVGP(6pr3JR%K$yivCBZM<~t z2F#US!4geeDA~y2tKF9l!JNR6_2uqv`v3VDhEfWzOp7RHsXE^Y4g^VR8Hc;4!9yu& zyh)AuBA01)<@!%$LN6GL`hU6wK|IzQ zou*O#V|Z2f^T-iS%+-HrW0e%+G!I8dEXgZxQRL@P>tLu45#%CXo_4|>7wWfi_}=QB zj|mDYA)Kx*mr`9ex%Xgd!G5c80safgQVioJ3+D>X>4d%My77Eki@eyqUQzFnl$%Fel|%J(HMLrMCrKl} z#?Z$aogT#?+?3uI_6t}Hf3WL z>R(6|$a35mnf?M5Y5dLzdJKo-0xC$PL1j-})PazGAIFbrd5g2kof5D}KHsiu0(r*> zfsD7}q&3=Loz5%-F>-atMa7OI=du3nV_^I}-pgk9w@#hTW4v+_K|)hU64Q-TuC?;E zBb){`gx^A_3J>8KX`OLrM~>Xt0PKL=ZFZ)yO7cybV`9wu@e4)25a#5+3IN(E!8WZF z#fFGNw(<(w%0f$1YoG-pp8SJs6iSSu6{$TbGK5b{XXV=Lb~!z~t0g;36hkLW)6hg# z3jgB6qv=bZ&(SYJS8h4Sq6u`%d3^6YY~;+z8u?r(xV`OEU!~x*fA`#o;=v#C)O$;N z(I{81_X<#~JDIkk=h!2?7-8ool{(9rHnw09aNof~ZpB3W6D96MVHZL2p2lp1 zi~g#S0SAxbuXWAm4{|RXmZwweGSO_+24KOd37Th=#15<){T3kP6jYM;ghRBm=2Jj~ z@Pg&Z?Q?TCr!ryP2zn)8MadUTJ&Ge#EQ0jXwBdO9?hDxvb zL7#T75&b^<&4gsG*E!Rm^)Jc4ZQp7i$mz({;(2gg!%n5zQmh9*q)P=v>=uA#Oxw+I zNXgqbmZf5j@(Y}_Oy?WWRQdE24uil%Htv)6TkD=->Jr}7XMQocL9EzD!%ua(FT9XN zn;`5%kz!GoA(5ziO74A?3x#(UD=_8m*(8P!hlKEic`Zw=bL;CAO}Zu@c&j-?8-h4w zNw6`ku43~()u*tdC61e1pEX6mFK8kHzk`GIgAVi`@z!tMhW8qA>0d^vx23F=1O*?d z8R!dg?Dit*==lQ*Ewl*g{ZQ#9kFTTLHav1bPlWJK+Z$?P+Sn<)KuYgN0_Y}(nvpZQvaKJXIW@W!FHh4B$yLo{AMWl3{i%mM)(e7E;@N5g!;u0K*hYN1{B<5X z4=ccHJy%XrR}E4tH?oyGQ%}}X44$-PthB4MR~1gS)gwL-q9H6{4W7|#!{CAN0&01o zCM-?72V{?!ey3bHAUvxh4BAn?s{XRnc)0eIfm$n z{z&S0El0sw|KEUgmY=y-*WfkpFE-sK(hcMsqdS9NpMLA=a+Me&)>E@V+0s8G_XlET zJwsDN52zptM#V*zv_Nx;Hgv?^rjWh*L^b9_tjU@Kw3v@A;BqC+=i$nE^bWh1`%B*5b}@|J&@8v-hPE9(;-S)DJlK zGv3Heo2Lx>_n{x)T{?kG171i-(kWNadRMllM+T+A`F&i09CINvi8tq7;Gw~>C7>14 zQ7Fjid7!fKuVw~NqLM|1@*6N#3JLd$vX-jt_yh*cVi^g&v7c?M>{e!gQHv`K%FQ;> zt6tAyEd|;Mkd48649e;X_FGzyA*#`BzIJMP6LgroC2twdv17u+T;Mc2Ua56Rl-pl94l^bFfos*T2*ehm*n>r<)D| z;`bXbV`ZOPLw%>cjtYh=}UT;jbi&vR3CIzWdc61_A7i%(9%(!Q65gC5ZPFU0JT8(eS)=NQ7tfs;~a zBLG(Q@nZ%0VSykCT5R3tySSqqTb5`PybdnNcf&vbxzBbW`kA$C3Apu~^6A+!XD})K zL$OxuQ>__>vkYX|uqKud+OF0$brog9P9&KnV`T`kYh^Fy9UfZ%G`ecBLN}hbcU%oAIV}x8&eFpdTv6bwAOP&kbOxZtrlJMT@wAjA6QhD6u9HCG zkORX0nv9o)ts${?BTZ!76bP;J~M{hdOj(mj6oEG1Nsus+N#1BCs1?oo3q1g3l)X4^E7s|oo2`vPS z`CU2dQdRI^X@%<@GUJO2Dp+JF7}HAD-KuO!y0xV?F}i~@5674}5AyB6raS2vp-qiO zdk@*F6)9VnLtkhFMEH|}WiAJ-m3$=Zw(`25xM(Q$Q`2on@zA zPSXdxP?quvenTy z7|YGL*ziRIZ@bZFxQEX{YyKYIVMQp`Ysb^F_RUJMNQ|S5qPWDc2h@RT@K^m&7E9Cl zpX8^QqbK#>p5@^Ll%`tpP-1OYfQwptFv@n-Rs^=NP+Qvjs!tRY)o>K2(_oWO$`HWU z4L?My-lRRGNbsCRVm2&M&%{tU+Vb%oXwa85!gF$xI#GUzr{lm<8(^iJFNjCy{cDh2 z7f|b~AdeX(X7K~CE>8rx=Fq?)jlW@~!xHTOl^9d@XO;DAJ8xRDZ)zsYsnF3tsD)tgBhWP` zb30_v;v0c-KOy3~Q6}POE$nt1V!YWAo<3B$h$9Nuhk87c?tI;kUp*Y&MXQ%Zzddui zM2gq8r-@|DmP=jlW>g$wpgn|nj4zMJ14YJ=k}NBz`*HsA;{Zxt4gKF>(qmwK@C~GH zfyqe_2PIK6_zQk-ha1$7wwa@a-w%Cr0xXeK$sjb7Z%B1#>s+W!Qg@F!I)nvZ(l8rs z;*L@+kV}&UaLwQ8RrFeF*eas?1*Da-iRiCF#h~;&$UG>w-SMrp<1XOuQhEu4pMIh2 zZdW`J!&6{=2NgHP9xpsrh}k;jrVN3~p|jM~XCHFI@mwn0tMOEEhfgK|vZ>rU2YgoU zL$zWidN+*^&cH7H1uL8BI^M>d#;Gv;BXdwD#_&t2Z(KmAm8o&WI2s$*Yd>?=u)GD+ zUqq7e@C2y9vIz4;7U}8O#QHi7O+Ip`&R$W{WO`-!YoAUdoZ~j^ECU^3@7O0m_FCCRWb8~gi>*060wB7@F!wckydq0=&}5c^H+ z``b94UM_n{0zHMT>Jb`Hr*i;RjK0EYQ@`i}Kju!HEleX>C@6(z2_qQY<;?H#(QgR8 zMrC-w{`qcPCgBy;pMkbN$2Ev+_c%v8uJhH9>C1b-1C=|+&94A)5J+*Eq|@N@ZA=tp z+PuhWc#}P;mGWtjCF*y}{ka#yy_xD#!8lM;({j~B;>9zU)!f+(Kw(MX`CoyJBc_Z1s%T$+^>T`wCBDoA*$U~!P5 zjs20I0hGj1XyaxY-pF|9W9eqrw(8^YAM>?u7yLx_qaH?5b!o(((?~-EW#x9_?nLdH zzh#r`4!)p8O7=s2Z)h|$UDU+Eb}*@@X~)|n;HJ-UuyViDyXdLl=%me;Ozm*n>~ZmDR^$ z5i=Aa^>kE0Yw1SWfQxW%cR8hg_

D_NS;t$alj-?%gi;nlcCiQ~!8PmuFG*9~vcZ zLDw2boASi;>RFX9dOS<;dIV={ur$y^MmP4 zUV;;V_j+f%{bCe{Dls@}PMSbz52lW-8qUj_AV&Q;`vjNl%p+2!NR(dZ3lxFvKDCB9 zV^^XEb6}gx6SsMb!DeSIIc1WNA8A*@jb)IX>qC20E+9D1^L)A-v^01Sb&_6Qz-dHc z5R^|9Y~$zpc6X=I`uhu(&FfpV!QYGqQ7DP#oZcS?X|RN&T%0htmVa9TWmifRIyghk zonI^u=W;m$YbQ8)##U?PCjSjjZHR1<3))$Ve4 z=l>Cpj+Gt|SX8CA#e57#Mf*Ql!7k7iJ)qKQb!0TlRwE`@YW8lb;}I@XS$H%XGQO%v zH#^1v={$DXVk4&S3G9#<3P#aP8H}CQX6J}O9H`FzJWbr@X4D$Jex{tw2dxY>wh!jd z5{d(>qzHK8d!>H>tv%YPsxcFcX9^N3BL5k~%%G<%;RUf%LBL0ah-*^(i5g&>+@tUM z$)ZdV#U+jmD^BD4()6Rz z_N&i!@q&Enky9cwY3i!bkX?WFC5vlwt83Wj;_NbQ8{N8XC&_BDrzd($gqrMXxyM(e zTV{Af1THD%BM||PW37r8bcbuGhB=;i4QV7cbynOW+C7h9BwxSq(xt)=A>dm2k5~&$ z>SSe|#EOp2vbh+D2iC#ybd_OEt8ZAA2CtLAyt?$KccXeDzlzEu01KWu_wF2q!rD^j z#sz_?``P^f+TlSPxqNmZzl~ni#Hks}$lkEdwO_(QOSriukQ6${WVp9?9@uNO%6}#c zG^xfDqGk%ZzfgguH0w>am21-Up+n`txi%@lTnitXkzY{85&vlnYxvhRl}Rfz+=5fA zjdq{b1(K`DESBSgU|lz|o3&p*#!fF6JAvuIj9FH&dxVoEOUvrZf%_3A_$E7#;?!NL z`yvnflLEnWn7^wtB1QOTj0_pl8=Anv-Sn(49V>EpT1NNF(d|qUoRwNqrWj&t*Ir|W zSmk6_k3eqTYM=^8_&xiGdLuopfF`+eJscq3dz2Hn61> z0$s_460^m433L#1!9NjL_>Hyo-`{L(+kHU;C;0Zp{bDw0w29q?`iENP?qek)zt+K0 zTvhR4iE3*F$7H|Aay=R%Z&BC!Zya}-mhK@8qF%e3VUq zn*K_JItLl}Pqf8q7;&4nwC<4YaLoKwYlXR^JB<)CK_ab}2c#3h3OrMb#aC;Qal-B2 z5c^2;HS(u5Ao(B@J%&hv^Ruf#^-5028f)RYLVO00?pCUjP;wpmLMk8&Z~iq?Krm8J8|Y|3-hwTm_9(ko6HAS`XQyL~bj*|blhC)4 z_Uo&`6_wBc3?JP^@V}^)?qzy{OT`auPxQm*Q=5mWyIpgM6B#xb35~}o5XknJI!wFW zo;|Jq0j=xSnhntez%&GZ9pcTTzzpsqI?E;6fb8C)sTuiI8>(|MpfZtMkFszf4PL0xw3-wX7NBo_mFK5*!H>ETFE7ea6O!iA)j0HR?^#ij^0ds%mR=Ggx zVw{dg3;hKbfRl)$vY$wJULe%h`TE8DsoNakq~Lx^IEQn9HF)g%PVi9F5|>`@O(L0kqJK@|2Q*eL+39(u4M=BCoT^4U6aN}%K2htK`tf= z8g)0cXPE*e-b=8CgCUc0h}uwGkdjb^hJ>HZ5?cQ+XEp$NZf%B}Z2DkG>@t^^;nWN( zV^f3;lT&%Z#B>@&oo+~~J3tbYvR10SNMzZVIu@4+ddN0AcMr844CrNTF7G9i-Ut^@ zHA0^+XLc**Q^=D)tFd=E1t#kGc|PsBGa6;#vJKZJGxlboU;*o#|)&J+=3?9uEs)6T)B%p2i7u4_rYF)>LzmCKAAMnr6738RD ze_e52e(7?av&ld#B91n?X=}+aNba>L6wD@SfOQ^a3|65!s9K|BH5HrKFF&lLBstoA zUD#$E5ASKg$aCSL#CRb!+N7zG)6W>`cHZ%ANp52eiUW1LZl>L+a#}3yly(Du%Ou6f zVZ2u(0m*?D(P|Ewf2Hsi$CTH!rQb7XiwcZ3d=GA(2ibeN8q%Ia4A?C0>CJ2>Bn*C8Fl0D7PbKSrlHpL!g8XA7qb5>UbnmEYB((b_Hf zSfkh8M)BQj=`Y8z&9{2Z{dz|@n3*i9S0d}ErOA&VP?gnBm7Me(=#`3e7iH)Y+X;6p zcXz7-fa$CxkH<2ITOzUPkM&0;( zqOF1E_C0;)tem-SXOTEr!_)Un2L{)b?>`BL1p^BcWV5Xf8ik1O=u<9f^O$Wd!13n` z`CJ4ZmOWP5U;Yph{ZI9w*Waltt7g4$eB5sSXMLJ`9aoz_O%MQhB{+9HKi^XWs{kP; zTP#C6kL3}+Yg}10 zMeUYfym{2d;?nXn3YYF7kPIL~M^Bf#g)VZCuOvPkS?F=2!t}VniF>sK;ZEl>H!XXF zc~`5Q*`&dz`fOfcIn~z&s4K`SK*{_p@9WkPlZkNj*@VZF5ZYsT5h^=EWN&P;XUmyGWR$)))#Vi2EP(zNN+f0XnH!{!J<&Z z$W_==8}FhvZy9PCv3ys$8Xi|XO1paDR!MZxW9Ax@1O2!w;eYkq`(F{`N#WF!vDPBL z5mIIkgWwmd@OEEvtyOG1-BDSj>Pq8q(fW+yeIM?H+5_4;aLkd35#hT_5G9{$l&BlG z9#Xp!REq9A7)>xSoQ`YT@9{HKwwdq^hcKfq0gJKK@iuX?A$EZI)@t4CT|3D zODog6V@KMW9#;lMb6n}exaImW8HQ1MwCozmtXXM0go4g#5`o|Kwpr^AY${<<|BOd= z6Rpn?4k_Ewd_a{CVX}qHf8KO~d?g~%L4YX*-NB`z&9$TN^&s--=Wy->zoQ?&@=R@q z#^!g%DXZxnSfK*9Ftj(RR%8}`{6bBDAkg9Pu%CCYw(JRg*+zf?RA}?ECmz*RPE9Sp zcVM~U^g1TY(ZeV$17b)N>lB9-cMB_asa4vR+>Y#6l`vgK>h5GikB3KI8pwrg^3V;M zJ}HkYv4%X@AaA%Eqg&cN{NG(Jin`f5qh8a-ZDZM%%xHemGLZ|gDPq) z)_4u*zfTgNhS=x7c1w!-)7wmEEyS_`^qSU0_~MAK9~hBcaW)0YgV9GoqFj`<_`1BS zF#psVNK*zc9f&>gR~OLh3c9sorzB)oS^(%;5%Z?SWVpltYj43|PLtiTv1kjP0-V(L zdmlX$@zCg}bD<1}91RC1sOb-}oweY5Iak2`6^f^k7*Y{UBE=K%z`0Rzmo<70$L}?P z=6PVc8A_#L&m!Pu*e|BuoTwywlTC*FK(pktF&;7APM1YBrxzThb{M#5y?tN|;*WU& zy2oM&bxzP4LtDH(H@GQ^vl%|Wt4R%1trs&(>|wu8l@-u+TdmWS))FwrplXZmrO%5! z_dGanUiSjKDU>l3Ul@O`EX{Ki%{NTd6(;SR@o*vZ@xvi|H_z_c7iC)GAN349My>_P zkeoUu4@Wy9O}&z`$i6^{Bo5(=UCW%Y+d&Wq5ytbQHJ!WMAzwmi3X*@(VRm(MYE`TC zCkh_(`5cS$%mrd+qs>oFna_bjL2<66Sxw+|+euzjH*b3VtFDv>^Hj$`_ZfhZ0e;iq zl6x`O>6f#Y8jTguWuuOv_sbOamzu7C!n5razGNH)sV?&GM-s1@uk_C9 z*gpmwZ0kFYmQJGnm(x5SS|CF8p@(~Uzh6B~R!-tiD>Qm02`UfX$ z*s7VBE0s2cGhYthXZY;=;?``er)?+Q!?G5EI-9K#oH|Qv{c0--bY*?1u)`ccCQkF?P9R#?0|f4^>e&5*mh z34)#(3NJQ-_7fsoBkEiH3sd_&vzl!QhF7BgQ z)YMi~Dbp(ozKvPGGHj_%w?r5pVhdu1zY*Q1axm&@qyubV{l|Dc&ifriRBrT9>W4o- zg@DPaXOo{}N}SW8@UHHb)BM>qG;rh;u}X1-Nm?GSVDlTRYA?8zJWj97io*UM7#Qg^ zAZPDM1!*sSt`{B<5(00&KuAW-8>vSpq)pnA^LNSy14a!YMv~A7dE^%;CkL1Hd^45-g^skY02r8~w)fSP$W86xqv{nMyI+Fn2;dzg3}+ZrEPWjd9)o87Z3P#X}Z)JWq3KM+fCNI*Dq=hA5nX& zNUWQ$A^)Ky?uf@6E7Iye>k}n=mc?#v)44@)N?oiiVPlhQRqr}m4V0EQfA{9`Q4>R? zo~R~&X4}ciJo7YBS6%he4}(>IP@3@#_0QSy?WfW(+iKoc-Q?gPD65p~yBDgy9CfO1 z?k>mbC6ZWQjTaE#kf7j0D_>veHph@JQ%#OApyZ5+CY%H!0_snB<1@8ryH^-b7PdT2 zr*Hg4YSwXxU~45++wT;c^>osNsZEkW_ueYXqv2ZVs;3!ZYe#?mLFr&WTW!oTAlpr6 z#I@2#vI77ffOLttGxWmqfpvTD05!tl@byQXqxkKg_?E%eke7VKjsmaEzpH#HG)fTm z;Mx0LBjt2jWTHR`caEK21fXD`KJ}Dm26W{)(_-c(@(?c4gDjj%zn%i8uu(bTq;Kgr z0tXS{Zo}O5B5Zk4*ZY{vSGA7kNTOE9HXvu05Q6BZm8U*7n^MW;n*;Ovn`Ne+lLnMy zB&jQXRrlwpnc*=7cWB9Wd2@|igG5>csn^{8o(4p*1_I!8;=cac?ZdVW?^8(W3Ty%M z3jh7GTE9&gpEKEUOclB!3$HyVN*a;?)gdy@B5oRJufvtOj8S-&Y2Ztz`7_|?XrP@g zY-aF9vbP@n_l<&6`U@~~%Bc5?{dx|RB^ zempu`QUkv;s{hc*>u=TuJw?7UuHHZCF>yAf_cXafHBZcJ0#Fq$cw|%v1VlVGggw z@^!+;vWPMvEfjrQ`p}$eE3Fe0!VA5RCRbTgYlB@?!;gORc?QA_nwj;%>pTtL*v*^N zyahmH9~R+LVu53D9lIqvrC`mjhnpKfE+e&!Q2+YJ^Z0&Vl{;(OncO+Av&=#MV>m3Y zHJscy*t9&Ew*$(oPZc9#Mhi+PTCG*$>TQQKoTD*gCBOR-JLNI-*YAJ$6?lNN{14En zRXW@16cnvbIha?hx|p{XP()Fw>cat8nz)X6^7S;^tqgCrc|=8lqtj&Rk9xdMPhzt! z1+ASOo3N8kpL4r+gaROLc!BPXYc$W1!a#my#hCC-S56wCORpA93*jsmJI;*R+VL^X zK#U%DU6sOhFF({s(m3lBLxW z8IyY!F7=Uc9N$hOmaCs=lm-U36`siTJdGcse-)JfAZ@Uq7EPK<(Nbknvr8pL0Yqn+ zIAAjzGPc;487I{J=09!!qw;w=mcS!jtlz*8ce^LCy1C1+$Z37^t;dK5Zmjd|`POCU zXg5b9AMVaFo;EX2qFVM~-S%D~byEX0#y6o^f#-t_`AKDmAAWw;2&-Q5SJ7xXC`su$p5ySIerKo*N=|1%} z?JUR3q`Po4i2pw%`}gfNIU8M7{uBWT8Ezh7;U)9rj4le9yF2`687MK)clh9waeKcJ zDYnL|QRx-l;gM(atCJx8V(^$5N}0tyTPOq#*Tt?5)cESi$YH9tAI@R#pm95C?p=AK z#%WCo51Ml{^#FBMo9Eq>Xp629ILT#kjVarP=mhbqNs$V-TwxhNo%qLPHwnm<%xSU_ ztf(ThC&ThhliBqZk28&}aPDY$2qF+u`_6$z3Rs_*MsZ7Ft-fBZWG1VQdb-O@Dmu-l z`MItFh8$Zo*UDdgb7Vj)F`jEl=9S2zU%Vr%kou+j`P5Bi`Om)MA8<~Y&Xzrp5ceCT zIl|1zJZ0zP7bYh)q}&a6W>LJ&J}3zmR@V*^DZS&oqMq@vPV(76hIFS9UNwsFe@k5k z&=okv5Fx9Ajyb|n_QHZ7Fm65zJBQS9zU(3mX@1i@)ryAdR(~+;@$WM=B4r^lHGc2Y zD3J?J@$7$7!#gxE*vwNg%@>l-q*J=p)nlT0m^7VY0<}{W935(0@|T~cZTCJXCbtb- zfjRNwtE;PZ(ed`y+F3Du+kdwfv88{+faN3~`~MvIf3Jaq+Us(XyN>-)gotFLefuCx#nb3-xXTD1P?O%?vGK;zB@;->9x1XQTdVC_5 zv&|xFYDOO%4!;EOmWF>#4bU38Q$1`1aE{h={#*@B?;xT&K>fTeCYpoc0v_tUta zKYh##sAB!He&~TE$FR&{crCEGYO_KC8LF#jgH*~XEDZ<0H+q2Uc^rm?aH+mJ={I^K z;&^Acto^C|q$gZj8zF{T=9~a|{IvW~v8VpS*q11lirEXVmKBE6Wyj@*grc<)A9;F3|AQ_ba zf{g%@2R*u+b1y^abKBdE-}o(DnY>>F`F!o_!OvVA!^BYo_EPplHg|X}Hpa+H9FiHv$!T8Rp~jY0R7o@5}Dt zN2(Q{U&9_)lCL@6cqUxKTf8ncMhh1zsS85&9CWF+>ZP?Id*r>8f2HkB%v3_v*-Wx7 zYr}glWLHRe1)2DMXbu3W{cxrUud<@{F;}WGI5RAM$@lBFP#LmM*Zs#f@vaDOlC zhWeC7G2F-Ivv$$aMYSRW7w|`@-Zp5sY8#g{4N4(tE{M|v$E*DsN7B^H+)I+4wv`2M zY);IF>r+dlX`G}oH>73QdRE95C~~b0h=sIuQVj(sLUhJdR`VWD1Up;?dpTNnmcLL) zE9y8<=g7gn0PHa2fx^jRCX<`YdDnK}kj>`aR9a^5ij|h6O%~`=JPk*fUVp-5vc%M+ zDG!|(0ONf1A5v#fF%*x8f148*M9ngX@|CINJP0%^`(zjj+6Jz*Zs%$t6lGDD)jWBK zo_iEo>>-8OYLe2I+(cUK5|bQ`KVTo<2{ajXoSx(Ma-nebRQM0|C|bIlFFRbwaE$%KCCWt**ZKK2co3O8oDR|0S|~yTEbn=_RaR_n zaUnO?AbW;jp96~yus_=mCR|QO2`SE9mquZVaUuf=TcH}Ngan(wH6gX~5o^XXm@AYH z4vLqv&lO6Zg`L-U4i7hd3fuE2piB8>l~&ii%f6Vv?C>f-O&cr(V zTr`K;{-jP&pi4Cxoeqcj2ZMq{FKnU{vez|3*_n+2+9O)y(t?;^6ac+IK(ST|6_}m8$^S>hQ~HR44aw*9a_cLj`e7ROb!B#*4UPl zt3MOrG3rvW6kZ!EhliUN`B)F0`Y14%f|Z*-EbiY*<<~?s4Sv?L64+`?qr=o2VObWQqPJKLpFuuYwCL#{v$bI^?m^L?tz1L z&q^6g8-=}juL%;#lkwP0H%bbXGbPA52k)p~af&o!JMP^!{3r&fI{I;>^9+ zf5+o_DvJ@s*4Qo5jH-)ZICCHyy>#Rpa`F^};{&lx_?MJj{GlWE`Gc512+5y^E9gDF zg2!Sx?rkt3fR?yeF?}O(2{yr)k2}m6NF09e(I2$jBM#yQ$Lro~!>2qC6Wf4J#e|+t z9d^CDxrHepdNi5AH8Hhi$JXa!0u*H%=7>$K*ZjRO=P)$gV=F=%{x-A9Pa(I)jK1Y7 z_y)`K^l}MOXQ$CiERm*J@6p3ev-PmH2&P7zL;$B;Q+y1pFB$xDP%F&XNbr;+YioMG@GxY48rt>={ zYHFWaWcSR+!h)GFoF9YwJj3x5FkLyTKFE%$q)=;()H5kCQnT;f6Eu&!DmVMI`PbJ^ z^)e)gB3$@k{8DBni`Uqq^ot}MgZRfys;yYeNBW9ic*0p&U_8nr<4oYO!UzTj29{o< zf7URykK=VdhB&l!yRmAZT6&reCTTx}@y~l~tJN&2XT(?_Oo>*Be+=uWHv&r=?SQa>cF)N{e}tZY1QE&0v%2deWuLp)I84XHfI zO6yC`dZW<@uO)Ga&vVHUTs?b0;JLeFPbgT%Vu^9En*J>vhB(i#qBHE%=1(@)%@8dS zrdR7>y>`ZWjn}l3Jv$EHHcb!5v}1P>ofj+-O=?P9j7kVB~ zaU!czY%jZd$p*)7{ydz%yP5ndX9@nc2-s|*E6g8_Q(C70XHRyNjge(iVi;kRv?~A| zKku7hTdvrWHFb#*Tue&njg1VxW1EOna*n|@ZNN(9LA^mP^TxhKBKW8CohBE zK9$SXWngO59S}z6kn`xXHp!*B$(qALsf7WptJX|sa5V0EpZU~Ru?%xK1z03@?=Hk_ zX*j|i-(>UgfxF4at9^3L=}^!^n>{2BN3^ZNmN#$|MgR@xTjm}kSCe7-4`@5` z_nkoAv_#_EHTIRwz1{dcML)F(z)OFkDWmS%F?7F)iIpI0chg|dqL`V_?Ez4u`ec3pdJEeEx6aPc5Zm6GyQ<1 z8qv$Qnf9;*E`}ZEtRmML*NOmP7+j!Ip z=?KqgVBtyoFU3E5VNGY|bG9kp2cRsr-P{5VLnc+}yVKLrR_teziJR7%beokup*X~y zm)6bTBZIDrlJlph0VOKK&uZV3i@Zilluo~A6MlAro1LOAG6nCA$BCS)2eay9*B2)% z0|Wm3Mi#ngQC|*kudrq?8K74x*zeXopELQX&`3HZe;YEQd%y4lX@0#;8-&XI!+JE; zv=Wr5LbeJbw6<6@c-7>yO0Oz}i2L?p75~wyU!`6)&mpffFRIbZ@AVkcV<}F{SIl5PNr$O zJRYWVrxK2G)c1!9rf(%m;po2-RdB<4IyOf7(%F!vve9&cWXubbOCFgV9|r@Vp{^nm z8uz>Ugkw-*zeg?^6GlI94xVe+Nx79?wOVr7>>^OF#06<-5pJzm;thS88z-aNqU$ai z0%j4nryDU6f#A+8?%#xy4cw0*aKo7Hw&-Lk3?7J6k`-^Bl~HG_z9P&^hO@`$A`=i; zXZ&}xZqId4sb&ns5D6N#>6xorYJC}Mc-+d{+*cZkZJVS_uq_Z~qp zX_l@N+q>2qmHu!m`%2Bj57LyQvQr4{;A3sO$Z|`h70B|3t+5(LTjMd6G&ZHR#E_(l zu=^4WpuIoK01IwBYBa5g>ltIbGi)MK}jkQ7%WhQ+M=%Hqf5Q)}OJC)zgsxe^e!$ ztj$)kF$j^8Ac36$60SLfIwvhJ;GMSKhuj4I=YRiynyMPnLU_^jUn8k9GR}TMP=W_o zy7XLQz?ILi8QgW5Xni760J<6^z12FL>!=fK_!~TwU#Ia$jj;Qxyr)hVfR@xY+e;J| zxmx!OEcf?uWpdw?U!rVMoVpxit17E#l?P0vC|mLN!KhdbpwRk) zQ)EQ-m0)57^Yncb+K=dhZEC;~GXEN8+GH)C>WkTPj^J{uwMxRI$>6fYRKE&8rddu0 z$}9no4a^KF9LBstL;5s+!oUz%ltqD%dj7yK1ggJ05A|>h?-vFEzYF1IvbMV2zc9{N z0SkbAbv9E7t;ULeo!+`z+Xe!Gkp-(WkX(S+Wt6Vhh`k7ag8LYm@-)%xaV0(-c61|b zNIZLf9z-sbbm6|opY;vgRw1tG8sGA0{N~fFSRpeA7$$vvR&Rc>;ukWY_S7b^5e0Mj zEXAFL>6tD^rHn+ayNUd*!g30Q)!1EnVXp9D?$G-`TglYT{;Oq2|}W^_(CMos4?>VavCKC0fcP9H85?c&Ul^ zq^wIy)otqeVK~=!u~I|1)`=buuOyaBCmQ1JGQp1Sqk5vUdiX@v`?UcN6RBbY)oWPN zvLCBYaJ{!OipbyPyl(zJ)8SN%oF#0Qw44!fwrvL627H+<*TVqwuvcp7FvN7^ZBWTY z@nhCbF=>QdtybMaeG;_{h-xLM?b>_vxkc@mMW0l}vJc_c-{6%Xvb@o|>4WhH>zVTI z!AYTgHjVwCXZgLipg1`??? zSQ0No^sil(XclM}C<_>3;~{<6>~Ct5lqK7(K?3~iD|wYvU5FvWm(4#)0coHe7_Qrt zi!a!v^`E$T#p3#QfbsnF@t?2X>Ld6*3P#Wm1%{KSL;;_-^Zd+=_t(pDfh2dw zxuP(*VV=q$luLmWE=BPkO|gy~_IR3qj3p@Sa$AZw-&745s+k)0&G zBz;vUN}Cxx!`0KIO9N`a=VgSZni}Qf#@K?pHN9LKNof3iJUvZIH?_AAC@}&zs*c)9 z^_c$m%aFuGZuhzoQ>U`|K-S$Z?7;%2Tmo@$ zztP(s?1kg8ud!;ot3fuLM0pmh*?`hPDow97NmXzXGn1yW(c5)#!Gv!(jOj*ElZs;wIn6LI}hk>OOyGH~oDKqjwx%$s0~J-ad`XiJq+9H=W2cv1U(r z@cWU@!pSB!UN2?#7-e}wsp*|89*f2B`9)u(fwR_gG*q?pz$_i8n{A({$fzD%P<-B-SBBo`vH8f;|eIP zOR5ifdv`|Hvk1ffrN<}%rLC_+*@94#3 z*({dR2$R{Rxikx9c>8u7yN7}rGs+&*Gn%6cj#zoiD)c7jp}rfm8R`qfR$z0e9F*wQ ziu5b0RdT=3*YoIIWzYx~UN{!c+U5PLFFS!<`2zrs=xrhcg)doDMJ-(W0IL zPVs1;X%_z)V$A|juJ|)PgB~{BaQ;BRt(GEpUYAX#okfR|duZVCmix4x5s~?O%RjOS z;R>woZknKLptbIQEZurb+(-vyjjWq zJKtp#KFb2X=!P#)?xZp~DP;&0G!e_VG_aCHnxt`~%p=?Umi7$K0pH29F9nD7MM}4UNo6 zFuBILrAj@=A^WRnUGDbMnyAD-$imFh%!;5jt7N6uoxdPj{Mv9Fi9Mv)1N$0{kfCwNcSBTbURr}guD5GNQmTz)S;Oc5arPAe zO{Sb_9sM)vh2%YJifZI=7pyS(I@XNsDHCN`R6b9~Nd$ol&Du`Vq%*-hhm(=~^~_yE z@@c2A-`^1nhX<@@rU!jF(vc;)+G#?=3-6EmZ%7cDTxJ$R%ULH9TcGOv(dha#uEb|( zzi`1@w+^6a@PLp{9pZbKeA<}eeK|gkT6WO$LGgTqZeq}RKTvUuvDbL8VO2pr`CNAs%VX{uu3Vl$}IRh^`;`?UyB+ZwE262E%&tSPTP(=$9qXpl`MX zT!q>$v?d8ssZN}aS?d~0#&{OYy+)vK@5hRMAIyiCk-esjLZ?B89)X3}9>9UKXR~!2 z@yoqToAlg{;#oYq50_+C=(e;KN}@hhUtX!|3ERzC@EYjkyUL1&?lRL4tp?B$C-<$W z-}w7e0D%OUTBNq$>o0$9rB1W+nmFg+XpyagcElcJE&tDQ?Yfx_t_?OM10m=pRdzzo zEIKSv>uY=Ap5Gb}>_Cif+h6ZMeF5MXLk2e%Yf$SqSZSLUZ|~o*7InU$o#Kw;sb6S3 z51_G&Z9)9jAgPz){EV7R8e6xqxvo*p_G%jiW-f&?cXoBGar4A#4PJn3!3yBn@acT0?%#g0zK&=3LyidX#9hS<++dwsxmy(Q1&J9u4=pMSE{$hnq&0R^U^Q zo@Pun`M@5~L`Xbd>#;0xu3?EC5;RfDUwEsK2o~E1-N72FL#hb5_B{^e^J@TSPGQGF zDT0j0Tw@H(k74v{}GO}0He zK=>^7HPAO&y5oz=*1EqQ0eDJjov-SrE#A`tQYL5xcso(~MP1%@CX$PB#XqLg`7+)v zZ$4|`Hyw_wS#0AvDE8%RfXomeNKY{fD<6)Rh`#{}PWVn>1DQLxYveA|2Et)rLlb-F zGP93{%|rdYk7XKpM8}{T5IGbEHAw%+gz#y23(C=O(?cj8U_Uh@jVuN`MA|@AM+~sX z;Yfte^KjW?l5(6+r9WWMVwd#IcIB`wAZ??J*Q@rSAsVl$8v zZD@Kx#KX$qD%vwW?5ZD3l^_hLbDfX9)AcZ3c9IRlc&wX`+92nq5>+xy9``^y|6{b^ zz499`|Jt|UxwPI{GG=vsoXwzt4@V?_qG-tLRuHUE@>oB&72KdRFS6Ms-_;ultTqgb zPD)e~flRVkXx&TWd$!St_oeb7)Rn>5@sJAjq$?BI>%qk~+OMDz85?V#DD^HRwFVM7 z>F%vzVg=KXH4$Jc@HGnx&i4R$w0Hd!I1wMy4+U zVtW$$uw4ymKyfy#j2RfO-hQcxx6{J!9ZL9*!@1>=f3Q^}Hu?UtL>rQ3Ok!=h%OZ7B z+cE2cQ<1;Ke57?f{ch6@;)$YYe^~{$9QOvmQL~xOEi27Gx;3(GN>`*+_tanCFj*-l z>1CyF*fYti$KJVGa_VAJZ9CgNV#?XpN=$71xc59F!_%g3O+mb^Nq9@P3t0+|&_cd0 z>UnhRTPK7g^BeeK->XF!teMq+1o#NLcgW&uC}P_rpqDuAJj{bOB6}Z*JG4ID!I@(0 zso)VK^H3&IGnRfjgLHK=AzF#9D%mpNbmG>SBbqy4xs8qN%)r%ipo>G@I;k*m*gVx} zz4X=&HEJi<6(KBm4*f4JP5P=aT{6m=q~Dtqgzf_%32ruZXbr+Mb(lR1Lp(%?y@HCUrxpe$286lNDgBHud3y>P&z zHXtOr_t(2eH1p&fhNq2*JnZK+zYIMt5||cM7Ylr2gEV&QAl5yJ5EnTa1Kg2SRkBC&Fj2I|ayib2 zn_x1ujyQ}Pj=xjr@9au^+#2LY?>N+Zm*uTBcf-z|?8R_3VNu)e_K8bCY zX#a>RpsvuJDrPTDyL}~-S85octDC%zLzxeVVTtc+c%lbAb^yE@thn${B@(}5nmy%u zJ)|i5zxg$5TVK&rW79Y45gjya~acTZ?vLI6h1}m3|%Aun=3ob084GJ^p;x ziw4>bE?hmv_cc0m1=_}SGi{6rq+~9!Jil2I`cypaIybxbbxotgzTKMY7j;tyf%EXLh4fnPP&TH25IV`dm4?I2dTqh7c`)>IKa)I_yn9_&N!c6Bx1HC(Z zBQAGf+bsZv#^rHG^;L(=y!l3fo-j+nJ1G^ST~^pbeua9D{F(;kA&6CBW- z;rW?srVxUp^UIhP>#sG^W6*c!Akk`dy--&A*U`~?^dnH7yuT*f$ppQvg-vdxaMW=P zrNvZrTIa~!9vXL~0B27#^CLVg`9J@Bb{rk}Q2ZIRqx;HS7?7H8mvC3l0G-R3FUGjA zM(Ht4C^(?!Yo~5^XmF_-OX>WHCLY1Y-DE@%jpc26QktaqVD+Y2;?zU&xGFL`A46l` zZj594*dL0v68Z`iD(O?UH>FP7)|wC3jz-F%KC2K$bm4ECIu6BuP>l055k`@3s43_k z+G$7;1ym0u5hg@s-f#Z(Inj7(l!dIMyFRHCn`4HWOHrXcrm}_?w^AsE#Ls(qFX`r+ z&J3}#Lebk_V{Hv?T8S84cpu@Bv72wl^2XL) z!J%=yH#s^oPCd<(_1HN4k@CT|`NS5Jkq;zoOOVm#yen8Mw@_cTBcje{TA8j314^rKaKP0Wg3oyGMV8eU^`mlyAe8W zny*V6Ex;?3z;!(G=Wcz57_OFy^5%Izz9;yU?GRIjUOyF#?WkXdX%cc9STho^i`Ki+#vyqXKldG9ZdeBtlP=nmNBi z;NQ2O3nBxh>MPcNG45LF*Cqq*d`;Dvx6gs_g z>tvznzH0veOy1j4mbB|5yhvS~`rH3)I8yJch@0c|oq&od`IXL9d0ez^wAhR|j>vYDkJ*=A4xMDoBt+5AGExcI0iWJV1*qvgc8}d~w?-faN zNEPfM{um74rUv~Q_SEdcp45LS;cnvXwUvuysFXxc%Nk&6+$P8PWlWS_VeSCS`R?Rf zP5J0~E1%OQi2wv9myRU$AxXd#FfDd?y@N+0DW3b>B{B~tN^Vz(ZB7g^qP~hqnD}XN z4A{)I!Grol>Jvfyx7 zzSZmg)`ip%$mx{u!q*}|i|&4`Tw=Ec9>{^a&3>6QR32Jc752D{2*++5$n{d%5E1Q2 zP1X%&p``|a1XYQFmfraj5R)8TDtY+;()^zxU^ zeZ}V)rlk#1remX-nfto!*jfa4lU*#PUR{gA(y?3f?RBVi-R1O+*x_l}QS#eu7VX2==8J}Z5Ji%u#OHX*+e4IdGmK0rLjpOZ8C9PMYrW!gQtG6FP1#KN$ z7AB>}26QgXNz8ocd>i`9<6LKJqZSbcqqS8msFo9pHhAv20%!v>+2D(dZbS#xv<^CP zoURl!*2F+KI8lEdT&;vbkgD(}BNdEtu~q~jH3pwzlKq|QYCb(pVC6f<4I$5dlN1+~ z*!!m;I}2S?j4M)YEBhzhOB!ygPLR`Kt;=hp$qq@bz6o|oeN3&m_8if~wRYR~R&I`% zC8Ei1yQcM4aUx59xW1^noMkDs0$T653x9#r4~^xIbdG){#*WD9lRA5GI4o#;W%tyX zbKGBM%FDI^Hu*e$#^YXP+QKHZro6(cwCm(e#o^>>crUH>I_;4EsMk}MN}O}9rck3j z&OU|Oljh0WeB%FN0R<1uz$6%(4D`XLp77U4vSe-n-NR|tfM&wM+m>_vNV^BKV)<6r z{kLs$oxAT*J(sA7tsp!#)}XkI{Je&22o;WR4y-ros>5#O%`T|qP_OvUL27Jd(>PPe zv%2i4VGAj8$!IV;H`+4NLYx_E0c`pTAvn~_ly_8&&*pm7!#*|tLhi0XERwz(QaNRr zpCDiiCx+1WwtwQ0?1bmY5hzy-^2g^wY5cp`Gw<)z+f&0Qc};aVx+;UZ{R!J{18WEqb4oXN)0*7(U1*Aa^1b>){h? z1;d$daWTDc=5XvmYmD#Q1OjwEM2m6t9j_hrCTbEiAd^}aRHorQ+I=M&#R`^zy2kOp6GZQxVkd#t&y>zSMR-Fa>A0?+(!hROWNT{Bi=Y4I4HC?7_>4oS0O3OrS0>RsRL0L_7k{Z(`^M*$k& z$8;YF1Y+(K>Q4$74Qv7U1JERT2x+3R>4eq4+y)W2e!`a9_=csMyWLQEbHur5swx4dRc zA)vr7uR}yzM?!2osbv$WcuAab>!837&166@%84nculvyHoM-PAh}LO*ZjAw^giD?& zWofEKZ^QY7VF*9?(C96V#2p^f;aBwqE*^Q6% zj|6M8NR5X3yqj=k+loWySD*oQ@*KjQ_Of{&g_?WQdZrzLTTPHai4@`C7TcOB0XVg( z12mIj_$ROj1_k^mA z{Fk_wGq>h^WJA`D%W22=gW~@9q7Q2B>U}H}B%RC?PsRY_4s1n>c1vs0(ZxZ60j8v> zU>Br@|1%ysNu2fX#)E}a-CaG*VD*Jn)vHn!PTwg!p>OO5KU7#&<(jcGE!7_G?(%zbXz=6uPgz{B^kJdUHrX@AhPpXncgOQ(pO&=)+#lv+y4bi? zJRN;1TIA0I5r4$xNJKABDR=FVps;Nf*Wy2y1Be82dPF{mfy-sswKk1yxrDP3nao04 zbvYd{N@GOXhQk5Ma%QSHSLASD_x3Srj5jx_J~T)bMEbl7zyk_Aj9gHAfj~7PlQ{tW z2%j$dX^6bVGOg{ToM;2JF_4*c$h!rRqsO2wWBsNnTC=trY^6virs!qQdJ6f?_jtO^ z9<&?3{U$i7`8A_*j&fJz?%T#=30RhJTd{9KH8chBM@@7bl+_}Bk2b&^}@be@Sh z9Y(jC@c4^P|1m`ply*Q~NI|BiQ>e=%bZLT65kcs342Wh1w1o0FYv&XP!F(j!I85UW z`;?%l_VKuD6Mft#bp@r@zioQ*gT`pdmvA(yd+Rw#mP0f z*R!^Xmacgs++6GD2mynV*ghT4AyXSI<%mHcrZKhI#6YX>@>qZL9deX+cP533r!S@2 z-G9CUaq8}=^>i(4(m((#&gWJ{`kE)r=DlB5l#v+PmRtco5`kabg#%rO@VianMmsm5 z2O12LP5LXYD1}sXxZnJAVEWM7JvCr2ArqU33Y!TnJ@(d)?ctV;wj0lqxBZnAw!V!U zjVYz^YQ5#tSMC$~hBc)fJ6WU>1gU5<_FU~@qkI5!XErIN4mj*mj@ZXV>3tiTGTrK7 zG7Pa-`FA-VtWT&tFA3Ew9o)FbM0Raj*Og@@B)9q72ny@+0VzV7Yh;ZG3RW_!F$7jo zWsS&PUvrB6bp1{Wg-Qc2;#5%&CyZ}%;{_tEVatRe8E^Cme68>0F$rtZ7fEtdr;OjI zh_o!HbygH#_m))Hg3~!HbC`oz%lsMD*yBQvMu)5 zNVitI{Y2^;yko1Dgr7!l=0VGbYoUoo_?LD!W(T_zq^42PJbYKn=eIZ34lod1&d1An zaW1MS@f2D^X?ex-#>65C=Ged-YD%+OOMZ@Noe5Xx8_*E53|E$hTB{ql)NJfRgUp-+ z-84SU+0MBzE3HWk`0~fOKGvGiB^J$nEKCWKWD81C^b9=$CLcA>5>&>?k^XwVVQi1% zjx{uy@sn@!dX>$UE|q9KGEb<>`+icbsgtm#3lN={eppZtiwTb=++s{Ot7XcwMOXL+ z_t6JPM@RAmLLi<9p4)j8a>;HIl|7K1^I7&iV=(fD!fa=L$sex&)jL`7ckv0m_8h1C z<8ti=1~^lKhoYlu*jRwgvPqmDFcB1Mhmm5~zYkX|?QP*`ma1gWx61Wdw5X1JJkfDI>43(@RPLc5l#7XQr3aFYBr%Q_8 zq``ojc(&^71+^j@5pe9oc$BNY^Mgm^tbT?z?TB`=nYd_E9^Ffz9`NoWx*STq^eRa8k=TovuOHm$}5s zeG2O~0E2a}eNHajy6vW1-M6iJCHX0m$~HDo5BR64ieud|W~?|OK6iD;wgWY98|GVr z7;9Vg1g{3YuSrY>j;*U_=2P$6G%x2qe`#L9(ciP6Y>6`|M1zA;y3(iFF)ed2gS09jhcxbTxRO=t}F#6$i{f_-Pf#=~p{99+lr{KUOzhJ-+7$P^O z#vego@6AmlXdONce~Bshk9tF1(V-;~aFTjW80grVIGR+UX+J_ACL9ekGjW$1_XR*7 zWJ`wcnF-^Y>`@%e8`Q^8S~*=NsVgi@;XARmXoylSNK|jl)SDVX4MKt(Jq^cmgBr@1 zfscE2EUY(vOXG1oA6bN)Mk)+BZd3bib`j7`q_k%Qrt*=^zAuI3uD4zNYZsDu_f7FZ zKfpiMN0v?P(lWTW>xtG=vWOhs&`X8F2bSWvge??mNYE1yMOT=#LuYt)Hi0kbzLzCL zF}62Qk^zK+6CIY|iR(7gu0~JhK>a?HT8`1)r%iXRFi~F_8FeP)UPp}^%=L#rI!qnt zh!G_+cm$PvL=6LgK2hSAJ4CQvu1Rom0}PGV`aQduUbANCO3e8@7r~XA{t{FOC{NiM z45>K&!9>CO+nUz&z;f=wPILU$u8ZSt>yx+>O+E28=Vgrw#VomT9dWqYj4OhvL1PZ4 zo|UB^kO_0R$41~-Njq%*)G3b$gUq+Ly`gd9Io6_9n^$CA++H3!TQ#)022k^rcf$3s_9bhXWlVxi z)(ELUgByoq*HoHoesk`CECvW^;)7OhP{cdWyHG2_jUV3V0lP}k*L;}!RVpDoIuAz} z8c*LxY_ybbPZ0QJ7bYdlthZ^HSG3x0qP8+XN2TRBDxRj)&D|aK?EZWL)sruPPi;#P z^VoQI_5TsH<3K?`Y~&8y{x6{R&&MyK5N9vm$hVriGw6PU*!8!_X-JQ#o1m%ha7YqP zUZksd03|+4uf_#w^3Q7sWJ#6tPA+X+kK->jo}J|Deq7C;+rf)G1q);Cu$8J>A@XzR zeMNNiM4Ro5wlwMdUZ>8ixu||jFY`$jTg)}yMypMAN_=(g3{T&cG3%L}8G4-l^-PS7 zIXL_AL=46blv5oeAm&)vqjWMUgwp(|F<79^^iHEH9&X1QRMu+U6t$OdHw)EHOjbY0 z4qvyQu&rq*ZzJC0^JPg+_vIL@$o=$8N?8y)?rAZ<>Og4DLYiLf!4@fwMzD0 zmzZ1WaIg6f_@?7-A5u%G+6r22qNcXvcnY!LNFY4^>}~Z@iDc}n@fyuOOoh*{O{P#I zu)vEi(c}4HJxz;}Y!f7)o2dbh`u@Z%sVizYGpnTO_%~GI6l6Y@wv{}@vyPRI(LTNH zxu;EP4$boH3~`;nuLO?|0v?4H*=oJOUDAYAN5J-LQP6+Z06%#-$Z{k5kn1AZ5vevL*krAUcel-JQC;o;QGJ-3;emR>hr;h@cnS%^KOqHCxrH^n=ZERpiE%mIzz zC8V#&x`WSIG>*6cy69(+6<#eutC00E7<~;zYvSE!Ivv$C1nL9VHHRg0>JZQMzR31S z9()W@JQ6L_m)0cFDngChWt@>*?^3Qz>G*VN9tLG~KWi*L5)F}I(`J&*z za*z&ce8G4+CApL|tT#5!>E?^pPtA;2D|K(xIuDR%?25+TO);gbp5tmvcz7lWUlq5@C z^&;9GH!5wKp3rUoOqy{~B?XWtCeP17FukIj3gC-<+Cr9#R}+p_@7%Qa6#J5f^*do? zkNh8EQVunb{IvPwe5sL&|MWF{nNMP)7~t_iS=vD6l!0OX{i~Uwo|Cuj()qbisl3_G z2zIdzmmRjfOX$tG132#qW`5gWJ6Ei+b1zSgyXI{scxRh-nSpdhV|G`o6qJh7X{g67 zPT3V3@VWEF2=1b1u+r+TM@xMrxPMaI;@zD_i3VDdn{^9vR0}+u>$BA>bvJn&$#}Be zZ?|1lOHUG;#Ub|))Cs4|7TtzpT)j%*UmmhX?p5k_*r-Vu!d13W=hzT!){+?iho-I_ zkZtz|EL97ai^YymBdw#LZ{uoVOj<<9fNG!wDZ337j4U4)7DC9(w|Pwb7`w5Mq=!Wp zT*zn`w>M8OUVg&aX?Hl#PK}i0qg|a1L;pdu*LtgONa9Z;nJL(l0VVosu~*4>4Cj@$ z3fI=e^_7|h`LK{r@1O6M3xv>MYjRBlo_%fOJ4~VS1Jj=QbKnIxH;F_;Ul3C(I)1JG5ge)%qBj2Y^?no;#gE%nnWxUCcpOiQ*2)I=Y{4|%Y!47=h9fISKz z)sQmLj}JQR(hY!wnWXj$8+YV2!wRQ)SGqx=?776GCH3n9D~KSCJSXNeVV~yy0~GNO zH&P2~XHa@bDb1Q@!Yi;+Bz_&NH|q>bUSTy#90(p(#)$p<;OMzZ;xmgXGUn2;f%ttS zfVX8Lohq$0Vs9DI|MS29U*BpZ|5gTn<0o1mD9JWsKt3T%kIc7xrz6P!Ei7=u(zCyr zIhLgWkb1y9&UfN^Qf3zR5g-6okra|7K(n0-?g#rejXM5qZSc?oIJ9~3x(1Pc?X|?j z%x(r)pVF{rsU;*L-r?7qSEw$cN`-^+U9$Rh1b$WS*omndgWD9TGAP0&dzRe))abd&BiY{aVzDt@U1E<>7H#@^ey@ik4W_I7d(Qv zN*#F^|D^T@8q(`AEY|6?l7^!6cK=dhv(N&;2=g0_``(6w$_0c$HcQ~%(Ue8N<5<7- zmtz_Sr{O?T4nH+CDmc@Ky-M z@V;7`#4c@F%25j0^qbHCqm_6=3b}sGX=lv++mwRy^N;KdCe_#!%tHzA{fF(iCN}7XDig@vpr19-9nwX{W7vEbPF*hkfPZah+{r1dcOFa z_Y(kmM=N%mEZ-!t8#Y~QO4qAr{!Q)g`WF(dOdZVqcW>UjH_wGK7yLjgM(Ne*&6Cuu z;&pO^W1F@BkJr-BWZL3*O!)+&E$^k$B!eDS{OGvdXXI)-;IRhaa{rTg&h)m&{T2VtOPN9+r9UlP&0(y0yG^U zQ{M!oZOP3j!^;$kT#VZ6Lxe=gdBvqngqfJWwcKV;?6Xr9DgWXtQu5}JJOl`&;AE25 zRoC%C_Mbsr?uRoeP>HjVSb|0mngeKOeXaRTeHtg=H~4&q)4{%>cXyAq9n${?Nnh-|a0WUyYCO`xAu z4+U=4znej)-*0N!rI5}x1=^A6t(CL~ZDbDHetVkF`(VSw4kz#mX*!W}7)@F|g^t2r zMsCH`{EBjSur)Z9*wgG7|8rdq*RDZ`PSZSk^N$36b3VN9J(}8Z4HGU5(3mUjqzJseRy1aVjiN zumiANod@HQ2J$FNC=JwbH~$NTTOr@KsQ5+o%$~*he`N@5h2-O4T@xBB{mLuk?@tT- z(7LoD#G>rGHY(8YtgV^Z^I&oIqb-(L%i2imUoR)ELZ`pD&SA%cm6H*=%li`^Nw#LI z|4fq6uk$-aU%vw-Qy=u#ZSDfqo%_+K{&w!?vn3$PbAKHA5BM`)I9O^Ud)I_}W-#@T z1B>-HY*0j;qq^IVw2|RdX`@}=D*22w6;2{x`ed9-V`Jx*v^=%q zyv`t?!v5tdO(w^ia}HmbN1Fj_V&ItzYA<{UYy)q0Dc{bbg%LUu+6XkM(>w+I14Rtw z&BEof%aD`w_vsmwEbjMg2nIKuV>Am%WWVKUPqHCCfD73)7IuCaP!4i2D_SM@tDs1W z^2xSx?y2VtjW5yi4_~ddT-s`IWb|R7y5X-u#WQWC!2y8y5nNXF$<-M-HBfK-aTPF z8j>1I9CET~n?n5k@f7Zsqi4wf@K#?&O+TakF$G^2`j{)G(mJ9*n zDiB0Bo21~|z-<|u@?79u?G~Amuu}Ua&qmDKH=(N*iM+#wI62-&_A(c0L;$?u)3|wb z79QSg_z_s+OwZ}Vq4i$=@&3W{3DyEngQ$8OsFlZkTqj2c$Rh)Jm4{QLMUxp|1f%sC zCGN_JXXMPB!Xd^*ymv&G-cvnlro9Qs^;`c1SF-ky(%39=55ZM^tVJw7B+8XXV|hc% zCTy+GDsKc7daF_LGOijt4aFgy&gacHbeOGHjta|g4^8hA**f5u3fdDFbcWat;1c>2 zHe<$I1tefR`gAN+4+rgE3DdVm+?Vq85E}`tZoS!P#@{OPbUVpk7kY=*qD)r0?mrIB z&l{1<=kKwocF}8Y+I>h5=C{~eHk=OZ#9!~T?~|CUM;4UJ*(FC;lawhfw5Ra7fyC6J zJ{O_9f2Z-_tBKZ)N8fSh%cIl0tjMNL0V?M$IdsE05-YWh(xzkSXUi5)4QG)1&7X1M zEOuGyh4ndjfbB*%-$rkWA{#}g>&bm(+A=E z&h*e7M7)Z!QVW;=Uut-(`|Fz_C|Z|{Y=2%f&9994f&=F(r?AYlHc3PL&B$qn-_ zx?G!$<2BWUv=bE@f*-If9#(9z&3Z;wJwkTVc)=PcG3D~c3S~S>|4~L0+f|9d#t9V< z@EpE2*oSUzEjT}%Ae}1oqZn1HLH2@N_J(}RGcn@hjZqg` zAS0Hzzg17EwC2-k6#9lWNTQm)hItJ_V|$?i#)ee6%0D}l_7NB$he*+Udb6~-O*b-1ny$|-8yfFt6rS!h;~0F zF~86WoR*#t|G4s8GYqUAZu{K>@zA){&O5R+)O0Le^+Uv<<%E$ZwxmM%>gFAt4`Yn@ zudknkybJ&)sytur7h0mT80@Oy-5_{$Hu*t=EXKyDPIemEs${9^gfOhiTs6BQIojLv zFnEiuW@R(L&HaS*EJL2KAVQ+D%RkSj&`v8hfPB36>VV2pQuEAi7eZRR!T5}T5Sk<$ zAajIjEwUrhB@npJlv~ySj-;jJEgC}@_ul#^iC=yClnz86!C|s2{KG zxCim^;?6~=4%U6JAEra%U}|7d#H>M@S+7zdt_E54)}JDg)B2eFEV^+TAyWs71%*MT zg$Qp;55Ra?W&Bix^?vhj_%tw|=5sHS z^}w(36RMwaRR{9L!16AG_EyA#5TA3iEhm-40l#oaVDa5+EyW!{&G?i-0qYFt;5Ofmv>!!iCit^ASMM7E>*zJSv?r zO_}{NaR{3{ZyCIcqW*w+2Ue0WJExhc$b+K3qEwjB^NfxoHp3se_yix}Rz1`+x}Fz{ zyFJmx=X#QH#vp1JoAC9YgIOYD_vbI0Uy!^>(Ww@!<3lD?=!WBS>a~4|m79>s+@62p zam#)@TVf4kM@Y#>CB%$z;n!9_gdE$_!P(jt_og15ZptNA`3nDGwmrkyB|u0Y(ft(@ z0ygXdIo6W0>{jw^G#eD&*KhRqJrPBX#DgOU9~kxFT-*Sf&Pfdmsfu*!3_ah#c$+;i z!|%v28^rO`C}9Ko!ybWm&}-vu)iu(k%AEVqE@&Ut45he)cc)keKLMJe9v-(N;QK^x zVaE9jV%YKJ{MxvG#SVcCf$bTI5!`_f>|E4lxT~j(@o8ysK5Gs`#E6Gw7RZ{k^^jSz zXs#esR51?Z?hZd82L!pxpkBs%3YIl$Wx$c|?!-0@l)fi81ABrwaLUsbl1S$b3ovonm7V^MnvJov%9}G?%lOz_?SUotz zwe%WlU08a=hfW+%P5Hm7@QsN?ZaRA(F{Y5BN$~xP$uw3gsDID!_3i7o(!qV3mc?}b zgdY;rlm36E(utwENDRdc{u9EhdXU;iLKxVLF#Bk>+tsYwr!DmlQw|@sOsur916{wO! zw-q{_361o>eQ6o-7H#rRT7Ea}@mPQLfB#kw?3gEZcJ*xC*3^Cbw$1nVqAq+`m4#QuGYINLv2_R#TXQ^n7Fo!EAwqyz)bXs-PqK`eZZr|TD zxR5sybW5P)G@>!`GaQ}+Y zOao@cCYPlP8P^`o4?#4rEs;q>?XaXQ-B{p_)JmbAVB<@^;=X?swlFj41 zAbn14nELwOP!`!^80XFKT|-)NhO~b zbA5I%J(Qpp_(!8pwp>O}j&dg-i^TIti^giW!eM-ST2b%6SzA5v_KSRF@z@-!|M=$PoN)QWU~oEHE6Q zI>pdT^7>XYyp^~6LLeganeBPiW}lW{f_#@$nzcLv*ArNuP>(-bbmLqa9z^*Sqd>xs zZBn_Ph~7;`SbF@a77T5oF`=w)?(dqn6fh)6{a_S!C# zu(+Hss%O~Js+R>9(D(Kq-)7t7G}&<_n$Go&$(XjD9JN|}6b%r+=W~3p3J(#ol<;Mg z#RCkN_nhB{xglIbfF5b56X-R3F1zh*sls!?6vy58JX#P*q82^#nVNQGZG3Tt{m)vB z()7dhv9=NkHVF24=o~a%EUNWk*k``rK~-z2jM!4?KgOUfz#X=U5f*lS13;`SukxSKq z{vF`Z9A7TLD&~CX+3E^`4T{pH#PMMdRkPU`+Uw1s02Y>QL^2+2v`AhFvPyO`0j5

zz+Ju9{rsl46^eq%313za(m?QmG{ZYxxz{E`lWR&BvgLpE9WUoSelkI%)Ds!%T0iZj ztn;xOpM?N_nL;D%);*-pfv|c{?I;obn9I?uKlVU9=c7LBAwW<(>Hi>zLZTsA6szp4J%82P{Xo;9kTin=7`Y$hz?a7_U8) zbhf>YLrGw^D7)6f=uVCLn{!Et=q+mfibuY^_EdD?mwByhN>kq`?U-l0^Km1&RvMVr47(Q=fPeKTsA`xg|

>}&UF~c4dDD<+)u}95AF?2P4v6^HyDT6fJXk{# z*!?ueVzC8E^Tjp?X=04mQPd^leaFHjdm|dC*@YE@SZIPUH8c2Rl&|2C9cQMHlBNCb^I6YD zhMZkx^3d6ZL5Np0kb!{@shi8yx4J>k+rAg3(l-4%>RR|4=E7DDsMX%c`?1?l&k$2o z!h|teZ7UESxHOk@1F_Hmlv|oSvexhDKrJ`$k@^w;ZGJrpP*|(%mC997b&uXAIw(s> zR9}{dySsO#)64v&lYR^_94S*DHC4XFyiS29!@mG{1U`wNsGJah~DrR>uD)%V%` z?2Dx)#@Kb6U^zL}9*n9Fge@WL4bx#P0-a3=#y4C@g~Zg-x3;n}2Qa zN~u$1@}?05-;wm9`Bu>-d)Q!IgU3MrW~c(4evHk!>2O(c@U9jx6I3>9t$vxQ@thnp zPOvahZS4Jc?f7Bdk=-r3-M70U{9v%x3^S6HD=9k))}+uXhz<|f%d)dU zLb~-dxw5%nl}cb(4lhg{}i2c%|eDtV|R zi)EgL?{j_ss+Ml>(O<|gC)z+`j7g^SH4FZCoYc{=s!7K8JX-#q2+3e)1V!iPK;y@g zgp4&eHd0F%Rju^@^GzcbI>qEdzOm&Ac~OBd%y^CTT5g)sbWJu2_g7RAxSBipY4}-rZ#x*Av_7<`4jgGw^HhyiJi+ zHlJ$09x$3^gbAAh2Yzn98uzv*_v!r?-e9E>)xyjIkE5*=l^Ks1MF@`G$?u10wBc`b z{tQ}izOXF2buINOYX$S91Iwie;&+iH=2^VYZP5GL)3A$1{qGO2T=l@UY5#ln_l0X2 zhfn&;jc4_A^dt5$lm~Uh@&F1b$vw{a|L`sv=S&YwvL0$Vq-0nL;dB)bbZN*Ib{6JT zzs@#=Czd@k+9UI>VI7TI4Wjw0(YzHaF&a;V3-UFhj}<^tAx^l#6TC*+!a{VJTOs^s zC}d z&r{DMz;omMvFlkk&6mwDi{#%2Oj`|B2$P(SP%E=Jf_pM$gNnNQ!y>_Q&{Tg-#|Ae> zc6Jj4vTagl-kUL9a#13wqmbzdOWc|W=-WLnu~#{72c#z?lC9D(uyH+0-2pI>UnLQ?%=L=KcR6AVs@zp_L)6dQ=6CU1IQB)7d!Cz`6Lu?F z6Ph{^TL6EU24Ppb!H_dGPU2wZ>z@u8jU+i!tCk|ygz3k;Zv;2@l&2K85oD(Ml zOdll|h5C?xO9AX=GsBcbnU@{;be7s_X1lsGy1={6&WUoE9PTR8f;v=tMO=rvn$o24>bOkMCYI}_o-*(A>%#^T1_C z;IDJqysGro`l-L0l4>_{3C$~ERa$IrN(&x^f}FQ(_*qk3el({!qkuK|f{CZ9%jRi` z6cdkJ`GuV%O?DT$g^<|GQ%o0HD=n_7yP|Gli>ulej!Gu0dD#fIcMto*Xu#{lJ6X1U zo9t`S!sCIYvOuje|2j%PuD#2CdDic$>-z<7h?<6Yvl#?U?4?b5`@6)>)P(HigjL3o zEE9(In>B?-$CTYQPBFlTqLVcnds4t&Ff&AUvmy}eR`oZpJ2Ke+talxD7HMu)LS=DG z6i8fis4?b1eZU5YzLd_#|LQ<_LR31DN6o)T`hBy-+zr=|pCu>+QLKsK_KszW%Qtm2 zo9v9ym@zlG6uR4ro;#ca(VG5hp|3hLxmv4q3J3$NPbaD zbCNie@HJ~mlAydrB6!wkov(;ZRQotbS?~^v^BAqe3K76;b4+R2f>HHjSU;$n+TfWU znBqFz)4JdMBa~``Hnf;6(7U|3bN4zEdcMC}GgW60V$LAj>um|o>n(+B!_b+Hw)@S$ zjr<07^1Yk_YaR(Xi-{&pxYh=*|F4`WY)l2MY-=k??vh6ci(|}JR?(56NvT&P5iOyv zTY&*qeovd}rS8Gu*qBgYVkS){l_*U&YtV@ggd)^)L&Pw1p($Kr71sOBUnU+oMn(Md z814CGcrRf_!GP2Ia;AS*AUj*5ZaV)5r7RJ~ziOJ$-Aqby)`VP7TXC8Ev~>6&taf$K zm-o(NBu;mAy5E5sn;EvxBt|jwqi#7%smRKsZ?nyaCqPtKZGE-o>8`C>LA<|iQ~7y1 zuU#lv{2>P5jqXP@mESIp`{~I7S$=z%^(bAhA5g2f;UurN$IHkBA?ytVf4kWa(Wmi|{|R-bH`Z*+(EDAM|wgP+kLzt|ahB)WX%4rOSP%&e~f> z!EggSUKo1}p$lUBN zt^6##*GIj-Z`f7e;-2amcp25LK%?nK1C{*A;S>T2Q89*%Q8%Sj15iDfkT|RN6>Pl~ z$6Y)EqM`&Ktve!sSld3qkSyd{|* zLv{n7t(xGqCpI;~p7pA01%LWRwU0oGsV$bx0uy9%Q9_j_=`zS+Xp;7Zx=ybhE+;=~ zl6VwgipVD3c*t-N>V+aUy8To)Oeq;RMds%f+znb?Xyj_N_nHLxSt|YHhpQCQWM*X9 zwStv-`F}k}i*~(v!l(Up?Gc1dKMbADU2Bwln4uaV)8aM(qw+s zUhWYk+WX6XgyeqbRT3~Hbnmw*ZmEQI++ zjT=Db(Gw0*2>4N4n2+<0^bRIGqI>WNt~DspW$Ldi-_5t{$5Ggk^invk`cJVFL+ zWi|1)%K-|fH86H~u5eB5Odk%MC@u9GzSkUmTLQh8`U*d=T^}#|RCuxw1dL)3UrrbK z1AvoTU(aO!hSDr?nZWW{?(_V7BfvONq%AlPl83xv;xY!i5_sCo-wx7(PJT>2- zrr@~dQl_Ke8*tHNo#RT=3blsp-l9ELTjBypIA(|lN70}713|Ya+>QdG3un>#H}z>A zx0W1zuVo~m^`jQNPq?&~H3CeB16F%IjCs1Aq(zays?H-uG8qi|^bsimi)BDp=#!l{ zycVSaBqsnq0L081%~}PN&K_>}?@q&eEQSC-K)}E4%7itVOU;8_c5+JdYh)dFVVZ#S z;6#?T6Fc9zlH2gYz~p|O;RSpNvRT^jICILbpu>gO2@BIN{H6(1D78CPPhF=_P&Kpj zJkYCy!o>Ade|e#y#nxZPT~T&uJ2}OE5Cx4K^>N&pJHk; zK?~&!&rkA+;g2v*wj*gny7ioF>;sW1U{& zd4y$Cn4aC8E;QO?BIZv9B=66xDDapI4+tW(#MG`|#S9t8jD3iC>@^2NkE%n#^%>8z z4cM5KOQnev_GOcfc5EQVpFQG%?e`&WV>W_7=2Q`;OAJ#O2H3xw!EHf*g?~;%@XKO% zl~gZfUb_c?Q3A0jU=6Z*R!U^_8($WcrBnrevdyp$ZQcTkwKHnnXn$vpJ z)*aj4$XP)v4ZRa%-O{pw&^EhOnwws3HpCL#rwL99 z;|_5T^ksi(TpmecA|6tb?yHw5Y(S`HmkXM9_lA)bX>;TMXC+JdlGGyhsg~Jw>Hcki3BoIlztRE6Z!5J5`My_@lK;UaAh^?bq5@04A6750P zp1L^hGg4`kZIaRIdDwcl9{q?8B_mP?2F;l8v-cJWWIeI)@V?2nly6O6Z-&qNxZ{BC zX^%TxRLC*z^8!K1bREWClSJ<5nZ*r4~Ae#rE|&X7PEjw2glifs*}3jt|E{ z8YygWuv2pGMeDgPI3z*d6Lx7By`lLy_;vD~c)6OXZC`{J+l6<%fN1Zd3#|99Xb=b} zI-cN`B16a$*M|ycZl}hG*kJm>W?|hE#9qhlg;p}pDIZ*);^W`=0+9=#p2M6b%(lNi8j_kd$6(3YBw&(js zV1SuYJS>zn3|+tzE@~EA_G@#NfqV<<=G#%_>F1geW8q`QelQTEnUMNst-y7gIICBj ztr5Q=twV8pgvVL)rdwjEIo-bG5%!#F41ii&8!3mA4beGXLuVl%K)^-JQd|Hz*5~Eb z$b(*e1%D*#ucxsadA(-hFxQfGObM7qfnGXwgDZoWg@5$Oao0rga_IbMYb=glYv}ss z4qev-$2aDw!ubHl;tkv`Gg7voa@yRP0kpRnRoXJ|rzZ*j_{}zd-T&!{Dxd_L5s?LEMRG=sgX4Cn-<@ma-9+Ak4dEFy$hDE!jh`=e%Ue z*!bSj-g08bIl66nI)0{AkUfw0ZVIaAwy6Q#7<8a4dp_mhN$}oF-9#x7Bz1OP7he(+ zA*}j;p^T#kyG?@K3thJnLKyXOmp8?S8^A9|OcyW)_^QO*{*h`s()nZB)ec=7fDNl{ zVYR=%nB*{QtRR1@OV{;EttcfELY|l5#(ppMuQeg-^EYSFW?+Y5KZSO}8*=m5=i3)a5~7A>Zo$+DrFkfpZL@6T?v39EUwHQ2UZUhLldpY}0)voy`(; z9vWGFFC3toc7$_G^!Rd!%Ftf}M_m;p+Rw+&&Oc&_Hl#Ju_rsu(#&m)b+BE)%vw@D? z*g60g645^$f6Q=HjZ?-zvCZi+9k-2(SnukRL_pj%88(k6%?ib%Va8fBkjr_h7uvb$ zGI2|}y)-<0xeO;AHAOAq2J;i1PRf4l_xUINR-F+&WXq=tjt?MAxJ4=3y_affJ=X&c+|p zzB^_PEl6`y9p?}y1wsOp*^>}@%2!Q)5B`W(kENG!qMzK#wek23)45aOpdRMf!j-AG zGYuCmM}d#Rmk$p{W_(j(7}*NO0|h4(y)&wKQ1M05AK~qX6UKWd>il(MF|_$wvJEK5 zz8JEn?Xtb?Zs*k7nr~>MfQXSg;OI5fCl}Rxm$NNsB8Ql?D3^_kzuh!MK0`yXS+LjSNnbdzh_I9JJdMvK~>ZWRPW3eXLS+NJjGM5|oLNV%EXpzPyfkQtu zvWdo5lj9*S^;`W-pym$#&WkVs&{#f6dX>G>OLnQ!@1Zr{(Aea*fdx?ZTNMWJvA2xV z&w+gy&Z}2w{lXWr#)KY`wJd8+etQ6ER&w(7Uge*9qo0+;zuXS2<-ctHUbcL!sn!nB zVQPxz@$<7*+wF_zV$l~G;k(u}Vf4if#Fo;b;CJx)1~byILVrJN!NzONhkl|chkC20(;b;{q0Oa6`Id~VDhhZ;y(JsdqJ#T z4J+iITKJ~v5iZ5%GbnG@s)S@rZJP(=w%vl3mMH*kh2(p{9M_67O?RXLJC9nb3FKtc z4d)4%k;@l~2k+}549B9&5z=sXcSC@NP>~>dTZp+A7Jhq^N=cI0;GCGk7eK=SyZg1P z+q|ndM2h;&q$oD-+0ra0nX|7 z0jz(J{c8s3HRKok91@$XzW;`j>s->1m2GR z8EmOt7;%7(Kh{IPt%ahsFdaoQ6CSsN+eHu71uRa5f^+tSK2mIQ_5OTRz1X1?Mtn>MhdQ zL$&W^za|9aKG&3{>*B_Vze%UtR|~WcA5FXmm>+F&t#2s z>IfS_7Xt&L9%HI&E?pjjTf1GdJ}`t*%w%kB*b5CML-Nb}^xSB;SW~CqGqlLpG|=m% z>-(8WGr}+%o6fE4@dlrp^W7r_yUC9q)ZpxkceX^9v8egcslI+LxmG+v4W+jH58cV9 zpFn|hT*r(y-5+tLhLBjgP)#A6>YUnIQjN1NgdbHx3RSca@T4K~?!7Cjf+!PW;kt1m z8hi9&BH1Z_4PIfM!7QLuJx24X5W^ln&P5U_iQQR<~giGMM9ac zW0xB9?3m7J#orEn9kmGBDiZB^DLi^p0r&QKVcdPi{+*uILcL#GLtjO|a>w!X zQ~5(eI^I*Z!Qu&D^MC=OL7jxcDP)%i%D(6iMyg1GFzeW%iNOXO8=z7B4eicnj3-}G`BJKx~&K8;u4 zMB?n}Ijd7)wmhJ|FZy~MzTSGt@pfPTjtZN0Bol5lDTQqKY$VQVGE z8sH_iyTSatE<;*3|M_*=Pj8EI`d-BiB!GqoW^<$giPMhZA+42`StPZ=A{>AfVEFTw z*LqQKC`%W{5b9;ib~G2xfGadzRmIYWjsN_|VRtQmygUs{TQ~V;KD&|3{wsH84WXtcMZWtOdhIm1CYs-h z#6OE{#94$W7KCTM5g78#fWSxKWmXnYwRlsHdfcQKc=K2n?S7t1Bj3!{;z`2bf}VVf zBfa$b!^Y1GTEDfH6zomfT7Qf&c~%lQPc9th4x|t@g{!~u0G5RvbeVc$3Ltw)2GGFI?WIlY%U4`%glOOOo!8HEi zZ3#3hvt|LOv@Td1*?j@AbfE#qIBr7Kw{AAV(i!O?x%PbH;sfR*-OZ3%MkChUp#=X$ znh{<3-qM_m&nKb5riG zj~9h=KA*ulElNU^fkIOsZh%AjX)mJ;ey!7kaQ+HCWEq)smSgP#EFmEs<2O0!91L4H zyF~Nm0$7e%mmtK;uZ*+5)vM#B-QU35REJnMN#LYq)x>1kk`B+2XEBVJsT~xNXd#Y*ExV*tJ4;42xbJm$4!vjaITpK+8t@VCingVqF_op1(Jnd-~*T351 z-1u8Jn0API_@(nD)FVB1@X&%2*KGv;jIL1Rgw!rpdM5x3t7LswU4n`>OmS* z057>0GuY$SLSZ;dD$XEdSQ!HMHL_#*=9KbAK`X+K>TK~DLKvL-%C?On-W}Fq!?eds z?N`%mRe^Q7(_CagS&Gt>-YYGfwmf1di>-}#1a1^&f=b^IEeX#EJlf0hSnncg za<@W-s?}(8cNzBqxX(8dRa8(zQhiYZ!XGRwgpe#mNTN70x~J=GMEqv?&ZfoF*s%6a zh&;gK?2}shi*mS@19t`!O7!`}JEUMFi9?A;nVDXgQ0Oxvg$1@?+|6=do&w6vjCI&A za{(yg96W*NQ%9#J>;!GsH?E}uTw@wBeo6REqHdcUAe!^sjab==pRZs~Q#w6Ig-r3F))@j8Tz zm)B4k@uc>b8pon<7v`wIoVi4Pthvkg@z@K>L|ld1z4-1%z;x~+uKjqXzl>~F?$y}- zRoi|yKJ7z=MSx~>3QOq8ftp<3z(IPJnM7?L|HFee^Qu418(<0baL&qlI(zK45CuXREm3$kHm+L7+E^ zEv+f4Tn*Kt4i^%$(3GbXi%~MCOHP zYN^6lvil6rJ+--`j`(mexMT-ALQjsH$!8N<>KxjeB@2*x#2gBm%&?1WipO_2hSqZ% znq1A>M&q@5JLJuLPTO6+lFsAdcGPheCqwcteY|CZreR~`+-OP?mQ056usfRS3LLX} z-wUJ&L(*#^3UIX#AQ163$H10TeLtE)Au2sSF<3b_C94)2SxiZ+>ZO zz+z^lN4Q)tnYb+XA6`={Zqtl{pMe5JPPx z>8b^q^wE+dw&}9yfw#BSJ?XtNpoV@vcGa|Id$({s``sRAp%_>4YFcDConpQUV@+9T zg~zU4Ezj51^3vAs20guIXis3ct%sI*9Y+U^^~X3mo7e7UXW6?CW_YPf!G~<0UOU`% zzPv^mhjUqJc!h1szY!0fXT%OJ*PPa0$qf%RnB0<$RxL1x_hy&`57K(|c5ITaxlE%J zk=ldYJU*NwnJ2}wRX7Ts_jGKAP-PNTjT;Gl_Q~Q^1HpJ&L7Xs%OJ^r1iZ zgR4LAN9@#a{_v133#ZS&V`^9;BoORC^Ez|s)!BHrdXLPv&_Ia{0BN`Da2yCt-R0%P zwswvoSM+g(y)<)Fr%;+42K8W?Zrlh~?uLd@*usc&5c-ubDMAe6I zkRK!)OD1#XYaO2m`NEr|k2`tdP~tL?HJssD-a!N(*9@Z-z$rg?``{R1OG+a3mO4du z5$9xcj$<7mP=RU4vo^Lo(ULjj?*dn&zxFZ8J_A7Bhx1z+m*3L_XNgjF%K~UAI!RqoxAY(h4 zQ&jxD(M@kVYLten{nXLJ4^%CygT)TQFr| zqsR?=FA)kIsO!4{!h+1_ZSM%LILi|Z(7Jk5Oh2?A(vTiimVJ`hQ9>g&{3+a^zXEF6 zAS-zui7_>)BKaHLD{81i4}^f{_Ci^Sy-11ytp}oNr(;$>zRom>qIBy4RFeDKwcJWt z8-xT(Zs0Mqn8pFuv51IkG0>+y?vA+d8vP_P*t3{xQcl(MLBVzh^qAL&8 zaw0y5ObU@(J8q3ijB+_P6_2lpm<&*-Jc^ittpb#*7su7mJ!u^lB?--9%40Xms;PUZ z*=q=|HxE6Q=4u{bA^(QRM2_kIfnE~Ums)M?XjpxwcQLn)pAMYH!`#M-yiVXo>|({c zlDnIZcl+U8oxvmJ7_TiXa5=R;b+e0L;iXc{f)b%()zr(At!Q@|>C=QM?(WC5V5N}g zYSMsvi@>h2Xgph*v+LqI2iWgIo_HJ0sN#Z_G_IKatVE$wZAhAF2<T7??ZP2Ok-5x#UX{5kjQp|* zaUlfYzgjqcmX5~S>^>b}(#3|oD_talz2hI%23j2>EhM*~C~>=^UH8SvXUO zqcjv+1jjkzTi2XU442_8mlHx3vIHVr>yw{IL2pF}bE_MwJ497owN;@^Q=b9h+C=DU?1zFa+giAs)DAB!jpIuIVZN@ASuJqe* z{r?Gjx7}8fD{b&TpMtuZ0l4)5rHhSfx7($^yWEDW{werMNRcTqYm-bhliV_WHIMNR zF!OTrBop6Zt%#)L9-z^1X-i}tB37)!cj#^iB|5WwG6>Jb{8F;}UxocKzV2}4qNlZu zjgOOL%5NGA@zFNR8KeyYsubW1#4?Dq6C|`&lQx00uVUxv?TOKk8FA#3egT>bV1m(VO1Fi_b%{ zN3xUrwI#Vo9OV+ddXEd*^q|pv5?w!`jH6glS9>=u41A)!vc;*GuDm)@kk?th>E!&X zCC;;%Szbw5zc(cIsv1bk;&M(2hPX0afFxJ0a=%; ztH^>wqE)cHpR)>BcTHAWN$_hk0IJwr9AG8y+%_U(Mm9lE?DA&C%7z!+h#qq@oegiS zD#wsDyuVQ;fNaIK1&&(QPf3IvijOjict9w8rR1S^PB-j_*VLp<+$hxJ(#{6zb0hWA zGEIDlpUvZ0yQbTZ&nME&wTXDJZ2C7V8oc|Ny=o43+`7qNg`^S&FD$obS+|x$dOD>M z?4$Gq&^~8rgD(~&`gA#J2US_)*pn$>SboP+cW?b$UstIGLabE*y1eufpB|sO;17P6 z+Tg;p;W0~kW2Wl7KuT(HDNiMIJ>C5IUH&SzEDW;LO# zozxxU8p{?wMGgaH_1XPN{30SDO|_IFBqN|4?|4v^uu^;CTbCCrg`a+J%_6ONDa$pYcQTU1FB6yI3)%zs*JcZku47&5Cn!s)F8*zqrwf(66 zRQ2WDIGlH<%kwjPd1}2ivig$h?ceLqB#3=cI;=!ODi)_ohoa}k@8z%x3yCg9 zR4gLv%!ZnlRcaG?g>r5^n;1^a7a%`52}xfUODwl}wkVR*GOUNkw-$2=*f>_ZlG6ey z!R}?836|4dH>#hBgr&hs3r*0RtYe3ety)#D46!AkJZ4QE_FNK84zxnw_?}CdK^X@? z1MAgKbi!LIwMX8V-&2zn1TWptjZNE69QTgL@G7566ddQB|cl-4CTc6Y?hG*2P&3z8$TiwJDKW?Jw+Gp9xC|cCp`vVIueVd zu^wzn^-v|Q-%W7Qgm9#R7Hns~**-Xh-CHH`+z?BhZdfKLYXysUA23$J#sZ2_V-Utn zwy>pueJ|ka0mUkoxAH^>gmx1L|70`01ts`@zEvNWoZdPzRM8h4I4rX2L#E~;Q+Nt~ zvM#}k=r<=0=IML--bg+*1Hh>LL8yAB(+*CkK*$i7)mMkS64?*dP>04WdBg9b0Am&(9nl4p$+rm;Motm3#N0D}@E- z&3$77pVGmMXiQ&6L-)UYX(rEK7ojVhI&(a&6eyzi*ToTgmL5>WtlLn8s1s0?S+7vL z+8L<4E|9kEY3Dx0AoUgz5;OEOb=v}7mcG$DShVZZB)CVfMhU@RJ(}(J@FvU)UM|-1 zd-B<~?=^8ms?)9BO3h{VL0hYj2IKbiQ}S%flU@Z+nkn%gq>Keg--X~t$UbK$yS@?6 zX2)FijRuyB#nuBc!RF&(QABno;jd(NLG9Ui5RW+hU(0U2z=w|v(NBFyG?2DSkKT4& z3^%QYKTnCi2?EttEa)gge+Kx%!kT*tInKda^Qb1qE_)R1=vc7{+;*Kx$qSE@dvCY9 z)YbL9Ek+T1ILJFmX(uKJ6X5ApU+T-bxCYP8V z;n6R(y}7EIxO!$ZHD^Id`K8Lj#y8yMA=Ah_6k4C*{xrQ+ zn?FT(#WP!Fkg_J$b=R*rDeq%&iAS+3k&V*~mRoj-izcG=msq?j^Pv9HYfCZz)u)#U z375v7VYxc1(0a_pmtL8vN%~Ww0{p_qS&jlBvz+)f!32^fbO){CuGx$6#98&R<2&E2 z{si?qb`^Ne9{S@_TR;=hZfo_<7P$JObEi^DHBu7gwS~p=eIv~CNjFiN9Sv`$FVSk+ zdw_}xcdo95P+v|J zkg^-!>g)aZg?B629MO+?+}_ceC`y%(N4mJz+tDz^$xbtriFt^QHv{Y%(=}K@P#ZG` zKaC0?+opApaZM+`8aBk`4+Y%a=6xmro++8WpAHYbt85)BCNf5 z``O7Rt(bEB73yHBJrW?L-(EP~uH$4!8?b8GICcdkF0wowd}VdwSp1xk z?)1#4w!Jx?U24OHQmZyj;%p>RbsAoKi@I>nStXm!#ii`%YLs_!RN=5Xe2Hyx-MN-( zyXHS(VB4$lcC#-+T|pn_++kgd`hNQzPg_bf_-SAox{5!GD&HKPg90L9^qh=(E~>rO z27^x5P>+n4cuV%|lvFAEt4g%nIHYi!6xxhk-BO)K3~-;DV?}G2#G^9uxT+0LE)^OI zbm60rJ|;ITl6W0%10Jd%T|-si3YbSA&NeMH zgk=6Jc)f!eyS8^LOHHBI7zKLmy#T3W#LO_zml2Qru2BHvkz5^IFdm}4YsDC21-LVk zM$Qr+&DDqi@q?O?6fBa~#AOaoBKFLOb6&M_36eh4Rv<^P#E;IVgQS&!F83H+e(7Xn zfoM}oH=WtLd?)<9iwG7g=y1WCXq<91UOH1-w%W8ju3q-CQc*bU>XmCZ;m9s)b4bqb zo@NTnyUE$c-|ABLQU$%Ga&UH9jE3fbUqBU6$Iw= zlxL&&IGwsjb%aXr16_2@y~d@amrQJj^s|-9Ow<)`0g;dL1#MHpdKw25GFyJpN5tYG z?01Aid{?n6<7>H|o?S#k3m{NnRXsI2+k9XZ!@sV<<46p|rh!ch(+sM_G7?)(NSnL$ zcnL^DjrN{&44FlYbR(oqtJGe}Jm=`v$xvi75m6e*2RFC=2Nrv;ehvZWt=7|(LW5WV zi^SJ0qUibPdh($76~CYON_;LJs7IW`xfxEjX2;Q%Uo^$?=Dswx3{}7~I=DGhxpmQ0 z_m%`4hO-8u`Gcup!O4Im@4J-@>KofKs?c`Cs@HuePD#hJPYJ*@i$ zNHpM0Y+E50?Zs?zX)t$@)ulTlKdC+Cq{C`A#tMkiB%gntHcb-QgV7$MD)Oh@y0TCj zh%Z7L_qPxZ#D~n~w7sAeOe)MTm3zuPa|AtqtYzT$abh~nw}>V(6}qI@$(@|{9{QW z8jEBliH{V+2#7d8(I&$9Z{Bw#Q)LU^MH8hNxgcf#!xa#|LVc5wiZB{~JUy2++qWjP zt?Z3MO~|nC-J3t~5GCzX`{dtfBc_B6w)5lj0$-Sx7TmLPP2mvCh^Fdq}6A;`g<0n(O z2aN?)BX6m#35I{*Vl@%6)@FQ--BO9HB{~?r%m=*R--RB(Q{j%jf`Xmle$O z-(l7Zf$m>w7H?pbGqEMVw?PIboX@9m(`s?@MeW@N@GaZ4L1Z)>*&~JCac%N5VtK6R z>~5<}Q3ys%VY79so1w&zu7IXQCacKg@cl5xHYP2W#*0`U@Q&K{S2qy$yBrjiHefiAuD90kBw9D74K%{Q4fDY>rOmnSGmc5%$ zaa;h|BI%%yKEi!#XIWO_R{dY+v4h^H&+Up?S)7O#9ZgTh$5oEL+({C!r=+hC|6ED@ zGriM69Jvo{T%MQYJ;%)qdg8l1Fw#Yvov0=87dwyxLnE{sHR- z-aeP$7^E@gCO4;^uc^yLg0536|;tlf>4oXdWZqk$m`%7}V2$-G6^F^upfnIbZ zOw2^&k-x4SI$~GfNH5T^c35W~e;iV~s`iQzz^tQP7Jn-=u$F5#8tthE0edrWnv$;ztg=V|LZpRgT=WW??+Yf?`$A?4^#JKZ(4NE0Poir5>!i>l zYm8twM5k@eCO}nKqW01KVj_W`TTS&$5oQ1g8Yvw4m5fu4Lp32IKc;IycK~K?6L~{f z$nhwtr{M7-=&*2sA3eH8^F=tiP6;e)QGhgWD;p0T{s&fZDkhD1|84~usFu%uX)53$~%bP^jaBOCFeIt~ynrkCR3{E1=kUr_22^oF{v z(J0I<2)v#A^{8A%@u8foQp4>YIejaNuMPmrh9Z0IWlCxxlOPCXvzOwp(7EYqQsF&i z$3EOO){5!W@qU+-+Bh{k;k`mfiVL7|&brZHrF7i6PVqw*S)(3H?8>u!Sw?&&9T@~G zC8{_3C)yV)DB4U1$z$w>eZ-w?&~`!xtA6FZc@2_EQf68GYC?WjS8jy#^hzp;Ah?icIH-V_I~wm zDCLrc)tNe7en^wfb7FTG8HJ0z4)MbCZV(_IG7;zkcq_f_Jxu_d?N#) z49t&Cc84O2DnGZfO)DGMX!nE_Mn`;--rw9yMxyPDS!#lOZi>I^)n|g&wVI%G!J)^) zc0imQLVSX{_0%;Tj^}K-qP}rLO$4+8X<35*HX3%dP%3B$#LsqK;-m!$_?|S9#g+Gx zl+5=nR7r|jl>jiElmo~k!wYCT>g_j)LFb;nZ)zJz)^`uzT?eQ-`&E!k;V+1FRUf$X z;1a|o#@t{5Iqt%DfysXWrPoy+g?s4w-l~4$c>0_6o z9ms^>dD%>&4o06R131a9tvsJ{h#>q8woFIR9A_G51nLZpw(r~|%C#eI3T;?6Ij*9@337nfI4~~eTp);w(rWP0}-HZ)lbn5 zoJ&ox8ALuq?&AGmOpkqZDQ+cHbbx;HNjf@5OUO}Kf}24Vex9gQ#O(%*DH8SRz3Sng zD~GM$XETCqD%!_u4F~pllHao-&`XjSQKs{{iF2tjl5dWgnx7vo5d$!}s2xb=b|Rd6 zwmx>zwZ<1_6QNYyj<8)zLVYR(+xczp!=A-l$+F%~Wfec9ibDw?!;5A>jj*5}EPFVW zG#pH^8$aSzBPM2ril!ZuJwQMONbt3H03FK6z||%|%Jh9y8kgw#@B*KxPX-lz`K6J~ zZcL&Z7OG>Y4>X2Jf~Kg=gapTG$8&=ik#PLxn@r*G@u;SvC+m&xvNEYs+m{p{NB9}< zci3DsLZAs4QR2qzedfej$9l3_UZkQm3_UFDX3NGoc~GwSYK10>(H+3de~mG{^#E#- z&#Og$>rFGX7P=3-(L>5|H0MT-=Ed7^1NBdj3XCjMzDkUFFAAR~{J8A^v*}7ozjDg* zJlKlm6GO0&%~@7Kl!Zo^In8~Q7H+EZ9CIB^ZVolaht-tw+8xV1bhi6F$m2bfkk+0X z50rJ^R+uM{K=Li3g$Ugp@i; zHl%fqm@A^C2BJQzk%VPjncO#;>mg#FB>tZBKq((c>EbpYQ)Y*@gGR1O?95H+)M5im zkiGj?l}FPzd&TtUC`fiLY1t*LBVDv_Eox!+W@Wp(5yQMcb61_E)QEBj7-Bi5p~-xB z%iu4M6;*thtWis?o}KmGJr9eRm_mS^5R-iSdK&48>~8crGpAdE5oGIteGWQTyoj8v z#1CK-Hj|NKFXx7CtoY?g9mzt)kck@f;tPXoE2JlSFa7D8KWaODY`aSOdi5uSF?k9yprQFo0G%S4_4Q?L zQvSy;=jq!Jy4mz<2k-!0HXU|*)U!STew=sg5dblTli-EIY*3;#7yci zqHeGiSo*`DQ}Y+}T1xDkwy!p;E-PW&%p7fnhxUEW17n-_R=RQHT`-vH@KJe`Qqhs3 zbFL8sod=1>HoP}lT9nVgb2##h4IXC0vz9jMK%yXtr5)lvaQ29Kh<53N@wFVB~2Oz9&P&3D5@iWo)9 z@w@NrGS(gMI!Pq0;6%Nc2EvwoJbOS|eeJ&m#vJ6Dz3n8mWf$RIot0EqH6{Wnn9jk7 z$*P8MbyRXwF0Cgu3&Zq+CDc|}gz=TPTw?D~gIv06rN{AYwFG(a)%Si6Hvx4ThHb4d ztE7=pfnCkdBT_Tf>WR>EMO_xs>xs4x)!pmeK(rG418Sl1dbyW953%DSTFX5pPY?nB zCn>7CH`wV^^>Trce7JpB1)8fY&32oFp%I^LZ0GN9RiR1Cqoo1dLwFpCSrCtA2asB3 zdcl@3Zoz1cvHzd?8|kIN8dUF!*V&^I!Z}95~dvuJd8f!RNmQ_%IOrdyw>^>8@4N(ykZKZNivihMOve|f)sXGy z?$kqP8>d^*yruA}KRpr1O3kW;MhCtOjIx&BX*n9{J29n4PIx9yN($L}g2!t-32<#{ zono0j)SqayU4mC(sGSx=FN1*-Ta%=LS?eG1aZkRadWrg+$7TEq%QGy>&;QJH@iB??g55mZPlanbm8IADZ6D1ZpS6U z`1J(7jyeyhiCl?0(QSknO9}lQ@nz`{WIG(H3YV9ofm{iQC_d5I|2O&ym|>Yp!%Zk8 z)^$2+7Ona_WVd!Dhy2F1+OZRYxh<_Dq6eY&=B=3tJzBJ0_;l1r>?}%$RH14guA8eU z|By4tR(jAgFyBS@yoYAF-P$s48$GyA@XG6nYPb*l0Su>)XBkR_Q+DhZ zj_Zgz_wUI2hv&$C6B2PH8;HWgc_3rg%X%N~C;s#S6X~ z_KeDa!X2SCQmc25HUNUD*Xylz9|b5_B38ydzqU@6eGj3yjI$)DM)X?g%oYe6O}ALn zZ%QN9hS_CuO9hW2bVhf2zF9Km=a(&;G3s5}#O-pjw-W}!b>DPLXE^E1W@CGQ&FTML zK(w(Qk|cEEA`aPMk!y1=Ic7(q7$X&#KnZ*_AYv~{&?haY?d&%P@lW)OA+$@LuiZw)ZPX0VINIH9EM{KZR^L) zTuPJ)#IKMQK_>u6CT%5En(*}BFB)I0m+~C=oy95FjO%7=5_TRD8z|x}n@+#zm)^?! zAD!E3FXd5F=ig@%9`!U10`eT-Gfg580t`;1mA*I`Lwt?6Lbg}+z_Bw(hNDZv*r+c*zT9;Sy1>98F7VP7@T z{Lq*D$_>FC%}k=jRu*M>t(sjLYEtuZ!JT0MGhL7qWj4l`5E8wK)r|&RGGu0fWf@M- zCq*^o+x(61gnop-+Xu0JjjH9Jei&+6z^m$6p*8q+7@>V6u+)eHLo3d_OB z6PNIyK2XF^6cuoipjY8-1QrRY)r^qbmrROIU)}$F)vzoe5sRLCUL+$aiu)~QKM69lQ zZvuRcRb)00{K3rl(aVRshZTBi0yYvX!bWS%)6HYq<^8GsW{1&w`R!yT{F;)DQvw&@ zL47$#2$~Fgn)sOJ{7>aIT-q^Mg=tOkuzKvYKtiK3Ew|1b&M_Z=PuNcow>m){j{lBV zRy}oe6jON1v>;9%hI9DhmKyb25EHFya3scwmr@PfEH|0L1Db}6C8Kw_kPXgQ-nW)s z>;&g|r5w$XCWSocLg&P`)EQRwTZ#K3U<6Pv(1Dl@WxAZrrpB0@nAz z))F#m>ZEeig2C^i2d=e|N8jnKmPUpR!brfAXN$*DvY1wK%3av z-2~~zmy20GWdLN|xj5lBK2w;ibhkbd7L1}|u^5^@U{Tn8;QSe_MD&8A-C7e-`vHj~7X3OjF4U_OS znl*3i)*fgH&g12on#i>`;_CZt4CfE!h!sR7nR9zr#p-sjjiS7E2xSwLezYFc_3Zs| zAf--S*#h%I-5DptB(N;sQv}$LGpKX!OOg(7r5h`N3M4@8$a{Yk>7_X|aaH~P#)uI^ zN1{m5$O6^WTos9wEm^Nt9Bylj&r`LM`gmY=@vJ;rYL>mvQ=S7}0n}+|%n<8Td7Wb4 zohB@yb|SSJ`LReVw{H0GL$6w`T(Dg=&^kLF;i$G;E}Z2F6Hs#hQE`%ft=#Ts zc_YNP%xE0fVSDxP4d()pXsWm4`op~Anf=*uW3h_1iR`k|@VpXWPo|&YO{7%XnxM^w zgYP~97T^fP`&y^4#wuYd*v@j=+0zFV7zIf$CwRWPnSUMbR_mbrc_jzY`w}uV>yn0%& zy*U>>o&zpeGP^lu-@(MMsM@8!jA8rn7P_#Q9B?EY1zVo%=2&HPj2>A!ywr8vDews# zwr%>|3%@simPIL=0hpLf>{wL4jig(tpbavKd~CCQaZOo{=-73)pdi7DTHHk2xgp$X zxwlNL`9dQ|IcVrfej=Gn-bL`VQwhJ)4U(JT-%(!pYeQ!8%h3bD&2*mEBJQnkWMa9| zh&LXk>a=CUtqr9d0G#PsV7ooND6>CXctk4tCy1yJ*S#EeTUMNi3LEUrly%?iXT>S8HTZ+ zuw&q7X}k?O{3M6D6-h;})lZd*SRVA-o8bKB#ezG*2ke*h+iRc-=UsTRu%i5Zx<*5&gswd# zhR)aQk@FLduuZ!>pX;%i&9mI>c>MyJq#AX+WHPUm6kkmQV5@zdDQ;3TSy+LODXE=Q zLBc_ahhUp|)HoNc;PIT)fSkm46Lne4s(RDJ6RoyutpJw^mfC%5YcmL}Yt1Cq;?ws@ zK4l*U+ZfQ}LbfTVaco0@0uhow9G>oMmJfISiVn%c3T9acY5(`ny?A0JecYmnG0sj; zG2Gmf@BT|le|9qN9>eTAm~Y$R;7Jyeo^GnbMK($8<~$ zY2;Iwx&`?V<9?+v%F|*wFBnGxT!`{FwO*}NDpIgUJp4b=( z1$Eym$cFcfw~g2$4~(4khD)Ry`*sYMOKc>{nY1X_F0|^hHe616V~$c;fzF~h#8DuO zztl3^k_=oWtGms<9}h`XX-BY&S&4N-hN`vDW(=V!;5;c!Igx#;4k;OJzpq(B>;~Jj zDvj&(_SG?sETD?hkl}SI6)^^P<6#si|K$=cwFXdC{1k$t{`*Y9*40y4!B4B_{Y6VX z&r1I!iSu1K#D;HkTEV|hHiVHqPb}>hibvR}Nx=%;sUUfjOnzTdE5)gbNC*@S?Nre} zQj~aR{d+?u_h3Olu+EZ_0^8vPOwug=Y!QM(B1)U{%x+jKf)~8v@^aMaB_T`5`^HW; zC|JvM*k-qcm4De?4pIX_i=Z8fGf`rli0GocTdTlDzGecmn?;@ix7wLi;D09}Gy=8{ zCAB>n(tIX244#01W>hP!L8@VrK`O6dudah|?d9$snt4x2ExOh?kZY95<=kMOY<$un z*_0>kHF}T^T-SIMQd9c4zq-ybPDbq#7?cbi@_wQ+t>E)mO%{KkRz-P0GgoK>g90I5 z=swp6rsSQ5Mb3;QFV2-*8j++i+58ibi$HuKU%69^Zmysq-gjXLYkPM)!s^bVo?-|7 zQ91_F%a)#yhq8}dq=qfIg^Pt;g_p|_h*4gVpHyANs?g zUx$H{@lSYLRP&gH8{m`92RWf~`Km|JS=zS&e3g~xQ&qeeo{hg@$WcBdpva-J3{5&P z2-lrtNEYPVX3oURh{Kb=3V^IMk9M|Mcr8^JkgtCai(6MW_>Jn}S3be3Q|Qepm3xG)0%1fY89uPHz)^p^ty{IX z7`DpYDbXLwYhdDXkx3Sk>M&w-&Y}5xl8n6e?QS&h7QV}z#@>M4+02i^f{>OkZzZe3 z>7cACn&ff5Ha>JAdu6FuGAvzJgyk4zepS2NQqWGu863C43g|6XKrfAMS2doT#Cf^? zxLc>xO?VcWMozKKRJ&j$_ll?aPO!2^(~_d@QYLLm$@Itn7ItRMAerNiO}SeBsgy4m z8qG5rhsr^hh#EsFg4_TtitOd7G{Pw*;0%NogZ@zh?ujjKy$&vn)Ei^jXh9Sq6SqyYj%7+`uUK{!UPburW_V{zd#}K}uKc--+!ZMF+H4LJs zC43s2p_o}rY?OS6%~Lf5O2(sAC}!aDw{z=^rI@wsG~XE2V0E_)JUZ4a%Jh`oZ3W`< zWJG9CkzE3vRt5$h9n#-}F8%nmQB_Kv8sm@g{loG5btN-MwU~%Y1EMQch$%LF@Da1W zY+_E8pRu%vGwNG{R-IP9&4+xoczpthaX>)zs|@n*blz5pz|T$@+5aOX=KY3W?6H3E z$m~4=F+0tvJQooFVmH*3>Rfs->T!EYmd1@Z3B11CuYSh;jc&@PU?wH5OVN7=?}CTb z`PQnyx!-*bBf~`^>C4C}?z_Kv>?N-n&-?v)?ASnFMM?l+PhBLRmMU~5`Tz6qUUrp= zxV>5D$8+fFm7C_2qQ3^d*JF-Zx%^`kZh=fjiprmJmJWm-+hbkXDUuv{xTA_`0 z2)(r3Yt!oTR{t#QxY#mvG+O@lG@O%);!)1Izt%K1aQqM1rXxK+!In`~(u(G)C&LT{=1! z>huMVkma~3xF%=;M76gT_*@O))81VJi6q1wpJAT{?)D-AnIAF6z|>0S>>2&qyL z5=y~04(HK&=NdingR$JJIxQ8;A|qp+-@Hi;&4wy&`)!Q2;PZIe&fT=2{l2VzDf^P} z5p5-lj>{s*-@!gKjY|JhBb_5lAWd8w3q22s5|=L0@VObK4Lo!@jg0biZjCZ)cgPh$ zS~sgC<~ZQ#3M*Y2`U&5sQ_Mp)`8>1tcfUejKtL)RgV%NR6wVWcf0-jK9EdnL**gpZ zW%8p>3TX0ye}Rf8vuR5UQ#T$W!qtZkw!A_Yf3E)iu=1g(C`M7haK@m(f_ljvAB2Dj&Yt!!{Axh&RR)N=ssL z$y`}*U1DAGMEf$6tdQj!@FB%AX8l7!ij{%xt&_mz7;toRDL5>jX%u#uYt2x8YI(NYv zeSCt+c(&yx4d#Eh@V%ApLa3ep4tn%~+95Gufg?CH4g9GKMv>-|itj?=Of;{rUdJE0SQbI-Ru~+;B{((;o;wzG?e0 zwm{JiDXaB*e3(98hjqG}o~IW77hu2T3e~y;3AsG9%Q4YMAKz7*W6DCcEd1)5(MeV~ zo<03cv4RdbdGLd!#kG5yp*Du3tE;IbI@6NKDruSM>4Hcse0J^@YA8xz`ASdW{6q-T z<-EXS46Fap+^Zyh{n=|2L&%RWXPrxLG4>9d=9o3F>tn^YyIH}d%XVGr{G6jpr^k)M z{b!1s)7I|Clomzs))r;_=d)zz0Od?Xkab&rMm~-sKz9)PzI$P5&hB-6J=KW^1EOgi zMNZyD3YCLtT<%2OyphgAvT9JBXB8aEhPQeORrwyW&HYtnd4~?dXbG^JMz^=-%9iG3 zaapT6W6!(F+!CHUIE1!azMWW9Rp*#iQpNkJWOh$VLQ`t4d!547Gqvwi?b4|Z;~ySR z<#skiB#KM*2;WUnoWFVJ!*rm6b2CeVLJ~O+p0Qk3^yQ)CUpLay>+6su>f&?|O)!u4} z1*i$s$^^lQ?ktg-^^nXCop*f{T|JoN8wkOeGMl#VgWIJ6u!w3S?`EN8PvG zo3s`Cq;}7+^nB6T%ID22Y9@36W{viNfM)?-z!8o;fS*jqx%EX<@1TB($0oMGO*hi* zx;pq($FoR-x;|nN*MCmYo2r?@R{G-q^vR#{7zbmj%C)wGop-62*uE`J`Agu!x;Hna zuhq>NJ}@G-!AH!dME2R)kd3}ja$p{E2+4$pl}Ot{(2wE?)_;yQd8W0=sfFj?H+wEC zNMWM7k8rir(~1c{aCJk-1m+LP^)SPu6X-bT$ZrCI?ytU9$3{(K_=D=9$#jn9wI+kj z7O9BNvj?jf37z`4aT`<`Ap9(=P>d1=$OqFGA4oB6HmCi(Z%7boi8(G_>AA_XX!!=<3Ch?mdhqS zSTc_5&Nxn+4pbW=mfFk)8{!I7iT@5ndTKqA0jW9`Z)SU%fXbre&*;jT74MlVC^_z- zhOktQ>PVE@4xTVa;9w59kS+Pd)8gG(Q$~U0@SZ20(JWPZd&A-U6m@<`^cUNFm)WLI zyy_D%(fcbD;(FztebhsIO7bj_Mu4M39ZiS!DAT+%Uk~hEIg4<4=5If2$>#N|mr31OI~0$8C{ z3!)1Hw&*g~$o-6tm?Rk)(n`b@nh0+fM3lpv63wI17~Ug}zA#gO$JLN(FaF+E4hTrW z5Ru{v!b7j|9eOyHAu)!uZX~RN7dAAP=K^WzY5Mu7&N{Dey1p zI2^Ed2r0I(@^uavosB;a*`Vt!O#+#N1xW9o;nS;>6PrGuphqxqXz8m5*PKM>y#?r7 zA1pS41X{;04Rgq>k7vp4bcoJZD=36R91Dic^fo6eFygchhEPMG8*2)Ag~~*r^lk*q zpUX+Ti;<{I%0i`II(-jGgV<+_v~et}S`ND)bCtK|6u%3bvc9aTBT$;1ge~#&!IE|7 zhMfz5&3m&fc=zejBK99AzqAt8#&<%O{vtdv109jeCOJ#fiAW*{UJg~)jdOGiR|M+% z-oIW{pI(m9iNWbJZW?o=%h7x8%*>z*A=GuQ`A@TTYa^NrXpHY|I&pxs>6b6dDY@(i z*f);!ik(9qU7;iU(xvYvdSU`cx@qbY*7NUM{hdIm>*fDWw}V}iV z$SFt=rPz2{#Ut&k{BB9j(`C2|k>f__4A={BwP&y1^JED*?nLmkI+ByENgPc9(>6J{ zlf2kl!Lus%2z{%cAWi4 zGm}$$C}S}CY3m&S?4kCGA9!woh*Ujs!t-1dpXbx~JbaMO5dgK5`9LsQG#iQwL5{{_ z^hiq@?UA?3(NdGaoEK}q1b6A|>Y1yaw57P@j%b9L-@kx2T34q7=t`wK_!CW5r3|-Cp_!Q7p+m+3x^IL@OW9-< zWc@1IF|tuaAA(4$MpTI@Nw)v~W#P%tuuAV%f8;x-=o0mhfcUc1+xKZNkHyYP*;ZGI z*S^lmSyq;m4c#>Eh&?smw#fnO!iw3d8G5l|c=oZlL!z4J5vlD}n@;AY#|dHL)RQ2M zjfcV2S#TQm^!a+fkN(IZfvqgJ0#dI0-oVFR@k>)Il&xmhQqk+C}w`-v!=PFZvrE5mJG4d0tl9S;vX z%M1f33>=f^bojB3ii85BsN!W}?XlMCXFPk$q)%>p*!E@s^0p zfT$qI``Ur6oqHjcB5{}ON^dzEcbCe)Qu+p(Aww8^DL?HbP zzf-&(c!g`meglI5(G;Zku}b!aNN$ND2&1R|z+5k?OJ$c|8!XhilQ)e4tci+3bE1}$ zA(<>;x_x$odJ$C;Y=pEwjNX;qRUw~+tZ+0hE=QBtJ_5PeS1c7!C0Ny_%l+!obBEW9zucv7ASvI6FBWVIbCuJM$AKqSQENO;Y>mch*MrAI#7>+ zHhcX$3oqaa2wnX)3~z2_XzKY@CUk!tuE+9AbKMhu-?pG;lG7bb>`xCT{&ozRtQAFt z5s%EuHRNje+u%Jg!q3sR-r5xlDa8}s{8I}suqCKulGvm}88Ob~oeL{9)a(DlV!lupMhQbgtW}<4 z)ApD^IrT|bN>N&GFqt(AZfM{uJop$!0(A1YTCmRpgRR&~9{syrSxKKEn8IEx!cKF3 z9by(pl4t5|OY1YdHsBdaHx*MAPc?S@#Tk}*7-1*5px(*U9bC(j75__FE77_u_fZ=_ z_bcgoMVPAKgo>h+bh;RxF*-g=Q=FxAS&Oi~$&34+M@u-um-=5v?}I^qFG^o7DfG%p z=%)K+9lSqw?wTspxhfA$82bt-RBYST%@nvS&@gGaq>_=M&|=XW-yOn>Ul+hGND73NN>74CGm{l-jW5 z`1`%_r>LV|2ic(9G0&?VekgTfE<=El(qX$X=2b$1(Pwp@PkI3d`Rxc2Gw0p^u_-VV z`5qTjg~o4WB-$Y~)Np8CT(@;{6LhO9)VHT&(^xYKUhcq)d2@*iWveilt-JGoZ(-w* zTy1F&F6haWCn+CYN_aW3Dvkve@#d*lTJ^MGTX%$M<2x|f z8&in}+kDYgTo}(8%v;jiQk2)=ct452M7YZU##pU+b#ltN(e%ETxP&-=$s~CdN62fEd zzui1T*|9XwJp^^bGO)UEzBhhJ0B{6<;2Nxf_sBebLsZA`@?Z#BDu%u5!%+QoE^vWW zQkDAPzPuG7*Ecr4tiGVww7yYdP&-C0@yN*_;9c^fl(M*?MYFhx?B)-&&uv%@>#Q!+ zi&N6)yuN~B{#6lQHYFX@B~Qb?v498L8U7?@!>{AydT%K<%6ZXyKMoywJ2m5{wdr&@ zr`*R6RIwhJSB)kmER;W9%cu6hCZ|OrpLx|5-$?)(%-cryrZp0kb(R$HT2Te)A%wr` zEp*^)G(cnj6&|5l{%SomIt=b%u*~pVw8~yBp&Hms2kCOBIib`V^08c$OT>5dE@BRv zGHHa~V|i2(C(4shM4-4h4uWl;qPAHMdU%SH>vD3s41n23D;9*(3V$irc^HL~DIw*} zjxUbM>y2>j1QaMcs#j3g@2AgM1S&KI`GngQ>~Bl$qHaYQ=aym5LKqt7`(d1daLd9+AXzt}sJ&DbBNUxuo}vjaS2Y=b8Ba-=C$YDS5uxHf>YZ|De3sw){e{PO zNqapu$wLaZdUFteQcsT5uXf8ymh@8IEl(;Jsvy2ab0tg##^bk-0*_7B(Qq|O$b6;; zg=$N~{kmu(x}~LLR>=kxB~Sb5R6V;0hpz1j3|r=KL#>L0t~fz+-QxU0Y8|eIjp5z@ zPd$7vqdZLFYnYFmC+O7lLx6-c3buXI<2xHIEF)AU>*}Uu`%s+A!;tllYfA4{`Nv3M z-M`QT$c3!4sRW4%ZN1lCx}_N}ElZ;AT*`&8%`*$Pp*yj0r11!QiNZT2b~1gguGhjf z2)Y7oD``Aox>?EyY3@|>KaOv#NR=XlY%jfrl`t|n^z4u(31a5v_yHK`|WB$+*Dp_q3O52b@&vADUh3oeztMqWU~*pR=1K3 zIFa~n*URUGdM%Iz)VYmEm!ey7wvDSc-;$Ru>+(f{e`QN#j^j$V5YDPD4 z2=?dhQ5@)iL`Sm5nA*e-hRy{Ki5#5Z+dn$FqI8iva*ba8zbse@yQ;n%PQ!`(p1NjX ztPN(DqAUqtf*kM~W;N@c%JKN*-S^cttC(@!T)WB+U3;X@$gCyCN_z#EsklU%qWJAT zK*poTu1<+oNx0GL&h-AiwiQwX-DbaK!8>b=DU6urTKsd_TGM_unjo6|YF&{y@`H{) zi@HivLp(H-jXyyH%WJx`A^&7TK@HT7L94m$0O9ZE$@K{L)l{{HI zPUh6+uj#R0=9|_EB)NjPURGbI5e0OzR#m#*NW5)&5{pS*#4!0Ai5BSTkgUc&4_is6 zNtklAblD5-7_yZe!JgFL1K*_v=X;3~$aTvzC@)POU6}(Y{q4&0Fw96IU^g&6r;LV< z64-d&RTW_-L1jO5d=U@T06FUGkSN-lD$OjjLHgD~Ye{yWv}>Q_X8b6?oYNhF`JH>f z$#TAxG5VDDbg;BELfB6Y>zgA!Ib3wD|Ja=3O77HJfd{|3>EHS=GSIqj7(2(Qvz7cf zysMnxHAv??-G>^mg4{TwABZn7NClxPk62gK-__$x@| z`BF;ra(ae}F8}*-qzJ^iDlzZ4SI_LXhA*Njv1r)bm~Mli+Mf=LNyqOR8aA$qV<-e|etU)i8rJ(c}mD&g{FPi!IxwA)}|ROP;` z?0hKulzG~W1EwSU;oMP?zZ4~k)k7w`vPLB%ZQ&3-Gp{UX23@BK+Ut_cMhysH5>j-` zxU+CVy@nEzF-)eZcFl9IL}6++O8jqb<~3GYGGG}@1Xuvw5KOR8jjz6~2;3Cyl`WXC z%2Z_j77G-hS>zCS4C}Ha%Yj*?2|4Aj{(Iak|1kxwLvlZaFwz&YP z=XaiI4}oWM3MG3DqS3F;o~xxKKBjr!I>sXjX@bpK$|l*n31WyaWZe^1#}jA9s~>gI zZ8O@{_!bQZbTn$JNekg@gru2r)w~K;Z?I419B6(CnY1J5jM)ZRK zEE8#jD_U8L(9QoD-Mo^A!7owy;Ug(SyFt;0dQmrB603Gqim@Ev@_}#%Cw3K&8WICZ z%>7*DM0h$>%O2R78e8@rnX#VMu}Uf@{;C3&2Wd7e*h?Lu;2~`R!XlTI<_!`-rurB- zI)kdZNR(r^-&M`)iUVm6NLw`AFHgi)OH)YZkW#0v6OU{RANaQm*J=KpsecvC#@uPoD{^Qa^#C+lG@R(fDobB>NBY;S#r4qTbNR zwT&maT`8wZM|qrkU%sF#=USq(ZxyzK{@OUT@K_j6+f-0#bez3`Ke`~PczovvCLlG$ zOJu5GK~|21q<+|Mx6{UxJhB0dT3$@@erjzHl*@&NyM-js_v;BqjZXRR)Ae2l{{jvFw8{N!*AJ0L%rifnvnyK|w= z#{pQQP}If+etlR(;TgC&LX4IcYVW!HWzJH7Q-pTm;b|ev@%)Nb#+NVJO=S3M(2`=W z65&@H*Tn8Pk;f`o|jR4-LDc^!Y{OcE{7(**Gf`3Mh$zL8#nMi zG=>-((qV?t?)@9~8BrT5XG9(&H?3KcJnWlDVVp#Kj9SS^3Oj{}%0lII!Ho3@%{=#z z|Ak=rkUbY+!>103!z!WgSAPWTa&gUGwY|tg2GfRMrBbgvW~!f4oX*Fbp^Y8qQNW{;$t@516x!e4wlY-bRQhCXcCDX}l9<^_ysG@Rj?%pP_rWYP+T zB>|NX*A0qcK^Y4XbE;G@=VP46E#UPg_F@ko%^*u!bWxqF68-p}T{PLI)nYi(x^wBC zT9<;5hS>$?Wi861f%&1A87V2>5zdFB_%TFVjpw5deI*QgVnCoo+kYQXwl1R7eK4-~ zWbiLLE87#@Vm)r)oKc z|ILw)_M%n=?)l9mHW$Kv^b1N)C)Y4VRJAEdCn+iWx1p6VX| zh;Lg#*vlh>YeE=i>d~F0|1Bl?stK|QnQ%LBv<_O8T^>SW&cD_FHCHKcQ$XiAw~IqG zPQ%apeXl{NR`j!VS!^1DMAB5~BYizt>O zjdBf@yrhec+mz!7UspWqLf<& zD22n(Wv%^O(ktUXxUUI%YXv{IR? zt;9GRohxZ&@qe(j#rDfyC!9yO!%>WUQR>iZlN!rja1c8t4UOzj_05iCgSvfP=u9)o ze#vp`Jlz}mj+gl8LwuE(3~pJ3#IJ3(#11j0uJ z374(U=7H?1rNEmJP3I|R)sG8oo>2z;*hW1e2S0(?`is1$#A=z#SCjijml=E#Y9X_F zk@}&UJlQns=Mri9VSni7hZ{Xwkq)}i$jr0BL_b_K}=^OcCC3@kEuUMUBEeW0{`^? z{XhTLuK))??KPU{Nyb(>n1wBzk&La(Qeu4>wYg(b8mJ`nysnPkn5nzg<(v& zT__mehq2S)nrxOu>jK`5_`=JZ8C0GVi<;}w2|R}`04+J(RTHd=4;$&yGGIqj)FR`m z@H?e<+t4&pdLtT1YfjqwS#JB2g>!GqCMxwdO=IQ03qS8Xg}V^mTLy%NQ5hJchD8{| zi;AV@a4J33ag*sS*JM38@440Ex(Pelo%BW6(X%W#pW==I0EcNYvmjfFT&+)Vw29A> z0@B+WN$nF_q4Q-0udQis+Ur$v-WYn$X@Fw_rw66AprOyB97Ve}0CnE_w z1=wykO;(@#TKG%O{a=SyB$g#jmg4OW_?qKgfOVdL{dr1ph)`^-6^9sp3UVxPx-HYq z;TFKPNLQpgRbg8tMViQ)e#d@@GXA>TXDX2|6ZFJGFA8~HH z3{c+~zNw%Kn393k!BXV{5VDg~aB&C9GNtRra`k^=;Iz&&R7VY(A~{J&z&Cq<@ow>Z zN~4gHmCtu&WyQVYbh>ZGGrNi|_)Zxb9(&BisC~p`Y%^}!3EEfGFoJ3rw3)sSSLvvw z^hB;IeK%J~ZFz*GYTI~N>n9V{%O((J0y%TH`Yl{s7(8D3k-$6luvz42+L5nFl(K%? zy{O7cb6_HkgqNy{OBElHGCS*31?ix|I~*INFfZZz@OK2~3xoA$&+>&@7s95fZcew^ zwI;5ldU2}kt%-$KI` z{l7VAVbf_yg-sjYcmnBePtG9ZpJfOU2Z#L&+9Sv|1IDTw*GLQk{9-`h`roO6Hb}^k zb=x@Kt=zcOwczxO!H4b1Vu)kgRxY)_SsqG@OmtZViccPRIDkwT(GJ&dPi#Xp@6eEeO`koW#56 zYG;cq;d>7&^Xb1(^)D;1xhvJG0hK9^wHRuY#QS`XQ9C*rcqA{4CmHO zw6vCD&8)cy&Vt<1U7kv==@B<=5;oA{GQ9j+?GEfostCj&Y3$ zN;tiM0}1zsDvE^+NH3-rOw(FN7;SxWV1*)dO7634&%(%3r&=W8naDFLnEjzLlYVb@ z4Q1MQ0=~u{J-D`Q~-YW*g)6t#@j|gl`@Do+Ek?FgP678prSgEd|iY$gA z1tdwz(Yqh*f(l!O7@`RA#kS#b=pq?L&%VF9jBZ3CW3>YX(>J!*XLXppF;reqpFELa z?Dnv=gXXP9?{gjX@>7~|6Q{o2bX?A)U3Muvzyj3*wqggMZg z$a(`JUH9W>{1&9*OIrtF0WMZt*`)5EA>%y`CGj> z0;UiW!rr|FRBWMHED*VRI5i7rYE=h6G0}A#C@D&bdZ~H$8_fH}YvXCcvfcF^Llw2@yc! zox_-=lniS);UwVs|`{pR)|bT=^~MQ6oC(e4AA;-t8@i|5Q%IsT3}u{4~DZ@`7K3wHc9;l zWYs4)K60q?t5iDRxo!ENA+3ISUd8bmgtVFsc`%Ik;TQBlPh>48K^5zilT!!6nXIK4 zqc&9;E2WTzE8aEwH$ghUJ6ufv78rG5Lyn|3FxYZWtnK*weN@f=pSiFWs!RJG4N!Wk zAsHDcl}quv$i^O8lk<*za+{y3gX$jI;CMt^sx{jgy&ii3mVNJ8vjrt5x+vLINtw*9ZOu~T1(~Pb z<5%_yMn-uT;-J;#9G(8)W`ZWZPY~q-Yq)H*GeTw=q}`uFa`H#xhVA2veu@IOrTm zYc)R9yso@I;)$a*r<>mmwP-5WF#`{6&?YOD91#RSfa5O10God!Wax*G7uyb-aqCey zA5ZE4`I7FJ4LL>e^J%;s-gay$)LPYvXW*Ti(KP0zy!TK@qA$@6P2m%#U-M-x*Bo#4 zgTFPPA&p}(x7*Uw*HDO=o#`&I1%_S46}RogK!Rwj7iDqTe_tXNc754VAkzIbwpq`D zzhR9|t9#256SxI~(f}B|m zl>w;$E_8ACD{aC3;*war zY+ShQ?5By7vCAAS;%R>z5BB4_B}_VNn@aBakP2s34meIV{ZQ0xt;QNqAQGVdR6%;4 z;tS2|8D%Jz97{lUqQ;zEOUX5Zefxi-ujORQ|<73sg24<|uQEGyWP3y>2* zT(m0R0$2u_%*OHNOfFJG-<2<_jHwj#O4+HpM@WQ5D%$|PvQDjxX!y65t1mdtSw)`Y9g|7h#o*CxSbT;L{Q5-8>X4*&}vxPek9b zHD-2?_qdgTWkwZ=GK>d>`0qn_Cp2^+y0)OJF*x*NEY&goCpHp&Gd1tmDY%U5yVoKCi`V92%41{G|A1E2YlW zOcsOG9*xeDswAm<0FI;iJkwe@`Kc{}Xwfai1YN*lX)2 zr0XKe;eFRA-2{bMxe2W9eFqUCWQ3S{M2J_YYB0(dEVKqaH;?08yR?22kSm$7$*&7_ z7~B&0h{vao78u+J`^>J->xtZ@x~foDZ$n@k%3(O) z8A!GriCZWM?P+?)dBs$u#?WwJY?Vc%3c^Ib6tD^{4W7+>?;OU*2%rgR)svt0ftg)Xj9$GhKKNuZ4LgLxfXK(Xg|0qNH_H5KS9c`A|~XK}H={Nw1`75swV#VF95_ zE~E~oj*n&kh7kNbZLcD1^4&HLz$o61Gz&?yo(4YE1$;`K&O1p9tg2zqEH7pDAI;CW zBRF9bNGq4}pxk3fm4{|$3>M6F%IR&*tx+<1UssReVom!dT8!$*`n$xqBGkh1iH2u! z^*;Jh1jo3E#<-Id2goT|jQ8k&^tJo7@qR|~{K}I6hOo9XI@j0~!W8!lAXs*VT`o4D zB;@_yldSijhlTMqB zDyl>^Q4lIJeksej)H7PalF+28k|fk_dXH*iOfLx&h>_;(j&?-Z^cC~TIwVq%)!)L= zPHnQwX2S+NbRVfxubQoSsn{PLThj*>m#84q zq*&(C2oIjIq#ZAnFdilftoxA>r0kxpJKU+CU3;#Z0|fe%985jjoCQuUnKsp=jjeL` z8@ZAX>cV!h4j{whE1}WL?!+SJ`Pz~J;gZQwQ>iBpj@d07{?xyfB-Nf#crtXSx~AY} zj2kb8G~nBA=j_hC6$H#t)4Y z*tTm#;Vf)u#kee!^$qhBGMmdsIl<(%#^iIreT`F)GP30_I;^AF(}2??(gKratON0pc||<6mB#QM zkMS{*{ycZEpem1!1d z^s#vgDK%F@gEhK3!7=U|c@|G$h4vJ5Px!E%if1Ng%}|(MD@vs;M6^gk?hbG$j0wI1 z0{i3cZrQs|F}A=n-U^ZWNVvlz|0N)y8V^`bPG47VfZE1vXd`XLeO|<0;)5UtR*MEV zavXuNVo%>TBa}-P8F1TaE3M=zb@oIrAsF_J9?=q;Z3(x~&UnLAe+=LsLvh*uB?2apiyKq|dhG(FncpS&5iVfbZiud~ZR8#Sgk$RmFsXydylEl#dU` z->2*9R*zqSx2sQ1(0uv$ISC7$432As!{~*h^P3;fnlE;r_yDXMtsS{C9cf%dbBNPW z1sfHtkQCq`(D_S$P5?}puwE-2n2%vJ%04~;Wy4;_3CqI}Bb-a9KF1)-thJ;r6A414 zy?IneshsK{C_xc#pVMrT8js;b+sb7GKuG5Gr^~Qk!d7DvEtR;c6SfzoPks|JU$m{V zI7aHUjFE;Zm0K9`>n4$vSS5B4Uf!;f+e+0bx=UwwV}a?ew? z^izM)#GkymPYV=Fy7K=COs#n^qf#Phm8Y-Sc}7Vs8LUsO@-RLOVjDB z-Y*jke-8!CfMny2KMKa1;W`|4zw-8De}p8WwZi%mB%KF zCd8QqV2u@6f6f4P0LFr5M4Z)D6d*lNBLC2N>2qkQyd40#R`D%A+$}f$H=x#Col?#2 z%mTe-a_{kF;qQ@nJfPLthcki#p`U{(jqF=tvf5$%V3xuKiTwH+V3HOT+0alu1*ewA}B&pwW#ijK7 zYN+m<7}n#os*t+KI#^f5-_%pQ(WAl?wJ*iZThe;_Qp>{QNqjW5p->tLt zjbtL2+1&gghOC#e7nvjIjQHQLyr@45M-C7zKY~ZU2cP4qQFcuB0l7O(@uu*%)agqW zQDbUH=_uT+2L;*hO`!T^uD(meN?zXPQc<$cRG}Rp1l(5xmpkOXuln>mpXZ_GF(G{N zhR5^Qm0XjaiJ654S^Td%H{8eQ!u50Ymie0Y3P!CZJ)DRZ0uF_RZq+>i1?`bfxh3{G zNmU`z6))d<4$mPu+p~|IG+Ub~A3QB{O)P}$s3TMPsR=OdaFhy2#{h-})jVY-leo%i zAD>U;a`gI0%E*-DZnAjU&pp-m+c)72r9+TM|LRxNfzRdNH~S%^z=da5@0kLM+23k+ zCk}<=E_3G{&0}s7Q~bjp$RZrgNa)j^Pt0n@iFmXglOXUzP9fk47WQpBW% zu()03by~M9^5)R^8jr_mBQ|w#DgW=^eCTt`*GpcB;%#47zo`@@){0%5dZQ~@`uKPH z10BxF?|!Y)yr#Wub|YrA=_vS^Agy;|ch5oRVNuE+Af7M#$4h#F(2{BiRd)J2gO;1` z<-y6JJh<#@DNuZU^Gz}%MrY$ysJ4jTw^twe9K3}u=SCk6#!Iwb-hl} zW}|yVw6CP<{V{Mf-W;EB*oa4SRIkVogY@8sRK7^N!Jz%Ln{LB{cnBSCtQCX2y${Zk zOICrMa(FX9n?ewILIrLN>r|tTVcXI5Y=3PpOrwu5WY-lou253sTWd=MDWOZoW;-ii zc3kD%XJDdw$dD!Nlg5!W#(J0|;c1$YDRi+Z1;&Bt2IVP!`Tc_aV3)kU zs88ju&;VOVm5uR%SQK^|aGr;A9$LDB02sho#r!~$;Xf_XU@)l%16GJ|j7Oo|IR>T2 zk{}Tp*5;bQkBE*p-RVM!r;;x^`!R6n&JWshP+kx=n^BLuYn10xX3-@MuEFc_ zM(#8jKhKF7=~RKRI)4e&$KIl50dZ4PZ?nIUb$L0GCz37ZE~VPiSs9}@D!`}^{6*SM z=-}lLE@+pl5YM9IzaV?{I+W9rYdhT&KkP}; z!~7$1W-o^~=_sEFK42`2+v9W4XtkP+&d}P$9vcXhD+l8%pCk=EFe7S4o}ppIHHa-+ z>c>slV8J!vD<3wljg3DU#boPL!}S>8e?x5^&G8O9~12!dMZi>xU&48%Te_Wey~DVrCeZG z4^=)YQgC7|JENdHIkz8AV~079=hQ{O@ONI{2r)f_F%xbEilC={(@Ivg2iCo+7OPIT z#snWuy>+c^eBYWux#sPxalEd{KmLu$qNHnEyJR2Ya55zVGoXQhxf0Ajq&)J7}$;kMnF)i9lmYFd|eV{P$_?2{ZZZEjo7-UaJwJg(*mnTbTw0>vN7g7JW{3w|9&yy{`uOe4*v{KVCaMK zdoNzHa)VGKkywJ{VqhCZCfcrg&5^7y2mt{X6TmU7rSezt6ov|rlDCA7ui54KP$sBO z*wiO^q~aE}GxWq4J3RpcTOzJ%SFH;ufe|3J@)!=!K;epvy!d1DoYQStXgZ_w>WzCg`k66XG9wea4+%&a^hFdSE5^N-`Tz zT7f{j1$?-NZrQxET=G&Iz_UgvxGnTmI|^DR3NG@(%)(+Bk&1d%(Z*L4MTR96RN4o4 zBm_mwzG5!&IyK8fWJf2-3o~LFRkR%kPHoT`ooJ(W8ejH9y}*Un?mX>F)H#=wOFBt$ zZ;eMq7s8>TQ6^`wXgR(bdU&{6j;VYIgut#`9Kl5^XI(W1d2i-*RUn!zKo-OjwPJt6|l*+$7H_5%0v}riRv8<@^YcX@59+=?4I&a$O12KB2<)+_=V+7 z9VnEzd0y0~e8j8}?>}hi(1aRuk4iUMOB#@)(NIJ>^=~DaauJN;OvojJnklP+zOj!` z${JWxEQX5x)?0}My48Nro%qhP(9VC94nf-K^3NhB;SP1w`SfuN;TDwi?p2{J?aWM8 z>uA1u-~^YlEzNH?-t{m2ejav&E=Zpqdci2%?y_naDSr*l(CmgY%h5*bx_tFrn2H$! z7E%|EiWQvJ9wW0yG}4oZ{n2?9SKd6SxYFa(Uey{&iJ!K+%Uk=}v_Im=wzc_f|HdD} zaZ=j7#DX%Zq-~88l@Q(c+W*c0kUDcaW3`pw@St%CiRC=$gxT!-@!(70NJL80Op&Vu zHNn`yyKAJ5C!{v}YnJOwOHzqZodyeIhag?3^dt+08y_dYZhW(pfsJbcIBDkD>&bt|mV8(x?EIn<|H*cr#(2|KC8v?=A?nUp85WS>iG zI<$O%dw2EZ-yTYhjhiwiuar+<-rYJq%gL*4(r%W$T?q&uXDe)XYQLI9203P4GIq#t zI?POU8sb>%8Ix{@T_eI#HEEkVxYa~kH^TOsaMdA1*K{BpA@xem?sBcG0IgLC31tl3 zRZ%D@cjR@hE{}q4SqinE)O2Kwo@Dg{18I?gDO;sBxc9YuH>^D12EK*!Du;(A#A-h^ zLqkV>&Ckj)lK9z`I%1YAOcscn=7At=(o^F&{}vdQ=OJKMx>b?`8XuPEe5v{2+-<%3 zHUyEJiMyHCE0#uR8Lc4k_H&?m1%dsSkTPG>y z*MP9e@A#v>K0LPY{dcRF_ScQcm(DbwYZum&iCN}Tho}*vfC4Qheb;qhXkVx@SVD&k zP8TXScI&9l*mN-Egzr}lD7P_SS#-44p+S+Or3LsFdV-Uhy>|=ur{q#$=G<$jh}gZy zz>=aK*vZoWFjsf1-g`va*^~`w)o9ByuIHptgnx750y7qOTKEM2ju-%7(|OV1Ix)Hf zC1!ye=Q?~07c#dYQc}+{Y-O)6tN>aQ)5l8eBlVd6<0k*-k~Ct+NQ>B9c)|krsi4=H zHJ=-BB6<0Cyr-y}CGb*am%_U1+6$C8R!M~#eKlhlDSUTULZB2l6z4e?u4;zLJXL#= ze69x>pwOQLmBs+Li*W%TXR1FFY7(wCwI;>KNHct?(bs6kA0n50xv-}}vri3|kUz80|z}2?8cc;;963u$9*#vQ)|8PjrOXKoX`rQ;xG`cweU+2{;Rbj6BO=q zsq~udtM5$#m^kp=L^x1{0)0;Dgjk|}BaxsPSDJD|rsTv)Kjjpj=Wsnu&nvxJNS^i-Sc%8DpnzrHC_Um| zR`nQ2u!-ym+UZmSD#8uAyEI$PJ3vA@#c2?OiCrpl?MM|CHrAQM^`=R{$EwUf`iS81 z2f9}s`%d7_Gqc9=^*EjSW0-!bn6h-2;_3W$a-^tJauYy4Pqafh+YkFaC4XiRxyuuJgbMy_ zjRz!26&YEwlMqTD=g4~Fg7fz~LhdUw<2+R7g%+|p9gV(ZwZs1Y4;AU$0qLCTUO`79 zgE+KWs|3dVE$&(%f*A$y3vtja(B6i@Z~s$2Bx8R>5@xG%*7(?Xu z;qSchP~f-t5oNA$Go0OQOSw%$Kvu8~l?7xQiFeBItpXqugzk0CGALT7YMIJr~i&r729--;c)bNo8)giG?jD4R8Mwz-^8g@+NQj^nYa_19Q*KJ{03(wGgn zr`RoAf_Pvhu0PMHT((iK$WbL$FxTaIF~L!Hi+Qrm8Tysmtqv$vGP1nB-!I!zz;~&X ztl4}_8cK>@165wj3!&Ype{+VEg;%MZ5cSw|RcX>`RN~6O%NA21Ar67il={QglKt1D zS`kJg{z@m@esqN>!1&lqcXFdS4Bg=uJQ}7`#KlOwP*gZbsR@5n9jjk#)}e619*vR> zG=LaK%SL-lN0N|XngT=5M(=(1>&U?aw+Ktfrl+>nPoki&q(p?b{EIG=T+B`}3s@&T zW(PwDDotLK^WVt$k1z4nRCng`1rGq<1fU*ySeDV!qd-CVhY6>!KzCzQyMzVj{1?q$ z4j2-eSLK4O%DYGvoXzx$Vxy^U|`xOT$79<3N1h+7oM+$`by2~q4#3gdX)3wyag%D;j1?d(qd4&m z8^FY!_Yk|+cBHBO?=0O|2U(i1N6T9Abdh1jy36LRWDSCJMKn!b1#oT_mgO24XKlxU z0)s;!z}6NM?Rg0XgtW&^i$lX=$dH3C?*f*lepv_uyp)2X0<9OWrZMhH1caZ=LMKoe zLruC>k9!mkSV*7-!@gSx(k?Bp`ecwlRAK81&yX;ThgR%wZ0#`VLEXN8Kc*iIhSy@N zcsaLRqYyO>EFW3BYXRN=>oUH97@E@FvMJ-G_fn{7<-F?B6EJHBNQ61hB`O&A6>?P; zVBk%zDOwei{Zo8q?QQKQ%gOmb@xN zRbO7J7G||6%oW*Gyr70xo0-a(!y1G4QZB7v`c1ZPX z|D!Jtu+jxey7MpPrN8d(yG9tIDfRbzd7^F`u?(Y{wm($Sgl=KL+w^6OuC2qO2OCEv zyI#iA)i*Jl7z?d#8(NsA8ggk~JoLYS)$j}Olc#j#ZHmodf3|FzpGYEOs!Q&8Y8aCA z*5p@(wA?x{*@p6=7Q22_j%yUlL_b~4F+_xDc^*@o4iEiyX)KzjhFJp|KHT3}v>p_j zJf%GLhhd2Z2v{55Hb#>UrqUa0j0;w5(ojyMi|kY_v^^;7hT{ zT}{&K&&|IW{ebxuP0`XRi{s?-8N(ix4}JlGS^zQ~@6|tX&4{Ww2~kqn3lr?k^?w{8nywEcFHzOR!^`{y>1c zIlbfNz`zAC-;v#H?TZ#|ThYO#rk^i|1N=p6)}nzI6m`nNpU)rEFYGw64c)o(QlKTo z@$}>jO-@`xCk$Q?Kzj(pkb3Lp^>ETqzMo^bS3}i8u!L|UTeIrwq6ENh{rXXo96!GI z$U{1mF=CA_G{-DAdlt>xz?I)>alENyub5{;aajj;f7q4W3?qrPJq1H$d1p_8Ff2x3 z9zFKuM7?K2G5l3Kf4Fp6qwf?4ABT+tcir3NAODaJ#Zr_7+sQzabXD;XpgjoEqXc`j zI}L;nB8uuPzsU~xNGZiBZ3uF#}gH*QdM5)A!t$eq;8!cU~FK5*aznrT2)wAVnN10K=$Cyeo1Zk3l zqBbaJr*M+W4f?04juV@4XM@i3qd-genrRJ_ySxwdd7n zgu1VM?&{)KTB67R1?2g` zf#YxTiC~OCIE4`2yh!G_HY%>{##sAs?S4m#2(68=0f!J(N4vS|cZiwOW>mV#Ga0iX zwd8i?pzA6QqrOK!3%;D{Ahv3YCZ`*XFD?#fx1f*C`5s!Ji}+Ncf5jZxqn|EvfF1S? ziIdiCqLt&&I&L)2sGTsiK}sUCM>$Zh&YZtl7biS)8*DT?vbL!~M~dKW-Mo%Fm-SeH z1j!Gpr}9xxX#8O$WIM6v!7f;GOo2BRLOw-afNA-eF2`W_5b8Py>IUM?!+v-^kynQY z0crPyHL%W zs3!Dj&`k6oT|#NnyiS4q0KKI`ij+gJ+Gs2RZuusfeu(0_oSsFFp8AX(pY)e6`~trE zx=5q#DXnumToCgy0^ht2q0Fu^5`=h)1Jl*f;L8_V-{?QrmH(K^|Bu}hmdjI%g6Y9t zJ#I^&yfmQdtS(!2z2iW+oQ2OyaS&o7n9 zO#0+H7PaP%@udzs7uVZKfJ?KW*5T;!<*o`N%D4_GQP>SNvwp~?{G`M`B63&0 zt1M~rcs149_jbLzva1HAfv>B7)i>bpYV6bnc7C9!rQg6rDe?U@_1iki57{+NeC5*0 zw%bD1cw_CNbSy!hUd7wajUMz9fg^lwPmY<*mzg!yJO`}g|BtYDTW%!Du0-GcE0AxJ z8ORT)d`PQSpH`=tN|IUCOq-r)K_W;-CxD<3K(a7T=QDK7=IiE5j<3t!?g3_Lp1LbZ z0&#KQzAx)SqAO;OEu3~Ka;&TS%|m_d4@L3|-%?pDtc)JMK6D|zg@3DytxDkVyO&Pd zd}5EHKjXi~B4yWrD>MY{?(w3^D<#F37OmXW<9U0uk1AcElpKCpE#2HJuguH%Fn68? zVPzaX;y6|=^EG^azD52e{O3e<*rmsMVW6xeMF%H>_vxV~2Is{Ps{Q3ODheHj>$c}s zaCsh|2M@+z^YncTS9YfTz%jmi5L{2>rSo?Qu1K3Qn*DT9< zi4@J*NK6a3j{LJ#Stq6qt|07dpnVAenpJiRkh=mwFUpu~aQ z+Gk=?YLv$~3vG$}&0mC{6fpW3ZV@>+4eV9A;Nl`WOi&K~*y9WktL?ALLw!sSO=zT% zh=rJ2aK2Cbpvywcx*o!NIsVmGcytk7F2@|jKSD8;0B<|GDNPIZTjlpovsm(@x*?nNE*e9}880vN!PNYa1lj$}f`s=CJE-VvVbl2tw!KVlbnkhP;ft(Ad z+0grlcDaD3Tf#CsC>&HuT(esnhv$#WqPj9@^69z=uiXNlO(Z@fmF#*N1_lTxf?|#RwA}l_E9I2xZbq=W}DmMoOoUZwVZatx=(d9B6dH< zO9g!sbM!)l9CETEojt^+MD^4pXL_p7{bz|*@_3Ri34fqJKMZ=7UyV4%lgT&zCo}1E zSs3~}&8N#@y9t`d-Q7|7fMDu(cMuzzB>Bs3d8dcieodxl&i3qHqAgtiVR{O4gvbZL z2J;SzmMJ2!W%{3*V~J2@`Pdc@lNu`t?1rcDxCl72c!IU{_Ngb*!XozCoMP&eKdOlm zlll9?91S7CnABi9LC@thK)*$6m!2q7+OSdw{&8$W)q*f3aKxn_!oaFypjy)K{n~+A zmj4ShS8?ZAh|1HwjdU6om&gD3d##ti5j_IgFGByCNl-5rE(kQyZP@CReuzY`4KSWm z$%mz$4868_l5zTg9H7L?umi85^w5}yfFJ;r`HYdF-k&QkcR2vupR`V7LFTEqn{`hyP z*!CzmX)XEGz-}_^D=p~sWbN8(QOQYaX zGJVC~NAsvNR%XS%2V7>izLj=xx0?-%KhaZb-eAdafuTE&xikpAlcKT2A)hDtY(Gh= z7JoqwJ&J#SijqCGo!7}%Kuz3hDUPZ5WlCxI7TX4Q1yU=L8mGT7pNML zY-<8lQ<^@;aRFLZg5P?3mN18mp~)O?ulD`Dsa?2d9#1N-FK=y;9{-zMCBG-AF~+BE z?0ImG9Hr=cLn35o+G(F11lMY6xzay}y~6t_-Y7aG?GQO#%qe}SKYaMZPwA4i_ra0X zc$1pp&&9mSlf2`XOrz9SVv|(vx7?Fm$4Y1|c!d$@+wlAKt0Z-N-E`}KX~^UFN9=>? zxqrko94u0d{kwAqP3u_uctF#62eBF5o()Qw1Nk>k7P18*$G5uaO3X}!+)w_2Xl>J_ zp`~O9kIm+C;Q|mTONRmxd?YLix!|K&^+9S(0Ih}`QQ@RlI1=HY!#JTv8j1X{9`%UZ zlidamVytB$Q^OY8cZYCWxG~o>cCnQ$Uv2_+3=QLh454AI`ordnc(1SX(>H4gLxPvupJ0%1U#Do&#Tl*TR9+(BD>|Xix`kn-tN67v7PWw*_gQFAmSDC3PCTH_jrE|W z2GJ`_1HTgEHgyxPpIITbisgY1^`ZB_3}=k4-8iA&Kke8zpeSJazAkesNZS+e1hK(d z26X9KeUU|uu=g1~=o`R=R(GN)K<-4lMnW#%)&vTRhqGkRvhEIX%30PFM4)`x{K|$& zy0-{Y(elW0NS%C=YUp<8otibsLMm@F=?cpU^AeG{97|W5Zhod7(;+*jjr^_`x+EaybK6Z!0K+A^jGe=5psr0|e$$e_NDlGG_-|NY+ zzvl22iA#_mt*Dc1ixttLG{f`pvi;5a6bhpbr zTNWT)UjpvlYv?ZdcoW&!i8-NJcYyZOxS!uf$+v%1(^is$ojr0w7p3hIDMv`(59JMA z*ScLVZ}sIYAry75SRLRjwUiH0yy1T1sYOma|35gE1 z#rR#X!%n4ZbLk9nO5KQWHB?^eKJG^=+(Zu?2}e!D&>D-)u_Hv|RLHo`-~;=9g#w+9 z%VmEyg>X8$MVIqexKYo$e%*IIE$${V*S39(v3Mss1qas&Nv~0dla}Mj>^p+JU z=Ku}J@^kp$tfmV^-xuT8)+yB*U>6j=%RL7q<{Fb>?dXb}$^&BB0~CIyb8EZiA;Wt- z-?}gyV3y=85@bbd$77+Dcj1V48(TQ@tffYQ@_}EAF_;q0b)4^8f>HJ66c2 z&2RmWsPY&a@6+agb~x=4N&WlUS*#`$H+AffntQ}W4Ts0cA*)kf;u6x>UoAksq0?1j zSYm#(@&$peW$wzQG$2s(bV4Ih8clcuY!9LxcOROK=Vp<2dm@|zcADHRldL+lX*j<( zO5MBmxHYLmGua%9rgEF*<=GXUw8fNF+>lwm;=S&yV^_D7Qf;+lgUq5(5bC%QM6-kI zK*~Y-5>oZ#4ZC*zBM%Fn(Vx>jikwgRTDZPQ$2#oB9<1&&IpM-Ql@rX{%r4})7skL) zoh$*4Y0s5R%hAfrI>sAXw@Ft1HF4Xp$f4&vzT_}UOMLy}q%GAud2BUC|Hd)Sy2;S( z*eAKcg*rQpIGZ)S(g3@s_Q;s&ymlsOg=78w&Ox+UK$+!ZUJS-niHv4>{wX!z%}Lm# zR(bD_TF5K-lm=8hO=VBkv6t|c(8jqa?KE2>Y{|Imi!UgY*Jp@c2V-c>pOFlj1ozCB zq_B^f(c4s~YU)Zc634yZE4<$WD{yOI-3o)y)3oJ|GyuN*_vSLq`X*TjO%4{f^Vt?~ zz1yO4D z`v$wndkSPWyZ0tsCBPXF_P)DEk*Pp(axkVhzxz1Vk4aoYE{0SswMjs){PBsWLNi4b zfZSl;Vw!9F3O&8~gjOR*Y3oJAd>bHj0xNaIeJ<(Ka~hxO&gu9_s{O++%-tv!a`L3C zcOK$h1TLu(Y`<^xaEdhH8l;l7)wNKtDsUg=#ypoy0Y;;AK5cpYrp7(I4#Ke=jg7jYpF_uqE${c5{xd#?nv3>hz{-zv|}cMmNpS~S*frPrm#F-s0~)@rKU&OYO98aAYt z_4cFXNsP(*YPun0isoAS6*km(!Lw0Nr)w|Nu}yVQLQiFWD!oPKObOBrguV8vyKFBf zGinZF+9!FLu-Py_#xn2h;TKyAp~#KPBwg@{CVwmk@xi42XvWoPKv|Zg$*e3lVgXwP zz=0bC&GEHV3eHq)B4tPNS;nGInUibIj_GR}w>>rIZbDH?P?Hr$r_M(`_dgQSTi337 zSWWnK6+Smr$MPnzT8fXcvnh}JfVspRM$e+R+1vJZm9i#cx}j=!n?FK%D`yY;M_5=D z%@dmmN%36TvnHCFw>}+nHhACJcXp*}kd^Wx;A;T7lJ(7gii%ip(`JeJYw|t=9{Soj zF|lrq2bT!@_fFr=C25Jl%d2RZCKXZl65ty;OEm zD+&(4Pv4q=0-!S70FR^I;|CcEj~Qz1G(AmkLw-{&{79eih%6H-f;9JoQI+?%&uYxM zl2fwh3C+~Ox`JBL0xu2cG~q6`#+|zM61Qw>wKV%T0eNkl7S_@YD^S-|kw`iBOlUm0 z0~8#kht_pCha(SERo>@^=4l>V1ck%5LUPafG#%z6 z>Qj*?ZuRo1uWKt~1JOAyAs(cK9P#wnzC2xc!(6^MR#C5%S943+^ut<&(PF07{L|CN zh1bL;0^=4SCj9-#Lt965&1+=n5!)EiKD+p#rExG)y%P|K{1pRa=Wnmx7D<2#H>*io?yo@A6_AUo_1b+epIpou7FqqI(o zFxzZ@x%t7T2GZkjQ&rpd@s|DaD|D4#)1pAJedW@#ZF*dUqdkrT-ojxTTLY1l6w5L%|Eso`^=JAof$`I05o_iaSDXQX5#^s+-*dM842J*r<&=^>F( zfjus3xX!{RNn-{^PxXmN0L=Dv9y>*aa&#z9tIICVsyw!`V=Vv!3rNAW^)q-jz#YLLzWUang^RY7>w0mmD{- zuv5Q_8tb(7)?#pb<(2l7OU=TW)pR0u(rHGs^w?^@x~s<`d|>mkx`x()Y?_8tvr1vs z305iM$9f)GlkFw@0Q-TUwEwRjjAF04k8q_5$TE5f`^It;ME6{NHmI^9Jf814HiaE|9*YPLC&zGSTrs0vIAb z^#CLS7ed4Xp!8&;{D){4&4bBjJF;AiMFyjAz0%%?6CJ|^s_I9fgT_~mxLjcrxkcjv zPY>^Bg3m9xR}^BbTa_J;r@cDKPG(?)x5A=|5MCpPi^VPuZA)*+B5KFj#B^GXo_YlJ zZ~0b}0)=rndV|~2z0mbl)VKD<`bfmzAVFM^i-KA4U%y;CO{MZQupbm?x!b_-w=r^< zU=Id9HcvJ-P0AnXBW?+9_2#N4R)eHJy+$sqI&FU~<8|cCKIFvs!CDsMxJ{vNj9$o& z^aE(;)&s8fp`_5FrWpUEQSF$+WJg;gSL<74mr2$D)KqiHsM+FldQwxJ$6Ke~9!_%N z#k6^Fdhpepe-ywo4c#St_@2683eLMPZAPvaD!eM5n0nsFYZ3OGC6!43!^Y))lKVE< zTxowVeI5ygOoABhr{|H%=6HKJuQ9#-bThI!pTrQ1VRo00Xq6)$s%!LgRXfa$`y-9Z zPEf!;j^Wq0zdSJ$q_ft2E&spUWlzR7=5g$5ba)a>l$|V2g_pT_Le#liv&U$_@2N2Z zcaTx&5mL~kzT#N@_tT&gbJIb$>t2vLq6tjVQESp2IZEk?nXC34C^_XQXHsO6xr6r+ z9qH-^+&g4`dLjn$v(iC6B-WJmKpjrzTEHM64$dc;yBBq@ zO*Ctuz#HYz42s*{xj*VoWEOUaJV2&|pL*IACH>ldK1wfS?`H7Z* zs)mAxN18*tj5*B{Q=+@FXgQatSanP|$27(sV`xF0m8TFjqAhv4 zD*7{$J-G=lP*IrW+ytMZG_6)|Biz;g;TXK^?7Yq!QCjhLrT> z)mz3%6KtX;WgCCvhpH1)95*E3*^ozY$V-UJ!_xaiZGlZ`19>Fh=qE+TF*C=Q)z8Cg6dKKgG@AEaEb7(_kE9`MB4^)U>Ua1EQ zfKq7R_3AUjb6u0<((^d2Dbzm3koTLv&`yZG9Ml-N%x#>sY@w?ZqgBkY0nN`XAl(^-c)X$3Kw-H#QeTM@ce45-pxv^%#vDJ^ z)NJ=@^NacCU$L?d9Q2p>6U70yE5#~qpjr`i{Q=vEpTO>$J)hGU{aVtuNuaBLWr^}i zR|ly9#lqGy-!{V?%w{)XyJ0BiTf?NX55k5=daNDt9^@^;Ce`81Zh- zl=Ja!I^OMvcg@+~PNa227LTJ2E#Gw}T|(z_TD)ApZ2m08Kw)**+$VswJF?C4n%8Ws zwFhS4L%GAFD~%t1Hgk!%NiR9O)Xr~kuS9rdrGQT^tb9tI6J!$1{0HBtDIsl z-_X2Gx_m?zra}0Ge3D|6qJG5NmtN3ypRToP34V^Y`>p|7+{G zQ7fE?g41Qh%0(?pT;#^W2CeO)@p|U(=0miN@;Up{G2rcec}iM6g~-;s8HZitPOMV3 zC6MwpJCN*ZGO@Q-Apofl%9w*AEjxIU<)AFoh4Vq+LKSa~9bQ4Xr5rN56JSsESGW5t1SQ zSeY`oGR87KotS3O8p4EUe|j(C%=yQ}Nn#-`k3;W3oP~&6Sb-Tf@SHw)t?R@I0q8=+ z7Gpu~PSmdN-N^aMn^^Cju%WQ+*oMH|&R{8Q`cMOBO&>WA>LYx}M>iIU|JhgFEO~## zc0=F9r!N2J$jkW+|97DCZM}Z9wBBrG zLE`s*p!)LTShvV7biTSX(;f%-BCnY+2m=y^_@AB^|3Bq(DyieaE}~Gx@GwjU&@|=@ z9QS}ER-hTJL|~C`v?E=LO96n0 zF6(9n?(5dP2*5AE2wVW8afCpuYTVo&&m9PfBjdNt9ehaOq}F=cEPd+=Pnh z@pra|Z-KL-F5&9Yz8{s0eH8)+G4gLz3OH|ia}d@8MVX7c2`eE0$>P;N++X8?CI5uG zQSFUR1A`91C-jT@bXWh>6IT2p*#$bg3mz>!b^^x3tWz<3{F1-f@`&kZos8%Uq|RSS zdg@+9R@bzn2VjEo`jM$xj0~**JOC$5J!@p@fbwRvm)__S$lzYPQSBR(=b(T7Y{@G;dl%-fITc?{;d2 z#-1`cf;Vr%qKf#Ms>uiKp_QXZ-*HN{!Ua*lVSg~H zMJF^ZtRS15#QKcO6LCB|*38M8^}_py!OqVsLHReFOfPsX zZ{ckz!_8TZm3c}iEFsnQoRZ2k-$QVj71#BM*o)p33_-obW>bQbOlGU;S&NcWI6TzR!D1+jke-H7vo{hIJIew~`1bLvq@Kd3n; z_T&v2O1*_gsEp^d4UA=4&qdT{SK1$FsIDEK=GE&)JUfq)l~7N&YknmyHuEV%mK1`*$MuUbIw=H_lk5q%=H2xd1sXsss>D3zwe`DbOt zxpI@~#yBIF^5@O&-Im|=lz9X9vPQ=iP$PCEuelS;DJf|lP;4Y=$WcTlgj&}~8QDe5a2A#V*BEh3Bcn!k zxaFC+EtiZpb-e4Vrp>4d#}#c;Pu~)9nd6bUkizh%34-Klv2`9qFwkHUt?ygp=&%^v zx6~@;3CTKh!W@cZuB2?bvx@qUK0W`8uDK%RMG1m2BL z6OeJCad{YfDKpxeceP8Xgx|mmD4gyv42zsFG;8ULAcj_9Vh2!;@XJlU$2NKjX6Fq9 zH1d%0><_wKwJMgUlZRcakC2jh zF;U#8tCjQ^>l1$4pf+;?I-_tgDftVA9FZG{01gT8p`(F7(uZ>Pqk`T+aW%`Ox(BiD zW!jdi+r=5L6JvJGWVLvb#RV-*zmHZ9ubXFnj-FMy9BbjR3*o1x%+nDhllT1iil%?> zH}JQyA-DeXQWMl#&PH)h@FOWg9uBMa0;u5kFb=9q)?RNv&}pLzslcj8*V!|(uq{Og z(6_?IEU4mv;@LJ_&x!Xr)I=$1@tG|h8;Wg4p6HPk7}IhBhF6}g$hzX$Pev14^G?Bb zJU&hPFcJ}k`?sD4={Wl?d9a31P6z<+Z7amf)z)zM`AJRS=aj8!cyzOBZ)L@xD2G|tkz^K#R!fur%mdc*ZgjTgK+BJ z7+z6QYe(H7yFG3~1ytU0ar`JPK3biwTOLKrV;hpz^2xJ|?Yd#r0M|sjX&f4i%OYyi z5HnF*2t*{7lv^h>^KV4A30=bJmiO5v_ZjbES?|ZC1+5(l)iTI>vAJC~O#b&VAgR&f zV`@LK=14K&Fg36~a`U*&Nj`(wpQ&TL$`FA3{ogzOdre$sn3#kjZcz|^%-IhV&;P8` zhM=PWAR`uVPK!rrO(m*IH15HdB%~y2-qk=RJgI9CiO)8lweF=JVK*&?JCA9ea*v$#G&xptO!7aKNYe>Vqzx zb}KZ{Q9yHTTiA}71Oa@Nv!4qintL;O%nmE3M22+)FdNk18pH~Rbb9toqI_`T)}?^U zFw}=|eYgrp#Fu(%#cSR`zqDV}zcqHwtwcS{W9o)XpT}<3Z?LJ3z{`iE59^!L_XL3) zuGajqy=z!@cZXqzGS%WcT#Kb~mUIMR!x;_+^$MjwoQN;R@qmCpm9{%9i~Bsn4XF^N zEOE=Gh!fCyu)!b;lOMFExwO)v%82)L*)Fx@IY?n_p7+76Kzw5kK<>&habZ3>0Dm}3 z%z7w%`#38f@sIT8Y`PUccvIbcDQ*ASpXO3-keVHx;LuP3Uxct<2sMJ;QMV-12nn#` zEtnme^;%owt>E|1tkcDrCLd%V;kzFs)N4BXt-bU3r*>p#=DCuLa2;(4FG00&j?@Iw zI8-@C9aO2>jMJm7-bpCM6Ld~-<4c$$cp7!ly}eHRd70l{X}!i=)U8tW%2_Egcxlf` zF%|KH2T+Mt*2nxUV~hi(kt>__jVqB^dO)2XYMxYsjMS_tE}T9{^{&ngDRI0;=Gp0k zItt@lD1+@}gCBJWy9EFa@*1|6HXfM`9ehO<0Qq2#*8cgB(%=Pu}&iFo_ zel$&oP?TkqkRrTf+c1-iex}&PLfM z?yvsZPV+5go;8#DA4f@vYv2jn7xE+PM*4@)mf2iKh;zoz9(Xqo{|K2=(O+lQ2a;tI zaG1P=IUftJ5f>qYwe98~ILhIqqK$>(U2-p^#@W|1L;y)Zwe%^(sL;H#tvqZQV5>sP=5 z$B%~08J2I!e!O-Cr+Q%T?z4cKUn{KC)3p`So|V%<;YH#UUz!kev%UPP`nQAYB@kil z&6yesD|xW!K}q9<=5@@>xNbXVAiT@+%e8Yz)>Nfy=BbHC6~eVtnat#^kqm+M=sVo@ z0h=vR%-|l7)oB_|6ZXs4uTWbNKqCG4@CQk-r#Ccb*OIf-POqX8-{ z)zXW^tUV+2JR~x#bmW5KWN9sUbx|95ozb5ZMgI)jHg#tougq87P=#1alfaB_WLG_= zhdWSoe`R92f=p|2%~i@R#|8|gpd z0(?5=yvtt=-}tbvc95t!sjwQT(mAz2T8xjzr2rzwmkk|?mlwE%aErV*pV<`7L%@V_ zxyDnw`)qQ=D0w{fQ|_0W#sbHX(H9*OQ&68Y5sb=_Fi;~a?0KL7G~D2yr6nK`D7)*m z@CLbV5o~HdPBlS_k`VKMu3-v8 zD^on1__wvJ<>lC#7r8Hyww)!!;6M?B%73%+xw3~ub6*>a$CK)hvNyy83~{D~z#Py5 z(J*VUrY!TDV=tShc8Nz2Eyi(YB|{yrPVO#j%H3hSwUVGnoR*l$EXLn}#$L~Np!)tw zbdd7_mV!tFs6f%U4&YU=1FyB&6Di_a4T?yLp%2fSwo^khvP*BHlIJV!Fa=`_ESvg=fjgT`+7c7C5Fxn+2b|8TG7 zz`-KWmd{o0#K}>%8-1Q8x@ILxF7cLh|8ZBkQz(8C#d5797$Q7SubB?L0zXGM5!$JC zw7}?IUq0?gsX^~t`vvQ85_1YWqZ!{eAx3L7mQ|STnw$7GI8Ps&AX_k2Lq%QZ5?txm6sVGQ{w1sl0chdB)K*)xs6^&akxYrJDj1ZtA1bllqWpCp)D>?tx(lJC&uH`gh#UD9k*D z%c6P91+1#Wcy?w&$p@)vm`7;2$-7$-JG41dC;~2RgcVslD$#oVw{d`9xfA+E!s@)? zK4)l&dq|7~+t8ulb2l!3xlAibCUtZa2m@B;VLi25M!0yDUK%||ES(M%K`-N3Rfq+i zAIAnc(JP4*=@hzRvnWXV?H3Mlj!nQU_IwMLHRb3iL+p|UygNX2R1~o%^lTDx6Z53} znPjA*E8IX_O=3Ufcbk)oq$6=H{vCD&W5fopNtYOVG5&M&zpJxl8YYm*{xb z-h7S-xNMOW_O$PXzT#)Ug>fV8csykOdUXEGG74?xYl^iAa8ATY$ScR=TB!O`2q@il zAU^!g`+#H=uT+f8(41B?6x~UcXz$08@Fjgi!yPXVv%^4xEqU`g==CiG&w+Z(6%IJy-h{2}5LBP} zy)}&YzpY#s5b1Mx^hSkG@^=p#o0_6yuI&0<&ybs;Zw$Q5CCt`~^}N`4-raHH>b9|& z9x&v6X!^wOp^aM9ctIla&?BcgIibWn=4CPoQIzd^2p7fqq z`hyKdfT@#U6rJ086M1W1vqs}Gmj9HuB6$nm9ZJ)BAOYVZNk=RpY!4myWGO#OBv*a< zkX!U#a1a{GgPGb7QSHy?=1pURRzK^MpjQ8ICl5gsq4j#EFcH8s+x4#WpXy`Ag<)|C zQk;u7MM7&ms~y|fZSc$pA{x1hoziB56hb8+-sRAQ#})wyur&uno7$FpV; z*#65VGMx93uBo;W)txdVH+3O0UXcMugAL&2KUIW&2pLj-#=mZNQjupRP{cs+i{%%@n=V%$=zM2m6BP(-CbDa3Ca=T{z-!qwW3I2FgU1RI&{QoY~Hzj z89InGO~O2=d)TArcUh39YoY?E)_%C~IUz&2p6WE6OBb{0xMU%3*fbApu~ zT?+RT!#FT59;D@M6Sf4UR+~Tt1)r5Ohm_e)OJ*(#XTqM)kLg)Q+{G6E5(Y~|*{*F< zC{MAQ!{3Qr;Mh5O8QeSqFn-x|dwn;zEF*H=dI%RfaJ4C6mY|!TSMQhsFJCbEsX5^h zXN~-sTY)c)JQ*C7$ypDMAJC6Ex+W2%Uf6LH!JiQ3{r(1AZ8G0DG%1(Ay>gLrf82~( z@)BxYxdGItIrFmv96y)!z{JlP?!j5bqB}==tko`Pyg5R~5MA^adVMWWId4-xn^8J<3rYT# zFs2xm9$lwh@Yc1Qiv%opMl3d64;Cg(*PMr=6v27SAC4W34TW6mow@7V^=KU5h`ZRY z-P;1X%j%$WgTzoO+R5*Fui8zFj34`31{buo*I5Y!h`iSo(n zNl;PjC8Nw9Kn6YK1hCU%j2WQFq-TQ7VnhZl3YZK0+Iw1Dj(Ec!-5C`ePpfIvUXF1H zE>t)5N3H}A$mbY#qHxkE1aKrm&UzYn8>IXcFSZvKj3!cv-j|wE<|B%zv69CN<+^*A zVXFe@>P{IpQEY}6p<3k4J;A=c(PckSEW{T7)a1B2v6PhjU(Wbt$`HNRbXq7ax&xGS z2dE!8c?yc@YN}HpV+nYZg-2+4ZS-2}V~F5>A)XBXXkmaCQ-=`)Y)Y zvfKBOXKCbL{U+n)N9%`12u&vBr8(L#qrV1XVmkIwpd{Vkb~xYzg!2KmCOjZmX1(|$ zK}ADK@qR{&u9$y8xTIJ>!F)b=;+wLFj#xqON5#NQ?O4V)r zOKM`&t-F_kO~KpvM*9apWqu84#q4`50t@!(D`(nRo!y|}vjsAJJ(vzlJce<9s8zDa zZmXsK8j8Y!@7>pPYrz&D+2LARvxcvf?`c0y zuM~XpTv}Zf;Q2Gx3{t&+8z8o}`3206m1{yI@I&4APoDGPkMsp?3%(M)PYIQfd;C6) z@BO5LV)wIxKhxTvkf~|1g1&UUXBtNaw7W2Ioz8itCchXT#`~>%W`;hr#_aqjJu?aH z)OZ^UGMu}dKi9B)&3U?StFG&3PjEF}EGsV$!CZ8!%?g1}Q4gpS1)4-yFIJ8NwD0Go zsr-byCt0lHnaqRw8heGQ-Z*1hmH|$?%>pB{5o^DA9b}T<_FOZF6weCCf>mS%{LjUH z9Q1*%Pf|v5YYx*<@|;{Kn7{B{AUU)4>bJ2mKR2!Owu{q*PI|jzvfL;3W?e4 zZ;zZdm=W}5sg7tmPREQ)!S4EV4)>;$S8t37(rr5@9-ZHfWqI?-6^Zk?7WGgdt<=~T z{%O@+cZQ?Etx=KV%Em49niLrEwb9Cs*=%P{MSr)ne$`ncJ@fNt9WfJhOQS=SK#Ax$ zzSo$1ehqW&<8Hik&eDw~7dD%kVkRB5HHMB_4e{|5`@;)ustnLA3*vFgkn<#Lzq7`R zaJR0#*t%mwW9W9RPK~ezhu!dY4#9`Uiz*h{SU%_iMi_n!oxd)O`BlH#nV_fuI@NEe zaVOKjzt7rO$1ts_zw>>BA!j4+8wmzI45vn4SLlpwcZPHekI!Ov8s9_7#&$-=d=2lV zT`YkQ!=6*wU7*}KEe{!I_47H&I7S&~sue~P@W;vZl5>y>H z>%AOR1~pXJ)NP$~GJ7?QcKK}MP|*0dMM@cv0bmOX;QDR%*CBHmNrJ1sr`MUpgSJDT zor&X@xU8NH+uzxmi>Iw8C2DrcNoweO$>lUYn@7=HkEAdtAHb`|Qte%&X`XW~`eT>y zG3Frj&^6ej&N&6h#0KgXZvPrCYIWaV&P*24GR4C<4%R#djE}9aN&6l*&g0GsRg`-( z-rE>Mi+d}_m`^n%GDje)t^PU$_LwC6^W0JDrR0b0K0v(2)Xr|165#Y}E)f_M5I2G* zN$GSy$;5K358>4qFvddzpm0pZ9<*Wp*thFlwY*uaSejq5<7IC!4a`THJqt!IqNz>> z*`t4v{R6gw`cy8b&wUF9WvW~KR1wzd5aS#ii;P;vHYKwz3r02$8$o$19gL1g$oK@@ z22|weAkc{i+i5Kzou#+fvLN&Pyk;c&wCm;Itcnz(8EOdiBCqdF+_iK5ioQ3IVa%gv z*;74VBpMRnmlbR&_la?8ClZGY^#w^>3Btf$LYhNGf)a2Fz)UDzqSv>i&|vt82#-e3 zU7VLsdwheQiT=TLu7RfPuNN_^Vl3a$HES^Y(g66KD@ZR%vjR4)1*iEPTy>U15$CO! ztnI+?$FTtv)xgmBMk-Q|M((4&=%{$hY?A>W$+7~IjH!Q*%jxl)xWl%nC z$2J9|Dp#!lns_72A)3WoI66A^X>Llah`cIx_TOE;k!}Jy_9Vt{{Z|Udr2nnKu<$K_!t|J|_6<+RBM&Bk8`9>{sFf8GW`xv)GFQ9JWEW9mlMq{7+dNVj!< zGaVYT7b&83^ccI}Qwd*W%EaEqLL>$y&rY_O+OwcXkswJwUXF|w+EG-*24OdwP%0v) z(s;M^y8a{Ys7dzJ|MxcHSvZg6?2iZjJ%-87OVVEJ92FEeDBj{qvx@v`Z8TuWrMNlbmwk|qBjdZz)VG`x-UvuC;vrZD40PdqiN-u72EFt+039bfQC zbU1^i!no_Z=QnwA*U`Zvp3^(5&%z$BAv+MTDo-Rr$u4D^3nq_=@#gzp3E^$6!eiz_ z;1QxQ@Jyks&83qxsyX2|gJK5z=8*MqHV1QaHt_+}&LYeo6ik4gK#H)nsc^G^mIGIP zX)o8Tr`R6Cs+wTaQA={C(lTQFkVyxYVhUO1)Rjs7V+i@u;KjN-ptc-5{axiep|URL z*5iLnt)$QaNm>7Zl9}P9br2aU+W| zcpsYhXkyl_GTPx5$vt;N(@EL3)G08)TK&25$Qt=x-#$dW0p|hCnpHE~*TOoENMuAC zLV0U@PtJYTV3A|Ow=hs3>0xPTk)OF|wqEjhk!_auxv0o_>&w}f*!*%8S0^Ql)PJP{x|dOsWLu@sTNluLxJu_R z{GF_N{IpjChUsvQQt2ka$SJ$W!i&xsku2U^q}xu!D77b!me(e9lW=(NTwd-|)6|fJ zQlw;D*mrduo-loSHMHh5n9f|^)#;!TpyX65S-GuO)aJKTRaj$GRvfM*!8=D@LCiXH z>Ohql>ujZua_fL06$az=S*-Vc@YhfQF;KZR=!#iAOt5EESJ&R_xaU%-?2&pdndW@z z2aR_}i0()`K02IRlrg*Dvr9Lz>;EyhPfZqK?e@xcokHb%D?bJ3RBsl*6;uIe&bWTG zAtOdWCe!h=2!7nOS~-py%(Qe<%Z&vAJkClF8l5z8lf(!k20i~I$L)0%1TLUeC2?MA zkv*QTO*a1ql?e-%dU2ed)wTIK7LhViPsb8Jk&>(Cfnh1w9MO%w!?P1@F{DvX09-gx z$-^x)yF+s&)UflnfE{||;U;g8ICk!4+~GOlm37``>pm}25X2rEpQPFE;cz*+jfi-W ziiY&c21t8_KK)z>!$ixG{i<+=LffrxGNa4kJ~DC)J%vnT%H?bo;_H5|wEbUalU&X^ zzj>r?El{TRMKc()Ja%L9HN!JT@mmdvKqx*APg$-yHNzjWTNq=6-pbKOuT9n6<|F9S z{fZi@-Xy7A_H)=n9Yc`dpthK()0r%p(`%N{Tp#2PoqBWnjuK;j9j5wI{5$-_mdQ}) zXbbVovEnw*%z(ZBV060q7bqm_*AC;S4s!EJ15Ny62dw-X)L{60ot1_`+!r2g6W>u} zai_~DKoRIR@8igX$lxM0no((&rVKSYC3PJ34E1d~-udj~d;I(OKFW@hkK5ID^WwyH zeQKit&23-ZkebeSck!Q53R}#+802+#x6CR!tOY@lWZ6Q{u981K?humsAJA<1BKo)LjuY$$z32At;=G`&cYc;oMc!H6z)Yq_2(Ou-u>ohyQ!IE z@~Mk!Vmnz82)6)xFi=pMF}Us4>JK?vqRNNUrgY)9kD>RsSTB`v(cvIE!Flr&d-|54h{Xs*@I78Z zxp@WJ$KPdq@@N8>+Bhg~nLSxgxIor)@8Q`V3flhQ{a@OOk zCRiF1jFgK^1NSgy`Ft}YZG_i62V&Ny!mj{1*~xG7qD}`Eyio_bSvfvkPI(ffrfa>z z7{G43q(J3#18FD`u;(E5~b7+7`KTIDO;( z*`*hoP-si^ukTA)n~BNV1z@}(^-lvT*gE}ZHRaH#i`jMw^@JGxB_Xz(gBkJBVAg?R zVJm{-T2{KIwu3msZZzE|K!PI<3sxz3)Tg@mv2NgZvtM-s3Aaw`xzJZBm=0Xqd@6?F z0e{twW$RQ%ebM5IjC>*Bq16A4v+qZ#c?ihx)~z2#U7q0@N(xRRlvW>Y`(-YWq%(=& zh*#(HObVlrAU&pDet72cjxo|uW+t`4m+APWhcn){=-8!htTQzStQQ=o*xp`%CCKqJ zNiF1;H;zV3kW;V#6Jt(m>Xw?i!D?ZD@5={)x992iFb=>1NYClzZ9eCJ;9$=G@Ss5= z2m3755pyz$7P)_3@!7qVxcjFa>?}iQw%FE%(et6+>jQ4^a_sTNM9f}ODL`aIJbVU# zZScUIfLN~R?(SCkS?xqd;Ly_Ybn3C*mYi7pIBMk&Q5ehAq}SN|Z7eq;QU<c`fMezY^elMKA&2cPcE}W}QS`gF_Lq1cXy69VQtOBHDAjY;RzAZuwv7b% zr}OHgoAHQU*}ix7V+exxWa}X`ZKkDz!I(t%#1lT$b12iCk&pj1Q!=X__ay_R+O+%t zcrw#}s8PS(c_>nmQ5su(JK2|GF744~l$Mj+s_^6-H~O5*CSPASFmDkjuaJ;D@{6Sd zWRo(B(7|p*VF*RcgW5vj+HKiZ8W4Nvaw7=e)TnSt)A^@l>byWgO$%IG{W4qR*hL~Jcx9{^xIpM6`+SvO^-h!-GEq<;M}_WG6!pEH0DH5b^)FBm?? zr*nAGla+FJ=T7J{q<|P{SZtuN`cuG+w=kWB@Y7QYAwGGNC9gvsKQB|KpIR4ZQzdPK z-Fg`!L5AM2Tm1EF;Zd*bkg59Jop8Klpz$*)^=Ub=Pe;Co{9l2C&ggUzjc(q&N0` zI-M^TIw+57o)ZAViORZ#ZmXc(XzR8&Xq$FfT?|ENdj|VLngH7PIh|sIUgPG~bhKZ` zRvl8rcA4eygPsw90)*yn0*%RDHafwncfZdW0Iy-Oz8k)ExR%FMf{5h(g(wq2A2=DY zjv6Ts&N$?VY) zp9Y7x6iek<%J}R+G6_S4mw<@J#2q z&?h&%CCT1W1;s6+-o}#6|M@iRd9xC&XOp%_1!jqk_Txz&rO)9e6YcVq_N_e@L(ko2 zJeYBQ5#>~9R}J*%)q2a$NI=xOr~F%7WsWZz>;c+F_q1CnD?43_oJ`M?y$gDGoMo&9 zJVM@ONu6c|Q@jOAvUdu-Y}71nqN$&{BZCRFF(i#^HsvAb42vBYq3`owEMW zTD$LL28kVey&QDJl=mTZmwOR*8+iUAb4CwF(YBagLYHO}Y9+SpXd*iuI>-rNQtR4a zs)0S$W;V&1!&!8!aev~I9M^KlofEc+U7WxyucP_eY7W%TplAmh18TZ#qwBIWTH-K< z4o2w?RXmd#ebq9zs1Z8Lll(lJ8?$T~`|$Ur%o45<+`)|kJ2ob*u2&OSzI3$+(wdai z3r)#!40ws8>!|o6#=A$s+eG?)Eefxa)U(M>6LA6Z1n|!t-o8>2ye|LVnG*x5fFBN#}|!wbtlyN8agIk z96LcAPD5dL*rvnqWG?#}YM}T~U;jz+YU`GlP5`J7=~NF^TR7E{OvS@F#>UCJbSuUg zz4AN{@B$cvbo!z&1lk!(x*MN<1=EgDF%ypov$9@&=p!nU;( zAplY1{V?sdqSH7xj4dQ1gu|NJ3n8J6{>{^A)&k%fwkmd5LpFy{55vaaCwd7J3;IxB z*F*erUsf=tzs@rl1@d7*V!VtUKKl4{-O_b-|6&WIy3lGVFvN7+raUfCHqR234U5e& zl}V}HKfsr87Zcco%60x|sw1gNchVHYdJVKdKzaq=tkbFUV9=%(qfCt4IjXC-nr|(_ zBAM-kwvqk!HZ`0OVF*E{u2Aw zIVY93`G{h1S~?vv!e$i?=grB31cH~o`Kj*)>jLmIn!toT%WotmLfS+@>usE&Lg8it znxb=w#jY(bOWR}?hP#x$P+xYEa8P28Ka)8ls{v}tVg(aja^F)@MsOA_o8YeI_x07P zo(Kn2!NW!2_tGES1@Fqsjz4|g{L|~H?tsIv+!s7-aq-6U(|yRMbd2#%X(W{6@r10b z5pbGdE1;+i$Ym3sPjiD+gM#7b=oX_U@QOfsY_rsGZrozGod>#*6{GFO)ySb;luu+M z@=Fhprt#De-RW6(_p*kYf=nN za;I@3bSlzif!%5zhrElwZ57}e=!$%w@%6t>CrZRj^$6ht+lHE*u(j0tG)x5(H~=(x zmGw>WB|V-(^_a+<=$x?U9kia_k9%QJw=@x97{U1=*Vo3HlAMk6=GQX_TcJ)goOLwU+Z&!6GHQQ z&9-Ez_1?;it$M%BA)}c@48Qx3LjDO7C<6B9PPcy3f5kS(P!5$L6eKJ;;EX{*k0HLW z+>(IH@lXdCbo}lG7|;g4bL*tp`A|(k?_5K?`Z@BX=7(?FvVRNs5ovV?Pcym8K?y z!9FhylQZa)Z{zqDPcxuG(WE%jYIFtk)3YWACtTc%)k(%2dDiQPn+TowYW9@v(5OpX<|iO#af*0I$_w(?H{L`S@&feA5W)BU)CFEYaTI0O zl^PFdo+Jga@#jOI0NW#kXmTK^kri@sTFNV->0mN1^SPX+)Y66~w}pq14H2XlC^^Vz z@x>44j#Eyy8KZkQ{-CUPO3`#}q)w#mmvlAQatd9ymR0=O>qJo!6bX6&1pE{_2e-)6 zOdv34M8Fm+A%5*98S)`l|D`U!?pJ|86gJfm(`mmukqBqatN=wCZigMO5-PXvK;|oI z&TY(ow~T6c$_+8drt4GO&8ag60YxbsczaNc;p<|HTE;eqa=&`eKLUK$*)edd^ z8U@O8{g)@2As#gcyDXTC4ouWL)RS6(JPe%nGkH1IrN9Y23y1G@o6-WDMI=v(Tu$Ix z_=H=)nb_s>=}ED&sA!C$Fpei5UycT_0J4?J>m1#?ruVx~YoguEbGxpDAFb?}?9tpJuo-8INh$!@v_%j8%WT){;43BB)Ylx~Jh5`cvs zI%|k1?^->yIB$t+?4dra2iXM3l8@>4L;a|SbzR4)zOv#STs}BSU0#jgD>_g!CLA?f zfWq5PcpA#qxU?ml#WBMX^aP~r7Hd47ugLP7z0K$Llx{?h@%OvkWIcU12dqm}Kk#WR z>#o?l_$PS72>gh6a)2Qp01@79F(zmDsY+Bbs09g5-FG4|D*L54WSageMc;EhO- zoNp`?1!z$`sgdHHc4{_;%AP zzB*at2K2nYg!o0%6@!`0+g_c^BGF~e!c7QHV2x;&!_gz)t%j~str|=y#eatzrVC2_ zN{wl6=pQ$;2g~(AeWT|Cx*qURsPJhzVc?#Zk&VpHWQ)uq`?uShci* z@7(Nuc=qm2!~3zl&bRtFF4H*xGVS16@b6A?rqMcZ1gUx!zETL8t&C?$?ocjKCK_7I4Wttwv(hHH+l)?iJ@5BT*jBSg zJa>DsGt0dZwY=~#Lw$@_`xXFA)ZDpai2BP*>r zEmT_n%GJTB0_7#IUH@^Wpq6Qtc|4|;t2+ju`ky<81ez$9U%kHTsn)eUhCc+P$AH6K ztS}NT!7K=#fYD6BcI}BDZ|E$Y1-M6pPG?OQp~LI&K%}T6x`}*f=?j zbq~AUACe0#Ty@@ivX^Iuqd5B1l8%!0r8Q7Jtcv`t`ck;(8wLZl{$69js5O}q%I0Xa z>C3$X>W9K^(6O#s^kj5n>FUd-FNF|w#84#2MNqb-%N?qbKh?nJ-QqkW++%P*8y*nnEPN^mYrThK zcwXSu=CRIg_ySjQzV-lMR#X}rRKhS#m&8(gbw$~@SP$Pc%k(=cGgrRmo*To_uldh>>V2A8nm>EJ$|Ab&!R94pjmY)eUfqOyfG{ImvQ$SF30wCUZ>t)z(zx z!qGaD7<5*Oh!bbT=k#(LBGI*@$VwGyy%gltCy6`Pn73s{GQqi6VNe*}&^1te05>7kwQ3omzB6;(!_A?wW)DF#pcZnME;pH|uEB8aqtJGSOh>dI0>Za+B8c9Pu~%lg6H~Zqw5*&51*MDb zD@J@;x+ta1lgD4c?1RZxwCCd-3R-hBwRYlt%Yx!seMBS&)W-0}z7e|Z>GKUOULVX9 X_J@c2{~rJV|NjF3P@u7{S3n8?q;DzF literal 0 HcmV?d00001 diff --git a/src/guidellm/dataset/__init__.py b/src/guidellm/dataset/__init__.py index 1980afd8..edf1f1c9 100644 --- a/src/guidellm/dataset/__init__.py +++ b/src/guidellm/dataset/__init__.py @@ -1,7 +1,7 @@ from .creator import ColumnInputTypes, DatasetCreator -from .datasets import HFDatasetsCreator from .entrypoints import load_dataset from .file import FileDatasetCreator +from .hf_datasets import HFDatasetsCreator from .in_memory import InMemoryDatasetCreator from .synthetic import SyntheticDatasetCreator diff --git a/src/guidellm/dataset/entrypoints.py b/src/guidellm/dataset/entrypoints.py index b7389f1e..f510933d 100644 --- a/src/guidellm/dataset/entrypoints.py +++ b/src/guidellm/dataset/entrypoints.py @@ -4,8 +4,8 @@ from transformers import PreTrainedTokenizerBase from guidellm.dataset.creator import ColumnInputTypes, DatasetCreator -from guidellm.dataset.datasets import HFDatasetsCreator from guidellm.dataset.file import FileDatasetCreator +from guidellm.dataset.hf_datasets import HFDatasetsCreator from guidellm.dataset.in_memory import InMemoryDatasetCreator from guidellm.dataset.synthetic import SyntheticDatasetCreator diff --git a/src/guidellm/dataset/datasets.py b/src/guidellm/dataset/hf_datasets.py similarity index 100% rename from src/guidellm/dataset/datasets.py rename to src/guidellm/dataset/hf_datasets.py diff --git a/src/guidellm/dataset/synthetic.py b/src/guidellm/dataset/synthetic.py index 6b7a57d2..dfb0627a 100644 --- a/src/guidellm/dataset/synthetic.py +++ b/src/guidellm/dataset/synthetic.py @@ -10,52 +10,63 @@ IterableDataset, IterableDatasetDict, ) -from pydantic import Field +from pydantic import BaseModel, Field from transformers import PreTrainedTokenizerBase -from guidellm.config import settings from guidellm.dataset.creator import ColumnInputTypes, DatasetCreator -from guidellm.objects import Serializable from guidellm.utils import EndlessTextCreator, IntegerRangeSampler, check_load_processor __all__ = ["SyntheticDatasetCreator"] -class SyntheticDatasetConfig(Serializable): +class SyntheticDatasetConfig(BaseModel): prompt_tokens: int = Field( - description="The average number of text tokens generated for prompts." + description="The average number of text tokens generated for prompts.", + gt=0, ) - prompt_tokens_variance: Optional[int] = Field( - description="The variance of the number of text tokens generated for prompts.", + prompt_tokens_stdev: Optional[int] = Field( + description="The standard deviation of the tokens generated for prompts.", + gt=0, default=None, ) prompt_tokens_min: Optional[int] = Field( description="The minimum number of text tokens generated for prompts.", + gt=0, default=None, ) prompt_tokens_max: Optional[int] = Field( description="The maximum number of text tokens generated for prompts.", + gt=0, default=None, ) output_tokens: int = Field( description="The average number of text tokens generated for outputs.", + gt=0, ) - output_tokens_variance: Optional[int] = Field( - description="The variance of the number of text tokens generated for outputs.", + output_tokens_stddev: Optional[int] = Field( + description="The standard deviation of the tokens generated for outputs.", + gt=0, default=None, ) output_tokens_min: Optional[int] = Field( description="The minimum number of text tokens generated for outputs.", + gt=0, default=None, ) output_tokens_max: Optional[int] = Field( description="The maximum number of text tokens generated for outputs.", + gt=0, default=None, ) samples: int = Field( description="The number of samples to generate for the dataset.", + gt=0, default=1000, ) + source: str = Field( + description="The source of the text data to be used for generation.", + default="data:prideandprejudice.txt.gz", + ) @staticmethod def parse_str(data: Union[str, Path]) -> "SyntheticDatasetConfig": @@ -114,27 +125,25 @@ def __init__( self.random_seed = random_seed self.tokens = [] self.text_creator = EndlessTextCreator( - data=settings.emulated_data.source, - filter_start=settings.emulated_data.filter_start, - filter_end=settings.emulated_data.filter_end, + data=config.source, ) def __iter__(self) -> Iterator[Tuple[str, int, int]]: prompt_tokens_sampler = IntegerRangeSampler( average=self.config.prompt_tokens, - variance=self.config.prompt_tokens_variance, + variance=self.config.prompt_tokens_stdev, min_value=self.config.prompt_tokens_min, max_value=self.config.prompt_tokens_max, random_seed=self.random_seed, ) output_tokens_sampler = IntegerRangeSampler( average=self.config.output_tokens, - variance=self.config.output_tokens_variance, + variance=self.config.output_tokens_stddev, min_value=self.config.output_tokens_min, max_value=self.config.output_tokens_max, - random_seed=self.random_seed, + random_seed=self.random_seed + 1, # ensure diff dist from prompts ) - rand = random.Random(self.random_seed) + rand = random.Random(self.random_seed + 2) # ensure diff distribution for _, prompt_tokens, output_tokens in zip( range(self.config.samples), @@ -149,12 +158,15 @@ def __iter__(self) -> Iterator[Tuple[str, int, int]]: } def _create_prompt(self, prompt_tokens: int, start_index: int) -> str: - left = max(1, start_index - 2 * prompt_tokens) - right = start_index + 2 * prompt_tokens + if prompt_tokens <= 0: + return "" + + left = start_index + right = start_index + 4 * prompt_tokens while left < right: mid = (left + right) // 2 - test_prompt = self.text_creator.create_text(start_index, mid) + test_prompt = self.text_creator.create_text(start_index, mid - start_index) test_tokens = len(self.processor.tokenize(test_prompt)) if test_tokens == prompt_tokens: diff --git a/src/guidellm/objects/statistics.py b/src/guidellm/objects/statistics.py index c9f25b96..ee3f47b6 100644 --- a/src/guidellm/objects/statistics.py +++ b/src/guidellm/objects/statistics.py @@ -353,50 +353,109 @@ def from_iterable_request_times( class StatusDistributionSummary(Serializable): """ A serializable model representing distribution summary statistics - based on groupings of status (e.g., completed, errored) for a given + based on groupings of status (e.g., successful, incomplete, error) for a given distribution of numerical values. - Handles the total, completed, and errored distributions where the total - is the combination of the completed and errored distributions. + Handles the total, successful, and errored dfistributions where the total + is the combination of the successful and errored distributions. """ total: DistributionSummary = Field( - description="The distribution summary for all statuses (errored, completed).", + description=( + "The dist summary for all statuses (successful, incomplete, error).", + ) ) - completed: DistributionSummary = Field( + successful: DistributionSummary = Field( description=( - "The distribution summary for completed statuses " + "The distribution summary for successful statuses " "(e.g., successful requests)." ) ) + incomplete: DistributionSummary = Field( + description=( + "The distribution summary for incomplete statuses " + "(e.g., requests that hit a timeout error and were unable to complete)." + ), + ) errored: DistributionSummary = Field( description=( - "The distribution summary for errored statuses " "(e.g., failed requests)." + "The distribution summary for errored statuses (e.g., failed requests)." ) ) @staticmethod def from_values( - completed_values: List[float], - errored_values: List[float], - completed_weights: Optional[List[float]] = None, - errored_weights: Optional[List[float]] = None, + value_types: List[Literal["successful", "incomplete", "error"]], + values: List[float], + weights: Optional[List[float]] = None, include_cdf: bool = False, ) -> "StatusDistributionSummary": - if completed_weights is None: - completed_weights = [1.0] * len(completed_values) + if any( + type_ not in {"successful", "incomplete", "error"} for type_ in value_types + ): + raise ValueError( + "value_types must be one of 'successful', 'incomplete', or 'error'. " + f"Got {value_types} instead.", + ) - if errored_weights is None: - errored_weights = [1.0] * len(errored_values) + if weights is None: + weights = [1.0] * len(values) + + if len(value_types) != len(values) or len(value_types) != len(weights): + raise ValueError( + "The length of value_types, values, and weights must be the same.", + ) + + _, successful_values, successful_weights = ( + zip(*successful) + if ( + successful := list( + filter( + lambda val: val[0] == "successful", + zip(value_types, values, weights), + ) + ) + ) + else ([], [], []) + ) + _, incomplete_values, incomplete_weights = ( + zip(*incomplete) + if ( + incomplete := list( + filter( + lambda val: val[0] == "incomplete", + zip(value_types, values, weights), + ) + ) + ) + else ([], [], []) + ) + _, errored_values, errored_weights = ( + zip(*errored) + if ( + errored := list( + filter( + lambda val: val[0] == "error", + zip(value_types, values, weights), + ) + ) + ) + else ([], [], []) + ) return StatusDistributionSummary( total=DistributionSummary.from_values( - values=[*completed_values, *errored_values], - weights=[*completed_weights, *errored_weights], + values=values, + weights=weights, include_cdf=include_cdf, ), - completed=DistributionSummary.from_values( - values=completed_values, - weights=completed_weights, + successful=DistributionSummary.from_values( + values=successful_values, + weights=successful_weights, + include_cdf=include_cdf, + ), + incomplete=DistributionSummary.from_values( + values=incomplete_values, + weights=incomplete_weights, include_cdf=include_cdf, ), errored=DistributionSummary.from_values( @@ -408,21 +467,64 @@ def from_values( @staticmethod def from_request_times( - completed_requests: List[Tuple[float, float]], - errored_requests: List[Tuple[float, float]], + request_types: List[Literal["successful", "incomplete", "error"]], + requests: List[Tuple[float, float]], distribution_type: Literal["concurrency", "rate"], include_cdf: bool = False, epsilon: float = 1e-6, ) -> "StatusDistributionSummary": + if distribution_type not in {"concurrency", "rate"}: + raise ValueError( + f"Invalid distribution_type '{distribution_type}'. " + "Must be 'concurrency' or 'rate'." + ) + + if any( + type_ not in {"successful", "incomplete", "error"} + for type_ in request_types + ): + raise ValueError( + "request_types must be one of 'successful', 'incomplete', or 'error'. " + f"Got {request_types} instead.", + ) + + if len(request_types) != len(requests): + raise ValueError( + "The length of request_types and requests must be the same. " + f"Got {len(request_types)} and {len(requests)} instead.", + ) + + _, successful_requests = ( + zip(*successful) + if (successful := list(zip(request_types, requests))) + else ([], []) + ) + _, incomplete_requests = ( + zip(*incomplete) + if (incomplete := list(zip(request_types, requests))) + else ([], []) + ) + _, errored_requests = ( + zip(*errored) + if (errored := list(zip(request_types, requests))) + else ([], []) + ) + return StatusDistributionSummary( total=DistributionSummary.from_request_times( - requests=[*completed_requests, *errored_requests], + requests=requests, + distribution_type=distribution_type, + include_cdf=include_cdf, + epsilon=epsilon, + ), + successful=DistributionSummary.from_request_times( + requests=successful_requests, distribution_type=distribution_type, include_cdf=include_cdf, epsilon=epsilon, ), - completed=DistributionSummary.from_request_times( - requests=completed_requests, + incomplete=DistributionSummary.from_request_times( + requests=incomplete_requests, distribution_type=distribution_type, include_cdf=include_cdf, epsilon=epsilon, @@ -437,43 +539,138 @@ def from_request_times( @staticmethod def from_iterable_request_times( - completed_requests: List[Tuple[float, float]], - errored_requests: List[Tuple[float, float]], - completed_first_iter_times: List[float], - errored_first_iter_times: List[float], - completed_iter_counts: List[int], - errored_iter_counts: List[int], - completed_first_iter_counts: Optional[List[int]] = None, - errored_first_iter_counts: Optional[List[int]] = None, + request_types: List[Literal["successful", "incomplete", "error"]], + requests: List[Tuple[float, float]], + first_iter_times: List[float], + iter_counts: Optional[List[int]] = None, + first_iter_counts: Optional[List[int]] = None, include_cdf: bool = False, epsilon: float = 1e-6, ) -> "StatusDistributionSummary": - if completed_first_iter_counts is None: - completed_first_iter_counts = [1] * len(completed_requests) + if any( + type_ not in {"successful", "incomplete", "error"} + for type_ in request_types + ): + raise ValueError( + "request_types must be one of 'successful', 'incomplete', or 'error'. " + f"Got {request_types} instead.", + ) - if errored_first_iter_counts is None: - errored_first_iter_counts = [1] * len(errored_requests) + if iter_counts is None: + iter_counts = [1] * len(requests) + + if first_iter_counts is None: + first_iter_counts = [1] * len(requests) + + if ( + len(request_types) != len(requests) + or len(requests) != len(first_iter_times) + or len(requests) != len(iter_counts) + or len(requests) != len(first_iter_counts) + ): + raise ValueError( + "request_types, requests, first_iter_times, iter_counts, and " + "first_iter_counts must be the same length." + f"Given {len(request_types)}, {len(requests)}, " + f"{len(first_iter_times)}, {len(iter_counts)}, " + f"{len(first_iter_counts)}", + ) + + ( + _, + successful_requests, + successful_first_iter_times, + successful_iter_counts, + successful_first_iter_counts, + ) = ( + zip(*successful) + if ( + successful := list( + filter( + lambda val: val[0] == "successful", + zip( + request_types, + requests, + first_iter_times, + iter_counts, + first_iter_counts, + ), + ) + ) + ) + else ([], [], [], [], []) + ) + ( + _, + incomplete_requests, + incomplete_first_iter_times, + incomplete_iter_counts, + incomplete_first_iter_counts, + ) = ( + zip(*incomplete) + if ( + incomplete := list( + filter( + lambda val: val[0] == "incomplete", + zip( + request_types, + requests, + first_iter_times, + iter_counts, + first_iter_counts, + ), + ) + ) + ) + else ([], [], [], [], []) + ) + ( + _, + errored_requests, + errored_first_iter_times, + errored_iter_counts, + errored_first_iter_counts, + ) = ( + zip(*errored) + if ( + errored := list( + filter( + lambda val: val[0] == "error", + zip( + request_types, + requests, + first_iter_times, + iter_counts, + first_iter_counts, + ), + ) + ) + ) + else ([], [], [], [], []) + ) return StatusDistributionSummary( total=DistributionSummary.from_iterable_request_times( - requests=[*completed_requests, *errored_requests], - first_iter_times=[ - *completed_first_iter_times, - *errored_first_iter_times, - ], - iter_counts=[*completed_iter_counts, *errored_iter_counts], - first_iter_counts=[ - *completed_first_iter_counts, - *errored_first_iter_counts, - ], + requests=requests, + first_iter_times=first_iter_times, + iter_counts=iter_counts, + first_iter_counts=first_iter_counts, + include_cdf=include_cdf, + epsilon=epsilon, + ), + successful=DistributionSummary.from_iterable_request_times( + requests=successful_requests, + first_iter_times=successful_first_iter_times, + iter_counts=successful_iter_counts, + first_iter_counts=successful_first_iter_counts, include_cdf=include_cdf, epsilon=epsilon, ), - completed=DistributionSummary.from_iterable_request_times( - requests=completed_requests, - first_iter_times=completed_first_iter_times, - iter_counts=completed_iter_counts, - first_iter_counts=completed_first_iter_counts, + incomplete=DistributionSummary.from_iterable_request_times( + requests=incomplete_requests, + first_iter_times=incomplete_first_iter_times, + iter_counts=incomplete_iter_counts, + first_iter_counts=incomplete_first_iter_counts, include_cdf=include_cdf, epsilon=epsilon, ), diff --git a/src/guidellm/scheduler/__init__.py b/src/guidellm/scheduler/__init__.py index 5b9c2cc8..e641dc1b 100644 --- a/src/guidellm/scheduler/__init__.py +++ b/src/guidellm/scheduler/__init__.py @@ -1,4 +1,9 @@ -from .result import SchedulerRequestInfo, SchedulerResult, SchedulerRunInfo +from .result import ( + SchedulerRequestInfo, + SchedulerRequestResult, + SchedulerResult, + SchedulerRunInfo, +) from .scheduler import Scheduler from .strategy import ( AsyncConstantStrategy, @@ -20,6 +25,7 @@ __all__ = [ "SchedulerRequestInfo", + "SchedulerRequestResult", "SchedulerResult", "SchedulerRunInfo", "Scheduler", diff --git a/src/guidellm/scheduler/result.py b/src/guidellm/scheduler/result.py index 48ec64f6..34de5770 100644 --- a/src/guidellm/scheduler/result.py +++ b/src/guidellm/scheduler/result.py @@ -10,6 +10,7 @@ __all__ = [ "SchedulerResult", + "SchedulerRequestResult", "SchedulerRunInfo", "SchedulerRequestInfo", ] @@ -69,16 +70,23 @@ class SchedulerRequestInfo(Serializable): :param process_id: The ID of the underlying process that handled the request. """ + requested: bool = False + completed: bool = False + errored: bool = False + canceled: bool = False + targeted_start_time: float = -1 queued_time: float = -1 dequeued_time: float = -1 scheduled_time: float = -1 worker_start: float = -1 + request_start: float = -1 + request_end: float = -1 worker_end: float = -1 process_id: int = -1 -class SchedulerResult(Serializable, Generic[REQ, RES]): +class SchedulerResult(Serializable): """ The yielded, iterative result for a scheduler run. These are triggered on the start and end of the run, @@ -110,8 +118,13 @@ class SchedulerResult(Serializable, Generic[REQ, RES]): "request_start", "request_complete", ] - request: REQ - response: Optional[RES] - request_info: Optional[SchedulerRequestInfo] run_info: SchedulerRunInfo - preempted: bool = False + + +class SchedulerRequestResult( + SchedulerResult, + Generic[REQ, RES], +): + request: REQ + request_info: SchedulerRequestInfo + response: Optional[RES] = None diff --git a/src/guidellm/scheduler/scheduler.py b/src/guidellm/scheduler/scheduler.py index e603a3be..88ee167a 100644 --- a/src/guidellm/scheduler/scheduler.py +++ b/src/guidellm/scheduler/scheduler.py @@ -13,12 +13,14 @@ List, Optional, Tuple, + Union, ) from loguru import logger from guidellm.config import settings from guidellm.scheduler.result import ( + SchedulerRequestResult, SchedulerResult, SchedulerRunInfo, ) @@ -72,7 +74,7 @@ async def run( scheduling_strategy: SchedulingStrategy, max_number: Optional[int] = None, max_duration: Optional[float] = None, - ) -> AsyncGenerator[SchedulerResult[REQ, RES], None]: + ) -> AsyncGenerator[Union[SchedulerResult, SchedulerRequestResult[REQ, RES]], None]: """ The main method that runs the scheduler. This method is a generator that yields SchedulerResult objects @@ -122,9 +124,6 @@ async def run( ) yield SchedulerResult( type_="run_start", - request=None, - response=None, - request_info=None, run_info=run_info, ) @@ -165,9 +164,6 @@ async def run( yield SchedulerResult( type_="run_complete", - request=None, - response=None, - request_info=None, run_info=run_info, ) @@ -309,7 +305,7 @@ def _check_result_ready( self, responses_queue: multiprocessing.Queue, run_info: SchedulerRunInfo, - ) -> Optional[SchedulerResult]: + ) -> Optional[SchedulerRequestResult[REQ, RES]]: try: process_response: WorkerProcessResult[REQ, RES] = ( responses_queue.get_nowait() @@ -321,37 +317,36 @@ def _check_result_ready( run_info.queued_requests -= 1 run_info.scheduled_requests += 1 - return SchedulerResult( + return SchedulerRequestResult( type_="request_scheduled", + run_info=run_info, request=process_response.request, - response=None, request_info=process_response.info, - run_info=run_info, + response=None, ) if process_response.type_ == "request_start": run_info.scheduled_requests -= 1 run_info.processing_requests += 1 - return SchedulerResult( + return SchedulerRequestResult( type_="request_start", + run_info=run_info, request=process_response.request, - response=None, request_info=process_response.info, - run_info=run_info, + response=None, ) if process_response.type_ == "request_complete": run_info.processing_requests -= 1 run_info.completed_requests += 1 - return SchedulerResult( + return SchedulerRequestResult( type_="request_complete", + run_info=run_info, request=process_response.request, - response=process_response.response, request_info=process_response.info, - run_info=run_info, - preempted=process_response.preempted, + response=process_response.response, ) raise ValueError(f"Invalid process response type: {process_response}") diff --git a/src/guidellm/scheduler/worker.py b/src/guidellm/scheduler/worker.py index 0102a6dd..2611b122 100644 --- a/src/guidellm/scheduler/worker.py +++ b/src/guidellm/scheduler/worker.py @@ -53,7 +53,17 @@ class WorkerProcessResult(Generic[REQ, RES]): request: REQ response: RES info: SchedulerRequestInfo - preempted: bool = False + + +@dataclass +class ResolveStatus: + requested: bool + completed: bool + errored: bool + canceled: bool + + request_start: float + request_end: float class RequestsWorker(ABC, Generic[REQ, RES]): @@ -92,7 +102,7 @@ async def resolve( self, request: REQ, timeout_time: float, - ) -> RES: + ) -> Tuple[ResolveStatus, RES]: """ An abstract method that must be implemented by subclasses. This method should handle the resolution of a request through asyncio, @@ -153,22 +163,18 @@ async def resolve_scheduler_request( info=info, ) asyncio.create_task(self.send_result(results_queue, result)) - time_til_timeout = timeout_time - time.time() - - if time_til_timeout > 0: - response = await self.resolve(request, timeout_time) - preempted = False - info.worker_end = time.time() - else: - response = None - preempted = True + status, response = await self.resolve(request, timeout_time) + info.worker_end = time.time() + info.requested = status.requested + info.completed = status.completed + info.errored = status.errored + info.canceled = status.canceled result = WorkerProcessResult( type_="request_complete", request=request, response=response, info=info, - preempted=preempted, ) asyncio.create_task(self.send_result(results_queue, result)) @@ -326,7 +332,7 @@ async def resolve( self, request: GenerationRequest, timeout_time: float, - ) -> ResponseSummary: + ) -> Tuple[ResolveStatus, ResponseSummary]: """ Resolve a request by sending it to the backend and handling the response. This method sends the request to the backend, waits for a response, @@ -341,8 +347,22 @@ async def resolve( resolve_start_time = time.time() response = None error: Optional[str] = None + status = ResolveStatus( + requested=False, + completed=False, + errored=False, + canceled=False, + request_start=-1, + request_end=-1, + ) try: + if timeout_time < time.time(): + raise asyncio.TimeoutError( + "The timeout time has already passed." + ) # exit early + + status.requested = True request_func, request_kwargs = self._create_request_func_kwargs(request) async def _runner(): @@ -367,12 +387,18 @@ async def _runner(): f"Received no ResponseSummary for request: {request} " f"and backend: {self.backend}, received: {response}" ) + + status.completed = True except asyncio.TimeoutError: error = "TimeoutError: The request timed out before completing." + status.errored = True + status.canceled = True except Exception as exc: # noqa: BLE001 error = str(exc) + status.errored = True return self._handle_response( + status=status, request=request, response=response, error=error, @@ -418,11 +444,12 @@ def _create_request_func_kwargs( def _handle_response( self, + status: ResolveStatus, request: GenerationRequest, response: Any, error: Optional[str], resolve_start_time: float, - ) -> ResponseSummary: + ) -> Tuple[ResolveStatus, ResponseSummary]: if response is None or not isinstance( response, (ResponseSummary, StreamingTextResponse) ): @@ -434,7 +461,7 @@ def _handle_response( ) ) + (error or "") - return ResponseSummary( + response = ResponseSummary( value="", request_args=RequestArgs( target=self.backend.target, @@ -442,15 +469,14 @@ def _handle_response( payload={}, ), start_time=resolve_start_time, - end_time=time.time(), + end_time=status.request_end, first_iter_time=None, last_iter_time=None, request_id=request.request_id, error=error or "Unknown error", ) - - if isinstance(response, StreamingTextResponse): - return ResponseSummary( + elif isinstance(response, StreamingTextResponse): + response = ResponseSummary( value=response.value, request_args=RequestArgs( target=self.backend.target, @@ -470,5 +496,7 @@ def _handle_response( ) response.error = error + status.request_start = response.start_time + status.request_end = response.end_time - return response + return status, response diff --git a/src/guidellm/utils/__init__.py b/src/guidellm/utils/__init__.py index a0e12b81..99411070 100644 --- a/src/guidellm/utils/__init__.py +++ b/src/guidellm/utils/__init__.py @@ -1,3 +1,4 @@ +from .colors import Colors from .hf_transformers import ( check_load_processor, ) @@ -5,6 +6,7 @@ from .text import EndlessTextCreator, clean_text, filter_text, load_text, split_text __all__ = [ + "Colors", "check_load_processor", "filter_text", "clean_text", diff --git a/src/guidellm/utils/colors.py b/src/guidellm/utils/colors.py new file mode 100644 index 00000000..4689c94f --- /dev/null +++ b/src/guidellm/utils/colors.py @@ -0,0 +1,24 @@ +__all__ = ["Colors"] + + +class Colors: + INFO: str = "light_steel_blue" + PROGRESS: str = "dark_slate_gray1" + SUCCESS: str = "chartreuse1" + ERROR: str = "orange_red1" + + +import gzip +from pathlib import Path + + +def compress_to_gz(input_path: str, encoding: str = "utf-8") -> str: + input_file = Path(input_path) + output_file = input_file.with_suffix(input_file.suffix + ".gz") + + with open(input_file, encoding=encoding) as f_in: + with gzip.open(output_file, "wt", encoding=encoding) as f_out: + f_out.writelines(f_in) + + print(f"Compressed file saved to: {output_file}") + return str(output_file) diff --git a/src/guidellm/utils/random.py b/src/guidellm/utils/random.py index 17873d12..830dab77 100644 --- a/src/guidellm/utils/random.py +++ b/src/guidellm/utils/random.py @@ -22,12 +22,12 @@ def __init__( def __iter__(self) -> Iterator[int]: calc_min = self.min_value - if not calc_min: + if calc_min is None: calc_min = max( - 0, self.average - 5 * self.variance if self.variance else self.average + 1, self.average - 5 * self.variance if self.variance else self.average ) calc_max = self.max_value - if not calc_max: + if calc_max is None: calc_max = ( self.average + 5 * self.variance if self.variance else self.average ) diff --git a/src/guidellm/utils/text.py b/src/guidellm/utils/text.py index 6a3b6042..131e87f3 100644 --- a/src/guidellm/utils/text.py +++ b/src/guidellm/utils/text.py @@ -1,4 +1,6 @@ +import gzip import re +from importlib.resources import as_file, files from pathlib import Path from typing import List, Optional, Union @@ -6,6 +8,7 @@ import httpx from loguru import logger +from guidellm import data as package_data from guidellm.config import settings __all__ = [ @@ -93,6 +96,19 @@ def load_text(data: Union[str, Path], encoding: Optional[str] = None) -> str: response.raise_for_status() return response.text + # check package data + if isinstance(data, str) and data.startswith("data:"): + resource_path = files(package_data).joinpath(data[5:]) + with as_file(resource_path) as resource_file, gzip.open( + resource_file, "rt", encoding=encoding + ) as file: + return file.read() + + # check gzipped files + if isinstance(data, str) and data.endswith(".gz"): + with gzip.open(data, "rt", encoding=encoding) as file: + return file.read() + # check if it's raw text by not being a path if isinstance(data, str) and ( len(data) > MAX_PATH_LENGTH or not Path(data).exists() From 0a6230bf64cf462dceaa972daa4f4ccedd157b3a Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Wed, 9 Apr 2025 11:13:35 -0400 Subject: [PATCH 16/43] Update src/guidellm/scheduler/scheduler.py Co-authored-by: Samuel Monson --- src/guidellm/scheduler/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/guidellm/scheduler/scheduler.py b/src/guidellm/scheduler/scheduler.py index 88ee167a..b5fa10ba 100644 --- a/src/guidellm/scheduler/scheduler.py +++ b/src/guidellm/scheduler/scheduler.py @@ -115,7 +115,7 @@ async def run( if max_duration is not None and max_duration < 0: raise ValueError(f"Invalid max_duration: {max_duration}") - with multiprocessing.Manager() as manager, ProcessPoolExecutor() as executor: + with multiprocessing.Manager() as manager, ProcessPoolExecutor(max_workers=scheduling_strategy.processes_limit) as executor: futures, requests_queue, responses_queue = await self._start_processes( manager, executor, scheduling_strategy ) From 62cd7e9d9b50d8c658edbffd892a52424f204ce5 Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Wed, 9 Apr 2025 16:13:28 +0000 Subject: [PATCH 17/43] Fix synthetic data generation edge case where text is much larger than requested --- src/guidellm/dataset/synthetic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/guidellm/dataset/synthetic.py b/src/guidellm/dataset/synthetic.py index dfb0627a..21b796e0 100644 --- a/src/guidellm/dataset/synthetic.py +++ b/src/guidellm/dataset/synthetic.py @@ -176,7 +176,7 @@ def _create_prompt(self, prompt_tokens: int, start_index: int) -> str: else: right = mid - return self.text_creator.create_text(start_index, left) + return self.text_creator.create_text(start_index, left - start_index) class SyntheticDatasetCreator(DatasetCreator): From 94efbd4d80895fa11145ab25c610109f2f28dd3e Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Thu, 10 Apr 2025 13:46:06 -0400 Subject: [PATCH 18/43] Update src/guidellm/dataset/synthetic.py Co-authored-by: David Gray <40244437+dagrayvid@users.noreply.github.com> --- src/guidellm/dataset/synthetic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/guidellm/dataset/synthetic.py b/src/guidellm/dataset/synthetic.py index 21b796e0..dbd73baa 100644 --- a/src/guidellm/dataset/synthetic.py +++ b/src/guidellm/dataset/synthetic.py @@ -43,7 +43,7 @@ class SyntheticDatasetConfig(BaseModel): description="The average number of text tokens generated for outputs.", gt=0, ) - output_tokens_stddev: Optional[int] = Field( + output_tokens_stdev: Optional[int] = Field( description="The standard deviation of the tokens generated for outputs.", gt=0, default=None, From a3e86d81d73016b8e6239a46812eb72bdc16382f Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Thu, 10 Apr 2025 13:46:21 -0400 Subject: [PATCH 19/43] Update src/guidellm/benchmark/benchmark.py Co-authored-by: David Gray <40244437+dagrayvid@users.noreply.github.com> --- src/guidellm/benchmark/benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index 580e2a23..5b3df6a8 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -524,7 +524,7 @@ class GenerativeBenchmark(Benchmark): requests_latency: StatusDistributionSummary = Field( description="The distribution of latencies for the completed requests.", ) - prompts_token_count: StatusDistributionSummary = Field( + prompt_token_count: StatusDistributionSummary = Field( description=( "The distribution of token counts in the prompts for completed, " "errored, and all requests." From c7476ab42a0e3b2b9b2d96d3e058f90b3a21ab9c Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Thu, 10 Apr 2025 13:46:26 -0400 Subject: [PATCH 20/43] Update src/guidellm/benchmark/benchmark.py Co-authored-by: David Gray <40244437+dagrayvid@users.noreply.github.com> --- src/guidellm/benchmark/benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index 5b3df6a8..8ea4932b 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -530,7 +530,7 @@ class GenerativeBenchmark(Benchmark): "errored, and all requests." ) ) - outputs_token_count: StatusDistributionSummary = Field( + output_token_count: StatusDistributionSummary = Field( description=( "The distribution of token counts in the outputs for completed, " "errored, and all requests." From 3d8cd6294fcb253d9fdd11c7867e2e7934fe89d0 Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Thu, 10 Apr 2025 13:46:36 -0400 Subject: [PATCH 21/43] Update src/guidellm/benchmark/benchmark.py Co-authored-by: David Gray <40244437+dagrayvid@users.noreply.github.com> --- src/guidellm/benchmark/benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index 8ea4932b..875a7a0e 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -555,7 +555,7 @@ class GenerativeBenchmark(Benchmark): "completed, errored, and all requests." ), ) - outputs_tokens_per_second: StatusDistributionSummary = Field( + output_tokens_per_second: StatusDistributionSummary = Field( description=( "The distribution of output tokens per second for completed, " "errored, and all requests." From dbc4789b1fc407cb1b5238433dbf1f4f164ea07f Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Thu, 10 Apr 2025 13:46:41 -0400 Subject: [PATCH 22/43] Update src/guidellm/benchmark/benchmark.py Co-authored-by: David Gray <40244437+dagrayvid@users.noreply.github.com> --- src/guidellm/benchmark/benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index 875a7a0e..26f9622a 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -542,7 +542,7 @@ class GenerativeBenchmark(Benchmark): "milliseconds for completed, errored, and all requests." ), ) - times_per_output_tokens_ms: StatusDistributionSummary = Field( + times_per_output_token_ms: StatusDistributionSummary = Field( description=( "The distribution of latencies per output token in milliseconds for " "completed, errored, and all requests. " From 649a86d788eeb1cc12d27f24bcf4b312b56e6806 Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Thu, 10 Apr 2025 17:48:52 +0000 Subject: [PATCH 23/43] Updates for pydantic export with polymorphism and general cleanup --- src/guidellm/backend/__init__.py | 4 +- src/guidellm/backend/openai.py | 2 +- src/guidellm/backend/response.py | 9 +- src/guidellm/benchmark/aggregator.py | 73 ++++++++-- src/guidellm/benchmark/benchmark.py | 18 +-- src/guidellm/benchmark/benchmarker.py | 10 +- src/guidellm/benchmark/output.py | 46 ++++--- src/guidellm/benchmark/profile.py | 4 +- src/guidellm/benchmark/test.py | 52 -------- src/guidellm/dataset/__init__.py | 8 +- src/guidellm/dataset/synthetic.py | 6 +- src/guidellm/objects/__init__.py | 5 +- src/guidellm/objects/pydantic.py | 28 ++++ src/guidellm/objects/serializable.py | 169 ------------------------ src/guidellm/objects/statistics.py | 10 +- src/guidellm/request/__init__.py | 2 + src/guidellm/request/loader.py | 12 +- src/guidellm/request/request.py | 4 +- src/guidellm/scheduler/__init__.py | 10 +- src/guidellm/scheduler/result.py | 15 ++- src/guidellm/scheduler/scheduler.py | 4 +- src/guidellm/scheduler/strategy.py | 4 +- src/guidellm/scheduler/worker.py | 16 ++- src/guidellm/utils/__init__.py | 10 +- src/guidellm/utils/colors.py | 16 --- src/guidellm/utils/injector.py | 70 ---------- src/guidellm/utils/text.py | 1 + tests/unit/objects/test_serializable.py | 4 +- 28 files changed, 218 insertions(+), 394 deletions(-) delete mode 100644 src/guidellm/benchmark/test.py create mode 100644 src/guidellm/objects/pydantic.py delete mode 100644 src/guidellm/objects/serializable.py delete mode 100644 src/guidellm/utils/injector.py diff --git a/src/guidellm/backend/__init__.py b/src/guidellm/backend/__init__.py index a45a66a7..8dc2ef8f 100644 --- a/src/guidellm/backend/__init__.py +++ b/src/guidellm/backend/__init__.py @@ -2,7 +2,7 @@ Backend, BackendType, ) -from .openai import OpenAIHTTPBackend +from .openai import CHAT_COMPLETIONS_PATH, TEXT_COMPLETIONS_PATH, OpenAIHTTPBackend from .response import ( RequestArgs, ResponseSummary, @@ -18,4 +18,6 @@ "Backend", "BackendType", "OpenAIHTTPBackend", + "TEXT_COMPLETIONS_PATH", + "CHAT_COMPLETIONS_PATH", ] diff --git a/src/guidellm/backend/openai.py b/src/guidellm/backend/openai.py index 3618465b..3b5ade98 100644 --- a/src/guidellm/backend/openai.py +++ b/src/guidellm/backend/openai.py @@ -16,7 +16,7 @@ ) from guidellm.config import settings -__all__ = ["OpenAIHTTPBackend"] +__all__ = ["OpenAIHTTPBackend", "TEXT_COMPLETIONS_PATH", "CHAT_COMPLETIONS_PATH"] TEXT_COMPLETIONS_PATH = "/v1/completions" diff --git a/src/guidellm/backend/response.py b/src/guidellm/backend/response.py index ec7e8e7c..9dc74578 100644 --- a/src/guidellm/backend/response.py +++ b/src/guidellm/backend/response.py @@ -1,8 +1,9 @@ from typing import Any, Dict, Literal, Optional -from pydantic import BaseModel, computed_field +from pydantic import computed_field from guidellm.config import settings +from guidellm.objects.pydantic import StandardBaseModel __all__ = [ "StreamingResponseType", @@ -15,7 +16,7 @@ StreamingResponseType = Literal["start", "iter"] -class StreamingTextResponse(BaseModel): +class StreamingTextResponse(StandardBaseModel): """ A model representing the response content for a streaming text request. @@ -40,7 +41,7 @@ class StreamingTextResponse(BaseModel): request_id: Optional[str] = None -class RequestArgs(BaseModel): +class RequestArgs(StandardBaseModel): """ A model representing the arguments for a request to a backend. Biases towards an HTTP request, but can be used for other types of backends. @@ -60,7 +61,7 @@ class RequestArgs(BaseModel): http2: Optional[bool] = None -class ResponseSummary(BaseModel): +class ResponseSummary(StandardBaseModel): """ A model representing a summary of a backend request. Always returned as the final iteration of a streaming request. diff --git a/src/guidellm/benchmark/aggregator.py b/src/guidellm/benchmark/aggregator.py index 10ff8d32..c2f375a6 100644 --- a/src/guidellm/benchmark/aggregator.py +++ b/src/guidellm/benchmark/aggregator.py @@ -13,7 +13,7 @@ Union, ) -from pydantic import BaseModel, Field +from pydantic import Field from guidellm.backend import ResponseSummary from guidellm.benchmark.benchmark import ( @@ -24,15 +24,33 @@ GenerativeTextErrorStats, GenerativeTextResponseStats, ) -from guidellm.benchmark.profile import Profile +from guidellm.benchmark.profile import ( + AsyncProfile, + ConcurrentProfile, + Profile, + SweepProfile, + SynchronousProfile, + ThroughputProfile, +) from guidellm.config import settings -from guidellm.objects import RunningStats, Serializable, TimeRunningStats -from guidellm.request import GenerationRequest +from guidellm.objects import RunningStats, StandardBaseModel, TimeRunningStats +from guidellm.request import ( + GenerationRequest, + GenerativeRequestLoaderDescription, + RequestLoaderDescription, +) from guidellm.scheduler import ( REQ, RES, + AsyncConstantStrategy, + AsyncPoissonStrategy, + ConcurrentStrategy, + GenerativeRequestsWorkerDescription, SchedulerRequestResult, SchedulingStrategy, + SynchronousStrategy, + ThroughputStrategy, + WorkerDescription, ) from guidellm.utils import check_load_processor @@ -43,7 +61,7 @@ ] -class BenchmarkAggregator(ABC, BaseModel, Generic[BENCH, REQ, RES]): +class BenchmarkAggregator(ABC, StandardBaseModel, Generic[BENCH, REQ, RES]): """ A pydantic base class representing the base class for aggregating benchmark results. The purpose is to receive and process results from a Benchmarker as it iterates @@ -55,25 +73,43 @@ class BenchmarkAggregator(ABC, BaseModel, Generic[BENCH, REQ, RES]): fully calculated. """ + type_: Literal["benchmark_aggregator"] = "benchmark_aggregator" run_id: str = Field( description=( "The unique identifier for the encompasing benchmark run that this " "benchmark was a part of." ) ) - profile: Profile = Field( + profile: Union[ + AsyncProfile, + SweepProfile, + ConcurrentProfile, + ThroughputProfile, + SynchronousProfile, + Profile, + ] = Field( description=( "The profile used for the entire benchamrk run that the strategy for " "the active benchmark was pulled from." - ) + ), + discriminator="type_", ) strategy_index: int = Field( description=( "The index of the strategy in the profile that was used for this benchmark." ) ) - strategy: SchedulingStrategy = Field( - description="The scheduling strategy used to run this benchmark. " + strategy: Union[ + ConcurrentStrategy, + SchedulingStrategy, + ThroughputStrategy, + SynchronousStrategy, + AsyncPoissonStrategy, + AsyncConstantStrategy, + SchedulingStrategy, + ] = Field( + description="The scheduling strategy used to run this benchmark. ", + discriminator="type_", ) max_number: Optional[int] = Field( description="The maximum number of requests to run for this benchmark, if any." @@ -105,17 +141,23 @@ class BenchmarkAggregator(ABC, BaseModel, Generic[BENCH, REQ, RES]): "if any. These are requests that were not included in the final results." ) ) - worker_description: Optional[Serializable] = Field( + worker_description: Optional[ + Union[GenerativeRequestsWorkerDescription, WorkerDescription] + ] = Field( description=( "The description and specifics for the worker used to resolve requests " "for this benchmark." - ) + ), + discriminator="type_", ) - request_loader_description: Optional[Serializable] = Field( + request_loader_description: Optional[ + Union[GenerativeRequestLoaderDescription, RequestLoaderDescription] + ] = Field( description=( "The description and specifics for the request loader used to create " "requests for this benchmark." - ) + ), + discriminator="type_", ) extras: Dict[str, Any] = Field( description=( @@ -123,7 +165,7 @@ class BenchmarkAggregator(ABC, BaseModel, Generic[BENCH, REQ, RES]): ) ) - results: List[SchedulerRequestResult[GenerationRequest, ResponseSummary]] = Field( + results: List[SchedulerRequestResult[REQ, RES]] = Field( default_factory=list, description=( "The list of all results from the benchmark (complete, incomplete, error), " @@ -423,6 +465,9 @@ def compile(self) -> BENCH: class GenerativeBenchmarkAggregator( BenchmarkAggregator[GenerativeBenchmark, GenerationRequest, ResponseSummary] ): + type_: Literal["generative_benchmark_aggregator"] = ( + "generative_benchmark_aggregator" + ) processor: Optional[Union[str, Path, Any]] = Field( description=( "The tokenizer to use for calculating token counts when none are " diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index 26f9622a..23dc8385 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -6,7 +6,7 @@ from guidellm.benchmark.profile import Profile from guidellm.objects import ( - Serializable, + StandardBaseModel, StatusDistributionSummary, ) from guidellm.scheduler import SchedulerRequestInfo, SchedulingStrategy @@ -22,7 +22,7 @@ ] -class BenchmarkArgs(Serializable): +class BenchmarkArgs(StandardBaseModel): """ A serializable model representing the arguments used to specify a benchmark run and how data was collected for it. @@ -74,7 +74,7 @@ class BenchmarkArgs(Serializable): ) -class BenchmarkRunStats(Serializable): +class BenchmarkRunStats(StandardBaseModel): """ A serializable model representing the run process statistics for the entire benchmark run across all requests including warmup and cooldown. @@ -196,7 +196,7 @@ def total(self) -> int: return self.total_successful + self.total_incomplete + self.total_errored -class Benchmark(Serializable): +class Benchmark(StandardBaseModel): """ The base serializable model representing a benchmark run and its results. Specific benchmarker implementations should extend this model to include @@ -228,13 +228,13 @@ class Benchmark(Serializable): "The process statistics for the entire benchmark run across all requests." ) ) - worker: Optional[Serializable] = Field( + worker: Optional[StandardBaseModel] = Field( description=( "The description and specifics for the worker used to resolve requests " "for this benchmark." ) ) - request_loader: Optional[Serializable] = Field( + request_loader: Optional[StandardBaseModel] = Field( description=( "The description and specifics for the request loader used to create " "requests for this benchmark." @@ -257,7 +257,7 @@ class Benchmark(Serializable): BENCH = TypeVar("BENCH", bound=Benchmark) -class GenerativeTextResponseStats(Serializable): +class GenerativeTextResponseStats(StandardBaseModel): """ A serializable model representing the request values, response values, and statistics for a generative text response. @@ -660,8 +660,8 @@ def from_stats( errored: List[GenerativeTextErrorStats], args: BenchmarkArgs, run_stats: BenchmarkRunStats, - worker: Optional[Serializable], - requests_loader: Optional[Serializable], + worker: Optional[StandardBaseModel], + requests_loader: Optional[StandardBaseModel], extras: Optional[Dict[str, Any]], ) -> "GenerativeBenchmark": """ diff --git a/src/guidellm/benchmark/benchmarker.py b/src/guidellm/benchmark/benchmarker.py index d471e3f1..b53ec664 100644 --- a/src/guidellm/benchmark/benchmarker.py +++ b/src/guidellm/benchmark/benchmarker.py @@ -20,7 +20,7 @@ from guidellm.benchmark.aggregator import AGG, BENCH, GenerativeBenchmarkAggregator from guidellm.benchmark.benchmark import GenerativeBenchmark from guidellm.benchmark.profile import Profile -from guidellm.objects import Serializable +from guidellm.objects import StandardBaseModel from guidellm.request import GenerationRequest from guidellm.scheduler import ( REQ, @@ -35,7 +35,7 @@ __all__ = ["Benchmarker", "BenchmarkerResult", "GenerativeBenchmarker"] -class BenchmarkerResult(Serializable, Generic[AGG, BENCH, REQ, RES]): +class BenchmarkerResult(StandardBaseModel, Generic[AGG, BENCH, REQ, RES]): type_: Literal[ "run_start", "run_complete", @@ -54,7 +54,7 @@ class BenchmarkerResult(Serializable, Generic[AGG, BENCH, REQ, RES]): current_result: Optional[SchedulerRequestResult[REQ, RES]] = None -class BenchmarkerStrategyLimits(Serializable): +class BenchmarkerStrategyLimits(StandardBaseModel): requests_loader_size: Optional[int] = Field( description="Size of the request loader.", ) @@ -125,7 +125,7 @@ def __init__( self, worker: RequestsWorker[REQ, RES], request_loader: Iterable[REQ], - requests_loader_description: Optional[Serializable] = None, + requests_loader_description: Optional[StandardBaseModel] = None, benchmark_save_extras: Optional[Dict[str, Any]] = None, ): self.worker = worker @@ -291,7 +291,7 @@ def __init__( self, backend: Backend, request_loader: Iterable[GenerationRequest], - request_loader_description: Optional[Serializable] = None, + request_loader_description: Optional[StandardBaseModel] = None, benchmark_save_extras: Optional[Dict[str, Any]] = None, processor: Optional[Union[str, Path, PreTrainedTokenizer]] = None, processor_args: Optional[Dict[str, Any]] = None, diff --git a/src/guidellm/benchmark/output.py b/src/guidellm/benchmark/output.py index aae0fe89..a7134d4b 100644 --- a/src/guidellm/benchmark/output.py +++ b/src/guidellm/benchmark/output.py @@ -15,7 +15,7 @@ SweepProfile, ThroughputProfile, ) -from guidellm.objects import Serializable +from guidellm.objects import StandardBaseModel from guidellm.scheduler import strategy_display_str from guidellm.utils import Colors @@ -26,7 +26,7 @@ ] -class GenerativeBenchmarksReport(Serializable): +class GenerativeBenchmarksReport(StandardBaseModel): benchmarks: List[GenerativeBenchmark] @@ -204,14 +204,13 @@ def print_benchmarks_info(self): headers = [ "Benchmark", "Start Time", + "End Time", "Duration (sec)", - "Requests / sec", - "Requests Concurrency", - "Requests Made \n(comp / err)", - "Prompt Tok / Req \n(comp / err)", - "Output Tok / Req \n(comp / err)", - "Prompt Tokens \n(comp / err)", - "Output Tokens \n(comp / err)", + "Requests Made \n(comp / inc / err)", + "Prompt Tok / Req \n(comp / inc / err)", + "Output Tok / Req \n(comp / inc / err)", + "Prompt Tok Total \n(comp / inc / err)", + "Output Tok Total \n(comp / inc / err)", ] rows = [] @@ -220,25 +219,32 @@ def print_benchmarks_info(self): [ strategy_display_str(benchmark.args.strategy), f"{datetime.fromtimestamp(benchmark.start_time).strftime("%H:%M:%S")}", + f"{datetime.fromtimestamp(benchmark.end_time).strftime("%H:%M:%S")}", f"{(benchmark.end_time - benchmark.start_time):.1f}", - f"{benchmark.requests_per_second.successful.mean:.2f}", - f"{benchmark.requests_concurrency.successful.mean:.2f}", - (f"{benchmark.successful_total:>5} / {benchmark.errored_total}"), ( - f"{benchmark.prompts_token_count.successful.total_sum:.0f} / " - f"{benchmark.prompts_token_count.errored.total_sum:.0f}" + f"{benchmark.successful_total:>5} / " + f"{benchmark.incomplete_total} / " + f"{benchmark.errored_total}" ), ( - f"{benchmark.prompts_token_count.successful.mean:.0f} / " - f"{benchmark.prompts_token_count.errored.mean:.0f}" + f"{benchmark.prompts_token_count.successful.mean:>5.1f} / " + f"{benchmark.prompts_token_count.incomplete.mean:.1f} / " + f"{benchmark.prompts_token_count.errored.mean:.1f}" ), ( - f"{benchmark.outputs_token_count.successful.total_sum:.0f} / " - f"{benchmark.outputs_token_count.errored.total_sum:.0f}" + f"{benchmark.outputs_token_count.successful.mean:>5.1f} / " + f"{benchmark.outputs_token_count.incomplete.mean:.1f} / " + f"{benchmark.outputs_token_count.errored.mean:.1f}" ), ( - f"{benchmark.outputs_token_count.successful.mean:.0f} / " - f"{benchmark.outputs_token_count.errored.mean:.0f}" + f"{benchmark.prompts_token_count.successful.total_sum:>6.0f} / " + f"{benchmark.prompts_token_count.incomplete.total_sum:.0f} / " + f"{benchmark.prompts_token_count.errored.total_sum:.0f}" + ), + ( + f"{benchmark.outputs_token_count.successful.total_sum:>6.0f} / " + f"{benchmark.outputs_token_count.incomplete.total_sum:.0f} / " + f"{benchmark.outputs_token_count.errored.total_sum:.0f}" ), ] ) diff --git a/src/guidellm/benchmark/profile.py b/src/guidellm/benchmark/profile.py index 23b8dfab..6b7c9070 100644 --- a/src/guidellm/benchmark/profile.py +++ b/src/guidellm/benchmark/profile.py @@ -4,7 +4,7 @@ from pydantic import Field, computed_field from guidellm.config import settings -from guidellm.objects import Serializable +from guidellm.objects import StandardBaseModel from guidellm.scheduler import ( AsyncConstantStrategy, AsyncPoissonStrategy, @@ -29,7 +29,7 @@ ProfileType = Literal["synchronous", "concurrent", "throughput", "async", "sweep"] -class Profile(Serializable): +class Profile(StandardBaseModel): type_: ProfileType = Field( description="The type of benchmarking profile to use.", ) diff --git a/src/guidellm/benchmark/test.py b/src/guidellm/benchmark/test.py deleted file mode 100644 index 3e73dfdc..00000000 --- a/src/guidellm/benchmark/test.py +++ /dev/null @@ -1,52 +0,0 @@ -import asyncio -import json - -from guidellm.benchmark.benchmark import GenerativeBenchmark -from guidellm.benchmark.entrypoints import benchmark_generative_text -from guidellm.benchmark.output import GenerativeBenchmarksConsole - - -def run_benchmark_synthetic(): - # logging.basicConfig(level=logging.DEBUG) - results = asyncio.run( - benchmark_generative_text( - target="http://192.168.4.13:8000", - backend_type="openai_http", - backend_args=None, - model="neuralmagic/Qwen2.5-7B-quantized.w8a8", - processor=None, - processor_args=None, - data='{"prompt_tokens": 128, "output_tokens": 64}', - data_args=None, - data_sampler=None, - rate_type="sweep", - rate=10, - max_seconds=None, - max_requests=50, - warmup_percent=None, - cooldown_percent=None, - show_progress=True, - show_progress_scheduler_stats=True, - output_console=True, - output_path="benchmarks.json", - output_extras=None, - random_seed=42, - ) - ) - - -def print_benchmark(): - with open("benchmarks.json") as file: - data = json.load(file) - - benchmarks = [ - GenerativeBenchmark.model_validate_json(json.dumps(bench)) - for bench in data["benchmarks"] - ] - console = GenerativeBenchmarksConsole(benchmarks) - console.print() - - -if __name__ == "__main__": - run_benchmark_synthetic() - # print_benchmark() diff --git a/src/guidellm/dataset/__init__.py b/src/guidellm/dataset/__init__.py index edf1f1c9..20d68e64 100644 --- a/src/guidellm/dataset/__init__.py +++ b/src/guidellm/dataset/__init__.py @@ -3,7 +3,11 @@ from .file import FileDatasetCreator from .hf_datasets import HFDatasetsCreator from .in_memory import InMemoryDatasetCreator -from .synthetic import SyntheticDatasetCreator +from .synthetic import ( + SyntheticDatasetConfig, + SyntheticDatasetCreator, + SyntheticTextItemsGenerator, +) __all__ = [ "DatasetCreator", @@ -13,4 +17,6 @@ "FileDatasetCreator", "InMemoryDatasetCreator", "SyntheticDatasetCreator", + "SyntheticDatasetConfig", + "SyntheticTextItemsGenerator", ] diff --git a/src/guidellm/dataset/synthetic.py b/src/guidellm/dataset/synthetic.py index dbd73baa..a8e94b08 100644 --- a/src/guidellm/dataset/synthetic.py +++ b/src/guidellm/dataset/synthetic.py @@ -16,7 +16,11 @@ from guidellm.dataset.creator import ColumnInputTypes, DatasetCreator from guidellm.utils import EndlessTextCreator, IntegerRangeSampler, check_load_processor -__all__ = ["SyntheticDatasetCreator"] +__all__ = [ + "SyntheticDatasetCreator", + "SyntheticDatasetConfig", + "SyntheticTextItemsGenerator", +] class SyntheticDatasetConfig(BaseModel): diff --git a/src/guidellm/objects/__init__.py b/src/guidellm/objects/__init__.py index c2a75891..9329a70f 100644 --- a/src/guidellm/objects/__init__.py +++ b/src/guidellm/objects/__init__.py @@ -1,4 +1,4 @@ -from .serializable import Serializable, SerializableFileType +from .pydantic import StandardBaseModel from .statistics import ( DistributionSummary, Percentiles, @@ -11,8 +11,7 @@ "Percentiles", "DistributionSummary", "StatusDistributionSummary", - "Serializable", - "SerializableFileType", + "StandardBaseModel", "RunningStats", "TimeRunningStats", ] diff --git a/src/guidellm/objects/pydantic.py b/src/guidellm/objects/pydantic.py new file mode 100644 index 00000000..1c5754cc --- /dev/null +++ b/src/guidellm/objects/pydantic.py @@ -0,0 +1,28 @@ +from typing import Any + +from loguru import logger +from pydantic import BaseModel, ConfigDict + +__all__ = ["StandardBaseModel"] + + +class StandardBaseModel(BaseModel): + """ + A base class for models that require YAML and JSON serialization and + deserialization. + """ + + model_config = ConfigDict( + extra="allow", + use_enum_values=True, + validate_assignment=True, + from_attributes=True, + ) + + def __init__(self, /, **data: Any) -> None: + super().__init__(**data) + logger.debug( + "Initialized new instance of {} with data: {}", + self.__class__.__name__, + data, + ) diff --git a/src/guidellm/objects/serializable.py b/src/guidellm/objects/serializable.py deleted file mode 100644 index 7977df62..00000000 --- a/src/guidellm/objects/serializable.py +++ /dev/null @@ -1,169 +0,0 @@ -from pathlib import Path -from typing import Any, Literal, Union, get_args - -import yaml -from loguru import logger -from pydantic import BaseModel, ConfigDict - -__all__ = ["Serializable", "SerializableFileType"] - - -SerializableFileType = Literal["yaml", "json"] - - -class Serializable(BaseModel): - """ - A base class for models that require YAML and JSON serialization and - deserialization. - """ - - model_config = ConfigDict( - extra="allow", - use_enum_values=True, - validate_assignment=True, - from_attributes=True, - ) - - def __init__(self, /, **data: Any) -> None: - super().__init__(**data) - logger.debug( - "Initialized new instance of {} with data: {}", - self.__class__.__name__, - data, - ) - - def to_yaml(self) -> str: - """ - Serialize the model to a YAML string. - - :return: YAML string representation of the model. - """ - logger.debug("Serializing to YAML... {}", self) - - return yaml.dump(self.model_dump()) - - @classmethod - def from_yaml(cls, data: str): - """ - Deserialize a YAML string to a model instance. - - :param data: YAML string to deserialize. - :return: An instance of the model. - """ - logger.debug("Deserializing from YAML... {}", data) - - return cls.model_validate(yaml.safe_load(data)) - - def to_json(self) -> str: - """ - Serialize the model to a JSON string. - - :return: JSON string representation of the model. - """ - logger.debug("Serializing to JSON... {}", self) - - return self.model_dump_json() - - @classmethod - def from_json(cls, data: str): - """ - Deserialize a JSON string to a model instance. - - :param data: JSON string to deserialize. - :return: An instance of the model. - """ - logger.debug("Deserializing from JSON... {}", data) - - return cls.model_validate_json(data) - - def save_file( - self, - path: Union[str, Path], - type_: SerializableFileType = "yaml", - ) -> str: - """ - Save the model to a file in either YAML or JSON format. - - :param path: Path to the exact file or the containing directory. - If it is a directory, the file name will be inferred from the class name. - :param type_: Optional type to save ('yaml' or 'json'). - If not provided and the path has an extension, - it will be inferred to save in that format. - If not provided and the path does not have an extension, - it will save in YAML format. - :return: The path to the saved file. - """ - logger.debug("Saving to file... {} with format: {}", path, type_) - - if isinstance(path, str): - path = Path(path) - - if path.suffix: - # is a file - ext = path.suffix[1:].lower() - if type_ not in get_args(SerializableFileType): - raise ValueError( - f"Unsupported file extension: {type_}. " - f"Expected one of {SerializableFileType} " - f"for {path}" - ) - type_ = ext # type: ignore # noqa: PGH003 - else: - # is a directory - file_name = f"{self.__class__.__name__.lower()}.{type_}" - path = path / file_name - - path.parent.mkdir(parents=True, exist_ok=True) - - with path.open("w") as file: - if type_ == "yaml": - file.write(self.to_yaml()) - elif type_ == "json": - file.write(self.to_json()) - else: - raise ValueError( - f"Unsupported file extension: {type_}" - f"Expected one of {SerializableFileType} " - f"for {path}" - ) - - logger.info("Successfully saved {} to {}", self.__class__.__name__, path) - - return str(path) - - @classmethod - def load_file(cls, path: Union[str, Path]): - """ - Load a model from a file in either YAML or JSON format. - - :param path: Path to the file. - :return: An instance of the model. - """ - logger.debug("Loading from file... {}", path) - - if isinstance(path, str): - path = Path(path) - - if not path.exists(): - raise FileNotFoundError(f"File not found: {path}") - - if not path.is_file(): - raise ValueError(f"Path is not a file: {path}") - - extension = path.suffix[1:].lower() - - with path.open() as file: - data = file.read() - - if extension == "yaml": - obj = cls.from_yaml(data) - elif extension == "json": - obj = cls.from_json(data) - else: - raise ValueError( - f"Unsupported file extension: {extension}" - f"Expected one of {SerializableFileType} " - f"for {path}" - ) - - return obj diff --git a/src/guidellm/objects/statistics.py b/src/guidellm/objects/statistics.py index ee3f47b6..05ccf974 100644 --- a/src/guidellm/objects/statistics.py +++ b/src/guidellm/objects/statistics.py @@ -6,7 +6,7 @@ import numpy as np from pydantic import Field, computed_field -from guidellm.objects import Serializable +from guidellm.objects import StandardBaseModel __all__ = [ "Percentiles", @@ -17,7 +17,7 @@ ] -class Percentiles(Serializable): +class Percentiles(StandardBaseModel): """ A serializable model representing percentiles of a distribution. """ @@ -54,7 +54,7 @@ class Percentiles(Serializable): ) -class DistributionSummary(Serializable): +class DistributionSummary(StandardBaseModel): """ A serializable model representing a statistical summary for a given distribution of numerical values. @@ -350,7 +350,7 @@ def from_iterable_request_times( ) -class StatusDistributionSummary(Serializable): +class StatusDistributionSummary(StandardBaseModel): """ A serializable model representing distribution summary statistics based on groupings of status (e.g., successful, incomplete, error) for a given @@ -685,7 +685,7 @@ def from_iterable_request_times( ) -class RunningStats(Serializable): +class RunningStats(StandardBaseModel): start_time: float = Field( default_factory=timer.time, ) diff --git a/src/guidellm/request/__init__.py b/src/guidellm/request/__init__.py index b0420694..bdd87389 100644 --- a/src/guidellm/request/__init__.py +++ b/src/guidellm/request/__init__.py @@ -2,11 +2,13 @@ GenerativeRequestLoader, GenerativeRequestLoaderDescription, RequestLoader, + RequestLoaderDescription, ) from .request import GenerationRequest __all__ = [ "RequestLoader", + "RequestLoaderDescription", "GenerativeRequestLoaderDescription", "GenerativeRequestLoader", "GenerationRequest", diff --git a/src/guidellm/request/loader.py b/src/guidellm/request/loader.py index 063b0b9e..61b9b500 100644 --- a/src/guidellm/request/loader.py +++ b/src/guidellm/request/loader.py @@ -16,16 +16,21 @@ from transformers import PreTrainedTokenizer from guidellm.dataset import ColumnInputTypes, load_dataset -from guidellm.objects import Serializable +from guidellm.objects import StandardBaseModel from guidellm.request.request import GenerationRequest __all__ = [ + "RequestLoaderDescription", "RequestLoader", "GenerativeRequestLoaderDescription", "GenerativeRequestLoader", ] +class RequestLoaderDescription(StandardBaseModel): + type_: Literal["request_loader"] = "request_loader" + + class RequestLoader(Iterable): @abstractmethod def __iter__(self): ... @@ -35,10 +40,11 @@ def __len__(self): ... @property @abstractmethod - def description(self) -> Serializable: ... + def description(self) -> RequestLoaderDescription: ... -class GenerativeRequestLoaderDescription(Serializable): +class GenerativeRequestLoaderDescription(RequestLoaderDescription): + type_: Literal["generative_request_loader"] = "generative_request_loader" data: str data_args: Optional[Dict[str, Any]] processor: str diff --git a/src/guidellm/request/request.py b/src/guidellm/request/request.py index 400c15c1..216ca0e9 100644 --- a/src/guidellm/request/request.py +++ b/src/guidellm/request/request.py @@ -3,12 +3,12 @@ from pydantic import Field -from guidellm.objects.serializable import Serializable +from guidellm.objects.pydantic import StandardBaseModel __all__ = ["GenerationRequest"] -class GenerationRequest(Serializable): +class GenerationRequest(StandardBaseModel): """ A class representing a request for generation. This class is used to encapsulate the details of a generation request, diff --git a/src/guidellm/scheduler/__init__.py b/src/guidellm/scheduler/__init__.py index e641dc1b..73256a60 100644 --- a/src/guidellm/scheduler/__init__.py +++ b/src/guidellm/scheduler/__init__.py @@ -18,7 +18,10 @@ from .types import REQ, RES from .worker import ( GenerativeRequestsWorker, + GenerativeRequestsWorkerDescription, RequestsWorker, + ResolveStatus, + WorkerDescription, WorkerProcessRequest, WorkerProcessResult, ) @@ -39,8 +42,11 @@ "strategy_display_str", "REQ", "RES", - "GenerativeRequestsWorker", - "RequestsWorker", "WorkerProcessRequest", "WorkerProcessResult", + "ResolveStatus", + "WorkerDescription", + "RequestsWorker", + "GenerativeRequestsWorkerDescription", + "GenerativeRequestsWorker", ] diff --git a/src/guidellm/scheduler/result.py b/src/guidellm/scheduler/result.py index 34de5770..282c9538 100644 --- a/src/guidellm/scheduler/result.py +++ b/src/guidellm/scheduler/result.py @@ -4,7 +4,7 @@ Optional, ) -from guidellm.objects import Serializable +from guidellm.objects import StandardBaseModel from guidellm.scheduler.strategy import SchedulingStrategy from guidellm.scheduler.types import REQ, RES @@ -16,7 +16,7 @@ ] -class SchedulerRunInfo(Serializable): +class SchedulerRunInfo(StandardBaseModel): """ Information about the current run of the scheduler. This class holds metadata about the scheduling run, @@ -54,7 +54,7 @@ class SchedulerRunInfo(Serializable): completed_requests: int = 0 -class SchedulerRequestInfo(Serializable): +class SchedulerRequestInfo(StandardBaseModel): """ Information about a specific request run through the scheduler. This class holds metadata about the request, including @@ -86,7 +86,7 @@ class SchedulerRequestInfo(Serializable): process_id: int = -1 -class SchedulerResult(Serializable): +class SchedulerResult(StandardBaseModel): """ The yielded, iterative result for a scheduler run. These are triggered on the start and end of the run, @@ -111,6 +111,7 @@ class SchedulerResult(Serializable): and completed during the run. """ + pydantic_type: Literal["scheduler_result"] = "scheduler_result" type_: Literal[ "run_start", "run_complete", @@ -125,6 +126,12 @@ class SchedulerRequestResult( SchedulerResult, Generic[REQ, RES], ): + pydantic_type: Literal["scheduler_request_result"] = "scheduler_request_result" + type_: Literal[ + "request_scheduled", + "request_start", + "request_complete", + ] request: REQ request_info: SchedulerRequestInfo response: Optional[RES] = None diff --git a/src/guidellm/scheduler/scheduler.py b/src/guidellm/scheduler/scheduler.py index b5fa10ba..56960fe3 100644 --- a/src/guidellm/scheduler/scheduler.py +++ b/src/guidellm/scheduler/scheduler.py @@ -115,7 +115,9 @@ async def run( if max_duration is not None and max_duration < 0: raise ValueError(f"Invalid max_duration: {max_duration}") - with multiprocessing.Manager() as manager, ProcessPoolExecutor(max_workers=scheduling_strategy.processes_limit) as executor: + with multiprocessing.Manager() as manager, ProcessPoolExecutor( + max_workers=scheduling_strategy.processes_limit + ) as executor: futures, requests_queue, responses_queue = await self._start_processes( manager, executor, scheduling_strategy ) diff --git a/src/guidellm/scheduler/strategy.py b/src/guidellm/scheduler/strategy.py index d0eab157..0060e27e 100644 --- a/src/guidellm/scheduler/strategy.py +++ b/src/guidellm/scheduler/strategy.py @@ -12,7 +12,7 @@ from pydantic import Field from guidellm.config import settings -from guidellm.objects import Serializable +from guidellm.objects import StandardBaseModel __all__ = [ "StrategyType", @@ -29,7 +29,7 @@ StrategyType = Literal["synchronous", "concurrent", "throughput", "constant", "poisson"] -class SchedulingStrategy(Serializable): +class SchedulingStrategy(StandardBaseModel): """ An abstract base class for scheduling strategies. This class defines the interface for scheduling requests and provides diff --git a/src/guidellm/scheduler/worker.py b/src/guidellm/scheduler/worker.py index 2611b122..47756e5b 100644 --- a/src/guidellm/scheduler/worker.py +++ b/src/guidellm/scheduler/worker.py @@ -26,7 +26,7 @@ ResponseSummary, StreamingTextResponse, ) -from guidellm.objects import Serializable +from guidellm.objects import StandardBaseModel from guidellm.request import GenerationRequest from guidellm.scheduler.result import SchedulerRequestInfo from guidellm.scheduler.types import REQ, RES @@ -34,7 +34,10 @@ __all__ = [ "WorkerProcessRequest", "WorkerProcessResult", + "ResolveStatus", + "WorkerDescription", "RequestsWorker", + "GenerativeRequestsWorkerDescription", "GenerativeRequestsWorker", ] @@ -66,6 +69,10 @@ class ResolveStatus: request_end: float +class WorkerDescription(StandardBaseModel): + type_: Literal["worker"] = "worker" + + class RequestsWorker(ABC, Generic[REQ, RES]): """ An abstract base class for a worker that processes requests. @@ -79,7 +86,7 @@ class RequestsWorker(ABC, Generic[REQ, RES]): @property @abstractmethod - def description(self) -> Serializable: + def description(self) -> WorkerDescription: """ An abstract property that must be implemented by subclasses. This property should return a Serializable class representing the information @@ -256,7 +263,8 @@ def _task_done(_: asyncio.Task): ) -class GenerativeRequestsWorkerDescription(Serializable): +class GenerativeRequestsWorkerDescription(WorkerDescription): + type_: Literal["generative_requests_worker"] = "generative_requests_worker" backend_type: BackendType backend_target: str backend_model: str @@ -279,7 +287,7 @@ def __init__(self, backend: Backend): self.backend = backend @property - def description(self) -> Serializable: + def description(self) -> StandardBaseModel: """ Get the description of the worker. :return: The description of the worker. diff --git a/src/guidellm/utils/__init__.py b/src/guidellm/utils/__init__.py index 99411070..3d538f37 100644 --- a/src/guidellm/utils/__init__.py +++ b/src/guidellm/utils/__init__.py @@ -3,7 +3,14 @@ check_load_processor, ) from .random import IntegerRangeSampler -from .text import EndlessTextCreator, clean_text, filter_text, load_text, split_text +from .text import ( + EndlessTextCreator, + clean_text, + filter_text, + is_puncutation, + load_text, + split_text, +) __all__ = [ "Colors", @@ -12,5 +19,6 @@ "clean_text", "split_text", "load_text", + "is_puncutation", "EndlessTextCreator", ] diff --git a/src/guidellm/utils/colors.py b/src/guidellm/utils/colors.py index 4689c94f..e4d60d52 100644 --- a/src/guidellm/utils/colors.py +++ b/src/guidellm/utils/colors.py @@ -6,19 +6,3 @@ class Colors: PROGRESS: str = "dark_slate_gray1" SUCCESS: str = "chartreuse1" ERROR: str = "orange_red1" - - -import gzip -from pathlib import Path - - -def compress_to_gz(input_path: str, encoding: str = "utf-8") -> str: - input_file = Path(input_path) - output_file = input_file.with_suffix(input_file.suffix + ".gz") - - with open(input_file, encoding=encoding) as f_in: - with gzip.open(output_file, "wt", encoding=encoding) as f_out: - f_out.writelines(f_in) - - print(f"Compressed file saved to: {output_file}") - return str(output_file) diff --git a/src/guidellm/utils/injector.py b/src/guidellm/utils/injector.py deleted file mode 100644 index fb5216aa..00000000 --- a/src/guidellm/utils/injector.py +++ /dev/null @@ -1,70 +0,0 @@ -from pathlib import Path -from typing import Union - -from pydantic import BaseModel - -from guidellm.config import settings -from guidellm.utils.text import load_text - -__all__ = ["create_report", "inject_data"] - - -def create_report(model: BaseModel, output_path: Union[str, Path]) -> Path: - """ - Creates a report from the model and saves it to the output path. - - :param model: the model to serialize and inject - :type model: BaseModel - :param output_path: the path, either a file or a directory, - to save the report to. If a directory, the report will be saved - as "report.html" inside of the directory. - :type output_path: str - :return: the path to the saved report - :rtype: str - """ - if not isinstance(output_path, Path): - output_path = Path(output_path) - - html_content = load_text(settings.report_generation.source) - report_content = inject_data( - model, - html_content, - settings.report_generation.report_html_match, - settings.report_generation.report_html_placeholder, - ) - - if not output_path.suffix: - # assume directory, save as report.html - output_path = output_path / "report.html" - - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(report_content) - - return output_path - - -def inject_data( - model: BaseModel, - html: str, - match: str, - placeholder: str, -) -> str: - """ - Injects the data from the model into the HTML while replacing the placeholder. - - :param model: the model to serialize and inject - :type model: BaseModel - :param html: the html to inject the data into - :type html: str - :param match: the string to match in the html to find the placeholder - :type match: str - :param placeholder: the placeholder to replace with the model data - inside of the placeholder - :type placeholder: str - :return: the html with the model data injected - :rtype: str - """ - model_str = model.json() - inject_str = match.replace(placeholder, model_str) - - return html.replace(match, inject_str) diff --git a/src/guidellm/utils/text.py b/src/guidellm/utils/text.py index 131e87f3..fb9abd6a 100644 --- a/src/guidellm/utils/text.py +++ b/src/guidellm/utils/text.py @@ -16,6 +16,7 @@ "clean_text", "split_text", "load_text", + "is_puncutation", "EndlessTextCreator", ] diff --git a/tests/unit/objects/test_serializable.py b/tests/unit/objects/test_serializable.py index b23ae062..4cf31d07 100644 --- a/tests/unit/objects/test_serializable.py +++ b/tests/unit/objects/test_serializable.py @@ -3,10 +3,10 @@ import pytest -from guidellm.objects.serializable import Serializable +from guidellm.objects.pydantic import StandardBaseModel -class ExampleModel(Serializable): +class ExampleModel(StandardBaseModel): name: str age: int From 418351262265c4581c1e89ef5d740a7eab2817cf Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Thu, 10 Apr 2025 19:13:25 +0000 Subject: [PATCH 24/43] Fixes for json / yaml output --- src/guidellm/benchmark/benchmark.py | 143 ++++++++++++++++++++-------- src/guidellm/benchmark/output.py | 96 ++++++++++++------- src/guidellm/benchmark/profile.py | 2 +- src/guidellm/benchmark/progress.py | 28 +++--- src/guidellm/dataset/synthetic.py | 2 +- src/guidellm/objects/statistics.py | 83 ++++++++++------ src/guidellm/scheduler/strategy.py | 2 +- 7 files changed, 236 insertions(+), 120 deletions(-) diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index 23dc8385..4646f141 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -1,15 +1,36 @@ import random import uuid -from typing import Any, Dict, List, Literal, Optional, TypeVar +from typing import Any, Dict, List, Literal, Optional, TypeVar, Union from pydantic import Field, computed_field -from guidellm.benchmark.profile import Profile +from guidellm.benchmark.profile import ( + AsyncProfile, + ConcurrentProfile, + Profile, + SweepProfile, + SynchronousProfile, + ThroughputProfile, +) from guidellm.objects import ( StandardBaseModel, StatusDistributionSummary, ) -from guidellm.scheduler import SchedulerRequestInfo, SchedulingStrategy +from guidellm.request import ( + GenerativeRequestLoaderDescription, + RequestLoaderDescription, +) +from guidellm.scheduler import ( + AsyncConstantStrategy, + AsyncPoissonStrategy, + ConcurrentStrategy, + GenerativeRequestsWorkerDescription, + SchedulerRequestInfo, + SchedulingStrategy, + SynchronousStrategy, + ThroughputStrategy, + WorkerDescription, +) __all__ = [ "BENCH", @@ -28,19 +49,36 @@ class BenchmarkArgs(StandardBaseModel): and how data was collected for it. """ - profile: Profile = Field( + profile: Union[ + AsyncProfile, + SweepProfile, + ConcurrentProfile, + ThroughputProfile, + SynchronousProfile, + Profile, + ] = Field( description=( "The profile used for the entire benchmark run that the strategy for " "this benchmark was pulled from." - ) + ), + discriminator="type_", ) strategy_index: int = Field( description=( "The index of the strategy in the profile that was used for this benchmark." ) ) - strategy: SchedulingStrategy = Field( - description="The scheduling strategy used to run this benchmark. " + strategy: Union[ + ConcurrentStrategy, + SchedulingStrategy, + ThroughputStrategy, + SynchronousStrategy, + AsyncPoissonStrategy, + AsyncConstantStrategy, + SchedulingStrategy, + ] = Field( + description="The scheduling strategy used to run this benchmark. ", + discriminator="type_", ) max_number: Optional[int] = Field( description="The maximum number of requests to run for this benchmark, if any." @@ -208,6 +246,7 @@ class Benchmark(StandardBaseModel): what rates and concurrency values to use for subsequent strategies. """ + type_: Literal["benchmark"] = "benchmark" id_: str = Field( default_factory=lambda: str(uuid.uuid4()), description="The unique identifier for the benchmark.", @@ -228,17 +267,23 @@ class Benchmark(StandardBaseModel): "The process statistics for the entire benchmark run across all requests." ) ) - worker: Optional[StandardBaseModel] = Field( - description=( - "The description and specifics for the worker used to resolve requests " - "for this benchmark." + worker: Optional[Union[GenerativeRequestsWorkerDescription, WorkerDescription]] = ( + Field( + description=( + "The description and specifics for the worker used to resolve requests " + "for this benchmark." + ), + discriminator="type_", ) ) - request_loader: Optional[StandardBaseModel] = Field( + request_loader: Optional[ + Union[GenerativeRequestLoaderDescription, RequestLoaderDescription] + ] = Field( description=( "The description and specifics for the request loader used to create " "requests for this benchmark." - ) + ), + discriminator="type_", ) extras: Dict[str, Any] = Field( description=( @@ -263,6 +308,7 @@ class GenerativeTextResponseStats(StandardBaseModel): statistics for a generative text response. """ + type_: Literal["generative_text_response"] = "generative_text_response" request_id: str = Field( description="The unique identifier for the request.", ) @@ -378,6 +424,7 @@ class GenerativeTextErrorStats(GenerativeTextResponseStats): error message and optional properties given the error occurred. """ + type_: Literal["generative_text_error"] = "generative_text_error" error: str = Field( description=( "The error message for the error that occurred while making the request." @@ -466,6 +513,7 @@ class GenerativeBenchmark(Benchmark): and end times for the benchmark, and the statistics for the requests and responses. """ + type_: Literal["generative_benchmark"] = "generative_benchmark" successful_total: int = Field( description=( "The total number of completed requests in the benchmark, " @@ -495,7 +543,7 @@ class GenerativeBenchmark(Benchmark): "the benchmark. None if no sampling was applied." ), ) - incomplete_requests: List[GenerativeTextResponseStats] = Field( + incomplete_requests: List[GenerativeTextErrorStats] = Field( description="The list of incomplete requests.", ) errored_total: int = Field( @@ -521,7 +569,7 @@ class GenerativeBenchmark(Benchmark): description="The end time of the last request for the benchmark.", ) - requests_latency: StatusDistributionSummary = Field( + request_latency: StatusDistributionSummary = Field( description="The distribution of latencies for the completed requests.", ) prompt_token_count: StatusDistributionSummary = Field( @@ -536,20 +584,20 @@ class GenerativeBenchmark(Benchmark): "errored, and all requests." ) ) - times_to_first_token_ms: StatusDistributionSummary = Field( + time_to_first_token_ms: StatusDistributionSummary = Field( description=( "The distribution of latencies to receiving the first token in " "milliseconds for completed, errored, and all requests." ), ) - times_per_output_token_ms: StatusDistributionSummary = Field( + time_per_output_token_ms: StatusDistributionSummary = Field( description=( "The distribution of latencies per output token in milliseconds for " "completed, errored, and all requests. " "This includes the time to generate the first token and all other tokens." ), ) - inter_token_latencies_ms: StatusDistributionSummary = Field( + inter_token_latency_ms: StatusDistributionSummary = Field( description=( "The distribution of latencies between tokens in milliseconds for " "completed, errored, and all requests." @@ -656,7 +704,7 @@ def create_sampled( def from_stats( run_id: str, successful: List[GenerativeTextResponseStats], - incomplete: List[GenerativeTextResponseStats], + incomplete: List[GenerativeTextErrorStats], errored: List[GenerativeTextErrorStats], args: BenchmarkArgs, run_stats: BenchmarkRunStats, @@ -695,23 +743,38 @@ def from_stats( ] start_time = min(req.start_time for req in total) end_time = max(req.end_time for req in total) - total_with_prompt, total_types_with_prompt = zip( - *filter( - lambda val: bool(val[0].prompt_tokens), - zip(total, total_types), + total_with_prompt, total_types_with_prompt = ( + zip(*filtered) + if ( + filtered := list( + filter(lambda val: bool(val[0].prompt), zip(total, total_types)) + ) ) - ) - total_with_output_first, total_types_with_output_first = zip( - *filter( - lambda val: bool(val[0].output_tokens), - zip(total, total_types), + else ([], []) + ) + total_with_output_first, total_types_with_output_first = ( + zip(*filtered) + if ( + filtered := list( + filter( + lambda val: bool(val[0].output_tokens > 0), + zip(total, total_types), + ) + ) ) - ) - total_with_output_multi, total_types_with_output_multi = zip( - *filter( - lambda val: bool(val[0].output_tokens > 1), - zip(total, total_types), + else ([], []) + ) + total_with_output_multi, total_types_with_output_multi = ( + zip(*filtered) + if ( + filtered := list( + filter( + lambda val: bool(val[0].output_tokens > 1), + zip(total, total_types), + ) + ) ) + else ([], []) ) return GenerativeBenchmark( @@ -739,35 +802,35 @@ def from_stats( requests=[(req.start_time, req.end_time) for req in total], distribution_type="concurrency", ), - requests_latency=StatusDistributionSummary.from_values( + request_latency=StatusDistributionSummary.from_values( value_types=total_types, values=[req.request_latency for req in total], ), - prompts_token_count=StatusDistributionSummary.from_values( + prompt_token_count=StatusDistributionSummary.from_values( value_types=list(total_types_with_prompt), values=[req.prompt_tokens for req in total_with_prompt], ), - outputs_token_count=StatusDistributionSummary.from_values( + output_token_count=StatusDistributionSummary.from_values( value_types=list(total_types_with_output_first), values=[req.output_tokens for req in total_with_output_first], ), - times_to_first_token_ms=StatusDistributionSummary.from_values( + time_to_first_token_ms=StatusDistributionSummary.from_values( value_types=list(total_types_with_output_first), values=[req.time_to_first_token_ms for req in total_with_output_first], ), - times_per_output_tokens_ms=StatusDistributionSummary.from_values( + time_per_output_token_ms=StatusDistributionSummary.from_values( value_types=list(total_types_with_output_first), values=[ req.time_per_output_token_ms for req in total_with_output_first ], weights=[req.output_tokens for req in total_with_output_first], ), - inter_token_latencies_ms=StatusDistributionSummary.from_values( + inter_token_latency_ms=StatusDistributionSummary.from_values( value_types=list(total_types_with_output_multi), values=[req.inter_token_latency_ms for req in total_with_output_multi], weights=[req.output_tokens - 1 for req in total_with_output_multi], ), - outputs_tokens_per_second=StatusDistributionSummary.from_iterable_request_times( + output_tokens_per_second=StatusDistributionSummary.from_iterable_request_times( request_types=total_types_with_output_first, requests=[ (req.start_time, req.end_time) for req in total_with_output_first diff --git a/src/guidellm/benchmark/output.py b/src/guidellm/benchmark/output.py index a7134d4b..8d146e89 100644 --- a/src/guidellm/benchmark/output.py +++ b/src/guidellm/benchmark/output.py @@ -1,5 +1,7 @@ from collections import OrderedDict from datetime import datetime +import json +import yaml from pathlib import Path from typing import Any, List, Optional @@ -29,20 +31,44 @@ class GenerativeBenchmarksReport(StandardBaseModel): benchmarks: List[GenerativeBenchmark] + def save_file(self, path: Path): + if path.is_dir(): + path = path / "benchmarks.json" -def save_generative_benchmarks(benchmarks: List[GenerativeBenchmark], path: str): - path_inst = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + extension = path.suffix.lower() + + if extension == ".json": + self.save_json(path) + elif extension in [".yaml", ".yml"]: + self.save_yaml(path) + elif extension in [".csv"]: + self.save_csv(path) + else: + raise ValueError(f"Unsupported file extension: {extension} for {path}.") + + def save_json(self, path: Path): + model_dict = self.model_dump() + model_json = json.dumps(model_dict) + + with path.open("w") as file: + file.write(model_json) - if path_inst.is_dir(): - path_inst = path_inst / "generative_benchmarks.json" + def save_yaml(self, path: Path): + model_dict = self.model_dump() + model_yaml = yaml.dump(model_dict) - extension = path_inst.suffix.lower() + with path.open("w") as file: + file.write(model_yaml) - if extension in [".json", ".yaml", ".yml"]: - report = GenerativeBenchmarksReport(benchmarks=benchmarks) - report.save_file(path_inst, type_="json" if extension == ".json" else "yaml") - else: - raise ValueError(f"Unsupported file extension: {extension} for {path_inst}. ") + def save_csv(self, path: Path): + raise NotImplementedError("CSV format is not implemented yet.") + + +def save_generative_benchmarks(benchmarks: List[GenerativeBenchmark], path: str): + path = Path(path) + report = GenerativeBenchmarksReport(benchmarks=benchmarks) + report.save_file(path) class GenerativeBenchmarksConsole: @@ -227,24 +253,24 @@ def print_benchmarks_info(self): f"{benchmark.errored_total}" ), ( - f"{benchmark.prompts_token_count.successful.mean:>5.1f} / " - f"{benchmark.prompts_token_count.incomplete.mean:.1f} / " - f"{benchmark.prompts_token_count.errored.mean:.1f}" + f"{benchmark.prompt_token_count.successful.mean:>5.1f} / " + f"{benchmark.prompt_token_count.incomplete.mean:.1f} / " + f"{benchmark.prompt_token_count.errored.mean:.1f}" ), ( - f"{benchmark.outputs_token_count.successful.mean:>5.1f} / " - f"{benchmark.outputs_token_count.incomplete.mean:.1f} / " - f"{benchmark.outputs_token_count.errored.mean:.1f}" + f"{benchmark.output_token_count.successful.mean:>5.1f} / " + f"{benchmark.output_token_count.incomplete.mean:.1f} / " + f"{benchmark.output_token_count.errored.mean:.1f}" ), ( - f"{benchmark.prompts_token_count.successful.total_sum:>6.0f} / " - f"{benchmark.prompts_token_count.incomplete.total_sum:.0f} / " - f"{benchmark.prompts_token_count.errored.total_sum:.0f}" + f"{benchmark.prompt_token_count.successful.total_sum:>6.0f} / " + f"{benchmark.prompt_token_count.incomplete.total_sum:.0f} / " + f"{benchmark.prompt_token_count.errored.total_sum:.0f}" ), ( - f"{benchmark.outputs_token_count.successful.total_sum:>6.0f} / " - f"{benchmark.outputs_token_count.incomplete.total_sum:.0f} / " - f"{benchmark.outputs_token_count.errored.total_sum:.0f}" + f"{benchmark.output_token_count.successful.total_sum:>6.0f} / " + f"{benchmark.output_token_count.incomplete.total_sum:.0f} / " + f"{benchmark.output_token_count.errored.total_sum:.0f}" ), ] ) @@ -279,27 +305,27 @@ def print_benchmarks_stats(self): strategy_display_str(benchmark.args.strategy), f"{benchmark.requests_per_second.successful.mean:.2f}", f"{benchmark.requests_concurrency.successful.mean:.2f}", - f"{benchmark.outputs_tokens_per_second.total.mean:.1f}", + f"{benchmark.output_tokens_per_second.total.mean:.1f}", f"{benchmark.tokens_per_second.total.mean:.1f}", ( - f"{benchmark.requests_latency.successful.mean:.2f} / " - f"{benchmark.requests_latency.successful.median:.2f} / " - f"{benchmark.requests_latency.successful.percentiles.p99:.2f}" + f"{benchmark.request_latency.successful.mean:.2f} / " + f"{benchmark.request_latency.successful.median:.2f} / " + f"{benchmark.request_latency.successful.percentiles.p99:.2f}" ), ( - f"{benchmark.times_to_first_token_ms.successful.mean:.1f} / " - f"{benchmark.times_to_first_token_ms.successful.median:.1f} / " - f"{benchmark.times_to_first_token_ms.successful.percentiles.p99:.1f}" + f"{benchmark.time_to_first_token_ms.successful.mean:.1f} / " + f"{benchmark.time_to_first_token_ms.successful.median:.1f} / " + f"{benchmark.time_to_first_token_ms.successful.percentiles.p99:.1f}" ), ( - f"{benchmark.inter_token_latencies_ms.successful.mean:.1f} / " - f"{benchmark.inter_token_latencies_ms.successful.median:.1f} / " - f"{benchmark.inter_token_latencies_ms.successful.percentiles.p99:.1f}" + f"{benchmark.inter_token_latency_ms.successful.mean:.1f} / " + f"{benchmark.inter_token_latency_ms.successful.median:.1f} / " + f"{benchmark.inter_token_latency_ms.successful.percentiles.p99:.1f}" ), ( - f"{benchmark.times_per_output_tokens_ms.successful.mean:.1f} / " - f"{benchmark.times_per_output_tokens_ms.successful.median:.1f} / " - f"{benchmark.times_per_output_tokens_ms.successful.percentiles.p99:.1f}" + f"{benchmark.time_per_output_token_ms.successful.mean:.1f} / " + f"{benchmark.time_per_output_token_ms.successful.median:.1f} / " + f"{benchmark.time_per_output_token_ms.successful.percentiles.p99:.1f}" ), ] ) diff --git a/src/guidellm/benchmark/profile.py b/src/guidellm/benchmark/profile.py index 6b7c9070..5020081f 100644 --- a/src/guidellm/benchmark/profile.py +++ b/src/guidellm/benchmark/profile.py @@ -30,7 +30,7 @@ class Profile(StandardBaseModel): - type_: ProfileType = Field( + type_: Literal["profile"] = Field( description="The type of benchmarking profile to use.", ) completed_strategies: int = Field( diff --git a/src/guidellm/benchmark/progress.py b/src/guidellm/benchmark/progress.py index 9fd1e956..0bcc6f31 100644 --- a/src/guidellm/benchmark/progress.py +++ b/src/guidellm/benchmark/progress.py @@ -45,7 +45,7 @@ class BenchmarkerTaskProgressState: in_cooldown: bool = False requests_rate: float = 0 - requests_latency: float = 0 + request_latency: float = 0 requests_processing: int = 0 requests_successful: int = 0 requests_incomplete: int = 0 @@ -111,18 +111,24 @@ def formatted_start_time(self) -> str: def formatted_progress_status(self) -> str: if self.ended: status = "complete" + color = Colors.SUCCESS elif self.compiling: status = "compiling" + color = Colors.PROGRESS elif self.started and self.in_warmup: status = "warmup" + color = Colors.PROGRESS elif self.started and self.in_cooldown: status = "cooldown" + color = Colors.PROGRESS elif self.started: status = "running" + color = Colors.PROGRESS else: status = "pending" + color = Colors.INFO - return status.ljust(9) + return f"[{color}]{status.ljust(8)}[/{color}]" @property def formatted_requests_summary(self) -> str: @@ -140,7 +146,7 @@ def formatted_requests_summary(self) -> str: ) + ", " + BenchmarkerTaskProgressState.format_progress_display( - value=self.requests_latency, + value=self.request_latency, label="Lat", units="s", total_characters=12, @@ -483,7 +489,7 @@ def handle_update_scheduler_update( progress_state.requests_rate = ( result.current_aggregator.successful_requests.rate ) - progress_state.requests_latency = result.current_aggregator.request_time.mean + progress_state.request_latency = result.current_aggregator.request_time.mean progress_state.requests_processing = ( result.current_aggregator.scheduler_processing_requests.last ) @@ -538,8 +544,8 @@ def handle_update_benchmark_compiled( progress_state.requests_rate = ( result.current_benchmark.requests_per_second.successful.mean ) - progress_state.requests_latency = ( - result.current_benchmark.requests_latency.successful.mean + progress_state.request_latency = ( + result.current_benchmark.request_latency.successful.mean ) progress_state.requests_processing = ( result.current_benchmark.requests_concurrency.successful.mean @@ -614,22 +620,22 @@ def handle_update_benchmark_compiled(self, progress_state, result): super().handle_update_benchmark_compiled(progress_state, result) progress_state.output_tokens = ( - result.current_benchmark.outputs_token_count.successful.mean + result.current_benchmark.output_token_count.successful.mean ) progress_state.prompt_tokens = ( - result.current_benchmark.prompts_token_count.successful.mean + result.current_benchmark.prompt_token_count.successful.mean ) progress_state.output_tokens_rate = ( - result.current_benchmark.outputs_tokens_per_second.successful.mean + result.current_benchmark.output_tokens_per_second.successful.mean ) progress_state.total_tokens_rate = ( result.current_benchmark.tokens_per_second.successful.mean ) progress_state.tokens_ttft = ( - result.current_benchmark.times_to_first_token_ms.successful.mean + result.current_benchmark.time_to_first_token_ms.successful.mean ) progress_state.tokens_itl = ( - result.current_benchmark.inter_token_latencies_ms.successful.mean + result.current_benchmark.inter_token_latency_ms.successful.mean ) def create_task_progress_state( diff --git a/src/guidellm/dataset/synthetic.py b/src/guidellm/dataset/synthetic.py index a8e94b08..5b3b8686 100644 --- a/src/guidellm/dataset/synthetic.py +++ b/src/guidellm/dataset/synthetic.py @@ -142,7 +142,7 @@ def __iter__(self) -> Iterator[Tuple[str, int, int]]: ) output_tokens_sampler = IntegerRangeSampler( average=self.config.output_tokens, - variance=self.config.output_tokens_stddev, + variance=self.config.output_tokens_stdev, min_value=self.config.output_tokens_min, max_value=self.config.output_tokens_max, random_seed=self.random_seed + 1, # ensure diff dist from prompts diff --git a/src/guidellm/objects/statistics.py b/src/guidellm/objects/statistics.py index 05ccf974..21f48679 100644 --- a/src/guidellm/objects/statistics.py +++ b/src/guidellm/objects/statistics.py @@ -444,23 +444,23 @@ def from_values( return StatusDistributionSummary( total=DistributionSummary.from_values( - values=values, - weights=weights, + values, + weights, include_cdf=include_cdf, ), successful=DistributionSummary.from_values( - values=successful_values, - weights=successful_weights, + successful_values, + successful_weights, include_cdf=include_cdf, ), incomplete=DistributionSummary.from_values( - values=incomplete_values, - weights=incomplete_weights, + incomplete_values, + incomplete_weights, include_cdf=include_cdf, ), errored=DistributionSummary.from_values( - values=errored_values, - weights=errored_weights, + errored_values, + errored_weights, include_cdf=include_cdf, ), ) @@ -496,41 +496,62 @@ def from_request_times( _, successful_requests = ( zip(*successful) - if (successful := list(zip(request_types, requests))) + if ( + successful := list( + filter( + lambda val: val[0] == "successful", + zip(request_types, requests), + ) + ) + ) else ([], []) ) _, incomplete_requests = ( zip(*incomplete) - if (incomplete := list(zip(request_types, requests))) + if ( + incomplete := list( + filter( + lambda val: val[0] == "incomplete", + zip(request_types, requests), + ) + ) + ) else ([], []) ) _, errored_requests = ( zip(*errored) - if (errored := list(zip(request_types, requests))) + if ( + errored := list( + filter( + lambda val: val[0] == "error", + zip(request_types, requests), + ) + ) + ) else ([], []) ) return StatusDistributionSummary( total=DistributionSummary.from_request_times( - requests=requests, + requests, distribution_type=distribution_type, include_cdf=include_cdf, epsilon=epsilon, ), successful=DistributionSummary.from_request_times( - requests=successful_requests, + successful_requests, distribution_type=distribution_type, include_cdf=include_cdf, epsilon=epsilon, ), incomplete=DistributionSummary.from_request_times( - requests=incomplete_requests, + incomplete_requests, distribution_type=distribution_type, include_cdf=include_cdf, epsilon=epsilon, ), errored=DistributionSummary.from_request_times( - requests=errored_requests, + errored_requests, distribution_type=distribution_type, include_cdf=include_cdf, epsilon=epsilon, @@ -651,34 +672,34 @@ def from_iterable_request_times( return StatusDistributionSummary( total=DistributionSummary.from_iterable_request_times( - requests=requests, - first_iter_times=first_iter_times, - iter_counts=iter_counts, - first_iter_counts=first_iter_counts, + requests, + first_iter_times, + iter_counts, + first_iter_counts, include_cdf=include_cdf, epsilon=epsilon, ), successful=DistributionSummary.from_iterable_request_times( - requests=successful_requests, - first_iter_times=successful_first_iter_times, - iter_counts=successful_iter_counts, - first_iter_counts=successful_first_iter_counts, + successful_requests, + successful_first_iter_times, + successful_iter_counts, + successful_first_iter_counts, include_cdf=include_cdf, epsilon=epsilon, ), incomplete=DistributionSummary.from_iterable_request_times( - requests=incomplete_requests, - first_iter_times=incomplete_first_iter_times, - iter_counts=incomplete_iter_counts, - first_iter_counts=incomplete_first_iter_counts, + incomplete_requests, + incomplete_first_iter_times, + incomplete_iter_counts, + incomplete_first_iter_counts, include_cdf=include_cdf, epsilon=epsilon, ), errored=DistributionSummary.from_iterable_request_times( - requests=errored_requests, - first_iter_times=errored_first_iter_times, - iter_counts=errored_iter_counts, - first_iter_counts=errored_first_iter_counts, + errored_requests, + errored_first_iter_times, + errored_iter_counts, + errored_first_iter_counts, include_cdf=include_cdf, epsilon=epsilon, ), diff --git a/src/guidellm/scheduler/strategy.py b/src/guidellm/scheduler/strategy.py index 0060e27e..e07e0b8e 100644 --- a/src/guidellm/scheduler/strategy.py +++ b/src/guidellm/scheduler/strategy.py @@ -41,7 +41,7 @@ class SchedulingStrategy(StandardBaseModel): This should be one of the predefined strategy types. """ - type_: StrategyType = Field( + type_: Literal["strategy"] = Field( description="The type of scheduling strategy schedule requests with.", ) From f8161ed31fb5b2a8257cb1b8e7916d2b2a1179ef Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Fri, 11 Apr 2025 00:04:50 +0000 Subject: [PATCH 25/43] Ensure style and types pass, remove tests that are no longer relevant, finish up the objects module with docs and tests --- src/guidellm/__init__.py | 2 +- src/guidellm/__main__.py | 4 +- src/guidellm/backend/openai.py | 11 +- src/guidellm/benchmark/aggregator.py | 64 +- src/guidellm/benchmark/benchmark.py | 77 +- src/guidellm/benchmark/benchmarker.py | 16 +- src/guidellm/benchmark/entrypoints.py | 4 +- src/guidellm/benchmark/output.py | 42 +- src/guidellm/benchmark/profile.py | 24 +- src/guidellm/benchmark/progress.py | 161 +- src/guidellm/config.py | 1 - src/guidellm/dataset/creator.py | 11 +- src/guidellm/dataset/entrypoints.py | 6 +- src/guidellm/dataset/file.py | 12 +- src/guidellm/dataset/hf_datasets.py | 12 +- src/guidellm/dataset/in_memory.py | 21 +- src/guidellm/dataset/synthetic.py | 30 +- src/guidellm/objects/pydantic.py | 4 +- src/guidellm/objects/statistics.py | 340 ++- src/guidellm/request/loader.py | 15 +- src/guidellm/scheduler/result.py | 2 +- src/guidellm/scheduler/scheduler.py | 7 +- src/guidellm/scheduler/strategy.py | 17 +- src/guidellm/scheduler/worker.py | 20 +- src/guidellm/utils/__init__.py | 1 + src/guidellm/utils/hf_transformers.py | 2 +- src/guidellm/utils/random.py | 2 +- src/guidellm/utils/text.py | 2 +- tests/dummy/__init__.py | 8 - tests/dummy/data/pride_and_prejudice.txt | 2015 ----------------- tests/dummy/data/transformers.py | 50 - tests/dummy/services/__init__.py | 5 - tests/dummy/services/requests.py | 31 - tests/e2e/test_guidellm.py | 8 - tests/integration/test_guidellm.py | 8 - tests/unit/backend/test_openai_backend.py | 14 +- tests/unit/backend/test_response.py | 31 +- tests/unit/cli/__init__.py | 0 tests/unit/cli/test_custom_type_params.py | 38 - tests/unit/core/__init__.py | 0 tests/unit/core/test_report.py | 106 - tests/unit/core/test_request.py | 79 - tests/unit/core/test_result.py | 279 --- tests/unit/executor/__init__.py | 0 tests/unit/executor/test_executor.py | 542 ----- tests/unit/executor/test_profile_generator.py | 204 -- tests/unit/mock_backend.py | 25 +- .../{dummy/data => unit/objects}/__init__.py | 0 tests/unit/objects/test_distribution.py | 337 --- tests/unit/objects/test_pydantic.py | 43 + tests/unit/objects/test_serializable.py | 151 -- tests/unit/objects/test_statistics.py | 707 ++++++ tests/unit/request/__init__.py | 0 tests/unit/request/test_base.py | 160 -- tests/unit/request/test_emulated.py | 373 --- tests/unit/request/test_file.py | 161 -- tests/unit/request/test_transformers.py | 132 -- tests/unit/scheduler/__init__.py | 0 tests/unit/scheduler/test_load_generator.py | 153 -- tests/unit/scheduler/test_scheduler.py | 199 -- tests/unit/test_type.py | 0 tests/unit/utils/__init__.py | 0 tests/unit/utils/test_injector.py | 70 - tests/unit/utils/test_progress.py | 116 - tests/unit/utils/test_text.py | 394 ---- tests/unit/utils/test_transformers.py | 236 -- 66 files changed, 1398 insertions(+), 6187 deletions(-) delete mode 100644 tests/dummy/__init__.py delete mode 100644 tests/dummy/data/pride_and_prejudice.txt delete mode 100644 tests/dummy/data/transformers.py delete mode 100644 tests/dummy/services/__init__.py delete mode 100644 tests/dummy/services/requests.py delete mode 100644 tests/e2e/test_guidellm.py delete mode 100644 tests/integration/test_guidellm.py delete mode 100644 tests/unit/cli/__init__.py delete mode 100644 tests/unit/cli/test_custom_type_params.py delete mode 100644 tests/unit/core/__init__.py delete mode 100644 tests/unit/core/test_report.py delete mode 100644 tests/unit/core/test_request.py delete mode 100644 tests/unit/core/test_result.py delete mode 100644 tests/unit/executor/__init__.py delete mode 100644 tests/unit/executor/test_executor.py delete mode 100644 tests/unit/executor/test_profile_generator.py rename tests/{dummy/data => unit/objects}/__init__.py (100%) delete mode 100644 tests/unit/objects/test_distribution.py create mode 100644 tests/unit/objects/test_pydantic.py delete mode 100644 tests/unit/objects/test_serializable.py create mode 100644 tests/unit/objects/test_statistics.py delete mode 100644 tests/unit/request/__init__.py delete mode 100644 tests/unit/request/test_base.py delete mode 100644 tests/unit/request/test_emulated.py delete mode 100644 tests/unit/request/test_file.py delete mode 100644 tests/unit/request/test_transformers.py delete mode 100644 tests/unit/scheduler/__init__.py delete mode 100644 tests/unit/scheduler/test_load_generator.py delete mode 100644 tests/unit/scheduler/test_scheduler.py delete mode 100644 tests/unit/test_type.py delete mode 100644 tests/unit/utils/__init__.py delete mode 100644 tests/unit/utils/test_injector.py delete mode 100644 tests/unit/utils/test_progress.py delete mode 100644 tests/unit/utils/test_text.py delete mode 100644 tests/unit/utils/test_transformers.py diff --git a/src/guidellm/__init__.py b/src/guidellm/__init__.py index fb531d49..929d046e 100644 --- a/src/guidellm/__init__.py +++ b/src/guidellm/__init__.py @@ -13,7 +13,7 @@ with open(os.devnull, "w") as devnull, contextlib.redirect_stderr( devnull ), contextlib.redirect_stdout(devnull): - from transformers.utils import logging as hf_logging + from transformers.utils import logging as hf_logging # type: ignore[import] # Set the log level for the transformers library to ERROR # to ignore None of PyTorch, TensorFlow found diff --git a/src/guidellm/__main__.py b/src/guidellm/__main__.py index f0d40d30..096614de 100644 --- a/src/guidellm/__main__.py +++ b/src/guidellm/__main__.py @@ -14,7 +14,7 @@ ) -def parse_json(ctx, param, value): +def parse_json(ctx, param, value): # noqa: ARG001 if value is None: return None try: @@ -23,7 +23,7 @@ def parse_json(ctx, param, value): raise click.BadParameter(f"{param.name} must be a valid JSON string.") from err -def parse_number_str(ctx, param, value): +def parse_number_str(ctx, param, value): # noqa: ARG001 if value is None: return None diff --git a/src/guidellm/backend/openai.py b/src/guidellm/backend/openai.py index 3b5ade98..48bde08b 100644 --- a/src/guidellm/backend/openai.py +++ b/src/guidellm/backend/openai.py @@ -92,7 +92,7 @@ def __init__( if max_output_tokens is not None else settings.openai.max_output_tokens ) - self._async_client: Optional[httpx.Client] = None + self._async_client: Optional[httpx.AsyncClient] = None @property def target(self) -> str: @@ -311,11 +311,12 @@ def _get_async_client(self) -> httpx.AsyncClient: :return: The async HTTP client. """ if self._async_client is None: - self._async_client = httpx.AsyncClient( - http2=self.http2, timeout=self.timeout - ) + client = httpx.AsyncClient(http2=self.http2, timeout=self.timeout) + self._async_client = client + else: + client = self._async_client - return self._async_client + return client def _headers(self) -> Dict[str, str]: headers = { diff --git a/src/guidellm/benchmark/aggregator.py b/src/guidellm/benchmark/aggregator.py index c2f375a6..d7273a02 100644 --- a/src/guidellm/benchmark/aggregator.py +++ b/src/guidellm/benchmark/aggregator.py @@ -141,8 +141,8 @@ class BenchmarkAggregator(ABC, StandardBaseModel, Generic[BENCH, REQ, RES]): "if any. These are requests that were not included in the final results." ) ) - worker_description: Optional[ - Union[GenerativeRequestsWorkerDescription, WorkerDescription] + worker_description: Union[ + GenerativeRequestsWorkerDescription, WorkerDescription ] = Field( description=( "The description and specifics for the worker used to resolve requests " @@ -150,8 +150,8 @@ class BenchmarkAggregator(ABC, StandardBaseModel, Generic[BENCH, REQ, RES]): ), discriminator="type_", ) - request_loader_description: Optional[ - Union[GenerativeRequestLoaderDescription, RequestLoaderDescription] + request_loader_description: Union[ + GenerativeRequestLoaderDescription, RequestLoaderDescription ] = Field( description=( "The description and specifics for the request loader used to create " @@ -391,10 +391,10 @@ def add_result( "completed, canceled, or errored." ) - self.queued_time += ( + self.queued_time += ( # type: ignore[misc] result.request_info.dequeued_time - result.request_info.queued_time ) - self.scheduled_time_delay += ( + self.scheduled_time_delay += ( # type: ignore[misc] result.request_info.scheduled_time - result.request_info.dequeued_time ) sleep_time = max( @@ -402,15 +402,15 @@ def add_result( result.request_info.targeted_start_time - result.request_info.scheduled_time, ) - self.scheduled_time_sleep += sleep_time - time_to_worker_start = ( + self.scheduled_time_sleep += sleep_time # type: ignore[misc] + time_to_worker_start = ( # type: ignore[misc] result.request_info.worker_start - result.request_info.scheduled_time ) - self.worker_start_delay += time_to_worker_start - sleep_time - self.worker_time += ( + self.worker_start_delay += time_to_worker_start - sleep_time # type: ignore[misc] + self.worker_time += ( # type: ignore[misc] result.request_info.worker_end - result.request_info.worker_start ) - self.worker_start_time_targeted_delay += ( + self.worker_start_time_targeted_delay += ( # type: ignore[misc] result.request_info.worker_start - result.request_info.targeted_start_time ) @@ -433,9 +433,11 @@ def add_result( if ( self.cooldown_number + and self.max_number and total_completed > self.max_number - self.cooldown_number ) or ( self.cooldown_duration + and self.max_duration and result.request_info.worker_start >= global_start_time + self.max_duration - self.cooldown_duration ): @@ -459,14 +461,14 @@ def compile(self) -> BENCH: ... -AGG = TypeVar("AGG", bound=BenchmarkAggregator[BENCH, REQ, RES]) +AGG = TypeVar("AGG", bound=BenchmarkAggregator) class GenerativeBenchmarkAggregator( BenchmarkAggregator[GenerativeBenchmark, GenerationRequest, ResponseSummary] ): type_: Literal["generative_benchmark_aggregator"] = ( - "generative_benchmark_aggregator" + "generative_benchmark_aggregator" # type: ignore[assignment] ) processor: Optional[Union[str, Path, Any]] = Field( description=( @@ -531,20 +533,26 @@ def add_result( if not super().add_result(result): return False - self.request_start_time_delay += ( + if result.request is None: + raise ValueError("Request is None, cannot add result.") + + if result.response is None: + raise ValueError("Response is None, cannot add result.") + + self.request_start_time_delay += ( # type: ignore[misc] result.response.start_time - result.request_info.worker_start ) - self.request_start_time_targeted_delay += ( + self.request_start_time_targeted_delay += ( # type: ignore[misc] result.response.start_time - result.request_info.targeted_start_time ) - self.request_time_delay += ( + self.request_time_delay += ( # type: ignore[misc] (result.response.start_time - result.request_info.worker_start) + result.request_info.worker_end - result.response.end_time ) - self.request_time += result.response.end_time - result.response.start_time + self.request_time += result.response.end_time - result.response.start_time # type: ignore[misc] - self.time_to_first_token += ( + self.time_to_first_token += ( # type: ignore[misc] (result.response.first_iter_time - result.response.start_time) * 1000.0 if result.response.first_iter_time else 0.0 @@ -590,9 +598,9 @@ def compile(self) -> GenerativeBenchmark: run_stats=BenchmarkRunStats( start_time=self.scheduler_created_requests.start_time, end_time=time.time(), - total_successful=self.successful_requests.total, - total_incomplete=self.incomplete_requests.total, - total_errored=self.errored_requests.total, + total_successful=int(self.successful_requests.total), + total_incomplete=int(self.incomplete_requests.total), + total_errored=int(self.errored_requests.total), queued_time_avg=self.queued_time.mean, scheduled_time_delay_avg=self.scheduled_time_delay.mean, scheduled_time_sleep_avg=self.scheduled_time_sleep.mean, @@ -621,6 +629,12 @@ def _compile_results( error: List[GenerativeTextErrorStats] = [] for result in self.results: + if result.request is None: + raise ValueError("Request is None, cannot compile results.") + + if result.response is None: + raise ValueError("Response is None, cannot compile results.") + prompt_tokens = self._compile_tokens_count( value=str(result.request.content), requests_tokens=result.response.request_prompt_tokens, @@ -639,7 +653,7 @@ def _compile_results( if result.request_info.canceled: incomplete.append( GenerativeTextErrorStats( - error=result.response.error, + error=result.response.error or "", request_id=result.request.request_id, request_type=result.request.request_type, scheduler_info=result.request_info, @@ -656,7 +670,7 @@ def _compile_results( elif result.request_info.errored: error.append( GenerativeTextErrorStats( - error=result.response.error, + error=result.response.error or "", request_id=result.request.request_id, request_type=result.request.request_type, scheduler_info=result.request_info, @@ -682,8 +696,8 @@ def _compile_results( output_tokens=output_tokens, start_time=result.response.start_time, end_time=result.response.end_time, - first_token_time=result.response.first_iter_time, - last_token_time=result.response.last_iter_time, + first_token_time=result.response.first_iter_time or -1, + last_token_time=result.response.last_iter_time or -1, ) ) diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index 4646f141..f0332c3e 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -224,7 +224,7 @@ class BenchmarkRunStats(StandardBaseModel): ) ) - @computed_field + @computed_field # type: ignore[misc] @property def total(self) -> int: """ @@ -309,7 +309,7 @@ class GenerativeTextResponseStats(StandardBaseModel): """ type_: Literal["generative_text_response"] = "generative_text_response" - request_id: str = Field( + request_id: Optional[str] = Field( description="The unique identifier for the request.", ) request_type: Literal["text_completions", "chat_completions"] = Field( @@ -345,7 +345,7 @@ class GenerativeTextResponseStats(StandardBaseModel): description="The time the last token was received.", ) - @computed_field + @computed_field # type: ignore[misc] @property def request_latency(self) -> float: """ @@ -353,7 +353,7 @@ def request_latency(self) -> float: """ return self.end_time - self.start_time - @computed_field + @computed_field # type: ignore[misc] @property def time_to_first_token_ms(self) -> float: """ @@ -362,7 +362,7 @@ def time_to_first_token_ms(self) -> float: """ return 1000 * (self.first_token_time - self.start_time) - @computed_field + @computed_field # type: ignore[misc] @property def time_per_output_token_ms(self) -> float: """ @@ -376,7 +376,7 @@ def time_per_output_token_ms(self) -> float: 1000 * (self.last_token_time - self.first_token_time) / self.output_tokens ) - @computed_field + @computed_field # type: ignore[misc] @property def inter_token_latency_ms(self) -> float: """ @@ -392,7 +392,7 @@ def inter_token_latency_ms(self) -> float: / (self.output_tokens - 1) ) - @computed_field + @computed_field # type: ignore[misc] @property def tokens_per_second(self) -> float: """ @@ -404,7 +404,7 @@ def tokens_per_second(self) -> float: return (self.prompt_tokens + self.output_tokens) / latency - @computed_field + @computed_field # type: ignore[misc] @property def output_tokens_per_second(self) -> float: """ @@ -424,35 +424,35 @@ class GenerativeTextErrorStats(GenerativeTextResponseStats): error message and optional properties given the error occurred. """ - type_: Literal["generative_text_error"] = "generative_text_error" + type_: Literal["generative_text_error"] = "generative_text_error" # type: ignore[assignment] error: str = Field( description=( "The error message for the error that occurred while making the request." ) ) - output: Optional[str] = Field( + output: Optional[str] = Field( # type: ignore[assignment] default=None, description=( "The generated text output from the generative request, if any, " "before the error occurred." ), ) - first_token_time: Optional[float] = Field( + first_token_time: Optional[float] = Field( # type: ignore[assignment] default=None, description=( "The time the first token was received, if any, before the error occurred." ), ) - last_token_time: Optional[float] = Field( + last_token_time: Optional[float] = Field( # type: ignore[assignment] default=None, description=( "The time the last token was received, if any, before the error occurred." ), ) - @computed_field + @computed_field # type: ignore[misc] @property - def time_to_first_token_ms(self) -> Optional[float]: + def time_to_first_token_ms(self) -> Optional[float]: # type: ignore[override] """ :return: The time in milliseconds from the start of the request to the first token received. None if the first token was not received. @@ -462,9 +462,9 @@ def time_to_first_token_ms(self) -> Optional[float]: return super().time_to_first_token_ms - @computed_field + @computed_field # type: ignore[misc] @property - def time_per_output_token_ms(self) -> Optional[float]: + def time_per_output_token_ms(self) -> Optional[float]: # type: ignore[override] """ :return: The average time in milliseconds per output token generated. This includes the time to generate the first token and all other tokens. @@ -475,9 +475,9 @@ def time_per_output_token_ms(self) -> Optional[float]: return super().time_per_output_token_ms - @computed_field + @computed_field # type: ignore[misc] @property - def inter_token_latency_ms(self) -> Optional[float]: + def inter_token_latency_ms(self) -> Optional[float]: # type: ignore[override] """ :return: The average time in milliseconds between generating tokens in the output text. Note, does not include the time to generate the first token. @@ -492,9 +492,9 @@ def inter_token_latency_ms(self) -> Optional[float]: return super().inter_token_latency_ms - @computed_field + @computed_field # type: ignore[misc] @property - def output_tokens_per_second(self) -> Optional[float]: + def output_tokens_per_second(self) -> Optional[float]: # type: ignore[override] """ :return: The average number of tokens generated per second in the output text. Note, does not include the time to generate the first token. None if there @@ -513,7 +513,7 @@ class GenerativeBenchmark(Benchmark): and end times for the benchmark, and the statistics for the requests and responses. """ - type_: Literal["generative_benchmark"] = "generative_benchmark" + type_: Literal["generative_benchmark"] = "generative_benchmark" # type: ignore[assignment] successful_total: int = Field( description=( "The total number of completed requests in the benchmark, " @@ -616,7 +616,7 @@ class GenerativeBenchmark(Benchmark): ), ) - @computed_field + @computed_field # type: ignore[misc] @property def duration(self) -> float: """ @@ -708,8 +708,8 @@ def from_stats( errored: List[GenerativeTextErrorStats], args: BenchmarkArgs, run_stats: BenchmarkRunStats, - worker: Optional[StandardBaseModel], - requests_loader: Optional[StandardBaseModel], + worker: WorkerDescription, + requests_loader: RequestLoaderDescription, extras: Optional[Dict[str, Any]], ) -> "GenerativeBenchmark": """ @@ -736,13 +736,14 @@ def from_stats( populated and calculated """ total = successful + incomplete + errored - total_types = [ - *["successful"] * len(successful), - *["incomplete"] * len(incomplete), - *["error"] * len(errored), + total_types: List[Literal["successful", "incomplete", "error"]] = [ + *["successful"] * len(successful), # type: ignore[list-item] + *["incomplete"] * len(incomplete), # type: ignore[list-item] + *["error"] * len(errored), # type: ignore[list-item] ] start_time = min(req.start_time for req in total) end_time = max(req.end_time for req in total) + total_with_prompt, total_types_with_prompt = ( zip(*filtered) if ( @@ -816,37 +817,43 @@ def from_stats( ), time_to_first_token_ms=StatusDistributionSummary.from_values( value_types=list(total_types_with_output_first), - values=[req.time_to_first_token_ms for req in total_with_output_first], + values=[ + req.time_to_first_token_ms or 0 for req in total_with_output_first + ], ), time_per_output_token_ms=StatusDistributionSummary.from_values( value_types=list(total_types_with_output_first), values=[ - req.time_per_output_token_ms for req in total_with_output_first + req.time_per_output_token_ms or 0 for req in total_with_output_first ], weights=[req.output_tokens for req in total_with_output_first], ), inter_token_latency_ms=StatusDistributionSummary.from_values( value_types=list(total_types_with_output_multi), - values=[req.inter_token_latency_ms for req in total_with_output_multi], + values=[ + req.inter_token_latency_ms or 0 for req in total_with_output_multi + ], weights=[req.output_tokens - 1 for req in total_with_output_multi], ), output_tokens_per_second=StatusDistributionSummary.from_iterable_request_times( - request_types=total_types_with_output_first, + request_types=list(total_types_with_output_first), requests=[ (req.start_time, req.end_time) for req in total_with_output_first ], first_iter_times=[ - req.first_token_time for req in total_with_output_first + req.first_token_time or req.start_time + for req in total_with_output_first ], iter_counts=[req.output_tokens for req in total_with_output_first], ), tokens_per_second=StatusDistributionSummary.from_iterable_request_times( - request_types=total_types_with_output_first, + request_types=list(total_types_with_output_first), requests=[ (req.start_time, req.end_time) for req in total_with_output_first ], first_iter_times=[ - req.first_token_time for req in total_with_output_first + req.first_token_time or req.start_time + for req in total_with_output_first ], iter_counts=[ req.prompt_tokens + req.output_tokens diff --git a/src/guidellm/benchmark/benchmarker.py b/src/guidellm/benchmark/benchmarker.py index b53ec664..332d2ad1 100644 --- a/src/guidellm/benchmark/benchmarker.py +++ b/src/guidellm/benchmark/benchmarker.py @@ -21,7 +21,7 @@ from guidellm.benchmark.benchmark import GenerativeBenchmark from guidellm.benchmark.profile import Profile from guidellm.objects import StandardBaseModel -from guidellm.request import GenerationRequest +from guidellm.request import GenerationRequest, RequestLoaderDescription from guidellm.scheduler import ( REQ, RES, @@ -125,7 +125,7 @@ def __init__( self, worker: RequestsWorker[REQ, RES], request_loader: Iterable[REQ], - requests_loader_description: Optional[StandardBaseModel] = None, + requests_loader_description: RequestLoaderDescription, benchmark_save_extras: Optional[Dict[str, Any]] = None, ): self.worker = worker @@ -144,8 +144,8 @@ async def run( cooldown_percent_per_strategy: Optional[float], ) -> AsyncGenerator[BenchmarkerResult[AGG, BENCH, REQ, RES], None]: try: - requests_loader_size = len(self.scheduler.request_loader) - except Exception: + requests_loader_size = len(self.scheduler.request_loader) # type: ignore[arg-type] + except Exception: # noqa: BLE001 requests_loader_size = None strategy_limits = BenchmarkerStrategyLimits( @@ -174,7 +174,7 @@ async def run( while scheduling_strategy := profile.next_strategy(): current_index += 1 - aggregator: AGG[BENCH, REQ, RES] = self.create_benchmark_aggregator( + aggregator = self.create_benchmark_aggregator( run_id=run_id, profile=profile, strategy_index=current_index, @@ -272,7 +272,7 @@ def create_benchmark_aggregator( strategy: SchedulingStrategy, max_number: Optional[int], max_duration: Optional[float], - warmup_number: Optional[float], + warmup_number: Optional[int], warmup_duration: Optional[float], cooldown_number: Optional[int], cooldown_duration: Optional[float], @@ -291,7 +291,7 @@ def __init__( self, backend: Backend, request_loader: Iterable[GenerationRequest], - request_loader_description: Optional[StandardBaseModel] = None, + request_loader_description: RequestLoaderDescription, benchmark_save_extras: Optional[Dict[str, Any]] = None, processor: Optional[Union[str, Path, PreTrainedTokenizer]] = None, processor_args: Optional[Dict[str, Any]] = None, @@ -313,7 +313,7 @@ def create_benchmark_aggregator( strategy: SchedulingStrategy, max_number: Optional[int], max_duration: Optional[float], - warmup_number: Optional[float], + warmup_number: Optional[int], warmup_duration: Optional[float], cooldown_number: Optional[int], cooldown_duration: Optional[float], diff --git a/src/guidellm/benchmark/entrypoints.py b/src/guidellm/benchmark/entrypoints.py index c321b01d..9b5a85a3 100644 --- a/src/guidellm/benchmark/entrypoints.py +++ b/src/guidellm/benchmark/entrypoints.py @@ -2,7 +2,7 @@ from typing import Any, Callable, Dict, Iterable, List, Literal, Optional, Union from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict -from transformers import PreTrainedTokenizer +from transformers import PreTrainedTokenizer # type: ignore[import] from guidellm.backend import Backend, BackendType from guidellm.benchmark.benchmark import GenerativeBenchmark @@ -111,6 +111,8 @@ async def benchmark_generative_text( progress.update(result) if result.type_ == "benchmark_compiled": + if result.current_benchmark is None: + raise ValueError("Current benchmark is None") benchmarks.append(result.current_benchmark) if output_console: diff --git a/src/guidellm/benchmark/output.py b/src/guidellm/benchmark/output.py index 8d146e89..de34d626 100644 --- a/src/guidellm/benchmark/output.py +++ b/src/guidellm/benchmark/output.py @@ -1,10 +1,10 @@ +import json from collections import OrderedDict from datetime import datetime -import json -import yaml from pathlib import Path -from typing import Any, List, Optional +from typing import Any, List, Optional, Union +import yaml from rich.console import Console from rich.padding import Padding from rich.table import Table @@ -65,8 +65,10 @@ def save_csv(self, path: Path): raise NotImplementedError("CSV format is not implemented yet.") -def save_generative_benchmarks(benchmarks: List[GenerativeBenchmark], path: str): - path = Path(path) +def save_generative_benchmarks( + benchmarks: List[GenerativeBenchmark], path: Union[Path, str] +): + path = Path(path) if isinstance(path, str) else path report = GenerativeBenchmarksReport(benchmarks=benchmarks) report.save_file(path) @@ -79,7 +81,11 @@ def __init__(self, enabled: bool = True): @property def benchmarks_profile_str(self) -> str: - profile = self.benchmarks[0].args.profile + profile = self.benchmarks[0].args.profile if self.benchmarks else None + + if profile is None: + return "None" + profile_args = OrderedDict( { "type": profile.type_, @@ -88,21 +94,25 @@ def benchmarks_profile_str(self) -> str: ) if isinstance(profile, ConcurrentProfile): - profile_args["streams"] = profile.streams + profile_args["streams"] = str(profile.streams) elif isinstance(profile, ThroughputProfile): - profile_args["max_concurrency"] = profile.max_concurrency + profile_args["max_concurrency"] = str(profile.max_concurrency) elif isinstance(profile, AsyncProfile): - profile_args["max_concurrency"] = profile.max_concurrency - profile_args["rate"] = profile.rate - profile_args["initial_burst"] = profile.initial_burst + profile_args["max_concurrency"] = str(profile.max_concurrency) + profile_args["rate"] = str(profile.rate) + profile_args["initial_burst"] = str(profile.initial_burst) elif isinstance(profile, SweepProfile): - profile_args["sweep_size"] = profile.sweep_size + profile_args["sweep_size"] = str(profile.sweep_size) return ", ".join(f"{key}={value}" for key, value in profile_args.items()) @property def benchmarks_args_str(self) -> str: - args = self.benchmarks[0].args + args = self.benchmarks[0].args if self.benchmarks else None + + if args is None: + return "None" + args_dict = OrderedDict( { "max_number": args.max_number, @@ -118,15 +128,15 @@ def benchmarks_args_str(self) -> str: @property def benchmarks_worker_desc_str(self) -> str: - return str(self.benchmarks[0].worker) + return str(self.benchmarks[0].worker) if self.benchmarks else "None" @property def benchmarks_request_loader_desc_str(self) -> str: - return str(self.benchmarks[0].request_loader) + return str(self.benchmarks[0].request_loader) if self.benchmarks else "None" @property def benchmarks_extras_str(self) -> str: - extras = self.benchmarks[0].extras + extras = self.benchmarks[0].extras if self.benchmarks else None if not extras: return "None" diff --git a/src/guidellm/benchmark/profile.py b/src/guidellm/benchmark/profile.py index 5020081f..99f01f2e 100644 --- a/src/guidellm/benchmark/profile.py +++ b/src/guidellm/benchmark/profile.py @@ -53,7 +53,7 @@ def completed_strategy(self, average_rate: float, average_concurrency: float): self.measured_concurrencies.append(average_concurrency) self.completed_strategies += 1 - @computed_field + @computed_field # type: ignore[misc] @property def strategy_types(self) -> List[StrategyType]: return [] @@ -63,7 +63,7 @@ def next_strategy(self) -> Optional[SchedulingStrategy]: class SynchronousProfile(Profile): - type_: Literal["synchronous"] = "synchronous" + type_: Literal["synchronous"] = "synchronous" # type: ignore[assignment] @property def strategy_types(self) -> List[StrategyType]: @@ -98,7 +98,7 @@ def from_standard_args( class ConcurrentProfile(Profile): - type_: Literal["concurrent"] = "concurrent" + type_: Literal["concurrent"] = "concurrent" # type: ignore[assignment] streams: Union[int, Sequence[int]] = Field( description="The number of concurrent streams to use.", ) @@ -144,11 +144,11 @@ def from_standard_args( "No additional arguments are allowed for concurrent profile." ) - return ConcurrentProfile(streams=rate) + return ConcurrentProfile(streams=[int(rat) for rat in rate]) class ThroughputProfile(Profile): - type_: Literal["throughput"] = "throughput" + type_: Literal["throughput"] = "throughput" # type: ignore[assignment] max_concurrency: Optional[int] = Field( default=None, description="The maximum number of concurrent requests that can be scheduled.", @@ -184,7 +184,7 @@ def from_standard_args( class AsyncProfile(ThroughputProfile): - type_: Literal["async"] = "async" + type_: Literal["async"] = "async" # type: ignore[assignment] strategy_type: Literal["constant", "poisson"] = Field( description="The type of asynchronous strategy to use.", ) @@ -235,7 +235,7 @@ def next_strategy(self) -> Optional[SchedulingStrategy]: raise ValueError(f"Invalid strategy type: {self.strategy_type}") @staticmethod - def from_standard_args( + def from_standard_args( # type: ignore[override] rate_type: Union[StrategyType, ProfileType], rate: Optional[Union[float, Sequence[float]]], random_seed: int, @@ -262,7 +262,7 @@ def from_standard_args( rate_type = "constant" # default to constant if not specified return AsyncProfile( - strategy_type=rate_type, + strategy_type=rate_type, # type: ignore[arg-type] rate=rate, random_seed=random_seed, **kwargs, @@ -270,7 +270,7 @@ def from_standard_args( class SweepProfile(AsyncProfile): - type_: Literal["sweep"] = "sweep" + type_: Literal["sweep"] = "sweep" # type: ignore[assignment] sweep_size: int = Field( description="The number of strategies to generate for the sweep.", ) @@ -280,7 +280,7 @@ class SweepProfile(AsyncProfile): @property def strategy_types(self) -> List[StrategyType]: return ( - ["synchronous"] + ["throughput"] + [self.rate_type] * (self.sweep_size - 2) + ["synchronous"] + ["throughput"] + [self.rate_type] * (self.sweep_size - 2) # type: ignore[return-value] ) def next_strategy(self) -> Optional[SchedulingStrategy]: @@ -315,7 +315,7 @@ def next_strategy(self) -> Optional[SchedulingStrategy]: raise ValueError(f"Invalid strategy type: {self.rate_type}") @staticmethod - def from_standard_args( + def from_standard_args( # type: ignore[override] rate_type: Union[StrategyType, ProfileType], rate: Optional[Union[float, Sequence[float]]], random_seed: int, @@ -350,7 +350,7 @@ def from_standard_args( if "strategy_type" not in kwargs: kwargs["strategy_type"] = "constant" - return SweepProfile(sweep_size=rate, random_seed=random_seed, **kwargs) + return SweepProfile(sweep_size=int(rate), random_seed=random_seed, **kwargs) def create_profile( diff --git a/src/guidellm/benchmark/progress.py b/src/guidellm/benchmark/progress.py index 0bcc6f31..b7b4042f 100644 --- a/src/guidellm/benchmark/progress.py +++ b/src/guidellm/benchmark/progress.py @@ -19,6 +19,11 @@ TimeRemainingColumn, ) +from guidellm.benchmark.aggregator import ( + BenchmarkAggregator, + GenerativeBenchmarkAggregator, +) +from guidellm.benchmark.benchmark import Benchmark, GenerativeBenchmark from guidellm.benchmark.benchmarker import BenchmarkerResult from guidellm.scheduler import ( SchedulingStrategy, @@ -39,17 +44,17 @@ class BenchmarkerTaskProgressState: ended: bool = False start_time: Optional[float] = None - max_number: Optional[int] = None + max_number: Optional[float] = None max_duration: Optional[float] = None in_warmup: bool = False in_cooldown: bool = False requests_rate: float = 0 request_latency: float = 0 - requests_processing: int = 0 - requests_successful: int = 0 - requests_incomplete: int = 0 - requests_errored: int = 0 + requests_processing: float = 0 + requests_successful: float = 0 + requests_incomplete: float = 0 + requests_errored: float = 0 worker_overheads_time_ms: float = 0.0 backend_overheads_time_ms: float = 0.0 @@ -81,7 +86,7 @@ def completed(self) -> int: ) duration_percent = ( (time.time() - self.start_time) / self.max_duration * 1000 - if self.max_duration + if self.max_duration and self.start_time else -math.inf ) @@ -250,10 +255,12 @@ def format_progress_display( formatted_number = f"{value:>{digits_places}.{decimal_places}f}" result = f"{formatted_number}{units} [{Colors.INFO}]{label}[/{Colors.INFO}]" - total_characters += len(Colors.INFO) * 2 + 5 - if total_characters is not None and len(result) < total_characters: - result = result.rjust(total_characters) + if total_characters is not None: + total_characters += len(Colors.INFO) * 2 + 5 + + if len(result) < total_characters: + result = result.rjust(total_characters) return result @@ -371,7 +378,7 @@ def __init__(self, display_scheduler_stats: bool): redirect_stderr=True, ) self.active_task: Optional[TaskID] = None - self.benchmarker_tasks: List[BenchmarkerTaskProgressState] = [] + self.benchmarker_tasks: List[BTPS] = [] self.progress_task: Optional[TaskID] = None def update(self, result: BenchmarkerResult): @@ -415,7 +422,7 @@ def handle_start(self, result: BenchmarkerResult): task_id, description=task_progress_state.description, visible=True, - **task_progress_state.fields, + **task_progress_state.fields, # type: ignore[arg-type] ) self.progress_task = self.benchmarker_progress.add_task( @@ -426,7 +433,7 @@ def handle_start(self, result: BenchmarkerResult): ) def handle_update(self, result: BenchmarkerResult): - current_state = self.benchmarker_tasks[result.current_index] + current_state: BTPS = self.benchmarker_tasks[result.current_index] if result.type_ == "scheduler_start": self.handle_update_scheduler_start(current_state, result) @@ -440,12 +447,15 @@ def handle_update(self, result: BenchmarkerResult): else: raise ValueError(f"Unknown result type: {result.type_}") + if self.progress_task is None: + raise RuntimeError("Progress task not set.") + self.benchmarker_tasks_progress.update( current_state.task_id, description=current_state.description, completed=current_state.completed, total=current_state.total, - **current_state.fields, + **current_state.fields, # type: ignore[arg-type] ) self.benchmarker_progress.update( self.progress_task, @@ -462,21 +472,22 @@ def handle_update(self, result: BenchmarkerResult): self.active_task = None def handle_update_scheduler_start( - self, progress_state: BenchmarkerTaskProgressState, result: BenchmarkerResult + self, progress_state: BTPS, result: BenchmarkerResult ): if self.active_task is not None: raise RuntimeError("Active task already set.") - progress_state.strategy = result.current_strategy + progress_state.strategy = result.current_strategy # type: ignore[assignment] progress_state.started = True + current_aggregator: BenchmarkAggregator = result.current_aggregator # type: ignore[assignment] progress_state.start_time = ( - result.current_aggregator.scheduler_created_requests.start_time + current_aggregator.scheduler_created_requests.start_time ) - progress_state.max_number = result.current_aggregator.max_number - progress_state.max_duration = result.current_aggregator.max_duration + progress_state.max_number = current_aggregator.max_number + progress_state.max_duration = current_aggregator.max_duration def handle_update_scheduler_update( - self, progress_state: BenchmarkerTaskProgressState, result: BenchmarkerResult + self, progress_state: BTPS, result: BenchmarkerResult ): if self.active_task is None: raise RuntimeError("Active task not set.") @@ -484,41 +495,40 @@ def handle_update_scheduler_update( if self.active_task != progress_state.task_id: raise RuntimeError("Active task does not match current task.") - progress_state.in_warmup = result.current_aggregator.in_warmup - progress_state.in_cooldown = result.current_aggregator.in_cooldown - progress_state.requests_rate = ( - result.current_aggregator.successful_requests.rate - ) - progress_state.request_latency = result.current_aggregator.request_time.mean + current_aggregator: BenchmarkAggregator = result.current_aggregator # type: ignore[assignment] + progress_state.in_warmup = current_aggregator.in_warmup + progress_state.in_cooldown = current_aggregator.in_cooldown + progress_state.requests_rate = current_aggregator.successful_requests.rate + progress_state.request_latency = current_aggregator.request_time.mean progress_state.requests_processing = ( - result.current_aggregator.scheduler_processing_requests.last + current_aggregator.scheduler_processing_requests.last ) progress_state.requests_successful = ( - result.current_aggregator.successful_requests.total + current_aggregator.successful_requests.total ) progress_state.requests_incomplete = ( - result.current_aggregator.incomplete_requests.total - ) - progress_state.requests_errored = ( - result.current_aggregator.errored_requests.total + current_aggregator.incomplete_requests.total ) + progress_state.requests_errored = current_aggregator.errored_requests.total progress_state.worker_overheads_time_ms = ( - result.current_aggregator.scheduled_time_delay.mean_ms - + result.current_aggregator.worker_start_delay.mean_ms + current_aggregator.scheduled_time_delay.mean_ms + + current_aggregator.worker_start_delay.mean_ms ) progress_state.backend_overheads_time_ms = ( - result.current_aggregator.request_time_delay.mean_ms + current_aggregator.request_time_delay.mean_ms ) progress_state.requests_sleep_time_ms = ( - result.current_aggregator.scheduled_time_sleep.mean_ms + current_aggregator.scheduled_time_sleep.mean_ms ) progress_state.requests_targeted_start_time_delay_ms = ( - result.current_aggregator.request_start_time_targeted_delay.mean_ms + current_aggregator.request_start_time_targeted_delay.mean_ms ) def handle_update_scheduler_complete( - self, progress_state: BenchmarkerTaskProgressState, result: BenchmarkerResult + self, + progress_state: BTPS, + result: BenchmarkerResult, # noqa: ARG002 ): if self.active_task is None: raise RuntimeError("Active task not set.") @@ -531,7 +541,7 @@ def handle_update_scheduler_complete( progress_state.compiling = True def handle_update_benchmark_compiled( - self, progress_state: BenchmarkerTaskProgressState, result: BenchmarkerResult + self, progress_state: BTPS, result: BenchmarkerResult ): if self.active_task is None: raise RuntimeError("Active task not set.") @@ -539,21 +549,20 @@ def handle_update_benchmark_compiled( if self.active_task != progress_state.task_id: raise RuntimeError("Active task does not match current task.") + current_benchmark: Benchmark = result.current_benchmark # type: ignore[assignment] progress_state.compiling = False progress_state.ended = True progress_state.requests_rate = ( - result.current_benchmark.requests_per_second.successful.mean - ) - progress_state.request_latency = ( - result.current_benchmark.request_latency.successful.mean + current_benchmark.requests_per_second.successful.mean ) progress_state.requests_processing = ( - result.current_benchmark.requests_concurrency.successful.mean + current_benchmark.requests_concurrency.successful.mean ) - progress_state.requests_successful = result.current_benchmark.successful_total - progress_state.requests_errored = result.current_benchmark.errored_total - def handle_end(self, result: BenchmarkerResult): + def handle_end(self, result: BenchmarkerResult): # noqa: ARG002 + if self.progress_task is None: + raise RuntimeError("Progress task not set.") + self.benchmarker_progress.update( self.progress_task, completed=len(self.benchmarker_tasks) * 1000, @@ -593,11 +602,11 @@ def create_task_progress_columns(self) -> List[ProgressColumn]: def create_task_progress_state( self, task_id: TaskID, - index: int, + index: int, # noqa: ARG002 strategy_type: StrategyType, - result: BenchmarkerResult, - ) -> BenchmarkerTaskProgressState: - return BenchmarkerTaskProgressState( + result: BenchmarkerResult, # noqa: ARG002 + ) -> BTPS: + return BenchmarkerTaskProgressState( # type: ignore[return-value] display_scheduler_stats=self.display_scheduler_stats, task_id=task_id, strategy=strategy_type, @@ -607,43 +616,61 @@ def create_task_progress_state( class GenerativeTextBenchmarkerProgressDisplay( BenchmarkerProgressDisplay[GenerativeTextBenchmarkerTaskProgressState] ): - def handle_update_scheduler_update(self, progress_state, result): + def handle_update_scheduler_update( + self, + progress_state: GenerativeTextBenchmarkerTaskProgressState, + result: BenchmarkerResult, + ): super().handle_update_scheduler_update(progress_state, result) - progress_state.output_tokens = result.current_aggregator.output_tokens.mean - progress_state.prompt_tokens = result.current_aggregator.prompt_tokens.mean - progress_state.output_tokens_rate = result.current_aggregator.output_tokens.rate - progress_state.total_tokens_rate = result.current_aggregator.total_tokens.rate - progress_state.tokens_ttft = result.current_aggregator.time_to_first_token.mean - progress_state.tokens_itl = result.current_aggregator.inter_token_latency.mean - - def handle_update_benchmark_compiled(self, progress_state, result): + current_aggregator: GenerativeBenchmarkAggregator = result.current_aggregator # type: ignore[assignment] + progress_state.output_tokens = current_aggregator.output_tokens.mean + progress_state.prompt_tokens = current_aggregator.prompt_tokens.mean + progress_state.output_tokens_rate = current_aggregator.output_tokens.rate + progress_state.total_tokens_rate = current_aggregator.total_tokens.rate + progress_state.tokens_ttft = current_aggregator.time_to_first_token.mean + progress_state.tokens_itl = current_aggregator.inter_token_latency.mean + + def handle_update_benchmark_compiled( + self, + progress_state: GenerativeTextBenchmarkerTaskProgressState, + result: BenchmarkerResult, + ): super().handle_update_benchmark_compiled(progress_state, result) + current_benchmark: GenerativeBenchmark = result.current_benchmark # type: ignore[assignment] + progress_state.request_latency = ( + current_benchmark.request_latency.successful.mean + ) + progress_state.requests_processing = ( + current_benchmark.requests_concurrency.successful.mean + ) + progress_state.requests_successful = current_benchmark.successful_total + progress_state.requests_errored = current_benchmark.errored_total progress_state.output_tokens = ( - result.current_benchmark.output_token_count.successful.mean + current_benchmark.output_token_count.successful.mean ) progress_state.prompt_tokens = ( - result.current_benchmark.prompt_token_count.successful.mean + current_benchmark.prompt_token_count.successful.mean ) progress_state.output_tokens_rate = ( - result.current_benchmark.output_tokens_per_second.successful.mean + current_benchmark.output_tokens_per_second.successful.mean ) progress_state.total_tokens_rate = ( - result.current_benchmark.tokens_per_second.successful.mean + current_benchmark.tokens_per_second.successful.mean ) progress_state.tokens_ttft = ( - result.current_benchmark.time_to_first_token_ms.successful.mean + current_benchmark.time_to_first_token_ms.successful.mean ) progress_state.tokens_itl = ( - result.current_benchmark.inter_token_latency_ms.successful.mean + current_benchmark.inter_token_latency_ms.successful.mean ) def create_task_progress_state( self, task_id: TaskID, - index: int, + index: int, # noqa: ARG002 strategy_type: StrategyType, - result: BenchmarkerResult, + result: BenchmarkerResult, # noqa: ARG002 ) -> GenerativeTextBenchmarkerTaskProgressState: return GenerativeTextBenchmarkerTaskProgressState( display_scheduler_stats=self.display_scheduler_stats, diff --git a/src/guidellm/config.py b/src/guidellm/config.py index 16378856..f3d0e09b 100644 --- a/src/guidellm/config.py +++ b/src/guidellm/config.py @@ -7,7 +7,6 @@ __all__ = [ "DatasetSettings", - "EmulatedDataSettings", "Environment", "LoggingSettings", "OpenAISettings", diff --git a/src/guidellm/dataset/creator.py b/src/guidellm/dataset/creator.py index 41eccd8a..42103a46 100644 --- a/src/guidellm/dataset/creator.py +++ b/src/guidellm/dataset/creator.py @@ -3,12 +3,15 @@ from typing import Any, Dict, List, Literal, Optional, Tuple, Union from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict -from transformers import PreTrainedTokenizerBase +from transformers import PreTrainedTokenizerBase # type: ignore[import] __all__ = ["DatasetCreator", "ColumnInputTypes"] ColumnInputTypes = Literal[ - "text_column", "prompt_tokens_count_column", "output_tokens_count_column" + "prompt_column", + "text_column", + "prompt_tokens_count_column", + "output_tokens_count_column", ] @@ -71,7 +74,7 @@ class DatasetCreator(ABC): "eval_dataset", "eval_data", ] - DEFAULT_SPLITS_DATASET = {} + DEFAULT_SPLITS_DATASET: Dict[str, str] = {} @classmethod def create( @@ -117,7 +120,7 @@ def extract_args_column_mappings( cls, data_args: Optional[Dict[str, Any]], ) -> Dict[ColumnInputTypes, str]: - columns = {} + columns: Dict[ColumnInputTypes, str] = {} if data_args: if "prompt_column" in data_args: diff --git a/src/guidellm/dataset/entrypoints.py b/src/guidellm/dataset/entrypoints.py index f510933d..643ea0f5 100644 --- a/src/guidellm/dataset/entrypoints.py +++ b/src/guidellm/dataset/entrypoints.py @@ -1,9 +1,9 @@ from typing import Any, Dict, List, Optional, Tuple, Union from datasets import Dataset, IterableDataset -from transformers import PreTrainedTokenizerBase +from transformers import PreTrainedTokenizerBase # type: ignore[import] -from guidellm.dataset.creator import ColumnInputTypes, DatasetCreator +from guidellm.dataset.creator import ColumnInputTypes from guidellm.dataset.file import FileDatasetCreator from guidellm.dataset.hf_datasets import HFDatasetsCreator from guidellm.dataset.in_memory import InMemoryDatasetCreator @@ -20,7 +20,7 @@ def load_dataset( random_seed: int = 42, split_pref_order: Optional[List[str]] = None, ) -> Tuple[Union[Dataset, IterableDataset], Dict[ColumnInputTypes, str]]: - creators: List[DatasetCreator] = [ + creators = [ InMemoryDatasetCreator, SyntheticDatasetCreator, FileDatasetCreator, diff --git a/src/guidellm/dataset/file.py b/src/guidellm/dataset/file.py index 1118db87..9f9cf696 100644 --- a/src/guidellm/dataset/file.py +++ b/src/guidellm/dataset/file.py @@ -1,7 +1,7 @@ from pathlib import Path from typing import Any, Dict, Optional, Union -import pandas as pd +import pandas as pd # type: ignore[import] from datasets import ( Dataset, DatasetDict, @@ -9,7 +9,7 @@ IterableDatasetDict, load_dataset, ) -from transformers import PreTrainedTokenizerBase +from transformers import PreTrainedTokenizerBase # type: ignore[import] from guidellm.dataset.creator import DatasetCreator @@ -30,7 +30,7 @@ class FileDatasetCreator(DatasetCreator): } @classmethod - def is_supported(cls, data: Any, data_args: Optional[Dict[str, Any]]) -> bool: + def is_supported(cls, data: Any, data_args: Optional[Dict[str, Any]]) -> bool: # noqa: ARG003 if isinstance(data, (str, Path)) and (path := Path(data)).exists(): # local folder or py file, assume supported return path.suffix.lower() in cls.SUPPORTED_TYPES @@ -42,9 +42,9 @@ def handle_create( cls, data: Any, data_args: Optional[Dict[str, Any]], - processor: Optional[Union[str, Path, PreTrainedTokenizerBase]], - processor_args: Optional[Dict[str, Any]], - random_seed: int, + processor: Optional[Union[str, Path, PreTrainedTokenizerBase]], # noqa: ARG003 + processor_args: Optional[Dict[str, Any]], # noqa: ARG003 + random_seed: int, # noqa: ARG003 ) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]: if not isinstance(data, (str, Path)): raise ValueError(f"Unsupported data type: {type(data)} given for {data}. ") diff --git a/src/guidellm/dataset/hf_datasets.py b/src/guidellm/dataset/hf_datasets.py index 4575d920..e0102538 100644 --- a/src/guidellm/dataset/hf_datasets.py +++ b/src/guidellm/dataset/hf_datasets.py @@ -9,7 +9,7 @@ get_dataset_config_info, load_dataset, ) -from transformers import PreTrainedTokenizerBase +from transformers import PreTrainedTokenizerBase # type: ignore[import] from guidellm.dataset.creator import DatasetCreator @@ -18,7 +18,7 @@ class HFDatasetsCreator(DatasetCreator): @classmethod - def is_supported(cls, data: Any, data_args: Optional[Dict[str, Any]]) -> bool: + def is_supported(cls, data: Any, data_args: Optional[Dict[str, Any]]) -> bool: # noqa: ARG003 if isinstance( data, (Dataset, DatasetDict, IterableDataset, IterableDatasetDict) ): @@ -33,7 +33,7 @@ def is_supported(cls, data: Any, data_args: Optional[Dict[str, Any]]) -> bool: try: # try to load dataset return get_dataset_config_info(data) is not None - except: + except Exception: # noqa: BLE001, S110 pass return False @@ -43,9 +43,9 @@ def handle_create( cls, data: Any, data_args: Optional[Dict[str, Any]], - processor: Optional[Union[str, Path, PreTrainedTokenizerBase]], - processor_args: Optional[Dict[str, Any]], - random_seed: int, + processor: Optional[Union[str, Path, PreTrainedTokenizerBase]], # noqa: ARG003 + processor_args: Optional[Dict[str, Any]], # noqa: ARG003 + random_seed: int, # noqa: ARG003 ) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]: if isinstance(data, (str, Path)): data = load_dataset(data, **(data_args or {})) diff --git a/src/guidellm/dataset/in_memory.py b/src/guidellm/dataset/in_memory.py index bd531b1f..dc173d2f 100644 --- a/src/guidellm/dataset/in_memory.py +++ b/src/guidellm/dataset/in_memory.py @@ -7,7 +7,7 @@ IterableDataset, IterableDatasetDict, ) -from transformers import PreTrainedTokenizerBase +from transformers import PreTrainedTokenizerBase # type: ignore[import] from guidellm.dataset.creator import DatasetCreator @@ -16,7 +16,7 @@ class InMemoryDatasetCreator(DatasetCreator): @classmethod - def is_supported(cls, data: Any, data_args: Optional[Dict[str, Any]]) -> bool: + def is_supported(cls, data: Any, data_args: Optional[Dict[str, Any]]) -> bool: # noqa: ARG003 return isinstance(data, Iterable) and not isinstance(data, str) @classmethod @@ -24,9 +24,9 @@ def handle_create( cls, data: Any, data_args: Optional[Dict[str, Any]], - processor: Optional[Union[str, Path, PreTrainedTokenizerBase]], - processor_args: Optional[Dict[str, Any]], - random_seed: int, + processor: Optional[Union[str, Path, PreTrainedTokenizerBase]], # noqa: ARG003 + processor_args: Optional[Dict[str, Any]], # noqa: ARG003 + random_seed: int, # noqa: ARG003 ) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]: if not isinstance(data, Iterable): raise TypeError( @@ -39,7 +39,7 @@ def handle_create( if isinstance(data, Dict): # assume data is a dictionary of columns and values: {"c1": ["i1", "i2"]} data_dict = cls.format_data_dict(data) - elif isinstance(data[0], Dict): + elif isinstance(data[0], Dict): # type: ignore[index] # assume data is a list of dictionaries: [{"c1": "i1"}, {"c1": "i2"}] data_dict = cls.format_data_iterable_dicts(data) else: @@ -90,13 +90,13 @@ def format_data_iterable_dicts( f"got {type(data)}" ) - if not all(isinstance(key, str) for key in data[0]): + if not all(isinstance(key, str) for key in data[0]): # type: ignore[index] raise TypeError( "Unsupported data format. Expected Dict[str, Any], " f"but one of the items had a non string column for {data}" ) - columns = list(data[0].keys()) + columns = list(data[0].keys()) # type: ignore[index] if not all( len(item) == len(columns) and all(key in item for key in columns) for item in data @@ -106,7 +106,7 @@ def format_data_iterable_dicts( f"for {data}" ) - data_dict = {key: [] for key in columns} + data_dict: Dict[str, Any] = {key: [] for key in columns} for item in data: for key, value in item.items(): data_dict[key].append(value) @@ -121,7 +121,8 @@ def format_data_iterable_values(cls, data: Iterable[Any]) -> Dict[str, Any]: f"got {type(data)}" ) - first_type = type(data[0]) + first_item = next(iter(data), None) + first_type = type(first_item) if not all(isinstance(item, first_type) for item in data): raise TypeError( f"Unsupported data format. Not all types are the same for {data}" diff --git a/src/guidellm/dataset/synthetic.py b/src/guidellm/dataset/synthetic.py index 5b3b8686..f2bf69d3 100644 --- a/src/guidellm/dataset/synthetic.py +++ b/src/guidellm/dataset/synthetic.py @@ -1,7 +1,7 @@ import json import random from pathlib import Path -from typing import Any, Dict, Iterable, Iterator, Optional, Tuple, Union +from typing import Any, Dict, Iterable, Iterator, Literal, Optional, Union import yaml from datasets import ( @@ -11,7 +11,7 @@ IterableDatasetDict, ) from pydantic import BaseModel, Field -from transformers import PreTrainedTokenizerBase +from transformers import PreTrainedTokenizerBase # type: ignore[import] from guidellm.dataset.creator import ColumnInputTypes, DatasetCreator from guidellm.utils import EndlessTextCreator, IntegerRangeSampler, check_load_processor @@ -107,7 +107,7 @@ def parse_key_value_pairs(data: str) -> "SyntheticDatasetConfig": int(value.strip()) if value.strip().isnumeric() else value.strip() ) - return SyntheticDatasetConfig(**config_dict) + return SyntheticDatasetConfig(**config_dict) # type: ignore[arg-type] @staticmethod def parse_config_file(data: Union[str, Path]) -> "SyntheticDatasetConfig": @@ -117,7 +117,14 @@ def parse_config_file(data: Union[str, Path]) -> "SyntheticDatasetConfig": return SyntheticDatasetConfig(**config_dict) -class SyntheticTextItemsGenerator(Iterable[Dict[str, Union[str, int]]]): +class SyntheticTextItemsGenerator( + Iterable[ + Dict[ + Literal["prompt", "prompt_tokens_count", "output_tokens_count"], + Union[str, int], + ] + ] +): def __init__( self, config: SyntheticDatasetConfig, @@ -127,12 +134,18 @@ def __init__( self.config = config self.processor = processor self.random_seed = random_seed - self.tokens = [] self.text_creator = EndlessTextCreator( data=config.source, ) - def __iter__(self) -> Iterator[Tuple[str, int, int]]: + def __iter__( + self, + ) -> Iterator[ + Dict[ + Literal["prompt", "prompt_tokens_count", "output_tokens_count"], + Union[str, int], + ] + ]: prompt_tokens_sampler = IntegerRangeSampler( average=self.config.prompt_tokens, variance=self.config.prompt_tokens_stdev, @@ -147,7 +160,8 @@ def __iter__(self) -> Iterator[Tuple[str, int, int]]: max_value=self.config.output_tokens_max, random_seed=self.random_seed + 1, # ensure diff dist from prompts ) - rand = random.Random(self.random_seed + 2) # ensure diff distribution + # ensure diff distribution from output tokens + rand = random.Random(self.random_seed + 2) # noqa: S311 for _, prompt_tokens, output_tokens in zip( range(self.config.samples), @@ -185,7 +199,7 @@ def _create_prompt(self, prompt_tokens: int, start_index: int) -> str: class SyntheticDatasetCreator(DatasetCreator): @classmethod - def is_supported(cls, data: Any, data_args: Optional[Dict[str, Any]]) -> bool: + def is_supported(cls, data: Any, data_args: Optional[Dict[str, Any]]) -> bool: # noqa: ARG003 if ( isinstance(data, Path) and data.exists() diff --git a/src/guidellm/objects/pydantic.py b/src/guidellm/objects/pydantic.py index 1c5754cc..68b87b97 100644 --- a/src/guidellm/objects/pydantic.py +++ b/src/guidellm/objects/pydantic.py @@ -8,8 +8,8 @@ class StandardBaseModel(BaseModel): """ - A base class for models that require YAML and JSON serialization and - deserialization. + A base class for Pydantic models throughout GuideLLM enabling standard + configuration and logging. """ model_config = ConfigDict( diff --git a/src/guidellm/objects/statistics.py b/src/guidellm/objects/statistics.py index 21f48679..e192dee6 100644 --- a/src/guidellm/objects/statistics.py +++ b/src/guidellm/objects/statistics.py @@ -1,7 +1,7 @@ import math import time as timer from collections import defaultdict -from typing import Any, List, Literal, Optional, Tuple +from typing import Any, Dict, List, Literal, Optional, Tuple import numpy as np from pydantic import Field, computed_field @@ -19,7 +19,7 @@ class Percentiles(StandardBaseModel): """ - A serializable model representing percentiles of a distribution. + A pydantic model representing the standard percentiles of a distribution. """ p001: float = Field( @@ -56,7 +56,7 @@ class Percentiles(StandardBaseModel): class DistributionSummary(StandardBaseModel): """ - A serializable model representing a statistical summary for a given + A pydantic model representing a statistical summary for a given distribution of numerical values. """ @@ -91,7 +91,7 @@ class DistributionSummary(StandardBaseModel): description="The percentiles of the distribution.", ) cumulative_distribution_function: Optional[List[Tuple[float, float]]] = Field( - description=("The cumulative distribution function (CDF) of the distribution."), + description="The cumulative distribution function (CDF) of the distribution.", default=None, ) @@ -101,20 +101,19 @@ def from_distribution_function( include_cdf: bool = False, ) -> "DistributionSummary": """ - Calculate a distribution summary from a values or - probability distribution function (PDF). - For a PDF, it is expected to be a list of tuples where each tuple - contains a value and its probability. - The probabilities across all elements should be normalized (sum to 1). - If the PDF is not normalized, it will be normalized. - The values distribution function is a list of tuples where each tuple - contains a value and some weighting for that value. - The weightings will be normalized to a probability distribution function. - - :param pdf: A list of tuples representing the PDF. - Each tuple contains a value and its probability. - :param include_cdf: Whether to include the cumulative distribution function - in the output DistributionSummary. + Create a statistical summary for a given distribution of weighted numerical + values or a probability distribution function (PDF). + 1. If the distribution is a PDF, it is expected to be a list of tuples + where each tuple contains (value, probability). The sum of the + probabilities should be 1. If it is not, it will be normalized. + 2. If the distribution is a values distribution function, it is expected + to be a list of tuples where each tuple contains (value, weight). + The weights are normalized to a probability distribution function. + + :param distribution: A list of tuples representing the distribution. + Each tuple contains (value, weight) or (value, probability). + :param include_cdf: Whether to include the calculated cumulative distribution + function (CDF) in the output DistributionSummary. :return: An instance of DistributionSummary with calculated values. """ values, weights = zip(*distribution) if distribution else ([], []) @@ -122,7 +121,7 @@ def from_distribution_function( weights = np.array(weights) # create the PDF - probabilities = weights / np.sum(weights) + probabilities = weights / np.sum(weights) # type: ignore[operator] pdf = np.column_stack((values, probabilities)) pdf = pdf[np.argsort(pdf[:, 0])] values = pdf[:, 0] @@ -133,15 +132,15 @@ def from_distribution_function( cdf = np.column_stack((values, cumulative_probabilities)) # calculate statistics - mean = np.sum(values * probabilities).item() - median = cdf[np.argmax(cdf[:, 1] >= 0.5), 0].item() if len(cdf) > 0 else 0 - mode = values[np.argmax(probabilities)].item() if len(values) > 0 else 0 - variance = np.sum((values - mean) ** 2 * probabilities).item() + mean = np.sum(values * probabilities).item() # type: ignore[attr-defined] + median = cdf[np.argmax(cdf[:, 1] >= 0.5), 0].item() if len(cdf) > 0 else 0 # noqa: PLR2004 + mode = values[np.argmax(probabilities)].item() if len(values) > 0 else 0 # type: ignore[call-overload] + variance = np.sum((values - mean) ** 2 * probabilities).item() # type: ignore[attr-defined] std_dev = math.sqrt(variance) minimum = values[0].item() if len(values) > 0 else 0 maximum = values[-1].item() if len(values) > 0 else 0 count = len(values) - total_sum = np.sum(values).item() + total_sum = np.sum(values).item() # type: ignore[attr-defined] return DistributionSummary( mean=mean, @@ -190,19 +189,16 @@ def from_values( include_cdf: bool = False, ) -> "DistributionSummary": """ - Calculate a distribution summary from a list of values. - If the list is empty, all stats are set to 0. - If weights are provided, they are used to weight the values - so that the probabilities are shifted accordingly and larger - weights are given more importance / weight in the distribution. - If the weights are not provided, all values are treated equally. - - :param values: A list of numerical values. - :param weights: A list of weights for each value. - If None, all values are treated equally. - :param include_cdf: Whether to include the cumulative distribution function - in the output DistributionSummary. - :return: An instance of DistributionSummary with calculated values. + Create a statistical summary for a given distribution of numerical values. + This is a wrapper around from_distribution_function to handle the optional case + of including weights for the values. If weights are not provided, they are + automatically set to 1.0 for each value, so each value is equally weighted. + + :param values: A list of numerical values representing the distribution. + :param weights: A list of weights for each value in the distribution. + If not provided, all values are equally weighted. + :param include_cdf: Whether to include the calculated cumulative distribution + function (CDF) in the output DistributionSummary. """ if weights is None: weights = [1.0] * len(values) @@ -224,9 +220,25 @@ def from_request_times( include_cdf: bool = False, epsilon: float = 1e-6, ) -> "DistributionSummary": + """ + Create a statistical summary for a given distribution of request times. + Specifically, this is used to measure concurrency or rate of requests + given an input list containing the start and end time of each request. + This will first convert the request times into a distribution function + and then calculate the statistics with from_distribution_function. + + :param requests: A list of tuples representing the start and end times of + each request. Example: [(start_1, end_1), (start_2, end_2), ...] + :param distribution_type: The type of distribution to calculate. + Either "concurrency" or "rate". + :param include_cdf: Whether to include the calculated cumulative distribution + function (CDF) in the output DistributionSummary. + :param epsilon: The epsilon value for merging close events. + :return: An instance of DistributionSummary with calculated values. + """ if distribution_type == "concurrency": # convert to delta changes based on when requests were running - time_deltas = defaultdict(int) + time_deltas: Dict[float, int] = defaultdict(int) for start, end in requests: time_deltas[start] += 1 time_deltas[end] -= 1 @@ -242,21 +254,30 @@ def from_request_times( # convert to events for when requests finished global_start = min(start for start, _ in requests) if requests else 0 events = [(global_start, 1)] + [(end, 1) for _, end in requests] + else: + raise ValueError( + f"Invalid distribution_type '{distribution_type}'. " + "Must be 'concurrency' or 'rate'." + ) # combine any events that are very close together - flattened_events = [] + flattened_events: List[Tuple[float, float]] = [] for time, val in sorted(events): last_time, last_val = ( flattened_events[-1] if flattened_events else (None, None) ) - if last_time is not None and abs(last_time - time) <= epsilon: + if ( + last_time is not None + and last_val is not None + and abs(last_time - time) <= epsilon + ): flattened_events[-1] = (last_time, last_val + val) else: flattened_events.append((time, val)) # convert to value distribution function - distribution = defaultdict(float) + distribution: Dict[float, float] = defaultdict(float) for ind in range(len(flattened_events) - 1): start_time, value = flattened_events[ind] @@ -271,10 +292,10 @@ def from_request_times( rate = value / duration distribution[rate] += duration - distribution = sorted(distribution.items()) + distribution_list: List[Tuple[float, float]] = sorted(distribution.items()) return DistributionSummary.from_distribution_function( - distribution=distribution, + distribution=distribution_list, include_cdf=include_cdf, ) @@ -287,6 +308,32 @@ def from_iterable_request_times( include_cdf: bool = False, epsilon: float = 1e-6, ) -> "DistributionSummary": + """ + Create a statistical summary for a given distribution of request times + for a request with iterable responses between the start and end. + For example, this is used to measure auto regressive requests where + a request is started and at some later point, iterative responses are + received. This will convert the request times and iterable values into + a distribution function and then calculate the statistics with + from_distribution_function. + + :param requests: A list of tuples representing the start and end times of + each request. Example: [(start_1, end_1), (start_2, end_2), ...] + :param first_iter_times: A list of times when the first iteration of + each request was received. Must be the same length as requests. + :param iter_counts: A list of the total number of iterations for each + request that occurred starting at the first iteration and ending + at the request end time. Must be the same length as requests. + :param first_iter_counts: A list of the number of iterations to log + for the first iteration of each request. For example, when calculating + total number of tokens processed, this is set to the prompt tokens number. + If not provided, defaults to 1 for each request. + :param include_cdf: Whether to include the calculated cumulative distribution + function (CDF) in the output DistributionSummary. + :param epsilon: The epsilon value for merging close events. + :return: An instance of DistributionSummary with calculated values. + """ + if first_iter_counts is None: first_iter_counts = [1] * len(requests) @@ -320,20 +367,24 @@ def from_iterable_request_times( events[first_iter + ind * iter_latency] += 1 # combine any events that are very close together - flattened_events = [] + flattened_events: List[Tuple[float, int]] = [] for time, count in sorted(events.items()): last_time, last_count = ( flattened_events[-1] if flattened_events else (None, None) ) - if last_time is not None and abs(last_time - time) <= epsilon: + if ( + last_time is not None + and last_count is not None + and abs(last_time - time) <= epsilon + ): flattened_events[-1] = (last_time, last_count + count) else: flattened_events.append((time, count)) # convert to value distribution function - distribution = defaultdict(float) + distribution: Dict[float, float] = defaultdict(float) for ind in range(len(flattened_events) - 1): start_time, count = flattened_events[ind] @@ -342,27 +393,26 @@ def from_iterable_request_times( rate = count / duration distribution[rate] += duration - distribution = sorted(distribution.items()) + distribution_list = sorted(distribution.items()) return DistributionSummary.from_distribution_function( - distribution=distribution, + distribution=distribution_list, include_cdf=include_cdf, ) class StatusDistributionSummary(StandardBaseModel): """ - A serializable model representing distribution summary statistics - based on groupings of status (e.g., successful, incomplete, error) for a given - distribution of numerical values. - Handles the total, successful, and errored dfistributions where the total - is the combination of the successful and errored distributions. + A pydantic model representing a statistical summary for a given + distribution of numerical values grouped by status. + Specifically used to represent the total, successful, incomplete, + and errored values for a benchmark or other statistical summary. """ total: DistributionSummary = Field( description=( - "The dist summary for all statuses (successful, incomplete, error).", - ) + "The dist summary for all statuses (successful, incomplete, error)." + ), ) successful: DistributionSummary = Field( description=( @@ -389,6 +439,24 @@ def from_values( weights: Optional[List[float]] = None, include_cdf: bool = False, ) -> "StatusDistributionSummary": + """ + Create a statistical summary by status for a given distribution of numerical + values. This is used to measure the distribution of values for different + statuses (e.g., successful, incomplete, error) and calculate the statistics + for each status. Weights are optional to weight the probability distribution + for each value by. If not provided, all values are equally weighted. + + :param value_types: A list of status types for each value in the distribution. + Must be one of 'successful', 'incomplete', or 'error'. + :param values: A list of numerical values representing the distribution. + Must be the same length as value_types. + :param weights: A list of weights for each value in the distribution. + If not provided, all values are equally weighted (set to 1). + Must be the same length as value_types. + :param include_cdf: Whether to include the calculated cumulative distribution + function (CDF) in the output StatusDistributionSummary. + :return: An instance of StatusDistributionSummary with calculated values. + """ if any( type_ not in {"successful", "incomplete", "error"} for type_ in value_types ): @@ -449,18 +517,18 @@ def from_values( include_cdf=include_cdf, ), successful=DistributionSummary.from_values( - successful_values, - successful_weights, + successful_values, # type: ignore[arg-type] + successful_weights, # type: ignore[arg-type] include_cdf=include_cdf, ), incomplete=DistributionSummary.from_values( - incomplete_values, - incomplete_weights, + incomplete_values, # type: ignore[arg-type] + incomplete_weights, # type: ignore[arg-type] include_cdf=include_cdf, ), errored=DistributionSummary.from_values( - errored_values, - errored_weights, + errored_values, # type: ignore[arg-type] + errored_weights, # type: ignore[arg-type] include_cdf=include_cdf, ), ) @@ -473,6 +541,25 @@ def from_request_times( include_cdf: bool = False, epsilon: float = 1e-6, ) -> "StatusDistributionSummary": + """ + Create a statistical summary by status for given distribution of request times. + This is used to measure the distribution of request times for different statuses + (e.g., successful, incomplete, error) for concurrency and rates. + This will call into DistributionSummary.from_request_times to calculate + the statistics for each status. + + :param request_types: List of status types for each request in the distribution. + Must be one of 'successful', 'incomplete', or 'error'. + :param requests: A list of tuples representing the start and end times of + each request. Example: [(start_1, end_1), (start_2, end_2), ...]. + Must be the same length as request_types. + :param distribution_type: The type of distribution to calculate. + Either "concurrency" or "rate". + :param include_cdf: Whether to include the calculated cumulative distribution + function (CDF) in the output StatusDistributionSummary. + :param epsilon: The epsilon value for merging close events. + :return: An instance of StatusDistributionSummary with calculated values. + """ if distribution_type not in {"concurrency", "rate"}: raise ValueError( f"Invalid distribution_type '{distribution_type}'. " @@ -539,19 +626,19 @@ def from_request_times( epsilon=epsilon, ), successful=DistributionSummary.from_request_times( - successful_requests, + successful_requests, # type: ignore[arg-type] distribution_type=distribution_type, include_cdf=include_cdf, epsilon=epsilon, ), incomplete=DistributionSummary.from_request_times( - incomplete_requests, + incomplete_requests, # type: ignore[arg-type] distribution_type=distribution_type, include_cdf=include_cdf, epsilon=epsilon, ), errored=DistributionSummary.from_request_times( - errored_requests, + errored_requests, # type: ignore[arg-type] distribution_type=distribution_type, include_cdf=include_cdf, epsilon=epsilon, @@ -568,6 +655,34 @@ def from_iterable_request_times( include_cdf: bool = False, epsilon: float = 1e-6, ) -> "StatusDistributionSummary": + """ + Create a statistical summary by status for given distribution of request times + for a request with iterable responses between the start and end. + For example, this is used to measure auto regressive requests where + a request is started and at some later point, iterative responses are + received. This will call into DistributionSummary.from_iterable_request_times + to calculate the statistics for each status. + + :param request_types: List of status types for each request in the distribution. + Must be one of 'successful', 'incomplete', or 'error'. + :param requests: A list of tuples representing the start and end times of + each request. Example: [(start_1, end_1), (start_2, end_2), ...]. + Must be the same length as request_types. + :param first_iter_times: A list of times when the first iteration of + each request was received. Must be the same length as requests. + :param iter_counts: A list of the total number of iterations for each + request that occurred starting at the first iteration and ending + at the request end time. Must be the same length as requests. + If not provided, defaults to 1 for each request. + :param first_iter_counts: A list of the number of iterations to log + for the first iteration of each request. For example, when calculating + total number of tokens processed, this is set to the prompt tokens number. + If not provided, defaults to 1 for each request. + :param include_cdf: Whether to include the calculated cumulative distribution + function (CDF) in the output StatusDistributionSummary. + :param epsilon: The epsilon value for merging close events. + :return: An instance of StatusDistributionSummary with calculated values. + """ if any( type_ not in {"successful", "incomplete", "error"} for type_ in request_types @@ -680,26 +795,26 @@ def from_iterable_request_times( epsilon=epsilon, ), successful=DistributionSummary.from_iterable_request_times( - successful_requests, - successful_first_iter_times, - successful_iter_counts, - successful_first_iter_counts, + successful_requests, # type: ignore[arg-type] + successful_first_iter_times, # type: ignore[arg-type] + successful_iter_counts, # type: ignore[arg-type] + successful_first_iter_counts, # type: ignore[arg-type] include_cdf=include_cdf, epsilon=epsilon, ), incomplete=DistributionSummary.from_iterable_request_times( - incomplete_requests, - incomplete_first_iter_times, - incomplete_iter_counts, - incomplete_first_iter_counts, + incomplete_requests, # type: ignore[arg-type] + incomplete_first_iter_times, # type: ignore[arg-type] + incomplete_iter_counts, # type: ignore[arg-type] + incomplete_first_iter_counts, # type: ignore[arg-type] include_cdf=include_cdf, epsilon=epsilon, ), errored=DistributionSummary.from_iterable_request_times( - errored_requests, - errored_first_iter_times, - errored_iter_counts, - errored_first_iter_counts, + errored_requests, # type: ignore[arg-type] + errored_first_iter_times, # type: ignore[arg-type] + errored_iter_counts, # type: ignore[arg-type] + errored_first_iter_counts, # type: ignore[arg-type] include_cdf=include_cdf, epsilon=epsilon, ), @@ -707,35 +822,66 @@ def from_iterable_request_times( class RunningStats(StandardBaseModel): + """ + Create a running statistics object to track the mean, rate, and other + statistics of a stream of values. + 1. The start time is set to the time the object is created. + 2. The count is set to 0. + 3. The total is set to 0. + 4. The last value is set to 0. + 5. The mean is calculated as the total / count. + """ + start_time: float = Field( default_factory=timer.time, + description=( + "The time the running statistics object was created. " + "This is used to calculate the rate of the statistics." + ), ) count: int = Field( default=0, + description="The number of values added to the running statistics.", ) total: float = Field( default=0.0, + description="The total sum of the values added to the running statistics.", ) last: float = Field( default=0.0, description="The last value added to the running statistics.", ) - @computed_field + @computed_field # type: ignore[misc] @property def mean(self) -> float: + """ + :return: The mean of the running statistics (total / count). + If count is 0, return 0.0. + """ if self.count == 0: return 0.0 return self.total / self.count - @computed_field + @computed_field # type: ignore[misc] @property def rate(self) -> float: + """ + :return: The rate of the running statistics + (total / (time.time() - start_time)). + If count is 0, return 0.0. + """ if self.count == 0: return 0.0 return self.total / (timer.time() - self.start_time) def __add__(self, value: Any) -> float: + """ + Enable the use of the + operator to add a value to the running statistics. + + :param value: The value to add to the running statistics. + :return: The mean of the running statistics. + """ if not isinstance(value, (int, float)): raise ValueError( f"Value must be an int or float, got {type(value)} instead.", @@ -746,6 +892,12 @@ def __add__(self, value: Any) -> float: return self.mean def __iadd__(self, value: Any) -> "RunningStats": + """ + Enable the use of the += operator to add a value to the running statistics. + + :param value: The value to add to the running statistics. + :return: The running statistics object. + """ if not isinstance(value, (int, float)): raise ValueError( f"Value must be an int or float, got {type(value)} instead.", @@ -758,7 +910,10 @@ def __iadd__(self, value: Any) -> "RunningStats": def update(self, value: float, count: int = 1) -> None: """ Update the running statistics with a new value. + :param value: The new value to add to the running statistics. + :param count: The number of times to 'count' for the value. + If not provided, defaults to 1. """ self.count += count self.total += value @@ -766,22 +921,43 @@ def update(self, value: float, count: int = 1) -> None: class TimeRunningStats(RunningStats): - @computed_field + """ + Create a running statistics object to track the mean, rate, and other + statistics of a stream of time values. This is used to track time values + in milliseconds and seconds. + + Adds time specific computed_fields such as measurements in milliseconds and seconds. + """ + + @computed_field # type: ignore[misc] @property def total_ms(self) -> float: + """ + :return: The total time multiplied by 1000.0 to convert to milliseconds. + """ return self.total * 1000.0 - @computed_field + @computed_field # type: ignore[misc] @property def last_ms(self) -> float: + """ + :return: The last time multiplied by 1000.0 to convert to milliseconds. + """ return self.last * 1000.0 - @computed_field + @computed_field # type: ignore[misc] @property def mean_ms(self) -> float: + """ + :return: The mean time multiplied by 1000.0 to convert to milliseconds. + """ return self.mean * 1000.0 - @computed_field + @computed_field # type: ignore[misc] @property def rate_ms(self) -> float: + """ + :return: The rate of the running statistics multiplied by 1000.0 + to convert to milliseconds. + """ return self.rate * 1000.0 diff --git a/src/guidellm/request/loader.py b/src/guidellm/request/loader.py index 61b9b500..ac0704a6 100644 --- a/src/guidellm/request/loader.py +++ b/src/guidellm/request/loader.py @@ -13,7 +13,7 @@ ) from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict -from transformers import PreTrainedTokenizer +from transformers import PreTrainedTokenizer # type: ignore[import] from guidellm.dataset import ColumnInputTypes, load_dataset from guidellm.objects import StandardBaseModel @@ -44,7 +44,7 @@ def description(self) -> RequestLoaderDescription: ... class GenerativeRequestLoaderDescription(RequestLoaderDescription): - type_: Literal["generative_request_loader"] = "generative_request_loader" + type_: Literal["generative_request_loader"] = "generative_request_loader" # type: ignore[assignment] data: str data_args: Optional[Dict[str, Any]] processor: str @@ -135,7 +135,7 @@ def description(self) -> GenerativeRequestLoaderDescription: def num_unique_items(self, raise_err: bool = True) -> int: try: return len(self.dataset) - except Exception: + except Exception: # noqa: BLE001, S110 pass dataset_size = self.dataset.info.dataset_size @@ -151,7 +151,7 @@ def _create_column_mappings( self, args_column_mappings: Dict[ColumnInputTypes, str], ) -> Dict[ColumnInputTypes, str]: - column_mappings = {} + column_mappings: Dict[ColumnInputTypes, str] = {} if "text_column" in args_column_mappings: column_mappings["prompt_column"] = args_column_mappings["text_column"] @@ -183,6 +183,13 @@ def _extract_text_column(self) -> str: ) ) + if not column_names: + raise ValueError( + "Unable to determine text column from dataset and it is required. " + "To specify the text column, set the 'text_column' key in the " + "'data_args' dictionary." + ) + if len(column_names) == 1: return column_names[0] diff --git a/src/guidellm/scheduler/result.py b/src/guidellm/scheduler/result.py index 282c9538..8a60b077 100644 --- a/src/guidellm/scheduler/result.py +++ b/src/guidellm/scheduler/result.py @@ -126,7 +126,7 @@ class SchedulerRequestResult( SchedulerResult, Generic[REQ, RES], ): - pydantic_type: Literal["scheduler_request_result"] = "scheduler_request_result" + pydantic_type: Literal["scheduler_request_result"] = "scheduler_request_result" # type: ignore[assignment] type_: Literal[ "request_scheduled", "request_start", diff --git a/src/guidellm/scheduler/scheduler.py b/src/guidellm/scheduler/scheduler.py index 56960fe3..9c801d15 100644 --- a/src/guidellm/scheduler/scheduler.py +++ b/src/guidellm/scheduler/scheduler.py @@ -118,6 +118,7 @@ async def run( with multiprocessing.Manager() as manager, ProcessPoolExecutor( max_workers=scheduling_strategy.processes_limit ) as executor: + requests_iter: Optional[Iterator[Any]] = None futures, requests_queue, responses_queue = await self._start_processes( manager, executor, scheduling_strategy ) @@ -239,10 +240,10 @@ def _run_setup( try: # update end number if the request loader is finite and less than max - iter_length = len(self.request_loader) + iter_length = len(self.request_loader) # type: ignore[arg-type] if 0 < iter_length < end_number: end_number = iter_length - except Exception: + except Exception: # noqa: BLE001, S110 pass if end_number == math.inf and end_time is None: @@ -312,7 +313,7 @@ def _check_result_ready( process_response: WorkerProcessResult[REQ, RES] = ( responses_queue.get_nowait() ) - except multiprocessing.queues.Empty: + except multiprocessing.queues.Empty: # type: ignore[attr-defined] return None if process_response.type_ == "request_scheduled": diff --git a/src/guidellm/scheduler/strategy.py b/src/guidellm/scheduler/strategy.py index e07e0b8e..7e8d253a 100644 --- a/src/guidellm/scheduler/strategy.py +++ b/src/guidellm/scheduler/strategy.py @@ -84,7 +84,7 @@ def queued_requests_limit(self) -> Optional[int]: return settings.max_concurrency @property - def processing_requests_limit(self) -> Optional[int]: + def processing_requests_limit(self) -> int: """ The maximum number of processing requests for the scheduling strategy. It determines how many requests can be processed at one time @@ -103,6 +103,7 @@ def request_times(self) -> Generator[float, None, None]: :return: A generator that yields timestamps for request scheduling or -1 for requests that should be sent immediately. """ + raise NotImplementedError("Subclasses must implement request_times() method.") class SynchronousStrategy(SchedulingStrategy): @@ -117,7 +118,7 @@ class SynchronousStrategy(SchedulingStrategy): :param type_: The synchronous StrategyType to schedule requests synchronously. """ - type_: Literal["synchronous"] = "synchronous" + type_: Literal["synchronous"] = "synchronous" # type: ignore[assignment] @property def processing_mode(self) -> Literal["sync"]: @@ -193,7 +194,7 @@ class ConcurrentStrategy(SchedulingStrategy): This must be a positive integer. """ - type_: Literal["concurrent"] = "concurrent" + type_: Literal["concurrent"] = "concurrent" # type: ignore[assignment] streams: int = Field( description=( "The number of concurrent streams to use for scheduling requests. " @@ -275,7 +276,7 @@ class ThroughputStrategy(SchedulingStrategy): :param type_: The throughput StrategyType to schedule requests asynchronously. """ - type_: Literal["throughput"] = "throughput" + type_: Literal["throughput"] = "throughput" # type: ignore[assignment] max_concurrency: Optional[int] = Field( default=None, description=( @@ -360,7 +361,7 @@ class AsyncConstantStrategy(ThroughputStrategy): False to not send an initial burst. """ - type_: Literal["constant"] = "constant" + type_: Literal["constant"] = "constant" # type: ignore[assignment] rate: float = Field( description=( "The rate at which to schedule requests asynchronously in " @@ -428,7 +429,7 @@ class AsyncPoissonStrategy(ThroughputStrategy): False to not send an initial burst. """ - type_: Literal["poisson"] = "poisson" + type_: Literal["poisson"] = "poisson" # type: ignore[assignment] rate: float = Field( description=( "The rate at which to schedule requests asynchronously in " @@ -483,9 +484,9 @@ def strategy_display_str(strategy: Union[StrategyType, SchedulingStrategy]) -> s strategy_instance = strategy if isinstance(strategy, SchedulingStrategy) else None if strategy_type == "concurrent": - rate = f"@{strategy_instance.streams}" if strategy_instance else "@##" + rate = f"@{strategy_instance.streams}" if strategy_instance else "@##" # type: ignore[attr-defined] elif strategy_type in ("constant", "poisson"): - rate = f"@{strategy_instance.rate:.2f}" if strategy_instance else "@#.##" + rate = f"@{strategy_instance.rate:.2f}" if strategy_instance else "@#.##" # type: ignore[attr-defined] else: rate = "" diff --git a/src/guidellm/scheduler/worker.py b/src/guidellm/scheduler/worker.py index 47756e5b..36c75fc8 100644 --- a/src/guidellm/scheduler/worker.py +++ b/src/guidellm/scheduler/worker.py @@ -46,7 +46,7 @@ class WorkerProcessRequest(Generic[REQ]): request: REQ start_time: float - timeout_time: Optional[float] + timeout_time: float queued_time: float @@ -54,7 +54,7 @@ class WorkerProcessRequest(Generic[REQ]): class WorkerProcessResult(Generic[REQ, RES]): type_: Literal["request_scheduled", "request_start", "request_complete"] request: REQ - response: RES + response: Optional[RES] info: SchedulerRequestInfo @@ -125,14 +125,14 @@ async def resolve( async def get_request( self, requests_queue: multiprocessing.Queue ) -> Optional[WorkerProcessRequest[REQ]]: - return await asyncio.to_thread(requests_queue.get) + return await asyncio.to_thread(requests_queue.get) # type: ignore[attr-defined] async def send_result( self, results_queue: multiprocessing.Queue, result: WorkerProcessResult[REQ, RES], ): - await asyncio.to_thread(results_queue.put, result) + await asyncio.to_thread(results_queue.put, result) # type: ignore[attr-defined] async def resolve_scheduler_request( self, @@ -264,7 +264,7 @@ def _task_done(_: asyncio.Task): class GenerativeRequestsWorkerDescription(WorkerDescription): - type_: Literal["generative_requests_worker"] = "generative_requests_worker" + type_: Literal["generative_requests_worker"] = "generative_requests_worker" # type: ignore[assignment] backend_type: BackendType backend_target: str backend_model: str @@ -287,7 +287,7 @@ def __init__(self, backend: Backend): self.backend = backend @property - def description(self) -> StandardBaseModel: + def description(self) -> GenerativeRequestsWorkerDescription: """ Get the description of the worker. :return: The description of the worker. @@ -295,7 +295,7 @@ def description(self) -> StandardBaseModel: return GenerativeRequestsWorkerDescription( backend_type=self.backend.type_, backend_target=self.backend.target, - backend_model=self.backend.model, + backend_model=self.backend.model or "None", backend_info=self.backend.info, ) @@ -376,7 +376,7 @@ async def resolve( async def _runner(): # wrap function so we can enforce timeout and # still return the latest state from the backend - async for resp in request_func(**request_kwargs): + async for resp in request_func(**request_kwargs): # type: ignore[operator] nonlocal response response = resp @@ -426,7 +426,7 @@ def _create_request_func_kwargs( request_kwargs: Dict[str, Any] if request.request_type == "text_completions": - request_func = self.backend.text_completions + request_func = self.backend.text_completions # type: ignore[assignment] request_kwargs = { "prompt": request.content, "request_id": request.request_id, @@ -435,7 +435,7 @@ def _create_request_func_kwargs( **request.params, } elif request.request_type == "chat_completions": - request_func = self.backend.chat_completions + request_func = self.backend.chat_completions # type: ignore[assignment] request_kwargs = { "content": request.content, "request_id": request.request_id, diff --git a/src/guidellm/utils/__init__.py b/src/guidellm/utils/__init__.py index 3d538f37..3620a3d3 100644 --- a/src/guidellm/utils/__init__.py +++ b/src/guidellm/utils/__init__.py @@ -13,6 +13,7 @@ ) __all__ = [ + "IntegerRangeSampler", "Colors", "check_load_processor", "filter_text", diff --git a/src/guidellm/utils/hf_transformers.py b/src/guidellm/utils/hf_transformers.py index b5b0f4db..2c298d2f 100644 --- a/src/guidellm/utils/hf_transformers.py +++ b/src/guidellm/utils/hf_transformers.py @@ -1,7 +1,7 @@ from pathlib import Path from typing import Any, Dict, Optional, Union -from transformers import AutoTokenizer, PreTrainedTokenizerBase +from transformers import AutoTokenizer, PreTrainedTokenizerBase # type: ignore[import] __all__ = [ "check_load_processor", diff --git a/src/guidellm/utils/random.py b/src/guidellm/utils/random.py index 830dab77..fefef4f1 100644 --- a/src/guidellm/utils/random.py +++ b/src/guidellm/utils/random.py @@ -18,7 +18,7 @@ def __init__( self.min_value = min_value self.max_value = max_value self.seed = random_seed - self.rng = random.Random(random_seed) + self.rng = random.Random(random_seed) # noqa: S311 def __iter__(self) -> Iterator[int]: calc_min = self.min_value diff --git a/src/guidellm/utils/text.py b/src/guidellm/utils/text.py index fb9abd6a..92a0284a 100644 --- a/src/guidellm/utils/text.py +++ b/src/guidellm/utils/text.py @@ -1,6 +1,6 @@ import gzip import re -from importlib.resources import as_file, files +from importlib.resources import as_file, files # type: ignore[attr-defined] from pathlib import Path from typing import List, Optional, Union diff --git a/tests/dummy/__init__.py b/tests/dummy/__init__.py deleted file mode 100644 index a0cccdbf..00000000 --- a/tests/dummy/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -""" -The tests.dummy package package represents dummy data factories and test services. - -test.dummy.data.openai_model_factory - openai.types.Model test factory -test.dummy.data.openai_completion_factory - openai.types.Completion test factory -""" - -from . import data, services # noqa: F401 diff --git a/tests/dummy/data/pride_and_prejudice.txt b/tests/dummy/data/pride_and_prejudice.txt deleted file mode 100644 index 3b93b50a..00000000 --- a/tests/dummy/data/pride_and_prejudice.txt +++ /dev/null @@ -1,2015 +0,0 @@ -*** START OF THE PROJECT GUTENBERG EBOOK 1342 *** - - PAGE - -Frontispiece iv - -Title-page v - -Dedication vii - -Heading to Preface ix - -Heading to List of Illustrations xxv - -Heading to Chapter I. 1 - -“He came down to see the place” 2 - -Mr. and Mrs. Bennet 5 - -“I hope Mr. Bingley will like it” 6 - -“I’m the tallest” 9 - -“He rode a black horse” 10 - -“When the party entered” 12 - -“She is tolerable” 15 - -Heading to Chapter IV. 18 - -Heading to Chapter V. 22 - -“Without once opening his lips” 24 - -Tailpiece to Chapter V. 26 - -Heading to Chapter VI. 27 - -“The entreaties of several” 31 - -“A note for Miss Bennet” 36 - -“Cheerful prognostics” 40 - -“The apothecary came” 43 - -“Covering a screen” 45 - -“Mrs. Bennet and her two youngest girls” 53 - -Heading to Chapter X. 60 - -“No, no; stay where you are” 67 - -“Piling up the fire” 69 - -Heading to Chapter XII. 75 - -Heading to Chapter XIII. 78 - -Heading to Chapter XIV. 84 - -“Protested that he never read novels” 87 - -Heading to Chapter XV. 89 - -Heading to Chapter XVI. 95 - -“The officers of the ----shire” 97 - -“Delighted to see their dear friend again” 108 - -Heading to Chapter XVIII. 113 - -“Such very superior dancing is not often seen” 118 - -“To assure you in the most animated language” 132 - -Heading to Chapter XX. 139 - -“They entered the breakfast-room” 143 - -Heading to Chapter XXI. 146 - -“Walked back with them” 148 - -Heading to Chapter XXII. 154 - -“So much love and eloquence” 156 - -“Protested he must be entirely mistaken” 161 - -“Whenever she spoke in a low voice” 166 - -Heading to Chapter XXIV. 168 - -Heading to Chapter XXV. 175 - -“Offended two or three young ladies” 177 - -“Will you come and see me?” 181 - -“On the stairs” 189 - -“At the door” 194 - -“In conversation with the ladies” 198 - -“Lady Catherine,” said she, “you have given me a treasure” 200 - -Heading to Chapter XXX. 209 - -“He never failed to inform them” 211 - -“The gentlemen accompanied him” 213 - -Heading to Chapter XXXI. 215 - -Heading to Chapter XXXII. 221 - -“Accompanied by their aunt” 225 - -“On looking up” 228 - -Heading to Chapter XXXIV. 235 - -“Hearing herself called” 243 - -Heading to Chapter XXXVI. 253 - -“Meeting accidentally in town” 256 - -“His parting obeisance” 261 - -“Dawson” 263 - -“The elevation of his feelings” 267 - -“They had forgotten to leave any message” 270 - -“How nicely we are crammed in!” 272 - -Heading to Chapter XL. 278 - -“I am determined never to speak of it again” 283 - -“When Colonel Miller’s regiment went away” 285 - -“Tenderly flirting” 290 - -The arrival of the Gardiners 294 - -“Conjecturing as to the date” 301 - -Heading to Chapter XLIV. 318 - -“To make herself agreeable to all” 321 - -“Engaged by the river” 327 - -Heading to Chapter XLVI. 334 - -“I have not an instant to lose” 339 - -“The first pleasing earnest of their welcome” 345 - -The Post 359 - -“To whom I have related the affair” 363 - -Heading to Chapter XLIX. 368 - -“But perhaps you would like to read it” 370 - -“The spiteful old ladies” 377 - -“With an affectionate smile” 385 - -“I am sure she did not listen” 393 - -“Mr. Darcy with him” 404 - -“Jane happened to look round” 415 - -“Mrs. Long and her nieces” 420 - -“Lizzy, my dear, I want to speak to you” 422 - -Heading to Chapter LVI. 431 - -“After a short survey” 434 - -“But now it comes out” 442 - -“The efforts of his aunt” 448 - -“Unable to utter a syllable” 457 - -“The obsequious civility” 466 - -Heading to Chapter LXI. 472 - -The End 476 - - - - -[Illustration: ·PRIDE AND PREJUDICE· - - - - -Chapter I.] - - -It is a truth universally acknowledged, that a single man in possession -of a good fortune must be in want of a wife. - -However little known the feelings or views of such a man may be on his -first entering a neighbourhood, this truth is so well fixed in the minds -of the surrounding families, that he is considered as the rightful -property of some one or other of their daughters. - -“My dear Mr. Bennet,” said his lady to him one day, “have you heard that -Netherfield Park is let at last?” - -Mr. Bennet replied that he had not. - -“But it is,” returned she; “for Mrs. Long has just been here, and she -told me all about it.” - -Mr. Bennet made no answer. - -“Do not you want to know who has taken it?” cried his wife, impatiently. - -“_You_ want to tell me, and I have no objection to hearing it.” - -[Illustration: - -“He came down to see the place” - -[_Copyright 1894 by George Allen._]] - -This was invitation enough. - -“Why, my dear, you must know, Mrs. Long says that Netherfield is taken -by a young man of large fortune from the north of England; that he came -down on Monday in a chaise and four to see the place, and was so much -delighted with it that he agreed with Mr. Morris immediately; that he is -to take possession before Michaelmas, and some of his servants are to be -in the house by the end of next week.” - -“What is his name?” - -“Bingley.” - -“Is he married or single?” - -“Oh, single, my dear, to be sure! A single man of large fortune; four or -five thousand a year. What a fine thing for our girls!” - -“How so? how can it affect them?” - -“My dear Mr. Bennet,” replied his wife, “how can you be so tiresome? You -must know that I am thinking of his marrying one of them.” - -“Is that his design in settling here?” - -“Design? Nonsense, how can you talk so! But it is very likely that he -_may_ fall in love with one of them, and therefore you must visit him as -soon as he comes.” - -“I see no occasion for that. You and the girls may go--or you may send -them by themselves, which perhaps will be still better; for as you are -as handsome as any of them, Mr. Bingley might like you the best of the -party.” - -“My dear, you flatter me. I certainly _have_ had my share of beauty, but -I do not pretend to be anything extraordinary now. When a woman has five -grown-up daughters, she ought to give over thinking of her own beauty.” - -“In such cases, a woman has not often much beauty to think of.” - -“But, my dear, you must indeed go and see Mr. Bingley when he comes into -the neighbourhood.” - -“It is more than I engage for, I assure you.” - -“But consider your daughters. Only think what an establishment it would -be for one of them. Sir William and Lady Lucas are determined to go, -merely on that account; for in general, you know, they visit no new -comers. Indeed you must go, for it will be impossible for _us_ to visit -him, if you do not.” - -“You are over scrupulous, surely. I dare say Mr. Bingley will be very -glad to see you; and I will send a few lines by you to assure him of my -hearty consent to his marrying whichever he chooses of the girls--though -I must throw in a good word for my little Lizzy.” - -“I desire you will do no such thing. Lizzy is not a bit better than the -others: and I am sure she is not half so handsome as Jane, nor half so -good-humoured as Lydia. But you are always giving _her_ the preference.” - -“They have none of them much to recommend them,” replied he: “they are -all silly and ignorant like other girls; but Lizzy has something more of -quickness than her sisters.” - -“Mr. Bennet, how can you abuse your own children in such a way? You take -delight in vexing me. You have no compassion on my poor nerves.” - -“You mistake me, my dear. I have a high respect for your nerves. They -are my old friends. I have heard you mention them with consideration -these twenty years at least.” - -“Ah, you do not know what I suffer.” - -“But I hope you will get over it, and live to see many young men of four -thousand a year come into the neighbourhood.” - -“It will be no use to us, if twenty such should come, since you will not -visit them.” - -“Depend upon it, my dear, that when there are twenty, I will visit them -all.” - -Mr. Bennet was so odd a mixture of quick parts, sarcastic humour, -reserve, and caprice, that the experience of three-and-twenty years had -been insufficient to make his wife understand his character. _Her_ mind -was less difficult to develope. She was a woman of mean understanding, -little information, and uncertain temper. When she was discontented, she -fancied herself nervous. The business of her life was to get her -daughters married: its solace was visiting and news. - -[Illustration: M^{r.} & M^{rs.} Bennet - -[_Copyright 1894 by George Allen._]] - - - - -[Illustration: - -“I hope Mr. Bingley will like it” - -[_Copyright 1894 by George Allen._]] - - - - -CHAPTER II. - - -[Illustration] - -Mr. Bennet was among the earliest of those who waited on Mr. Bingley. He -had always intended to visit him, though to the last always assuring his -wife that he should not go; and till the evening after the visit was -paid she had no knowledge of it. It was then disclosed in the following -manner. Observing his second daughter employed in trimming a hat, he -suddenly addressed her with,-- - -“I hope Mr. Bingley will like it, Lizzy.” - -“We are not in a way to know _what_ Mr. Bingley likes,” said her mother, -resentfully, “since we are not to visit.” - -“But you forget, mamma,” said Elizabeth, “that we shall meet him at the -assemblies, and that Mrs. Long has promised to introduce him.” - -“I do not believe Mrs. Long will do any such thing. She has two nieces -of her own. She is a selfish, hypocritical woman, and I have no opinion -of her.” - -“No more have I,” said Mr. Bennet; “and I am glad to find that you do -not depend on her serving you.” - -Mrs. Bennet deigned not to make any reply; but, unable to contain -herself, began scolding one of her daughters. - -“Don’t keep coughing so, Kitty, for heaven’s sake! Have a little -compassion on my nerves. You tear them to pieces.” - -“Kitty has no discretion in her coughs,” said her father; “she times -them ill.” - -“I do not cough for my own amusement,” replied Kitty, fretfully. “When -is your next ball to be, Lizzy?” - -“To-morrow fortnight.” - -“Ay, so it is,” cried her mother, “and Mrs. Long does not come back till -the day before; so, it will be impossible for her to introduce him, for -she will not know him herself.” - -“Then, my dear, you may have the advantage of your friend, and introduce -Mr. Bingley to _her_.” - -“Impossible, Mr. Bennet, impossible, when I am not acquainted with him -myself; how can you be so teasing?” - -“I honour your circumspection. A fortnight’s acquaintance is certainly -very little. One cannot know what a man really is by the end of a -fortnight. But if _we_ do not venture, somebody else will; and after -all, Mrs. Long and her nieces must stand their chance; and, therefore, -as she will think it an act of kindness, if you decline the office, I -will take it on myself.” - -The girls stared at their father. Mrs. Bennet said only, “Nonsense, -nonsense!” - -“What can be the meaning of that emphatic exclamation?” cried he. “Do -you consider the forms of introduction, and the stress that is laid on -them, as nonsense? I cannot quite agree with you _there_. What say you, -Mary? For you are a young lady of deep reflection, I know, and read -great books, and make extracts.” - -Mary wished to say something very sensible, but knew not how. - -“While Mary is adjusting her ideas,” he continued, “let us return to Mr. -Bingley.” - -“I am sick of Mr. Bingley,” cried his wife. - -“I am sorry to hear _that_; but why did you not tell me so before? If I -had known as much this morning, I certainly would not have called on -him. It is very unlucky; but as I have actually paid the visit, we -cannot escape the acquaintance now.” - -The astonishment of the ladies was just what he wished--that of Mrs. -Bennet perhaps surpassing the rest; though when the first tumult of joy -was over, she began to declare that it was what she had expected all the -while. - -“How good it was in you, my dear Mr. Bennet! But I knew I should -persuade you at last. I was sure you loved your girls too well to -neglect such an acquaintance. Well, how pleased I am! And it is such a -good joke, too, that you should have gone this morning, and never said a -word about it till now.” - -“Now, Kitty, you may cough as much as you choose,” said Mr. Bennet; and, -as he spoke, he left the room, fatigued with the raptures of his wife. - -“What an excellent father you have, girls,” said she, when the door was -shut. “I do not know how you will ever make him amends for his kindness; -or me either, for that matter. At our time of life, it is not so -pleasant, I can tell you, to be making new acquaintances every day; but -for your sakes we would do anything. Lydia, my love, though you _are_ -the youngest, I dare say Mr. Bingley will dance with you at the next -ball.” - -“Oh,” said Lydia, stoutly, “I am not afraid; for though I _am_ the -youngest, I’m the tallest.” - -The rest of the evening was spent in conjecturing how soon he would -return Mr. Bennet’s visit, and determining when they should ask him to -dinner. - -[Illustration: “I’m the tallest”] - - - - -[Illustration: - - “He rode a black horse” -] - - - - -CHAPTER III. - - -[Illustration] - -Not all that Mrs. Bennet, however, with the assistance of her five -daughters, could ask on the subject, was sufficient to draw from her -husband any satisfactory description of Mr. Bingley. They attacked him -in various ways, with barefaced questions, ingenious suppositions, and -distant surmises; but he eluded the skill of them all; and they were at -last obliged to accept the second-hand intelligence of their neighbour, -Lady Lucas. Her report was highly favourable. Sir William had been -delighted with him. He was quite young, wonderfully handsome, extremely -agreeable, and, to crown the whole, he meant to be at the next assembly -with a large party. Nothing could be more delightful! To be fond of -dancing was a certain step towards falling in love; and very lively -hopes of Mr. Bingley’s heart were entertained. - -“If I can but see one of my daughters happily settled at Netherfield,” -said Mrs. Bennet to her husband, “and all the others equally well -married, I shall have nothing to wish for.” - -In a few days Mr. Bingley returned Mr. Bennet’s visit, and sat about ten -minutes with him in his library. He had entertained hopes of being -admitted to a sight of the young ladies, of whose beauty he had heard -much; but he saw only the father. The ladies were somewhat more -fortunate, for they had the advantage of ascertaining, from an upper -window, that he wore a blue coat and rode a black horse. - -An invitation to dinner was soon afterwards despatched; and already had -Mrs. Bennet planned the courses that were to do credit to her -housekeeping, when an answer arrived which deferred it all. Mr. Bingley -was obliged to be in town the following day, and consequently unable to -accept the honour of their invitation, etc. Mrs. Bennet was quite -disconcerted. She could not imagine what business he could have in town -so soon after his arrival in Hertfordshire; and she began to fear that -he might always be flying about from one place to another, and never -settled at Netherfield as he ought to be. Lady Lucas quieted her fears a -little by starting the idea of his - -[Illustration: - - “When the Party entered” - -[_Copyright 1894 by George Allen._]] - -being gone to London only to get a large party for the ball; and a -report soon followed that Mr. Bingley was to bring twelve ladies and -seven gentlemen with him to the assembly. The girls grieved over such a -number of ladies; but were comforted the day before the ball by hearing -that, instead of twelve, he had brought only six with him from London, -his five sisters and a cousin. And when the party entered the -assembly-room, it consisted of only five altogether: Mr. Bingley, his -two sisters, the husband of the eldest, and another young man. - -Mr. Bingley was good-looking and gentlemanlike: he had a pleasant -countenance, and easy, unaffected manners. His sisters were fine women, -with an air of decided fashion. His brother-in-law, Mr. Hurst, merely -looked the gentleman; but his friend Mr. Darcy soon drew the attention -of the room by his fine, tall person, handsome features, noble mien, and -the report, which was in general circulation within five minutes after -his entrance, of his having ten thousand a year. The gentlemen -pronounced him to be a fine figure of a man, the ladies declared he was -much handsomer than Mr. Bingley, and he was looked at with great -admiration for about half the evening, till his manners gave a disgust -which turned the tide of his popularity; for he was discovered to be -proud, to be above his company, and above being pleased; and not all his -large estate in Derbyshire could save him from having a most forbidding, -disagreeable countenance, and being unworthy to be compared with his -friend. - -Mr. Bingley had soon made himself acquainted with all the principal -people in the room: he was lively and unreserved, danced every dance, -was angry that the ball closed so early, and talked of giving one -himself at Netherfield. Such amiable qualities must speak for -themselves. What a contrast between him and his friend! Mr. Darcy danced -only once with Mrs. Hurst and once with Miss Bingley, declined being -introduced to any other lady, and spent the rest of the evening in -walking about the room, speaking occasionally to one of his own party. -His character was decided. He was the proudest, most disagreeable man in -the world, and everybody hoped that he would never come there again. -Amongst the most violent against him was Mrs. Bennet, whose dislike of -his general behaviour was sharpened into particular resentment by his -having slighted one of her daughters. - -Elizabeth Bennet had been obliged, by the scarcity of gentlemen, to sit -down for two dances; and during part of that time, Mr. Darcy had been -standing near enough for her to overhear a conversation between him and -Mr. Bingley, who came from the dance for a few minutes to press his -friend to join it. - -“Come, Darcy,” said he, “I must have you dance. I hate to see you -standing about by yourself in this stupid manner. You had much better -dance.” - -“I certainly shall not. You know how I detest it, unless I am -particularly acquainted with my partner. At such an assembly as this, it -would be insupportable. Your sisters are engaged, and there is not -another woman in the room whom it would not be a punishment to me to -stand up with.” - -“I would not be so fastidious as you are,” cried Bingley, “for a -kingdom! Upon my honour, I never met with so many pleasant girls in my -life as I have this evening; and there are several of them, you see, -uncommonly pretty.” - -“_You_ are dancing with the only handsome girl in the room,” said Mr. -Darcy, looking at the eldest Miss Bennet. - -“Oh, she is the most beautiful creature I ever beheld! But there is one -of her sisters sitting down just behind you, who is very pretty, and I -dare say very agreeable. Do let me ask my partner to introduce you.” - -[Illustration: - -“She is tolerable” - -[_Copyright 1894 by George Allen._]] - -“Which do you mean?” and turning round, he looked for a moment at -Elizabeth, till, catching her eye, he withdrew his own, and coldly said, -“She is tolerable: but not handsome enough to tempt _me_; and I am in no -humour at present to give consequence to young ladies who are slighted -by other men. You had better return to your partner and enjoy her -smiles, for you are wasting your time with me.” - -Mr. Bingley followed his advice. Mr. Darcy walked off; and Elizabeth -remained with no very cordial feelings towards him. She told the story, -however, with great spirit among her friends; for she had a lively, -playful disposition, which delighted in anything ridiculous. - -The evening altogether passed off pleasantly to the whole family. Mrs. -Bennet had seen her eldest daughter much admired by the Netherfield -party. Mr. Bingley had danced with her twice, and she had been -distinguished by his sisters. Jane was as much gratified by this as her -mother could be, though in a quieter way. Elizabeth felt Jane’s -pleasure. Mary had heard herself mentioned to Miss Bingley as the most -accomplished girl in the neighbourhood; and Catherine and Lydia had been -fortunate enough to be never without partners, which was all that they -had yet learnt to care for at a ball. They returned, therefore, in good -spirits to Longbourn, the village where they lived, and of which they -were the principal inhabitants. They found Mr. Bennet still up. With a -book, he was regardless of time; and on the present occasion he had a -good deal of curiosity as to the event of an evening which had raised -such splendid expectations. He had rather hoped that all his wife’s -views on the stranger would be disappointed; but he soon found that he -had a very different story to hear. - -“Oh, my dear Mr. Bennet,” as she entered the room, “we have had a most -delightful evening, a most excellent ball. I wish you had been there. -Jane was so admired, nothing could be like it. Everybody said how well -she looked; and Mr. Bingley thought her quite beautiful, and danced with -her twice. Only think of _that_, my dear: he actually danced with her -twice; and she was the only creature in the room that he asked a second -time. First of all, he asked Miss Lucas. I was so vexed to see him stand -up with her; but, however, he did not admire her at all; indeed, nobody -can, you know; and he seemed quite struck with Jane as she was going -down the dance. So he inquired who she was, and got introduced, and -asked her for the two next. Then, the two third he danced with Miss -King, and the two fourth with Maria Lucas, and the two fifth with Jane -again, and the two sixth with Lizzy, and the _Boulanger_----” - -“If he had had any compassion for _me_,” cried her husband impatiently, -“he would not have danced half so much! For God’s sake, say no more of -his partners. O that he had sprained his ancle in the first dance!” - -“Oh, my dear,” continued Mrs. Bennet, “I am quite delighted with him. He -is so excessively handsome! and his sisters are charming women. I never -in my life saw anything more elegant than their dresses. I dare say the -lace upon Mrs. Hurst’s gown----” - -Here she was interrupted again. Mr. Bennet protested against any -description of finery. She was therefore obliged to seek another branch -of the subject, and related, with much bitterness of spirit, and some -exaggeration, the shocking rudeness of Mr. Darcy. - -“But I can assure you,” she added, “that Lizzy does not lose much by not -suiting _his_ fancy; for he is a most disagreeable, horrid man, not at -all worth pleasing. So high and so conceited, that there was no enduring -him! He walked here, and he walked there, fancying himself so very -great! Not handsome enough to dance with! I wish you had been there, my -dear, to have given him one of your set-downs. I quite detest the man.” - - - - -[Illustration] - - - - -CHAPTER IV. - - -[Illustration] - -When Jane and Elizabeth were alone, the former, who had been cautious in -her praise of Mr. Bingley before, expressed to her sister how very much -she admired him. - -“He is just what a young-man ought to be,” said she, “sensible, -good-humoured, lively; and I never saw such happy manners! so much ease, -with such perfect good breeding!” - -“He is also handsome,” replied Elizabeth, “which a young man ought -likewise to be if he possibly can. His character is thereby complete.” - -“I was very much flattered by his asking me to dance a second time. I -did not expect such a compliment.” - -“Did not you? _I_ did for you. But that is one great difference between -us. Compliments always take _you_ by surprise, and _me_ never. What -could be more natural than his asking you again? He could not help -seeing that you were about five times as pretty as every other woman in -the room. No thanks to his gallantry for that. Well, he certainly is -very agreeable, and I give you leave to like him. You have liked many a -stupider person.” - -“Dear Lizzy!” - -“Oh, you are a great deal too apt, you know, to like people in general. -You never see a fault in anybody. All the world are good and agreeable -in your eyes. I never heard you speak ill of a human being in my life.” - -“I would wish not to be hasty in censuring anyone; but I always speak -what I think.” - -“I know you do: and it is _that_ which makes the wonder. With _your_ -good sense, to be so honestly blind to the follies and nonsense of -others! Affectation of candour is common enough; one meets with it -everywhere. But to be candid without ostentation or design,--to take the -good of everybody’s character and make it still better, and say nothing -of the bad,--belongs to you alone. And so, you like this man’s sisters, -too, do you? Their manners are not equal to his.” - -“Certainly not, at first; but they are very pleasing women when you -converse with them. Miss Bingley is to live with her brother, and keep -his house; and I am much mistaken if we shall not find a very charming -neighbour in her.” - -Elizabeth listened in silence, but was not convinced: their behaviour at -the assembly had not been calculated to please in general; and with more -quickness of observation and less pliancy of temper than her sister, and -with a judgment, too, unassailed by any attention to herself, she was -very little disposed to approve them. They were, in fact, very fine -ladies; not deficient in good-humour when they were pleased, nor in the -power of being agreeable where they chose it; but proud and conceited. -They were rather handsome; had been educated in one of the first private -seminaries in town; had a fortune of twenty thousand pounds; were in the -habit of spending more than they ought, and of associating with people -of rank; and were, therefore, in every respect entitled to think well of -themselves and meanly of others. They were of a respectable family in -the north of England; a circumstance more deeply impressed on their -memories than that their brother’s fortune and their own had been -acquired by trade. - -Mr. Bingley inherited property to the amount of nearly a hundred -thousand pounds from his father, who had intended to purchase an estate, -but did not live to do it. Mr. Bingley intended it likewise, and -sometimes made choice of his county; but, as he was now provided with a -good house and the liberty of a manor, it was doubtful to many of those -who best knew the easiness of his temper, whether he might not spend the -remainder of his days at Netherfield, and leave the next generation to -purchase. - -His sisters were very anxious for his having an estate of his own; but -though he was now established only as a tenant, Miss Bingley was by no -means unwilling to preside at his table; nor was Mrs. Hurst, who had -married a man of more fashion than fortune, less disposed to consider -his house as her home when it suited her. Mr. Bingley had not been of -age two years when he was tempted, by an accidental recommendation, to -look at Netherfield House. He did look at it, and into it, for half an -hour; was pleased with the situation and the principal rooms, satisfied -with what the owner said in its praise, and took it immediately. - -Between him and Darcy there was a very steady friendship, in spite of a -great opposition of character. Bingley was endeared to Darcy by the -easiness, openness, and ductility of his temper, though no disposition -could offer a greater contrast to his own, and though with his own he -never appeared dissatisfied. On the strength of Darcy’s regard, Bingley -had the firmest reliance, and of his judgment the highest opinion. In -understanding, Darcy was the superior. Bingley was by no means -deficient; but Darcy was clever. He was at the same time haughty, -reserved, and fastidious; and his manners, though well bred, were not -inviting. In that respect his friend had greatly the advantage. Bingley -was sure of being liked wherever he appeared; Darcy was continually -giving offence. - -The manner in which they spoke of the Meryton assembly was sufficiently -characteristic. Bingley had never met with pleasanter people or prettier -girls in his life; everybody had been most kind and attentive to him; -there had been no formality, no stiffness; he had soon felt acquainted -with all the room; and as to Miss Bennet, he could not conceive an angel -more beautiful. Darcy, on the contrary, had seen a collection of people -in whom there was little beauty and no fashion, for none of whom he had -felt the smallest interest, and from none received either attention or -pleasure. Miss Bennet he acknowledged to be pretty; but she smiled too -much. - -Mrs. Hurst and her sister allowed it to be so; but still they admired -her and liked her, and pronounced her to be a sweet girl, and one whom -they should not object to know more of. Miss Bennet was therefore -established as a sweet girl; and their brother felt authorized by such -commendation to think of her as he chose. - - - - -[Illustration: [_Copyright 1894 by George Allen._]] - - - - -CHAPTER V. - - -[Illustration] - -Within a short walk of Longbourn lived a family with whom the Bennets -were particularly intimate. Sir William Lucas had been formerly in trade -in Meryton, where he had made a tolerable fortune, and risen to the -honour of knighthood by an address to the king during his mayoralty. The -distinction had, perhaps, been felt too strongly. It had given him a -disgust to his business and to his residence in a small market town; -and, quitting them both, he had removed with his family to a house about -a mile from Meryton, denominated from that period Lucas Lodge; where he -could think with pleasure of his own importance, and, unshackled by -business, occupy himself solely in being civil to all the world. For, -though elated by his rank, it did not render him supercilious; on the -contrary, he was all attention to everybody. By nature inoffensive, -friendly, and obliging, his presentation at St. James’s had made him -courteous. - -Lady Lucas was a very good kind of woman, not too clever to be a -valuable neighbour to Mrs. Bennet. They had several children. The eldest -of them, a sensible, intelligent young woman, about twenty-seven, was -Elizabeth’s intimate friend. - -That the Miss Lucases and the Miss Bennets should meet to talk over a -ball was absolutely necessary; and the morning after the assembly -brought the former to Longbourn to hear and to communicate. - -“_You_ began the evening well, Charlotte,” said Mrs. Bennet, with civil -self-command, to Miss Lucas. “_You_ were Mr. Bingley’s first choice.” - -“Yes; but he seemed to like his second better.” - -“Oh, you mean Jane, I suppose, because he danced with her twice. To be -sure that _did_ seem as if he admired her--indeed, I rather believe he -_did_--I heard something about it--but I hardly know what--something -about Mr. Robinson.” - -“Perhaps you mean what I overheard between him and Mr. Robinson: did not -I mention it to you? Mr. Robinson’s asking him how he liked our Meryton -assemblies, and whether he did not think there were a great many pretty -women in the room, and _which_ he thought the prettiest? and his -answering immediately to the last question, ‘Oh, the eldest Miss Bennet, -beyond a doubt: there cannot be two opinions on that point.’” - -“Upon my word! Well, that was very decided, indeed--that does seem as -if--but, however, it may all come to nothing, you know.” - -“_My_ overhearings were more to the purpose than _yours_, Eliza,” said -Charlotte. “Mr. Darcy is not so well worth listening to as his friend, -is he? Poor Eliza! to be only just _tolerable_.” - -“I beg you will not put it into Lizzy’s head to be vexed by his -ill-treatment, for he is such a disagreeable man that it would be quite -a misfortune to be liked by him. Mrs. Long told me last night that he -sat close to her for half an hour without once opening his lips.” - -[Illustration: “Without once opening his lips” - -[_Copyright 1894 by George Allen._]] - -“Are you quite sure, ma’am? Is not there a little mistake?” said Jane. -“I certainly saw Mr. Darcy speaking to her.” - -“Ay, because she asked him at last how he liked Netherfield, and he -could not help answering her; but she said he seemed very angry at being -spoke to.” - -“Miss Bingley told me,” said Jane, “that he never speaks much unless -among his intimate acquaintance. With _them_ he is remarkably -agreeable.” - -“I do not believe a word of it, my dear. If he had been so very -agreeable, he would have talked to Mrs. Long. But I can guess how it -was; everybody says that he is eat up with pride, and I dare say he had -heard somehow that Mrs. Long does not keep a carriage, and had to come -to the ball in a hack chaise.” - -“I do not mind his not talking to Mrs. Long,” said Miss Lucas, “but I -wish he had danced with Eliza.” - -“Another time, Lizzy,” said her mother, “I would not dance with _him_, -if I were you.” - -“I believe, ma’am, I may safely promise you _never_ to dance with him.” - -“His pride,” said Miss Lucas, “does not offend _me_ so much as pride -often does, because there is an excuse for it. One cannot wonder that so -very fine a young man, with family, fortune, everything in his favour, -should think highly of himself. If I may so express it, he has a _right_ -to be proud.” - -“That is very true,” replied Elizabeth, “and I could easily forgive -_his_ pride, if he had not mortified _mine_.” - -“Pride,” observed Mary, who piqued herself upon the solidity of her -reflections, “is a very common failing, I believe. By all that I have -ever read, I am convinced that it is very common indeed; that human -nature is particularly prone to it, and that there are very few of us -who do not cherish a feeling of self-complacency on the score of some -quality or other, real or imaginary. Vanity and pride are different -things, though the words are often used synonymously. A person may be -proud without being vain. Pride relates more to our opinion of -ourselves; vanity to what we would have others think of us.” - -“If I were as rich as Mr. Darcy,” cried a young Lucas, who came with his -sisters, “I should not care how proud I was. I would keep a pack of -foxhounds, and drink a bottle of wine every day.” - -“Then you would drink a great deal more than you ought,” said Mrs. -Bennet; “and if I were to see you at it, I should take away your bottle -directly.” - -The boy protested that she should not; she continued to declare that she -would; and the argument ended only with the visit. - -[Illustration] - - - - -[Illustration] - - - - -CHAPTER VI. - - -[Illustration] - -The ladies of Longbourn soon waited on those of Netherfield. The visit -was returned in due form. Miss Bennet’s pleasing manners grew on the -good-will of Mrs. Hurst and Miss Bingley; and though the mother was -found to be intolerable, and the younger sisters not worth speaking to, -a wish of being better acquainted with _them_ was expressed towards the -two eldest. By Jane this attention was received with the greatest -pleasure; but Elizabeth still saw superciliousness in their treatment of -everybody, hardly excepting even her sister, and could not like them; -though their kindness to Jane, such as it was, had a value, as arising, -in all probability, from the influence of their brother’s admiration. It -was generally evident, whenever they met, that he _did_ admire her; and -to _her_ it was equally evident that Jane was yielding to the preference -which she had begun to entertain for him from the first, and was in a -way to be very much in love; but she considered with pleasure that it -was not likely to be discovered by the world in general, since Jane -united with great strength of feeling, a composure of temper and an -uniform cheerfulness of manner, which would guard her from the -suspicions of the impertinent. She mentioned this to her friend, Miss -Lucas. - -“It may, perhaps, be pleasant,” replied Charlotte, “to be able to impose -on the public in such a case; but it is sometimes a disadvantage to be -so very guarded. If a woman conceals her affection with the same skill -from the object of it, she may lose the opportunity of fixing him; and -it will then be but poor consolation to believe the world equally in the -dark. There is so much of gratitude or vanity in almost every -attachment, that it is not safe to leave any to itself. We can all -_begin_ freely--a slight preference is natural enough; but there are -very few of us who have heart enough to be really in love without -encouragement. In nine cases out of ten, a woman had better show _more_ -affection than she feels. Bingley likes your sister undoubtedly; but he -may never do more than like her, if she does not help him on.” - -“But she does help him on, as much as her nature will allow. If _I_ can -perceive her regard for him, he must be a simpleton indeed not to -discover it too.” - -“Remember, Eliza, that he does not know Jane’s disposition as you do.” - -“But if a woman is partial to a man, and does not endeavor to conceal -it, he must find it out.” - -“Perhaps he must, if he sees enough of her. But though Bingley and Jane -meet tolerably often, it is never for many hours together; and as they -always see each other in large mixed parties, it is impossible that -every moment should be employed in conversing together. Jane should -therefore make the most of every half hour in which she can command his -attention. When she is secure of him, there will be leisure for falling -in love as much as she chooses.” - -“Your plan is a good one,” replied Elizabeth, “where nothing is in -question but the desire of being well married; and if I were determined -to get a rich husband, or any husband, I dare say I should adopt it. But -these are not Jane’s feelings; she is not acting by design. As yet she -cannot even be certain of the degree of her own regard, nor of its -reasonableness. She has known him only a fortnight. She danced four -dances with him at Meryton; she saw him one morning at his own house, -and has since dined in company with him four times. This is not quite -enough to make her understand his character.” - -“Not as you represent it. Had she merely _dined_ with him, she might -only have discovered whether he had a good appetite; but you must -remember that four evenings have been also spent together--and four -evenings may do a great deal.” - -“Yes: these four evenings have enabled them to ascertain that they both -like Vingt-un better than Commerce, but with respect to any other -leading characteristic, I do not imagine that much has been unfolded.” - -“Well,” said Charlotte, “I wish Jane success with all my heart; and if -she were married to him to-morrow, I should think she had as good a -chance of happiness as if she were to be studying his character for a -twelvemonth. Happiness in marriage is entirely a matter of chance. If -the dispositions of the parties are ever so well known to each other, or -ever so similar beforehand, it does not advance their felicity in the -least. They always continue to grow sufficiently unlike afterwards to -have their share of vexation; and it is better to know as little as -possible of the defects of the person with whom you are to pass your -life.” - -“You make me laugh, Charlotte; but it is not sound. You know it is not -sound, and that you would never act in this way yourself.” - -Occupied in observing Mr. Bingley’s attention to her sister, Elizabeth -was far from suspecting that she was herself becoming an object of some -interest in the eyes of his friend. Mr. Darcy had at first scarcely -allowed her to be pretty: he had looked at her without admiration at the -ball; and when they next met, he looked at her only to criticise. But no -sooner had he made it clear to himself and his friends that she had -hardly a good feature in her face, than he began to find it was rendered -uncommonly intelligent by the beautiful expression of her dark eyes. To -this discovery succeeded some others equally mortifying. Though he had -detected with a critical eye more than one failure of perfect symmetry -in her form, he was forced to acknowledge her figure to be light and -pleasing; and in spite of his asserting that her manners were not those -of the fashionable world, he was caught by their easy playfulness. Of -this she was perfectly unaware: to her he was only the man who made -himself agreeable nowhere, and who had not thought her handsome enough -to dance with. - -He began to wish to know more of her; and, as a step towards conversing -with her himself, attended to her conversation with others. His doing so -drew her notice. It was at Sir William Lucas’s, where a large party were -assembled. - -“What does Mr. Darcy mean,” said she to Charlotte, “by listening to my -conversation with Colonel Forster?” - -“That is a question which Mr. Darcy only can answer.” - -“But if he does it any more, I shall certainly let him know that I see -what he is about. He has a very satirical eye, and if I do not begin by -being impertinent myself, I shall soon grow afraid of him.” - -[Illustration: “The entreaties of several” [_Copyright 1894 by George -Allen._]] - -On his approaching them soon afterwards, though without seeming to have -any intention of speaking, Miss Lucas defied her friend to mention such -a subject to him, which immediately provoking Elizabeth to do it, she -turned to him and said,-- - -“Did not you think, Mr. Darcy, that I expressed myself uncommonly well -just now, when I was teasing Colonel Forster to give us a ball at -Meryton?” - -“With great energy; but it is a subject which always makes a lady -energetic.” - -“You are severe on us.” - -“It will be _her_ turn soon to be teased,” said Miss Lucas. “I am going -to open the instrument, Eliza, and you know what follows.” - -“You are a very strange creature by way of a friend!--always wanting me -to play and sing before anybody and everybody! If my vanity had taken a -musical turn, you would have been invaluable; but as it is, I would -really rather not sit down before those who must be in the habit of -hearing the very best performers.” On Miss Lucas’s persevering, however, -she added, “Very well; if it must be so, it must.” And gravely glancing -at Mr. Darcy, “There is a very fine old saying, which everybody here is -of course familiar with--‘Keep your breath to cool your porridge,’--and -I shall keep mine to swell my song.” - -Her performance was pleasing, though by no means capital. After a song -or two, and before she could reply to the entreaties of several that she -would sing again, she was eagerly succeeded at the instrument by her -sister Mary, who having, in consequence of being the only plain one in -the family, worked hard for knowledge and accomplishments, was always -impatient for display. - -Mary had neither genius nor taste; and though vanity had given her -application, it had given her likewise a pedantic air and conceited -manner, which would have injured a higher degree of excellence than she -had reached. Elizabeth, easy and unaffected, had been listened to with -much more pleasure, though not playing half so well; and Mary, at the -end of a long concerto, was glad to purchase praise and gratitude by -Scotch and Irish airs, at the request of her younger sisters, who with -some of the Lucases, and two or three officers, joined eagerly in -dancing at one end of the room. - -Mr. Darcy stood near them in silent indignation at such a mode of -passing the evening, to the exclusion of all conversation, and was too -much engrossed by his own thoughts to perceive that Sir William Lucas -was his neighbour, till Sir William thus began:-- - -“What a charming amusement for young people this is, Mr. Darcy! There is -nothing like dancing, after all. I consider it as one of the first -refinements of polished societies.” - -“Certainly, sir; and it has the advantage also of being in vogue amongst -the less polished societies of the world: every savage can dance.” - -Sir William only smiled. “Your friend performs delightfully,” he -continued, after a pause, on seeing Bingley join the group; “and I doubt -not that you are an adept in the science yourself, Mr. Darcy.” - -“You saw me dance at Meryton, I believe, sir.” - -“Yes, indeed, and received no inconsiderable pleasure from the sight. Do -you often dance at St. James’s?” - -“Never, sir.” - -“Do you not think it would be a proper compliment to the place?” - -“It is a compliment which I never pay to any place if I can avoid it.” - -“You have a house in town, I conclude?” - -Mr. Darcy bowed. - -“I had once some thoughts of fixing in town myself, for I am fond of -superior society; but I did not feel quite certain that the air of -London would agree with Lady Lucas.” - -He paused in hopes of an answer: but his companion was not disposed to -make any; and Elizabeth at that instant moving towards them, he was -struck with the notion of doing a very gallant thing, and called out to -her,-- - -“My dear Miss Eliza, why are not you dancing? Mr. Darcy, you must allow -me to present this young lady to you as a very desirable partner. You -cannot refuse to dance, I am sure, when so much beauty is before you.” -And, taking her hand, he would have given it to Mr. Darcy, who, though -extremely surprised, was not unwilling to receive it, when she instantly -drew back, and said with some discomposure to Sir William,-- - -“Indeed, sir, I have not the least intention of dancing. I entreat you -not to suppose that I moved this way in order to beg for a partner.” - -Mr. Darcy, with grave propriety, requested to be allowed the honour of -her hand, but in vain. Elizabeth was determined; nor did Sir William at -all shake her purpose by his attempt at persuasion. - -“You excel so much in the dance, Miss Eliza, that it is cruel to deny me -the happiness of seeing you; and though this gentleman dislikes the -amusement in general, he can have no objection, I am sure, to oblige us -for one half hour.” - -“Mr. Darcy is all politeness,” said Elizabeth, smiling. - -“He is, indeed: but considering the inducement, my dear Miss Eliza, we -cannot wonder at his complaisance; for who would object to such a -partner?” - -Elizabeth looked archly, and turned away. Her resistance had not injured -her with the gentleman, and he was thinking of her with some -complacency, when thus accosted by Miss Bingley,-- - -“I can guess the subject of your reverie.” - -“I should imagine not.” - -“You are considering how insupportable it would be to pass many -evenings in this manner,--in such society; and, indeed, I am quite of -your opinion. I was never more annoyed! The insipidity, and yet the -noise--the nothingness, and yet the self-importance, of all these -people! What would I give to hear your strictures on them!” - -“Your conjecture is totally wrong, I assure you. My mind was more -agreeably engaged. I have been meditating on the very great pleasure -which a pair of fine eyes in the face of a pretty woman can bestow.” - -Miss Bingley immediately fixed her eyes on his face, and desired he -would tell her what lady had the credit of inspiring such reflections. -Mr. Darcy replied, with great intrepidity,-- - -“Miss Elizabeth Bennet.” - -“Miss Elizabeth Bennet!” repeated Miss Bingley. “I am all astonishment. -How long has she been such a favourite? and pray when am I to wish you -joy?” - -“That is exactly the question which I expected you to ask. A lady’s -imagination is very rapid; it jumps from admiration to love, from love -to matrimony, in a moment. I knew you would be wishing me joy.” - -“Nay, if you are so serious about it, I shall consider the matter as -absolutely settled. You will have a charming mother-in-law, indeed, and -of course she will be always at Pemberley with you.” - -He listened to her with perfect indifference, while she chose to -entertain herself in this manner; and as his composure convinced her -that all was safe, her wit flowed along. - - - - -[Illustration: - - “A note for Miss Bennet” - -[_Copyright 1894 by George Allen._]] - - - - -CHAPTER VII. - - -[Illustration] - -Mr. Bennet’s property consisted almost entirely in an estate of two -thousand a year, which, unfortunately for his daughters, was entailed, -in default of heirs male, on a distant relation; and their mother’s -fortune, though ample for her situation in life, could but ill supply -the deficiency of his. Her father had been an attorney in Meryton, and -had left her four thousand pounds. - -She had a sister married to a Mr. Philips, who had been a clerk to their -father and succeeded him in the business, and a brother settled in -London in a respectable line of trade. - -The village of Longbourn was only one mile from Meryton; a most -convenient distance for the young ladies, who were usually tempted -thither three or four times a week, to pay their duty to their aunt, and -to a milliner’s shop just over the way. The two youngest of the family, -Catherine and Lydia, were particularly frequent in these attentions: -their minds were more vacant than their sisters’, and when nothing -better offered, a walk to Meryton was necessary to amuse their morning -hours and furnish conversation for the evening; and, however bare of -news the country in general might be, they always contrived to learn -some from their aunt. At present, indeed, they were well supplied both -with news and happiness by the recent arrival of a militia regiment in -the neighbourhood; it was to remain the whole winter, and Meryton was -the head-quarters. - -Their visits to Mrs. Philips were now productive of the most interesting -intelligence. Every day added something to their knowledge of the -officers’ names and connections. Their lodgings were not long a secret, -and at length they began to know the officers themselves. Mr. Philips -visited them all, and this opened to his nieces a source of felicity -unknown before. They could talk of nothing but officers; and Mr. -Bingley’s large fortune, the mention of which gave animation to their -mother, was worthless in their eyes when opposed to the regimentals of -an ensign. - -After listening one morning to their effusions on this subject, Mr. -Bennet coolly observed,-- - -“From all that I can collect by your manner of talking, you must be two -of the silliest girls in the country. I have suspected it some time, but -I am now convinced.” - -Catherine was disconcerted, and made no answer; but Lydia, with perfect -indifference, continued to express her admiration of Captain Carter, and -her hope of seeing him in the course of the day, as he was going the -next morning to London. - -“I am astonished, my dear,” said Mrs. Bennet, “that you should be so -ready to think your own children silly. If I wished to think slightingly -of anybody’s children, it should not be of my own, however.” - -“If my children are silly, I must hope to be always sensible of it.” - -“Yes; but as it happens, they are all of them very clever.” - -“This is the only point, I flatter myself, on which we do not agree. I -had hoped that our sentiments coincided in every particular, but I must -so far differ from you as to think our two youngest daughters uncommonly -foolish.” - -“My dear Mr. Bennet, you must not expect such girls to have the sense of -their father and mother. When they get to our age, I dare say they will -not think about officers any more than we do. I remember the time when I -liked a red coat myself very well--and, indeed, so I do still at my -heart; and if a smart young colonel, with five or six thousand a year, -should want one of my girls, I shall not say nay to him; and I thought -Colonel Forster looked very becoming the other night at Sir William’s in -his regimentals.” - -“Mamma,” cried Lydia, “my aunt says that Colonel Forster and Captain -Carter do not go so often to Miss Watson’s as they did when they first -came; she sees them now very often standing in Clarke’s library.” - -Mrs. Bennet was prevented replying by the entrance of the footman with a -note for Miss Bennet; it came from Netherfield, and the servant waited -for an answer. Mrs. Bennet’s eyes sparkled with pleasure, and she was -eagerly calling out, while her daughter read,-- - -“Well, Jane, who is it from? What is it about? What does he say? Well, -Jane, make haste and tell us; make haste, my love.” - -“It is from Miss Bingley,” said Jane, and then read it aloud. - - /* NIND “My dear friend, */ - - “If you are not so compassionate as to dine to-day with Louisa and - me, we shall be in danger of hating each other for the rest of our - lives; for a whole day’s _tête-à -tête_ between two women can never - end without a quarrel. Come as soon as you can on the receipt of - this. My brother and the gentlemen are to dine with the officers. - Yours ever, - -“CAROLINE BINGLEY.” - -“With the officers!” cried Lydia: “I wonder my aunt did not tell us of -_that_.” - -“Dining out,” said Mrs. Bennet; “that is very unlucky.” - -“Can I have the carriage?” said Jane. - -“No, my dear, you had better go on horseback, because it seems likely to -rain; and then you must stay all night.” - -“That would be a good scheme,” said Elizabeth, “if you were sure that -they would not offer to send her home.” - -“Oh, but the gentlemen will have Mr. Bingley’s chaise to go to Meryton; -and the Hursts have no horses to theirs.” - -“I had much rather go in the coach.” - -“But, my dear, your father cannot spare the horses, I am sure. They are -wanted in the farm, Mr. Bennet, are not they?” - -[Illustration: Cheerful prognostics] - -“They are wanted in the farm much oftener than I can get them.” - -“But if you have got them to-day,” said Elizabeth, “my mother’s purpose -will be answered.” - -She did at last extort from her father an acknowledgment that the horses -were engaged; Jane was therefore obliged to go on horseback, and her -mother attended her to the door with many cheerful prognostics of a bad -day. Her hopes were answered; Jane had not been gone long before it -rained hard. Her sisters were uneasy for her, but her mother was -delighted. The rain continued the whole evening without intermission; -Jane certainly could not come back. - -“This was a lucky idea of mine, indeed!” said Mrs. Bennet, more than -once, as if the credit of making it rain were all her own. Till the next -morning, however, she was not aware of all the felicity of her -contrivance. Breakfast was scarcely over when a servant from Netherfield -brought the following note for Elizabeth:-- - - /* NIND “My dearest Lizzie, */ - - “I find myself very unwell this morning, which, I suppose, is to be - imputed to my getting wet through yesterday. My kind friends will - not hear of my returning home till I am better. They insist also on - my seeing Mr. Jones--therefore do not be alarmed if you should hear - of his having been to me--and, excepting a sore throat and a - headache, there is not much the matter with me. - -“Yours, etc.” - -“Well, my dear,” said Mr. Bennet, when Elizabeth had read the note -aloud, “if your daughter should have a dangerous fit of illness--if she -should die--it would be a comfort to know that it was all in pursuit of -Mr. Bingley, and under your orders.” - -“Oh, I am not at all afraid of her dying. People do not die of little -trifling colds. She will be taken good care of. As long as she stays -there, it is all very well. I would go and see her if I could have the -carriage.” - -Elizabeth, feeling really anxious, determined to go to her, though the -carriage was not to be had: and as she was no horsewoman, walking was -her only alternative. She declared her resolution. - -“How can you be so silly,” cried her mother, “as to think of such a -thing, in all this dirt! You will not be fit to be seen when you get -there.” - -“I shall be very fit to see Jane--which is all I want.” - -“Is this a hint to me, Lizzy,” said her father, “to send for the -horses?” - -“No, indeed. I do not wish to avoid the walk. The distance is nothing, -when one has a motive; only three miles. I shall be back by dinner.” - -“I admire the activity of your benevolence,” observed Mary, “but every -impulse of feeling should be guided by reason; and, in my opinion, -exertion should always be in proportion to what is required.” - -“We will go as far as Meryton with you,” said Catherine and Lydia. -Elizabeth accepted their company, and the three young ladies set off -together. - -“If we make haste,” said Lydia, as they walked along, “perhaps we may -see something of Captain Carter, before he goes.” - -In Meryton they parted: the two youngest repaired to the lodgings of one -of the officers’ wives, and Elizabeth continued her walk alone, crossing -field after field at a quick pace, jumping over stiles and springing -over puddles, with impatient activity, and finding herself at last -within view of the house, with weary ancles, dirty stockings, and a face -glowing with the warmth of exercise. - -She was shown into the breakfast parlour, where all but Jane were -assembled, and where her appearance created a great deal of surprise. -That she should have walked three miles so early in the day in such -dirty weather, and by herself, was almost incredible to Mrs. Hurst and -Miss Bingley; and Elizabeth was convinced that they held her in contempt -for it. She was received, however, very politely by them; and in their -brother’s manners there was something better than politeness--there was -good-humour and kindness. Mr. Darcy said very little, and Mr. Hurst -nothing at all. The former was divided between admiration of the -brilliancy which exercise had given to her complexion and doubt as to -the occasion’s justifying her coming so far alone. The latter was -thinking only of his breakfast. - -Her inquiries after her sister were not very favourably answered. Miss -Bennet had slept ill, and though up, was very feverish, and not well -enough to leave her room. Elizabeth was glad to be taken to her -immediately; and Jane, who had only been withheld by the fear of giving -alarm or inconvenience, from expressing in her note how much she longed -for such a visit, was delighted at her entrance. She was not equal, -however, to much conversation; and when Miss Bingley left them together, -could attempt little beside expressions of gratitude for the -extraordinary kindness she was treated with. Elizabeth silently attended -her. - -When breakfast was over, they were joined by the sisters; and Elizabeth -began to like them herself, when she saw how much affection and -solicitude they showed for Jane. The apothecary came; and having -examined his patient, said, as might be supposed, that she had caught a -violent cold, and that they must endeavour to get the better of it; -advised her to return to bed, and promised her some draughts. The advice -was followed readily, for the feverish symptoms increased, and her head -ached acutely. Elizabeth did not quit her room for a moment, nor were -the other ladies often absent; the gentlemen being out, they had in fact -nothing to do elsewhere. - -When the clock struck three, Elizabeth felt that she must go, and very -unwillingly said so. Miss Bingley offered her the carriage, and she only -wanted a little pressing to accept it, when Jane testified such concern -at parting with her that Miss Bingley was obliged to convert the offer -of the chaise into an invitation to remain at Netherfield for the -present. Elizabeth most thankfully consented, and a servant was -despatched to Longbourn, to acquaint the family with her stay, and bring -back a supply of clothes. - -[Illustration: - -“The Apothecary came” -] - - - - -[Illustration: - -“covering a screen” -] - - - - -CHAPTER VIII. - - -[Illustration] - -At five o’clock the two ladies retired to dress, and at half-past six -Elizabeth was summoned to dinner. To the civil inquiries which then -poured in, and amongst which she had the pleasure of distinguishing the -much superior solicitude of Mr. Bingley, she could not make a very -favourable answer. Jane was by no means better. The sisters, on hearing -this, repeated three or four times how much they were grieved, how -shocking it was to have a bad cold, and how excessively they disliked -being ill themselves; and then thought no more of the matter: and their -indifference towards Jane, when not immediately before them, restored -Elizabeth to the enjoyment of all her original dislike. - -Their brother, indeed, was the only one of the party whom she could -regard with any complacency. His anxiety for Jane was evident, and his -attentions to herself most pleasing; and they prevented her feeling -herself so much an intruder as she believed she was considered by the -others. She had very little notice from any but him. Miss Bingley was -engrossed by Mr. Darcy, her sister scarcely less so; and as for Mr. -Hurst, by whom Elizabeth sat, he was an indolent man, who lived only to -eat, drink, and play at cards, who, when he found her prefer a plain -dish to a ragout, had nothing to say to her. - -When dinner was over, she returned directly to Jane, and Miss Bingley -began abusing her as soon as she was out of the room. Her manners were -pronounced to be very bad indeed,--a mixture of pride and impertinence: -she had no conversation, no style, no taste, no beauty. Mrs. Hurst -thought the same, and added,-- - -“She has nothing, in short, to recommend her, but being an excellent -walker. I shall never forget her appearance this morning. She really -looked almost wild.” - -“She did indeed, Louisa. I could hardly keep my countenance. Very -nonsensical to come at all! Why must _she_ be scampering about the -country, because her sister had a cold? Her hair so untidy, so blowzy!” - -“Yes, and her petticoat; I hope you saw her petticoat, six inches deep -in mud, I am absolutely certain, and the gown which had been let down to -hide it not doing its office.” - -“Your picture may be very exact, Louisa,” said Bingley; “but this was -all lost upon me. I thought Miss Elizabeth Bennet looked remarkably well -when she came into the room this morning. Her dirty petticoat quite -escaped my notice.” - -“_You_ observed it, Mr. Darcy, I am sure,” said Miss Bingley; “and I am -inclined to think that you would not wish to see _your sister_ make such -an exhibition.” - -“Certainly not.” - -“To walk three miles, or four miles, or five miles, or whatever it is, -above her ancles in dirt, and alone, quite alone! what could she mean by -it? It seems to me to show an abominable sort of conceited independence, -a most country-town indifference to decorum.” - -“It shows an affection for her sister that is very pleasing,” said -Bingley. - -“I am afraid, Mr. Darcy,” observed Miss Bingley, in a half whisper, -“that this adventure has rather affected your admiration of her fine -eyes.” - -“Not at all,” he replied: “they were brightened by the exercise.” A -short pause followed this speech, and Mrs. Hurst began again,-- - -“I have an excessive regard for Jane Bennet,--she is really a very sweet -girl,--and I wish with all my heart she were well settled. But with such -a father and mother, and such low connections, I am afraid there is no -chance of it.” - -“I think I have heard you say that their uncle is an attorney in -Meryton?” - -“Yes; and they have another, who lives somewhere near Cheapside.” - -“That is capital,” added her sister; and they both laughed heartily. - -“If they had uncles enough to fill _all_ Cheapside,” cried Bingley, “it -would not make them one jot less agreeable.” - -“But it must very materially lessen their chance of marrying men of any -consideration in the world,” replied Darcy. - -To this speech Bingley made no answer; but his sisters gave it their -hearty assent, and indulged their mirth for some time at the expense of -their dear friend’s vulgar relations. - -With a renewal of tenderness, however, they repaired to her room on -leaving the dining-parlour, and sat with her till summoned to coffee. -She was still very poorly, and Elizabeth would not quit her at all, till -late in the evening, when she had the comfort of seeing her asleep, and -when it appeared to her rather right than pleasant that she should go -down stairs herself. On entering the drawing-room, she found the whole -party at loo, and was immediately invited to join them; but suspecting -them to be playing high, she declined it, and making her sister the -excuse, said she would amuse herself, for the short time she could stay -below, with a book. Mr. Hurst looked at her with astonishment. - -“Do you prefer reading to cards?” said he; “that is rather singular.” - -“Miss Eliza Bennet,” said Miss Bingley, “despises cards. She is a great -reader, and has no pleasure in anything else.” - -“I deserve neither such praise nor such censure,” cried Elizabeth; “I -am _not_ a great reader, and I have pleasure in many things.” - -“In nursing your sister I am sure you have pleasure,” said Bingley; “and -I hope it will soon be increased by seeing her quite well.” - -Elizabeth thanked him from her heart, and then walked towards a table -where a few books were lying. He immediately offered to fetch her -others; all that his library afforded. - -“And I wish my collection were larger for your benefit and my own -credit; but I am an idle fellow; and though I have not many, I have more -than I ever looked into.” - -Elizabeth assured him that she could suit herself perfectly with those -in the room. - -“I am astonished,” said Miss Bingley, “that my father should have left -so small a collection of books. What a delightful library you have at -Pemberley, Mr. Darcy!” - -“It ought to be good,” he replied: “it has been the work of many -generations.” - -“And then you have added so much to it yourself--you are always buying -books.” - -“I cannot comprehend the neglect of a family library in such days as -these.” - -“Neglect! I am sure you neglect nothing that can add to the beauties of -that noble place. Charles, when you build _your_ house, I wish it may be -half as delightful as Pemberley.” - -“I wish it may.” - -“But I would really advise you to make your purchase in that -neighbourhood, and take Pemberley for a kind of model. There is not a -finer county in England than Derbyshire.” - -“With all my heart: I will buy Pemberley itself, if Darcy will sell it.” - -“I am talking of possibilities, Charles.” - -“Upon my word, Caroline, I should think it more possible to get -Pemberley by purchase than by imitation.” - -Elizabeth was so much caught by what passed, as to leave her very little -attention for her book; and, soon laying it wholly aside, she drew near -the card-table, and stationed herself between Mr. Bingley and his eldest -sister, to observe the game. - -“Is Miss Darcy much grown since the spring?” said Miss Bingley: “will -she be as tall as I am?” - -“I think she will. She is now about Miss Elizabeth Bennet’s height, or -rather taller.” - -“How I long to see her again! I never met with anybody who delighted me -so much. Such a countenance, such manners, and so extremely accomplished -for her age! Her performance on the pianoforte is exquisite.” - -“It is amazing to me,” said Bingley, “how young ladies can have patience -to be so very accomplished as they all are.” - -“All young ladies accomplished! My dear Charles, what do you mean?” - -“Yes, all of them, I think. They all paint tables, cover screens, and -net purses. I scarcely know any one who cannot do all this; and I am -sure I never heard a young lady spoken of for the first time, without -being informed that she was very accomplished.” - -“Your list of the common extent of accomplishments,” said Darcy, “has -too much truth. The word is applied to many a woman who deserves it no -otherwise than by netting a purse or covering a screen; but I am very -far from agreeing with you in your estimation of ladies in general. I -cannot boast of knowing more than half-a-dozen in the whole range of my -acquaintance that are really accomplished.” - -“Nor I, I am sure,” said Miss Bingley. - -“Then,” observed Elizabeth, “you must comprehend a great deal in your -idea of an accomplished woman.” - -“Yes; I do comprehend a great deal in it.” - -“Oh, certainly,” cried his faithful assistant, “no one can be really -esteemed accomplished who does not greatly surpass what is usually met -with. A woman must have a thorough knowledge of music, singing, drawing, -dancing, and the modern languages, to deserve the word; and, besides all -this, she must possess a certain something in her air and manner of -walking, the tone of her voice, her address and expressions, or the word -will be but half deserved.” - -“All this she must possess,” added Darcy; “and to all she must yet add -something more substantial in the improvement of her mind by extensive -reading.” - -“I am no longer surprised at your knowing _only_ six accomplished women. -I rather wonder now at your knowing _any_.” - -“Are you so severe upon your own sex as to doubt the possibility of all -this?” - -“_I_ never saw such a woman. _I_ never saw such capacity, and taste, and -application, and elegance, as you describe, united.” - -Mrs. Hurst and Miss Bingley both cried out against the injustice of her -implied doubt, and were both protesting that they knew many women who -answered this description, when Mr. Hurst called them to order, with -bitter complaints of their inattention to what was going forward. As all -conversation was thereby at an end, Elizabeth soon afterwards left the -room. - -“Eliza Bennet,” said Miss Bingley, when the door was closed on her, “is -one of those young ladies who seek to recommend themselves to the other -sex by undervaluing their own; and with many men, I daresay, it -succeeds; but, in my opinion, it is a paltry device, a very mean art.” - -“Undoubtedly,” replied Darcy, to whom this remark was chiefly addressed, -“there is meanness in _all_ the arts which ladies sometimes condescend -to employ for captivation. Whatever bears affinity to cunning is -despicable.” - -Miss Bingley was not so entirely satisfied with this reply as to -continue the subject. - -Elizabeth joined them again only to say that her sister was worse, and -that she could not leave her. Bingley urged Mr. Jones’s being sent for -immediately; while his sisters, convinced that no country advice could -be of any service, recommended an express to town for one of the most -eminent physicians. This she would not hear of; but she was not so -unwilling to comply with their brother’s proposal; and it was settled -that Mr. Jones should be sent for early in the morning, if Miss Bennet -were not decidedly better. Bingley was quite uncomfortable; his sisters -declared that they were miserable. They solaced their wretchedness, -however, by duets after supper; while he could find no better relief to -his feelings than by giving his housekeeper directions that every -possible attention might be paid to the sick lady and her sister. - - - - -[Illustration: - -M^{rs} Bennet and her two youngest girls - -[_Copyright 1894 by George Allen._]] - - - - -CHAPTER IX. - - -[Illustration] - -Elizabeth passed the chief of the night in her sister’s room, and in the -morning had the pleasure of being able to send a tolerable answer to the -inquiries which she very early received from Mr. Bingley by a housemaid, -and some time afterwards from the two elegant ladies who waited on his -sisters. In spite of this amendment, however, she requested to have a -note sent to Longbourn, desiring her mother to visit Jane, and form her -own judgment of her situation. The note was immediately despatched, and -its contents as quickly complied with. Mrs. Bennet, accompanied by her -two youngest girls, reached Netherfield soon after the family breakfast. - -Had she found Jane in any apparent danger, Mrs. Bennet would have been -very miserable; but being satisfied on seeing her that her illness was -not alarming, she had no wish of her recovering immediately, as her -restoration to health would probably remove her from Netherfield. She -would not listen, therefore, to her daughter’s proposal of being carried -home; neither did the apothecary, who arrived about the same time, think -it at all advisable. After sitting a little while with Jane, on Miss -Bingley’s appearance and invitation, the mother and three daughters all -attended her into the breakfast parlour. Bingley met them with hopes -that Mrs. Bennet had not found Miss Bennet worse than she expected. - -“Indeed I have, sir,” was her answer. “She is a great deal too ill to be -moved. Mr. Jones says we must not think of moving her. We must trespass -a little longer on your kindness.” - -“Removed!” cried Bingley. “It must not be thought of. My sister, I am -sure, will not hear of her removal.” - -“You may depend upon it, madam,” said Miss Bingley, with cold civility, -“that Miss Bennet shall receive every possible attention while she -remains with us.” - -Mrs. Bennet was profuse in her acknowledgments. - -“I am sure,” she added, “if it was not for such good friends, I do not -know what would become of her, for she is very ill indeed, and suffers a -vast deal, though with the greatest patience in the world, which is -always the way with her, for she has, without exception, the sweetest -temper I ever met with. I often tell my other girls they are nothing to -_her_. You have a sweet room here, Mr. Bingley, and a charming prospect -over that gravel walk. I do not know a place in the country that is -equal to Netherfield. You will not think of quitting it in a hurry, I -hope, though you have but a short lease.” - -“Whatever I do is done in a hurry,” replied he; “and therefore if I -should resolve to quit Netherfield, I should probably be off in five -minutes. At present, however, I consider myself as quite fixed here.” - -“That is exactly what I should have supposed of you,” said Elizabeth. - -“You begin to comprehend me, do you?” cried he, turning towards her. - -“Oh yes--I understand you perfectly.” - -“I wish I might take this for a compliment; but to be so easily seen -through, I am afraid, is pitiful.” - -“That is as it happens. It does not necessarily follow that a deep, -intricate character is more or less estimable than such a one as yours.” - -“Lizzy,” cried her mother, “remember where you are, and do not run on in -the wild manner that you are suffered to do at home.” - -“I did not know before,” continued Bingley, immediately, “that you were -a studier of character. It must be an amusing study.” - -“Yes; but intricate characters are the _most_ amusing. They have at -least that advantage.” - -“The country,” said Darcy, “can in general supply but few subjects for -such a study. In a country neighbourhood you move in a very confined and -unvarying society.” - -“But people themselves alter so much, that there is something new to be -observed in them for ever.” - -“Yes, indeed,” cried Mrs. Bennet, offended by his manner of mentioning a -country neighbourhood. “I assure you there is quite as much of _that_ -going on in the country as in town.” - -Everybody was surprised; and Darcy, after looking at her for a moment, -turned silently away. Mrs. Bennet, who fancied she had gained a complete -victory over him, continued her triumph,-- - -“I cannot see that London has any great advantage over the country, for -my part, except the shops and public places. The country is a vast deal -pleasanter, is not it, Mr. Bingley?” - -“When I am in the country,” he replied, “I never wish to leave it; and -when I am in town, it is pretty much the same. They have each their -advantages, and I can be equally happy in either.” - -“Ay, that is because you have the right disposition. But that -gentleman,” looking at Darcy, “seemed to think the country was nothing -at all.” - -“Indeed, mamma, you are mistaken,” said Elizabeth, blushing for her -mother. “You quite mistook Mr. Darcy. He only meant that there was not -such a variety of people to be met with in the country as in town, which -you must acknowledge to be true.” - -“Certainly, my dear, nobody said there were; but as to not meeting with -many people in this neighbourhood, I believe there are few -neighbourhoods larger. I know we dine with four-and-twenty families.” - -Nothing but concern for Elizabeth could enable Bingley to keep his -countenance. His sister was less delicate, and directed her eye towards -Mr. Darcy with a very expressive smile. Elizabeth, for the sake of -saying something that might turn her mother’s thoughts, now asked her if -Charlotte Lucas had been at Longbourn since _her_ coming away. - -“Yes, she called yesterday with her father. What an agreeable man Sir -William is, Mr. Bingley--is not he? so much the man of fashion! so -genteel and so easy! He has always something to say to everybody. _That_ -is my idea of good breeding; and those persons who fancy themselves very -important and never open their mouths quite mistake the matter.” - -“Did Charlotte dine with you?” - -“No, she would go home. I fancy she was wanted about the mince-pies. For -my part, Mr. Bingley, _I_ always keep servants that can do their own -work; _my_ daughters are brought up differently. But everybody is to -judge for themselves, and the Lucases are a very good sort of girls, I -assure you. It is a pity they are not handsome! Not that _I_ think -Charlotte so _very_ plain; but then she is our particular friend.” - -“She seems a very pleasant young woman,” said Bingley. - -“Oh dear, yes; but you must own she is very plain. Lady Lucas herself -has often said so, and envied me Jane’s beauty. I do not like to boast -of my own child; but to be sure, Jane--one does not often see anybody -better looking. It is what everybody says. I do not trust my own -partiality. When she was only fifteen there was a gentleman at my -brother Gardiner’s in town so much in love with her, that my -sister-in-law was sure he would make her an offer before we came away. -But, however, he did not. Perhaps he thought her too young. However, he -wrote some verses on her, and very pretty they were.” - -“And so ended his affection,” said Elizabeth, impatiently. “There has -been many a one, I fancy, overcome in the same way. I wonder who first -discovered the efficacy of poetry in driving away love!” - -“I have been used to consider poetry as the _food_ of love,” said Darcy. - -“Of a fine, stout, healthy love it may. Everything nourishes what is -strong already. But if it be only a slight, thin sort of inclination, I -am convinced that one good sonnet will starve it entirely away.” - -Darcy only smiled; and the general pause which ensued made Elizabeth -tremble lest her mother should be exposing herself again. She longed to -speak, but could think of nothing to say; and after a short silence Mrs. -Bennet began repeating her thanks to Mr. Bingley for his kindness to -Jane, with an apology for troubling him also with Lizzy. Mr. Bingley was -unaffectedly civil in his answer, and forced his younger sister to be -civil also, and say what the occasion required. She performed her part, -indeed, without much graciousness, but Mrs. Bennet was satisfied, and -soon afterwards ordered her carriage. Upon this signal, the youngest of -her daughters put herself forward. The two girls had been whispering to -each other during the whole visit; and the result of it was, that the -youngest should tax Mr. Bingley with having promised on his first coming -into the country to give a ball at Netherfield. - -Lydia was a stout, well-grown girl of fifteen, with a fine complexion -and good-humoured countenance; a favourite with her mother, whose -affection had brought her into public at an early age. She had high -animal spirits, and a sort of natural self-consequence, which the -attentions of the officers, to whom her uncle’s good dinners and her -own easy manners recommended her, had increased into assurance. She was -very equal, therefore, to address Mr. Bingley on the subject of the -ball, and abruptly reminded him of his promise; adding, that it would be -the most shameful thing in the world if he did not keep it. His answer -to this sudden attack was delightful to her mother’s ear. - -“I am perfectly ready, I assure you, to keep my engagement; and, when -your sister is recovered, you shall, if you please, name the very day of -the ball. But you would not wish to be dancing while she is ill?” - -Lydia declared herself satisfied. “Oh yes--it would be much better to -wait till Jane was well; and by that time, most likely, Captain Carter -would be at Meryton again. And when you have given _your_ ball,” she -added, “I shall insist on their giving one also. I shall tell Colonel -Forster it will be quite a shame if he does not.” - -Mrs. Bennet and her daughters then departed, and Elizabeth returned -instantly to Jane, leaving her own and her relations’ behaviour to the -remarks of the two ladies and Mr. Darcy; the latter of whom, however, -could not be prevailed on to join in their censure of _her_, in spite of -all Miss Bingley’s witticisms on _fine eyes_. - - - - -[Illustration] - - - - -CHAPTER X. - - -[Illustration] - -The day passed much as the day before had done. Mrs. Hurst and Miss -Bingley had spent some hours of the morning with the invalid, who -continued, though slowly, to mend; and, in the evening, Elizabeth joined -their party in the drawing-room. The loo table, however, did not appear. -Mr. Darcy was writing, and Miss Bingley, seated near him, was watching -the progress of his letter, and repeatedly calling off his attention by -messages to his sister. Mr. Hurst and Mr. Bingley were at piquet, and -Mrs. Hurst was observing their game. - -Elizabeth took up some needlework, and was sufficiently amused in -attending to what passed between Darcy and his companion. The perpetual -commendations of the lady either on his hand-writing, or on the evenness -of his lines, or on the length of his letter, with the perfect unconcern -with which her praises were received, formed a curious dialogue, and was -exactly in unison with her opinion of each. - -“How delighted Miss Darcy will be to receive such a letter!” - -He made no answer. - -“You write uncommonly fast.” - -“You are mistaken. I write rather slowly.” - -“How many letters you must have occasion to write in the course of a -year! Letters of business, too! How odious I should think them!” - -“It is fortunate, then, that they fall to my lot instead of to yours.” - -“Pray tell your sister that I long to see her.” - -“I have already told her so once, by your desire.” - -“I am afraid you do not like your pen. Let me mend it for you. I mend -pens remarkably well.” - -“Thank you--but I always mend my own.” - -“How can you contrive to write so even?” - -He was silent. - -“Tell your sister I am delighted to hear of her improvement on the harp, -and pray let her know that I am quite in raptures with her beautiful -little design for a table, and I think it infinitely superior to Miss -Grantley’s.” - -“Will you give me leave to defer your raptures till I write again? At -present I have not room to do them justice.” - -“Oh, it is of no consequence. I shall see her in January. But do you -always write such charming long letters to her, Mr. Darcy?” - -“They are generally long; but whether always charming, it is not for me -to determine.” - -“It is a rule with me, that a person who can write a long letter with -ease cannot write ill.” - -“That will not do for a compliment to Darcy, Caroline,” cried her -brother, “because he does _not_ write with ease. He studies too much -for words of four syllables. Do not you, Darcy?” - -“My style of writing is very different from yours.” - -“Oh,” cried Miss Bingley, “Charles writes in the most careless way -imaginable. He leaves out half his words, and blots the rest.” - -“My ideas flow so rapidly that I have not time to express them; by which -means my letters sometimes convey no ideas at all to my correspondents.” - -“Your humility, Mr. Bingley,” said Elizabeth, “must disarm reproof.” - -“Nothing is more deceitful,” said Darcy, “than the appearance of -humility. It is often only carelessness of opinion, and sometimes an -indirect boast.” - -“And which of the two do you call _my_ little recent piece of modesty?” - -“The indirect boast; for you are really proud of your defects in -writing, because you consider them as proceeding from a rapidity of -thought and carelessness of execution, which, if not estimable, you -think at least highly interesting. The power of doing anything with -quickness is always much prized by the possessor, and often without any -attention to the imperfection of the performance. When you told Mrs. -Bennet this morning, that if you ever resolved on quitting Netherfield -you should be gone in five minutes, you meant it to be a sort of -panegyric, of compliment to yourself; and yet what is there so very -laudable in a precipitance which must leave very necessary business -undone, and can be of no real advantage to yourself or anyone else?” - - - CHISWICK PRESS:--CHARLES WHITTINGHAM AND CO. - TOOKS COURT, CHANCERY LANE, LONDON. - - -*** END OF THE PROJECT GUTENBERG EBOOK 1342 *** diff --git a/tests/dummy/data/transformers.py b/tests/dummy/data/transformers.py deleted file mode 100644 index 7d8911bb..00000000 --- a/tests/dummy/data/transformers.py +++ /dev/null @@ -1,50 +0,0 @@ -from typing import Iterable - -from datasets import ( # type: ignore - Dataset, - DatasetDict, - IterableDataset, - IterableDatasetDict, -) - - -def create_sample_dataset( - column: str = "text", pattern: str = "sample text {}" -) -> Dataset: - return Dataset.from_dict({column: [pattern.format(ind) for ind in range(1, 4)]}) - - -def create_sample_iterable_dataset( - column: str = "text", pattern: str = "sample text {}" -) -> IterableDataset: - def _generator(): - for ind in range(1, 4): - yield {column: pattern.format(ind)} - - return IterableDataset.from_generator(_generator) - - -def create_sample_dataset_dict( - splits: Iterable[str] = ("train", "test"), - column: str = "text", - pattern: str = "sample text {}", -): - return DatasetDict( - { - split: create_sample_dataset(column=column, pattern=pattern) - for split in splits - } - ) - - -def create_sample_iterable_dataset_dict( - splits: Iterable[str] = ("train", "test"), - column: str = "text", - pattern: str = "sample text {}", -): - return IterableDatasetDict( - { - split: create_sample_iterable_dataset(column=column, pattern=pattern) - for split in splits - } - ) diff --git a/tests/dummy/services/__init__.py b/tests/dummy/services/__init__.py deleted file mode 100644 index 8c63c5c4..00000000 --- a/tests/dummy/services/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .requests import TestRequestGenerator - -__all__ = [ - "TestRequestGenerator", -] diff --git a/tests/dummy/services/requests.py b/tests/dummy/services/requests.py deleted file mode 100644 index e7e29402..00000000 --- a/tests/dummy/services/requests.py +++ /dev/null @@ -1,31 +0,0 @@ -from typing import Optional - -from guidellm.core import TextGenerationRequest -from guidellm.request import GenerationMode, RequestGenerator - - -class TestRequestGenerator(RequestGenerator): - """ - This class represents the Testing Request Generator. - The purpose - to be used for testing. - """ - - def __init__( - self, - tokenizer: Optional[str] = None, - mode: GenerationMode = "async", - async_queue_size: int = 50, - ): - super().__init__( - type_="test", - source="test", - tokenizer=tokenizer, - mode=mode, - async_queue_size=async_queue_size, - ) - - def create_item(self) -> TextGenerationRequest: - return TextGenerationRequest(prompt="Test prompt") - - def __len__(self) -> int: - raise NotImplementedError diff --git a/tests/e2e/test_guidellm.py b/tests/e2e/test_guidellm.py deleted file mode 100644 index 75ab2212..00000000 --- a/tests/e2e/test_guidellm.py +++ /dev/null @@ -1,8 +0,0 @@ -import pytest - -from guidellm.config import settings - - -@pytest.mark.smoke() -def test_import(): - assert settings diff --git a/tests/integration/test_guidellm.py b/tests/integration/test_guidellm.py deleted file mode 100644 index 75ab2212..00000000 --- a/tests/integration/test_guidellm.py +++ /dev/null @@ -1,8 +0,0 @@ -import pytest - -from guidellm.config import settings - - -@pytest.mark.smoke() -def test_import(): - assert settings diff --git a/tests/unit/backend/test_openai_backend.py b/tests/unit/backend/test_openai_backend.py index db03c259..0749e9db 100644 --- a/tests/unit/backend/test_openai_backend.py +++ b/tests/unit/backend/test_openai_backend.py @@ -42,24 +42,26 @@ def test_openai_http_backend_intialization(): @pytest.mark.smoke() -def test_openai_http_backend_available_models(httpx_openai_mock): +@pytest.mark.asyncio() +async def test_openai_http_backend_available_models(httpx_openai_mock): backend = OpenAIHTTPBackend(target="http://target.mock") - models = backend.available_models() + models = await backend.available_models() assert models == ["mock-model"] @pytest.mark.smoke() -def test_openai_http_backend_validate(httpx_openai_mock): +@pytest.mark.asyncio() +async def test_openai_http_backend_validate(httpx_openai_mock): backend = OpenAIHTTPBackend(target="http://target.mock", model="mock-model") - backend.validate() + await backend.validate() backend = OpenAIHTTPBackend(target="http://target.mock") - backend.validate() + await backend.validate() assert backend.model == "mock-model" backend = OpenAIHTTPBackend(target="http://target.mock", model="invalid-model") with pytest.raises(ValueError): - backend.validate() + await backend.validate() @pytest.mark.smoke() diff --git a/tests/unit/backend/test_response.py b/tests/unit/backend/test_response.py index 8de78925..c4773083 100644 --- a/tests/unit/backend/test_response.py +++ b/tests/unit/backend/test_response.py @@ -20,6 +20,9 @@ def test_streaming_response_types(): def test_streaming_text_response_default_initilization(): response = StreamingTextResponse( type_="start", + value="", + start_time=0.0, + first_iter_time=None, iter_count=0, delta="", time=0.0, @@ -31,13 +34,19 @@ def test_streaming_text_response_default_initilization(): def test_streaming_text_response_initialization(): response = StreamingTextResponse( type_="start", - iter_count=0, + value="Hello, world!", + start_time=0.0, + first_iter_time=0.0, + iter_count=1, delta="Hello, world!", time=1.0, request_id="123", ) assert response.type_ == "start" - assert response.iter_count == 0 + assert response.value == "Hello, world!" + assert response.start_time == 0.0 + assert response.first_iter_time == 0.0 + assert response.iter_count == 1 assert response.delta == "Hello, world!" assert response.time == 1.0 assert response.request_id == "123" @@ -47,6 +56,9 @@ def test_streaming_text_response_initialization(): def test_streaming_text_response_marshalling(): response = StreamingTextResponse( type_="start", + value="Hello, world!", + start_time=0.0, + first_iter_time=0.0, iter_count=0, delta="Hello, world!", time=1.0, @@ -117,7 +129,18 @@ def test_response_summary_default_initialization(): ), start_time=0.0, end_time=0.0, + first_iter_time=None, + last_iter_time=None, ) + assert summary.value == "Hello, world!" + assert summary.request_args.target == "http://example.com" + assert summary.request_args.headers == {} + assert summary.request_args.payload == {} + assert summary.start_time == 0.0 + assert summary.end_time == 0.0 + assert summary.first_iter_time is None + assert summary.last_iter_time is None + assert summary.iterations == 0 assert summary.request_prompt_tokens is None assert summary.request_output_tokens is None assert summary.response_prompt_tokens is None @@ -137,6 +160,8 @@ def test_response_summary_initialization(): start_time=1.0, end_time=2.0, iterations=3, + first_iter_time=1.0, + last_iter_time=2.0, request_prompt_tokens=5, request_output_tokens=10, response_prompt_tokens=5, @@ -150,6 +175,8 @@ def test_response_summary_initialization(): assert summary.start_time == 1.0 assert summary.end_time == 2.0 assert summary.iterations == 3 + assert summary.first_iter_time == 1.0 + assert summary.last_iter_time == 2.0 assert summary.request_prompt_tokens == 5 assert summary.request_output_tokens == 10 assert summary.response_prompt_tokens == 5 diff --git a/tests/unit/cli/__init__.py b/tests/unit/cli/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/unit/cli/test_custom_type_params.py b/tests/unit/cli/test_custom_type_params.py deleted file mode 100644 index 1e66311d..00000000 --- a/tests/unit/cli/test_custom_type_params.py +++ /dev/null @@ -1,38 +0,0 @@ -import pytest -from click import BadParameter - -from guidellm.utils import cli_params - - -@pytest.fixture() -def max_requests_param_type(): - return cli_params.MaxRequestsType() - - -def test_valid_integer_input(max_requests_param_type): - assert max_requests_param_type.convert(10, None, None) == 10 - assert max_requests_param_type.convert("42", None, None) == 42 - - -def test_valid_dataset_input(max_requests_param_type): - assert max_requests_param_type.convert("dataset", None, None) == "dataset" - - -def test_invalid_string_input(max_requests_param_type): - with pytest.raises(BadParameter): - max_requests_param_type.convert("invalid", None, None) - - -def test_invalid_float_input(max_requests_param_type): - with pytest.raises(BadParameter): - max_requests_param_type.convert("10.5", None, None) - - -def test_invalid_non_numeric_string_input(max_requests_param_type): - with pytest.raises(BadParameter): - max_requests_param_type.convert("abc", None, None) - - -def test_invalid_mixed_string_input(max_requests_param_type): - with pytest.raises(BadParameter): - max_requests_param_type.convert("123abc", None, None) diff --git a/tests/unit/core/__init__.py b/tests/unit/core/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/unit/core/test_report.py b/tests/unit/core/test_report.py deleted file mode 100644 index c9e4ef3a..00000000 --- a/tests/unit/core/test_report.py +++ /dev/null @@ -1,106 +0,0 @@ -import tempfile -from pathlib import Path - -import pytest - -from guidellm.core import ( - GuidanceReport, - TextGenerationBenchmark, - TextGenerationBenchmarkReport, - TextGenerationRequest, - TextGenerationResult, -) - - -@pytest.fixture() -def sample_benchmark_report() -> TextGenerationBenchmarkReport: - sample_request = TextGenerationRequest(prompt="sample prompt") - sample_result = TextGenerationResult( - request=sample_request, - prompt_token_count=2, - output="sample output", - output_token_count=2, - start_time=None, - end_time=None, - first_token_time=None, - last_token_time=None, - ) - sample_benchmark = TextGenerationBenchmark( - mode="asynchronous", - rate=1.0, - results=[sample_result], - errors=[], - concurrencies=[], - ) - return TextGenerationBenchmarkReport( - benchmarks=[sample_benchmark], args={"arg1": "value1"} - ) - - -def compare_guidance_reports(report1: GuidanceReport, report2: GuidanceReport) -> bool: - return report1.benchmarks == report2.benchmarks - - -@pytest.mark.smoke() -def test_guidance_report_initialization(): - report = GuidanceReport() - assert report.benchmarks == [] - - -@pytest.mark.smoke() -def test_guidance_report_initialization_with_params(sample_benchmark_report): - report = GuidanceReport(benchmarks=[sample_benchmark_report]) - assert report.benchmarks == [sample_benchmark_report] - - -@pytest.mark.sanity() -def test_guidance_report_print(sample_benchmark_report): - report = GuidanceReport(benchmarks=[sample_benchmark_report]) - report.print() # This will output to the console - - -@pytest.mark.sanity() -def test_guidance_report_json(sample_benchmark_report): - report = GuidanceReport(benchmarks=[sample_benchmark_report]) - json_str = report.to_json() - loaded_report = GuidanceReport.from_json(json_str) - assert compare_guidance_reports(report, loaded_report) - - -@pytest.mark.sanity() -def test_guidance_report_yaml(sample_benchmark_report): - report = GuidanceReport(benchmarks=[sample_benchmark_report]) - yaml_str = report.to_yaml() - loaded_report = GuidanceReport.from_yaml(yaml_str) - assert compare_guidance_reports(report, loaded_report) - - -@pytest.mark.sanity() -def test_guidance_report_save_load_file(sample_benchmark_report): - report = GuidanceReport(benchmarks=[sample_benchmark_report]) - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "report.yaml" - report.save_file(file_path) - loaded_report = GuidanceReport.load_file(file_path) - assert compare_guidance_reports(report, loaded_report) - - -@pytest.mark.regression() -def test_empty_guidance_report(): - report = GuidanceReport() - assert len(report.benchmarks) == 0 - report.print() # Ensure it doesn't raise error with no benchmarks - - -@pytest.mark.regression() -def test_compare_guidance_reports(sample_benchmark_report): - report1 = GuidanceReport(benchmarks=[sample_benchmark_report]) - report2 = GuidanceReport(benchmarks=[sample_benchmark_report]) - assert compare_guidance_reports(report1, report2) - - -@pytest.mark.regression() -def test_compare_guidance_reports_inequality(sample_benchmark_report): - report1 = GuidanceReport(benchmarks=[sample_benchmark_report]) - report2 = GuidanceReport(benchmarks=[]) - assert not compare_guidance_reports(report1, report2) diff --git a/tests/unit/core/test_request.py b/tests/unit/core/test_request.py deleted file mode 100644 index 8550eb28..00000000 --- a/tests/unit/core/test_request.py +++ /dev/null @@ -1,79 +0,0 @@ -import pytest - -from guidellm.core import TextGenerationRequest - - -@pytest.mark.smoke() -def test_text_generation_request_initialization(): - prompt = "Generate a story" - request = TextGenerationRequest(prompt=prompt) - assert request.prompt == prompt - assert request.prompt_token_count is None - assert request.output_token_count is None - assert request.params == {} - - -@pytest.mark.sanity() -def test_text_generation_request_initialization_with_params(): - prompt = "Generate a story" - prompt_token_count = 50 - output_token_count = 100 - params = {"temperature": 0.7} - request = TextGenerationRequest( - prompt=prompt, - prompt_token_count=prompt_token_count, - output_token_count=output_token_count, - params=params, - ) - assert request.prompt == prompt - assert request.prompt_token_count == prompt_token_count - assert request.output_token_count == output_token_count - assert request.params == params - - -@pytest.mark.regression() -def test_request_json(): - prompt = "Generate text" - prompt_token_count = 10 - output_token_count = 50 - params = {"temperature": 0.7} - request = TextGenerationRequest( - prompt=prompt, - prompt_token_count=prompt_token_count, - output_token_count=output_token_count, - params=params, - ) - json_str = request.to_json() - assert '"prompt":"Generate text"' in json_str - assert '"id":' in json_str - - request_restored = TextGenerationRequest.from_json(json_str) - assert request.id == request_restored.id - assert request_restored.prompt == prompt - assert request_restored.prompt_token_count == prompt_token_count - assert request_restored.output_token_count == output_token_count - assert request_restored.params == params - - -@pytest.mark.regression() -def test_request_yaml(): - prompt = "Generate text" - prompt_token_count = 15 - output_token_count = 55 - params = {"temperature": 0.8} - request = TextGenerationRequest( - prompt=prompt, - prompt_token_count=prompt_token_count, - output_token_count=output_token_count, - params=params, - ) - yaml_str = request.to_yaml() - assert "prompt: Generate text" in yaml_str - assert "id:" in yaml_str - - request_restored = TextGenerationRequest.from_yaml(yaml_str) - assert request.id == request_restored.id - assert request_restored.prompt == prompt - assert request_restored.prompt_token_count == prompt_token_count - assert request_restored.output_token_count == output_token_count - assert request_restored.params == params diff --git a/tests/unit/core/test_result.py b/tests/unit/core/test_result.py deleted file mode 100644 index ddd62d7f..00000000 --- a/tests/unit/core/test_result.py +++ /dev/null @@ -1,279 +0,0 @@ -import time - -import pytest - -from guidellm.core import ( - RequestConcurrencyMeasurement, - TextGenerationBenchmark, - TextGenerationBenchmarkReport, - TextGenerationError, - TextGenerationRequest, - TextGenerationResult, -) - - -def create_sample_request(): - return TextGenerationRequest(prompt="Hello, world!") - - -def create_sample_result(): - start_time = time.time() - - return TextGenerationResult( - request=create_sample_request(), - prompt_token_count=4, - output="Generated text", - output_token_count=3, - start_time=start_time, - end_time=start_time + 1.5, - first_token_time=start_time + 0.5, - last_token_time=start_time + 1.4, - ) - - -@pytest.mark.smoke() -def test_text_generation_result_default_initialization(): - result = TextGenerationResult(request=create_sample_request()) - assert result.request.prompt == "Hello, world!" - assert result.prompt_token_count is None - assert result.output == "" - assert result.output_token_count is None - assert result.start_time is None - assert result.end_time is None - assert result.first_token_time is None - assert result.last_token_time is None - - -@pytest.mark.smoke() -def test_text_generation_result_initialization(): - result = create_sample_result() - assert result.request.prompt == "Hello, world!" - assert result.prompt_token_count == 4 - assert result.output == "Generated text" - assert result.output_token_count == 3 - assert result.start_time >= 0.0 - assert result.end_time == result.start_time + 1.5 - assert result.first_token_time == result.start_time + 0.5 - assert result.last_token_time == result.start_time + 1.4 - - # computed fields - assert result.request_latency == 1.5 - assert result.time_to_first_token == 0.5 * 1000 - assert result.inter_token_latency == pytest.approx((1.4 - 0.5) * 1000 / 2) - assert result.output_tokens_per_second == pytest.approx(2 / (1.4 - 0.5)) - - -@pytest.mark.smoke() -def test_text_generation_result_marshalling(): - result = create_sample_result() - serialized = result.model_dump() - deserialized = TextGenerationResult.model_validate(serialized) - - for key, value in vars(result).items(): - assert getattr(deserialized, key) == value - - -@pytest.mark.smoke() -def test_text_generation_error_initialization(): - error = TextGenerationError( - request=create_sample_request(), message="Error message" - ) - assert error.request.prompt == "Hello, world!" - assert error.message == "Error message" - - -@pytest.mark.smoke() -def test_text_generation_error_marshalling(): - error = TextGenerationError( - request=create_sample_request(), message="Error message" - ) - serialized = error.model_dump() - deserialized = TextGenerationError.model_validate(serialized) - - for key, value in vars(error).items(): - assert getattr(deserialized, key) == value - - -@pytest.mark.smoke() -def test_request_concurrency_measurement_initialization(): - start_time = time.time() - measurement = RequestConcurrencyMeasurement( - time=start_time, - completed=8, - errored=2, - processing=3, - ) - assert measurement.time == start_time - assert measurement.completed == 8 - assert measurement.errored == 2 - assert measurement.processing == 3 - - -@pytest.mark.smoke() -def test_request_concurrency_measurement_marshalling(): - start_time = time.time() - measurement = RequestConcurrencyMeasurement( - time=start_time, - completed=8, - errored=2, - processing=3, - ) - serialized = measurement.model_dump() - deserialized = RequestConcurrencyMeasurement.model_validate(serialized) - - for key, value in vars(measurement).items(): - assert getattr(deserialized, key) == value - - -@pytest.mark.smoke() -def test_text_generation_benchmark_default_initialization(): - benchmark = TextGenerationBenchmark(mode="asynchronous") - assert benchmark.mode == "asynchronous" - assert benchmark.rate is None - assert benchmark.results == [] - assert benchmark.errors == [] - assert benchmark.concurrencies == [] - - # computed - assert benchmark.request_count == 0 - assert benchmark.error_count == 0 - assert benchmark.total_count == 0 - assert benchmark.start_time is None - assert benchmark.end_time is None - assert benchmark.duration == 0.0 - assert benchmark.completed_request_rate == 0.0 - assert benchmark.request_latency_distribution is not None - assert benchmark.request_latency == 0.0 - assert benchmark.request_latency_percentiles == {} - assert benchmark.ttft_distribution is not None - assert benchmark.time_to_first_token == 0.0 - assert benchmark.time_to_first_token_percentiles == {} - assert benchmark.itl_distribution is not None - assert benchmark.inter_token_latency == 0.0 - assert benchmark.inter_token_latency_percentiles == {} - assert benchmark.output_token_throughput == 0.0 - assert benchmark.prompt_token_distribution is not None - assert benchmark.prompt_token == 0.0 - assert benchmark.prompt_token_percentiles == {} - assert benchmark.output_token_distribution is not None - assert benchmark.output_token == 0.0 - assert benchmark.output_token_percentiles == {} - - -@pytest.mark.smoke() -def test_text_generation_benchmark_initialization(): - benchmark = TextGenerationBenchmark(mode="asynchronous", rate=10) - assert benchmark.mode == "asynchronous" - assert benchmark.rate == 10 - - for _ in range(5): - benchmark.request_started() - benchmark.request_completed(create_sample_result()) - time.sleep(1.5) - - for _ in range(2): - benchmark.request_started() - benchmark.request_completed( - TextGenerationError( - request=create_sample_request(), message="Error message" - ) - ) - - def _test_percentiles(percentiles, value=None): - assert len(percentiles) == 7 - assert list(percentiles.keys()) == ["1", "5", "10", "50", "90", "95", "99"] - - if value is None: - assert all(per >= 0.0 for per in percentiles.values()) - else: - assert all(per == pytest.approx(value) for per in percentiles.values()) - - assert len(benchmark.results) == 5 - assert len(benchmark.errors) == 2 - assert len(benchmark.concurrencies) == 14 - assert benchmark.request_count == 5 - assert benchmark.error_count == 2 - assert benchmark.total_count == 7 - assert benchmark.start_time == pytest.approx(time.time() - 1.5 * 5, abs=0.01) - assert benchmark.end_time == pytest.approx(time.time(), abs=0.01) - assert benchmark.duration == benchmark.end_time - benchmark.start_time # type: ignore - assert benchmark.completed_request_rate == pytest.approx(5 / benchmark.duration) - assert benchmark.request_latency_distribution is not None - assert benchmark.request_latency == pytest.approx(1.5) - _test_percentiles(benchmark.request_latency_percentiles, 1.5) - assert benchmark.ttft_distribution is not None - assert benchmark.time_to_first_token == pytest.approx(500) - _test_percentiles(benchmark.time_to_first_token_percentiles, 500) - assert benchmark.itl_distribution is not None - assert benchmark.inter_token_latency == pytest.approx(450) - _test_percentiles(benchmark.inter_token_latency_percentiles, 450) - assert benchmark.output_token_throughput == pytest.approx(3.0 / 1.5, abs=0.01) - assert benchmark.prompt_token_distribution is not None - assert benchmark.prompt_token == pytest.approx(4.0) - _test_percentiles(benchmark.prompt_token_percentiles, 4.0) - assert benchmark.output_token_distribution is not None - assert benchmark.output_token == pytest.approx(3.0) - _test_percentiles(benchmark.output_token_percentiles, 3.0) - - -@pytest.mark.smoke() -def test_text_generation_benchmark_marshalling(): - benchmark = TextGenerationBenchmark(mode="asynchronous", rate=10) - for _ in range(5): - benchmark.request_started() - benchmark.request_completed(create_sample_result()) - - for _ in range(2): - benchmark.request_started() - benchmark.request_completed( - TextGenerationError( - request=create_sample_request(), message="Error message" - ) - ) - - serialized = benchmark.model_dump() - deserialized = TextGenerationBenchmark.model_validate(serialized) - - for key, value in vars(benchmark).items(): - assert getattr(deserialized, key) == value - - -@pytest.mark.smoke() -def test_text_generation_benchmark_report_initialization(): - report = TextGenerationBenchmarkReport( - benchmarks=[ - TextGenerationBenchmark(mode="asynchronous", rate=10), - TextGenerationBenchmark(mode="asynchronous", rate=20), - ], - args={ - "backend_type": "http", - "target": "http://example.com", - "model": "test-model", - }, - ) - assert len(report.benchmarks) == 2 - assert report.args == { - "backend_type": "http", - "target": "http://example.com", - "model": "test-model", - } - - -@pytest.mark.smoke() -def test_text_generation_benchmark_report_marshalling(): - report = TextGenerationBenchmarkReport( - benchmarks=[ - TextGenerationBenchmark(mode="asynchronous", rate=10), - TextGenerationBenchmark(mode="asynchronous", rate=20), - ], - args={ - "backend_type": "http", - "target": "http://example.com", - "model": "test-model", - }, - ) - serialized = report.model_dump() - deserialized = TextGenerationBenchmarkReport.model_validate(serialized) - - for key, value in vars(report).items(): - assert getattr(deserialized, key) == value diff --git a/tests/unit/executor/__init__.py b/tests/unit/executor/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/unit/executor/test_executor.py b/tests/unit/executor/test_executor.py deleted file mode 100644 index 58c0a9d4..00000000 --- a/tests/unit/executor/test_executor.py +++ /dev/null @@ -1,542 +0,0 @@ -from typing import List, Optional, Union -from unittest.mock import create_autospec, patch - -import pytest - -from guidellm.backend import Backend -from guidellm.config import settings -from guidellm.core import ( - TextGenerationBenchmarkReport, -) -from guidellm.executor import ( - Executor, - ExecutorResult, - Profile, - ProfileGenerationMode, - ProfileGenerator, -) -from guidellm.request import RequestGenerator -from guidellm.scheduler import Scheduler, SchedulerResult - - -@pytest.fixture() -def mock_scheduler(): - with patch("guidellm.executor.executor.Scheduler") as mock_scheduler: - - def scheduler_constructor(*args, **kwargs): - mock_instance = create_autospec(Scheduler, instance=True) - mock_instance.args = args - mock_instance.kwargs = kwargs - num_requests = kwargs.get("max_number", 10) - - async def run(): - benchmark = create_autospec( - TextGenerationBenchmarkReport, instance=True - ) - benchmark.completed_request_rate = kwargs.get("rate", None) - yield SchedulerResult( - completed=False, - count_total=10, - count_completed=0, - benchmark=benchmark, - current_result=None, - ) - - for index in range(num_requests): - yield SchedulerResult( - completed=False, - count_total=10, - count_completed=index + 1, - benchmark=benchmark, - current_result=create_autospec( - TextGenerationBenchmarkReport, instance=True - ), - ) - - yield SchedulerResult( - completed=True, - count_total=num_requests, - count_completed=num_requests, - benchmark=benchmark, - current_result=None, - ) - - mock_instance.run.side_effect = run - - return mock_instance - - mock_scheduler.side_effect = scheduler_constructor - yield mock_scheduler - - -@pytest.mark.smoke() -def test_executor_result_instantiation(): - report = create_autospec(TextGenerationBenchmarkReport, instance=True) - scheduler_result = create_autospec(SchedulerResult, instance=True) - executor_result = ExecutorResult( - completed=True, - count_total=10, - count_completed=5, - generation_modes=["synchronous", "throughput", "constant"], - report=report, - scheduler_result=scheduler_result, - ) - - assert executor_result.completed is True - assert executor_result.count_total == 10 - assert executor_result.count_completed == 5 - assert executor_result.report == report - assert executor_result.scheduler_result == scheduler_result - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("mode", "rate"), - [ - ("sweep", None), - ("synchronous", None), - ("throughput", None), - ("constant", 10), - ("constant", [10, 20, 30]), - ("poisson", 10), - ("poisson", [10, 20, 30]), - ], -) -def test_executor_instantiation(mode, rate): - backend = create_autospec(Backend, instance=True) - request_generator = create_autospec(RequestGenerator, instance=True) - executor = Executor( - backend=backend, - request_generator=request_generator, - mode=mode, - rate=rate, - max_number=100, - max_duration=60.0, - ) - - assert executor.backend == backend - assert executor.request_generator == request_generator - assert executor.profile_generator is not None - assert isinstance(executor.profile_generator, ProfileGenerator) - assert executor.profile_generator.mode == mode - assert ( - executor.profile_generator.rates == rate - if not rate or isinstance(rate, list) - else [rate] - ) - assert executor.max_number == 100 - assert executor.max_duration == 60.0 - - -def _check_executor_result_base( - result: ExecutorResult, - expected_completed: bool, - expected_count_total: int, - expected_count_completed: int, - expected_generation_modes: List[ProfileGenerationMode], -): - assert result.completed == expected_completed - assert result.count_total == expected_count_total - assert result.count_completed == expected_count_completed - assert result.generation_modes == expected_generation_modes - - -def _check_executor_result_report( - result: ExecutorResult, - mode: ProfileGenerationMode, - rate: Optional[Union[float, List[float]]], - max_number: Optional[int], - max_duration: Optional[float], - benchmarks_count: int, -): - assert result.report is not None - assert isinstance(result.report, TextGenerationBenchmarkReport) - - # check args - for expected in ( - "backend_type", - "target", - "model", - "data_type", - "data", - "tokenizer", - "mode", - "rate", - "max_number", - "max_duration", - ): - assert expected in result.report.args - - assert result.report.args["mode"] == mode - assert ( - result.report.args["rate"] == rate - if rate is None or not isinstance(rate, (float, int)) - else [rate] - ) - assert result.report.args["max_number"] == max_number - assert result.report.args["max_duration"] == max_duration - - # check benchmarks - assert len(result.report.benchmarks) == benchmarks_count - for benchmark in result.report.benchmarks: - assert isinstance(benchmark, TextGenerationBenchmarkReport) - - -def _check_executor_result_scheduler( - result: ExecutorResult, - expected_scheduler_result: bool, - expected_generation_modes: List[ProfileGenerationMode], - expected_index: Optional[int], - expected_profile_mode: Optional[ProfileGenerationMode], - expected_profile_rate: Optional[float], -): - if not expected_scheduler_result: - assert result.scheduler_result is None - assert result.current_index is None - assert result.current_profile is None - - return - - assert result.scheduler_result is not None - assert isinstance(result.scheduler_result, SchedulerResult) - assert result.current_index == expected_index - assert result.current_profile is not None - assert isinstance(result.current_profile, Profile) - assert result.current_profile.load_gen_mode == expected_profile_mode - assert result.current_profile.load_gen_rate == expected_profile_rate - assert ( - result.current_profile.load_gen_mode - == expected_generation_modes[expected_index] # type: ignore - ) - - -@pytest.mark.smoke() -@pytest.mark.asyncio() -async def test_executor_run_sweep(mock_scheduler): - num_requests = 15 - - backend = create_autospec(Backend, instance=True) - request_generator = create_autospec(RequestGenerator, instance=True) - executor = Executor( - backend=backend, - request_generator=request_generator, - mode="sweep", - rate=None, - max_number=num_requests, - ) - - num_profiles = 2 + settings.num_sweep_profiles - generation_modes = ["synchronous", "throughput"] + [ - "constant" - ] * settings.num_sweep_profiles - generation_rates = [None, None] + list(range(2, settings.num_sweep_profiles + 2)) - output_rates = [1, settings.num_sweep_profiles + 1] + list( - range(2, settings.num_sweep_profiles + 2) - ) - - iterator = executor.run() - - # Check start result - result = await iterator.__anext__() - _check_executor_result_base( - result=result, - expected_completed=False, - expected_count_total=num_profiles, - expected_count_completed=0, - expected_generation_modes=generation_modes, # type: ignore - ) - _check_executor_result_report( - result=result, - mode="sweep", - rate=None, - max_number=num_requests, - max_duration=None, - benchmarks_count=0, - ) - _check_executor_result_scheduler( - result=result, - expected_scheduler_result=False, - expected_generation_modes=generation_modes, # type: ignore - expected_index=None, - expected_profile_mode=None, - expected_profile_rate=None, - ) - - for scheduler_index in range(num_profiles): - for request_index in range(num_requests + 2): - result = await iterator.__anext__() - _check_executor_result_base( - result=result, - expected_completed=False, - expected_count_total=num_profiles, - expected_count_completed=scheduler_index - if request_index < num_requests + 1 - else scheduler_index + 1, - expected_generation_modes=generation_modes, # type: ignore - ) - _check_executor_result_report( - result=result, - mode="sweep", - rate=None, - max_number=num_requests, - max_duration=None, - benchmarks_count=scheduler_index - if request_index < num_requests + 1 - else scheduler_index + 1, - ) - _check_executor_result_scheduler( - result=result, - expected_scheduler_result=True, - expected_generation_modes=generation_modes, # type: ignore - expected_index=scheduler_index, - expected_profile_mode=generation_modes[scheduler_index], # type: ignore - expected_profile_rate=generation_rates[scheduler_index], - ) - # set the rate for the benchmark for sweep profile generation - result.report.benchmarks[-1].completed_request_rate = output_rates[ # type: ignore - scheduler_index - ] - result.report.benchmarks[-1].request_count = num_requests # type: ignore - - # Check end result - result = await iterator.__anext__() - _check_executor_result_base( - result=result, - expected_completed=True, - expected_count_total=num_profiles, - expected_count_completed=num_profiles, - expected_generation_modes=generation_modes, # type: ignore - ) - _check_executor_result_report( - result=result, - mode="sweep", - rate=None, - max_number=num_requests, - max_duration=None, - benchmarks_count=num_profiles, - ) - _check_executor_result_scheduler( - result=result, - expected_scheduler_result=False, - expected_generation_modes=generation_modes, # type: ignore - expected_index=None, - expected_profile_mode=None, - expected_profile_rate=None, - ) - - -@pytest.mark.smoke() -@pytest.mark.asyncio() -@pytest.mark.parametrize( - "mode", - [ - "synchronous", - "throughput", - ], -) -async def test_executor_run_non_rate_modes(mock_scheduler, mode): - num_requests = 15 - - backend = create_autospec(Backend, instance=True) - request_generator = create_autospec(RequestGenerator, instance=True) - executor = Executor( - backend=backend, - request_generator=request_generator, - mode=mode, - rate=None, - max_number=num_requests, - ) - - iterator = executor.run() - - # Check start result - result = await iterator.__anext__() - _check_executor_result_base( - result=result, - expected_completed=False, - expected_count_total=1, - expected_count_completed=0, - expected_generation_modes=[mode], - ) - _check_executor_result_report( - result=result, - mode=mode, - rate=None, - max_number=num_requests, - max_duration=None, - benchmarks_count=0, - ) - _check_executor_result_scheduler( - result=result, - expected_scheduler_result=False, - expected_generation_modes=[mode], - expected_index=None, - expected_profile_mode=None, - expected_profile_rate=None, - ) - - for request_index in range(num_requests + 2): - result = await iterator.__anext__() - _check_executor_result_base( - result=result, - expected_completed=False, - expected_count_total=1, - expected_count_completed=0 if request_index < num_requests + 1 else 1, - expected_generation_modes=[mode], - ) - _check_executor_result_report( - result=result, - mode=mode, - rate=None, - max_number=num_requests, - max_duration=None, - benchmarks_count=0 if request_index < num_requests + 1 else 1, - ) - _check_executor_result_scheduler( - result=result, - expected_scheduler_result=True, - expected_generation_modes=[mode], - expected_index=0, - expected_profile_mode=mode, - expected_profile_rate=None, - ) - - # Check end result - result = await iterator.__anext__() - _check_executor_result_base( - result=result, - expected_completed=True, - expected_count_total=1, - expected_count_completed=1, - expected_generation_modes=[mode], - ) - _check_executor_result_report( - result=result, - mode=mode, - rate=None, - max_number=num_requests, - max_duration=None, - benchmarks_count=1, - ) - _check_executor_result_scheduler( - result=result, - expected_scheduler_result=False, - expected_generation_modes=[mode], - expected_index=None, - expected_profile_mode=None, - expected_profile_rate=None, - ) - - -@pytest.mark.smoke() -@pytest.mark.asyncio() -@pytest.mark.parametrize( - ("mode", "rate"), - [ - ("constant", 10), - ("constant", [10, 20, 30]), - ("poisson", 10), - ("poisson", [10, 20, 30]), - ], -) -async def test_executor_run_rate_modes(mock_scheduler, mode, rate): - num_requests = 15 - - backend = create_autospec(Backend, instance=True) - request_generator = create_autospec(RequestGenerator, instance=True) - executor = Executor( - backend=backend, - request_generator=request_generator, - mode=mode, - rate=rate, - max_number=num_requests, - ) - - num_profiles = len(rate) if isinstance(rate, list) else 1 - generation_modes = [mode] * num_profiles - generation_rates = rate if isinstance(rate, list) else [rate] - - iterator = executor.run() - - # Check start result - result = await iterator.__anext__() - _check_executor_result_base( - result=result, - expected_completed=False, - expected_count_total=num_profiles, - expected_count_completed=0, - expected_generation_modes=generation_modes, - ) - _check_executor_result_report( - result=result, - mode=mode, - rate=rate, - max_number=num_requests, - max_duration=None, - benchmarks_count=0, - ) - _check_executor_result_scheduler( - result=result, - expected_scheduler_result=False, - expected_generation_modes=generation_modes, - expected_index=None, - expected_profile_mode=None, - expected_profile_rate=None, - ) - - for scheduler_index in range(num_profiles): - for request_index in range(num_requests + 2): - result = await iterator.__anext__() - _check_executor_result_base( - result=result, - expected_completed=False, - expected_count_total=num_profiles, - expected_count_completed=scheduler_index - if request_index < num_requests + 1 - else scheduler_index + 1, - expected_generation_modes=generation_modes, - ) - _check_executor_result_report( - result=result, - mode=mode, - rate=rate, - max_number=num_requests, - max_duration=None, - benchmarks_count=scheduler_index - if request_index < num_requests + 1 - else scheduler_index + 1, - ) - _check_executor_result_scheduler( - result=result, - expected_scheduler_result=True, - expected_generation_modes=generation_modes, - expected_index=scheduler_index, - expected_profile_mode=generation_modes[scheduler_index], - expected_profile_rate=generation_rates[scheduler_index], - ) - - # Check end result - result = await iterator.__anext__() - _check_executor_result_base( - result=result, - expected_completed=True, - expected_count_total=num_profiles, - expected_count_completed=num_profiles, - expected_generation_modes=generation_modes, - ) - _check_executor_result_report( - result=result, - mode=mode, - rate=rate, - max_number=num_requests, - max_duration=None, - benchmarks_count=num_profiles, - ) - _check_executor_result_scheduler( - result=result, - expected_scheduler_result=False, - expected_generation_modes=generation_modes, - expected_index=None, - expected_profile_mode=None, - expected_profile_rate=None, - ) diff --git a/tests/unit/executor/test_profile_generator.py b/tests/unit/executor/test_profile_generator.py deleted file mode 100644 index 9c91d574..00000000 --- a/tests/unit/executor/test_profile_generator.py +++ /dev/null @@ -1,204 +0,0 @@ -from typing import get_args -from unittest.mock import create_autospec - -import pytest - -from guidellm import settings -from guidellm.core import ( - TextGenerationBenchmark, - TextGenerationBenchmarkReport, -) -from guidellm.executor import Profile, ProfileGenerationMode, ProfileGenerator - - -@pytest.mark.smoke() -def test_profile_generator_mode(): - assert set(get_args(ProfileGenerationMode)) == { - "sweep", - "synchronous", - "throughput", - "constant", - "poisson", - } - - -@pytest.mark.smoke() -def test_profile_instantiation(): - profile = Profile(load_gen_mode="constant", load_gen_rate=10) - assert profile.load_gen_mode == "constant" - assert profile.load_gen_rate == 10 - assert profile.args == {} - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("mode", "rate"), - [ - ("sweep", None), - ("synchronous", None), - ("throughput", None), - ("constant", 10), - ("constant", [10, 20, 30]), - ("poisson", 10), - ("poisson", [10, 20, 30]), - ], -) -def test_profile_generator_instantiation(mode, rate): - generator = ProfileGenerator(mode=mode, rate=rate) - assert generator.mode == mode - - if rate is None: - assert generator.rates is None - elif isinstance(rate, list): - assert generator.rates == rate - else: - assert generator.rates == [rate] - - if mode == "sweep": - assert len(generator) == settings.num_sweep_profiles + 2 - assert ( - generator.profile_generation_modes - == ["synchronous", "throughput"] - + ["constant"] * settings.num_sweep_profiles - ) - elif mode in ("throughput", "synchronous"): - assert len(generator) == 1 - assert generator.profile_generation_modes == [mode] - else: - assert len(generator) == len(rate) if isinstance(rate, list) else 1 - assert generator.profile_generation_modes == [mode] * ( - len(rate) if isinstance(rate, list) else 1 - ) - - assert generator.generated_count == 0 - - -@pytest.mark.sanity() -@pytest.mark.parametrize( - ("mode", "rate"), - [ - # invalid modes - ("invalid_mode", None), - # rates supplied for non-applicable modes - ("sweep", 10), - ("sweep", [10, 20, 30]), - ("synchronous", 10), - ("synchronous", [10, 20, 30]), - ("throughput", 10), - ("throughput", [10, 20, 30]), - # invalid rates supplied for applicable modes - ("constant", None), - ("constant", -1), - ("constant", 0), - ("poisson", None), - ("poisson", -1), - ("poisson", 0), - ], -) -def test_profile_generator_invalid_instantiation(mode, rate): - with pytest.raises(ValueError): - ProfileGenerator(mode=mode, rate=rate) - - -@pytest.mark.sanity() -def test_profile_generator_next_sweep(): - generator = ProfileGenerator(mode="sweep") - current_report = TextGenerationBenchmarkReport() - - for index in range(settings.num_sweep_profiles + 2): - profile: Profile = generator.next(current_report) # type: ignore - - if index == 0: - assert profile.load_gen_mode == "synchronous" - assert profile.load_gen_rate is None - mock_benchmark = create_autospec(TextGenerationBenchmark, instance=True) - mock_benchmark.completed_request_rate = 1 - current_report.add_benchmark(mock_benchmark) - elif index == 1: - assert profile.load_gen_mode == "throughput" - assert profile.load_gen_rate is None - mock_benchmark = create_autospec(TextGenerationBenchmark, instance=True) - mock_benchmark.completed_request_rate = 10 - current_report.add_benchmark(mock_benchmark) - else: - assert profile.load_gen_mode == "constant" - assert profile.load_gen_rate == index - - assert generator.generated_count == index + 1 - - for _ in range(3): - assert generator.next(current_report) is None - - -@pytest.mark.sanity() -def test_profile_generator_next_synchronous(): - generator = ProfileGenerator(mode="synchronous") - current_report = TextGenerationBenchmarkReport() - - profile: Profile = generator.next(current_report) # type: ignore - assert profile.load_gen_mode == "synchronous" - assert profile.load_gen_rate is None - assert generator.generated_count == 1 - - for _ in range(3): - assert generator.next(current_report) is None - - -@pytest.mark.sanity() -def test_profile_generator_next_throughput(): - generator = ProfileGenerator(mode="throughput") - current_report = TextGenerationBenchmarkReport() - - profile: Profile = generator.next(current_report) # type: ignore - assert profile.load_gen_mode == "throughput" - assert profile.load_gen_rate is None - assert generator.generated_count == 1 - - for _ in range(3): - assert generator.next(current_report) is None - - -@pytest.mark.sanity() -@pytest.mark.parametrize( - "rate", - [ - 10, - [10, 20, 30], - ], -) -def test_profile_generator_next_constant(rate): - generator = ProfileGenerator(mode="constant", rate=rate) - test_rates = rate if isinstance(rate, list) else [rate] - current_report = TextGenerationBenchmarkReport() - - for index, test_rate in enumerate(test_rates): - profile: Profile = generator.next(current_report) # type: ignore - assert profile.load_gen_mode == "constant" - assert profile.load_gen_rate == test_rate - assert generator.generated_count == index + 1 - - for _ in range(3): - assert generator.next(current_report) is None - - -@pytest.mark.sanity() -@pytest.mark.parametrize( - "rate", - [ - 10, - [10, 20, 30], - ], -) -def test_profile_generator_next_poisson(rate): - generator = ProfileGenerator(mode="poisson", rate=rate) - test_rates = rate if isinstance(rate, list) else [rate] - current_report = TextGenerationBenchmarkReport() - - for index, test_rate in enumerate(test_rates): - profile: Profile = generator.next(current_report) # type: ignore - assert profile.load_gen_mode == "poisson" - assert profile.load_gen_rate == test_rate - assert generator.generated_count == index + 1 - - for _ in range(3): - assert generator.next(current_report) is None diff --git a/tests/unit/mock_backend.py b/tests/unit/mock_backend.py index 9eb4d6ee..e7335f61 100644 --- a/tests/unit/mock_backend.py +++ b/tests/unit/mock_backend.py @@ -36,10 +36,17 @@ def target(self) -> str: def model(self) -> Optional[str]: return self._model + @property + def info(self) -> Dict[str, Any]: + return {} + + async def prepare_multiprocessing(self): + pass + def check_setup(self): pass - def available_models(self) -> List[str]: + async def available_models(self) -> List[str]: return [self.model] # type: ignore async def text_completions( # type: ignore @@ -97,24 +104,38 @@ async def _text_prompt_response_generator( yield StreamingTextResponse( type_="start", + value="", + start_time=start_time, + first_iter_time=None, iter_count=0, delta="", time=start_time, request_id=request_id, ) + first_iter_time = None + last_iter_time = None + for index, token in enumerate(tokens): if self._iter_delay: await asyncio.sleep(self._iter_delay) + if first_iter_time is None: + first_iter_time = time.time() + yield StreamingTextResponse( type_="iter", + value="".join(tokens[: index + 1]), + start_time=start_time, + first_iter_time=first_iter_time, iter_count=index + 1, delta=token, time=time.time(), request_id=request_id, ) + last_iter_time = time.time() + yield ResponseSummary( value="".join(tokens), request_args=RequestArgs( @@ -125,6 +146,8 @@ async def _text_prompt_response_generator( iterations=len(tokens), start_time=start_time, end_time=time.time(), + first_iter_time=first_iter_time, + last_iter_time=last_iter_time, request_prompt_tokens=prompt_token_count, request_output_tokens=output_token_count, response_prompt_tokens=len(prompt.split()) + prompt.count(" "), diff --git a/tests/dummy/data/__init__.py b/tests/unit/objects/__init__.py similarity index 100% rename from tests/dummy/data/__init__.py rename to tests/unit/objects/__init__.py diff --git a/tests/unit/objects/test_distribution.py b/tests/unit/objects/test_distribution.py deleted file mode 100644 index e721e0c1..00000000 --- a/tests/unit/objects/test_distribution.py +++ /dev/null @@ -1,337 +0,0 @@ -import math - -import numpy as np -import pytest - -from guidellm.objects import DistributionSummary, Percentiles - - -def create_default_percentiles() -> Percentiles: - return Percentiles( - p001=0.1, - p01=1.0, - p05=5.0, - p10=10.0, - p25=25.0, - p75=75.0, - p90=90.0, - p95=95.0, - p99=99.0, - p999=99.9, - ) - - -def create_default_distribution_summary() -> DistributionSummary: - return DistributionSummary( - mean=50.0, - median=50.0, - variance=835, - std_dev=math.sqrt(835), - min=0.0, - max=100.0, - count=1001, - percentiles=create_default_percentiles(), - ) - - -@pytest.mark.smoke() -def test_percentiles_initialization(): - percentiles = create_default_percentiles() - assert percentiles.p001 == 0.1 - assert percentiles.p01 == 1.0 - assert percentiles.p05 == 5.0 - assert percentiles.p10 == 10.0 - assert percentiles.p25 == 25.0 - assert percentiles.p75 == 75.0 - assert percentiles.p90 == 90.0 - assert percentiles.p95 == 95.0 - assert percentiles.p99 == 99.0 - assert percentiles.p999 == 99.9 - - -@pytest.mark.smoke() -def test_percentiles_invalid_initialization(): - test_kwargs = { - "p001": 0.1, - "p01": 1.0, - "p05": 5.0, - "p10": 10.0, - "p25": 25.0, - "p75": 75.0, - "p90": 90.0, - "p95": 95.0, - "p99": 99.0, - "p999": 99.9, - } - test_missing_keys = list(test_kwargs.keys()) - - for missing_key in test_missing_keys: - kwargs = {key: val for key, val in test_kwargs.items() if key != missing_key} - with pytest.raises(ValueError): - Percentiles(**kwargs) - - -@pytest.mark.smoke() -def test_percentiles_from_values(): - values = [val / 10 for val in range(1001)] - percentiles = Percentiles.from_values(values) - true_percentiles = np.percentile( - values, [0.1, 1.0, 5.0, 10.0, 25.0, 75.0, 90.0, 95.0, 99.0, 99.9] - ) - assert percentiles.p001 == pytest.approx(true_percentiles[0]) - assert percentiles.p01 == pytest.approx(true_percentiles[1]) - assert percentiles.p05 == pytest.approx(true_percentiles[2]) - assert percentiles.p10 == pytest.approx(true_percentiles[3]) - assert percentiles.p25 == pytest.approx(true_percentiles[4]) - assert percentiles.p75 == pytest.approx(true_percentiles[5]) - assert percentiles.p90 == pytest.approx(true_percentiles[6]) - assert percentiles.p95 == pytest.approx(true_percentiles[7]) - assert percentiles.p99 == pytest.approx(true_percentiles[8]) - assert percentiles.p999 == pytest.approx(true_percentiles[9]) - - -@pytest.mark.smoke() -def test_percentiles_marshalling(): - percentiles = create_default_percentiles() - serialized = percentiles.model_dump() - deserialized = Percentiles.model_validate(serialized) - - for key, value in vars(percentiles).items(): - assert getattr(deserialized, key) == value - - -@pytest.mark.smoke() -def test_distribution_summary_initialization(): - distribution_summary = DistributionSummary( - mean=50.0, - median=50.0, - variance=835, - std_dev=math.sqrt(835), - min=0.0, - max=100.0, - count=1001, - percentiles=create_default_percentiles(), - ) - assert distribution_summary.mean == 50.0 - assert distribution_summary.median == 50.0 - assert distribution_summary.variance == 835 - assert distribution_summary.std_dev == math.sqrt(835) - assert distribution_summary.min == 0.0 - assert distribution_summary.max == 100.0 - assert distribution_summary.count == 1001 - assert distribution_summary.percentiles.p001 == 0.1 - assert distribution_summary.percentiles.p01 == 1.0 - assert distribution_summary.percentiles.p05 == 5.0 - assert distribution_summary.percentiles.p10 == 10.0 - assert distribution_summary.percentiles.p25 == 25.0 - assert distribution_summary.percentiles.p75 == 75.0 - assert distribution_summary.percentiles.p90 == 90.0 - assert distribution_summary.percentiles.p95 == 95.0 - assert distribution_summary.percentiles.p99 == 99.0 - assert distribution_summary.percentiles.p999 == 99.9 - - -@pytest.mark.smoke() -def test_distribution_summary_invalid_initialization(): - test_kwargs = { - "mean": 50.0, - "median": 50.0, - "variance": 835, - "std_dev": math.sqrt(835), - "min": 0.0, - "max": 100.0, - "count": 1001, - "percentiles": create_default_percentiles(), - } - test_missing_keys = list(test_kwargs.keys()) - for missing_key in test_missing_keys: - kwargs = {key: val for key, val in test_kwargs.items() if key != missing_key} - with pytest.raises(ValueError): - DistributionSummary(**kwargs) - - -@pytest.mark.smoke() -def test_distribution_summary_from_values(): - values = [val / 10 for val in range(1001)] - distribution_summary = DistributionSummary.from_values(values) - assert distribution_summary.mean == pytest.approx(np.mean(values)) - assert distribution_summary.median == pytest.approx(np.median(values)) - assert distribution_summary.variance == pytest.approx(np.var(values, ddof=1)) - assert distribution_summary.std_dev == pytest.approx(np.std(values, ddof=1)) - assert distribution_summary.min == min(values) - assert distribution_summary.max == max(values) - assert distribution_summary.count == len(values) - assert distribution_summary.percentiles.p001 == pytest.approx( - np.percentile(values, 0.1) - ) - assert distribution_summary.percentiles.p01 == pytest.approx( - np.percentile(values, 1.0) - ) - assert distribution_summary.percentiles.p05 == pytest.approx( - np.percentile(values, 5.0) - ) - assert distribution_summary.percentiles.p10 == pytest.approx( - np.percentile(values, 10.0) - ) - assert distribution_summary.percentiles.p25 == pytest.approx( - np.percentile(values, 25.0) - ) - assert distribution_summary.percentiles.p75 == pytest.approx( - np.percentile(values, 75.0) - ) - assert distribution_summary.percentiles.p90 == pytest.approx( - np.percentile(values, 90.0) - ) - assert distribution_summary.percentiles.p95 == pytest.approx( - np.percentile(values, 95.0) - ) - assert distribution_summary.percentiles.p99 == pytest.approx( - np.percentile(values, 99.0) - ) - assert distribution_summary.percentiles.p999 == pytest.approx( - np.percentile(values, 99.9) - ) - - -@pytest.mark.smoke() -def test_distribution_summary_from_time_measurements_count(): - # create bimodal distribution to test count comes out to average - # ie, 1 is active for 50 seconds, 2 is active for 100 seconds - values = [(val / 10, 1) for val in range(500)] - values += [(val / 5 + 50, 2) for val in range(500)] - distribution_summary = DistributionSummary.from_time_measurements( - values, time_weighting="count" - ) - assert distribution_summary.mean == pytest.approx( - (1 * 50 + 2 * 100) / 150, abs=0.001 - ) - assert distribution_summary.median == pytest.approx(2) - assert distribution_summary.variance == pytest.approx(0.2223, abs=0.001) - assert distribution_summary.std_dev == pytest.approx(0.4715, abs=0.001) - assert distribution_summary.min == 1 - assert distribution_summary.max == 2 - assert distribution_summary.count == pytest.approx(100000, abs=1000) - assert distribution_summary.percentiles.p001 == pytest.approx(1) - assert distribution_summary.percentiles.p01 == pytest.approx(1) - assert distribution_summary.percentiles.p05 == pytest.approx(1) - assert distribution_summary.percentiles.p10 == pytest.approx(1) - assert distribution_summary.percentiles.p25 == pytest.approx(1) - assert distribution_summary.percentiles.p75 == pytest.approx(2) - assert distribution_summary.percentiles.p90 == pytest.approx(2) - assert distribution_summary.percentiles.p95 == pytest.approx(2) - assert distribution_summary.percentiles.p99 == pytest.approx(2) - assert distribution_summary.percentiles.p999 == pytest.approx(2) - - -@pytest.mark.smoke() -def test_distribution_summary_from_time_measurements_multiply(): - # create consistent timestamped values matching a rate of 10 per second - values = [(val / 10, 1) for val in range(1001)] - distribution_summary = DistributionSummary.from_time_measurements( - values, time_weighting="multiply" - ) - assert distribution_summary.mean == pytest.approx(0.1) - assert distribution_summary.median == pytest.approx(0.1) - assert distribution_summary.variance == pytest.approx(0) - assert distribution_summary.std_dev == pytest.approx(0) - assert distribution_summary.min == pytest.approx(0.1) - assert distribution_summary.max == pytest.approx(0.1) - assert distribution_summary.count == len(values) - 1 - assert distribution_summary.percentiles.p001 == pytest.approx(0.1) - assert distribution_summary.percentiles.p01 == pytest.approx(0.1) - assert distribution_summary.percentiles.p05 == pytest.approx(0.1) - assert distribution_summary.percentiles.p10 == pytest.approx(0.1) - assert distribution_summary.percentiles.p25 == pytest.approx(0.1) - assert distribution_summary.percentiles.p75 == pytest.approx(0.1) - assert distribution_summary.percentiles.p90 == pytest.approx(0.1) - assert distribution_summary.percentiles.p95 == pytest.approx(0.1) - assert distribution_summary.percentiles.p99 == pytest.approx(0.1) - assert distribution_summary.percentiles.p999 == pytest.approx(0.1) - - -@pytest.mark.smoke() -def test_distribution_summary_from_time_measurements_divide(): - # create consistent timestamped values matching a rate of 10 per second - values = [(val / 10, 1) for val in range(1001)] - distribution_summary = DistributionSummary.from_time_measurements( - values, time_weighting="divide" - ) - assert distribution_summary.mean == pytest.approx(10.0) - assert distribution_summary.median == pytest.approx(10.0) - assert distribution_summary.variance == pytest.approx(0) - assert distribution_summary.std_dev == pytest.approx(0) - assert distribution_summary.min == pytest.approx(10.0) - assert distribution_summary.max == pytest.approx(10.0) - assert distribution_summary.count == len(values) - 1 - assert distribution_summary.percentiles.p001 == pytest.approx(10.0) - assert distribution_summary.percentiles.p01 == pytest.approx(10.0) - assert distribution_summary.percentiles.p05 == pytest.approx(10.0) - assert distribution_summary.percentiles.p10 == pytest.approx(10.0) - assert distribution_summary.percentiles.p25 == pytest.approx(10.0) - assert distribution_summary.percentiles.p75 == pytest.approx(10.0) - assert distribution_summary.percentiles.p90 == pytest.approx(10.0) - assert distribution_summary.percentiles.p95 == pytest.approx(10.0) - assert distribution_summary.percentiles.p99 == pytest.approx(10.0) - assert distribution_summary.percentiles.p999 == pytest.approx(10.0) - - -@pytest.mark.smoke() -def test_distribution_summary_from_time_ranges_count(): - # create consistent time ranges representing 10 concurrent requests constant - values = [(val / 10, val / 10 + 1, 1) for val in range(10001)] - distribution_summary = DistributionSummary.from_time_ranges( - values, time_weighting="count" - ) - assert distribution_summary.mean == pytest.approx(10.0, abs=0.01) - assert distribution_summary.median == pytest.approx(10.0) - assert distribution_summary.variance == pytest.approx(0, abs=0.1) - assert distribution_summary.std_dev == pytest.approx(0, abs=0.3) - assert distribution_summary.min == pytest.approx(1) - assert distribution_summary.max == pytest.approx(10.0) - assert distribution_summary.count == pytest.approx(100000, abs=1000) - assert distribution_summary.percentiles.p001 == pytest.approx(10, abs=5) - assert distribution_summary.percentiles.p01 == pytest.approx(10) - assert distribution_summary.percentiles.p05 == pytest.approx(10) - assert distribution_summary.percentiles.p10 == pytest.approx(10) - assert distribution_summary.percentiles.p25 == pytest.approx(10) - assert distribution_summary.percentiles.p75 == pytest.approx(10) - assert distribution_summary.percentiles.p90 == pytest.approx(10) - assert distribution_summary.percentiles.p95 == pytest.approx(10) - assert distribution_summary.percentiles.p99 == pytest.approx(10) - assert distribution_summary.percentiles.p999 == pytest.approx(10) - - -@pytest.mark.smoke() -def test_distribution_summary_from_time_ranges_multiply(): - # create consistent time ranges representing 10 concurrent requests constant - values = [(val / 10, val / 10 + 1, 1) for val in range(10001)] - distribution_summary = DistributionSummary.from_time_ranges( - values, time_weighting="multiply" - ) - assert distribution_summary.mean == pytest.approx(1.0, abs=0.01) - assert distribution_summary.median == pytest.approx(1.0) - assert distribution_summary.variance == pytest.approx(0, abs=0.1) - assert distribution_summary.std_dev == pytest.approx(0, abs=0.3) - assert distribution_summary.min == pytest.approx(0.1) - assert distribution_summary.max == pytest.approx(1.0) - assert distribution_summary.count == pytest.approx(10000, abs=10) - assert distribution_summary.percentiles.p001 == pytest.approx(1.0, abs=0.5) - assert distribution_summary.percentiles.p01 == pytest.approx(1.0) - assert distribution_summary.percentiles.p05 == pytest.approx(1.0) - assert distribution_summary.percentiles.p10 == pytest.approx(1.0) - assert distribution_summary.percentiles.p25 == pytest.approx(1.0) - assert distribution_summary.percentiles.p75 == pytest.approx(1.0) - assert distribution_summary.percentiles.p90 == pytest.approx(1.0) - assert distribution_summary.percentiles.p95 == pytest.approx(1.0) - assert distribution_summary.percentiles.p99 == pytest.approx(1.0) - assert distribution_summary.percentiles.p999 == pytest.approx(1.0) - - -@pytest.mark.smoke() -def test_distribution_summary_marshalling(): - distribution_summary = create_default_distribution_summary() - serialized = distribution_summary.model_dump() - deserialized = DistributionSummary.model_validate(serialized) - - for key, value in vars(distribution_summary).items(): - assert getattr(deserialized, key) == value diff --git a/tests/unit/objects/test_pydantic.py b/tests/unit/objects/test_pydantic.py new file mode 100644 index 00000000..a27fac5a --- /dev/null +++ b/tests/unit/objects/test_pydantic.py @@ -0,0 +1,43 @@ +import pytest +from pydantic import computed_field + +from guidellm.objects.pydantic import StandardBaseModel + + +class ExampleModel(StandardBaseModel): + name: str + age: int + + @computed_field # type: ignore[misc] + @property + def computed(self) -> str: + return self.name + " " + str(self.age) + + +@pytest.mark.smoke() +def test_standard_base_model_initialization(): + example = ExampleModel(name="John Doe", age=30) + assert example.name == "John Doe" + assert example.age == 30 + assert example.computed == "John Doe 30" + + +@pytest.mark.smoke() +def test_standard_base_model_invalid_initialization(): + with pytest.raises(ValueError): + ExampleModel(name="John Doe", age="thirty") # type: ignore[arg-type] + + +@pytest.mark.smoke() +def test_standard_base_model_marshalling(): + example = ExampleModel(name="John Doe", age=30) + serialized = example.model_dump() + assert serialized["name"] == "John Doe" + assert serialized["age"] == 30 + assert serialized["computed"] == "John Doe 30" + + serialized["computed"] = "Jane Doe 40" + deserialized = ExampleModel.model_validate(serialized) + assert deserialized.name == "John Doe" + assert deserialized.age == 30 + assert deserialized.computed == "John Doe 30" diff --git a/tests/unit/objects/test_serializable.py b/tests/unit/objects/test_serializable.py deleted file mode 100644 index 4cf31d07..00000000 --- a/tests/unit/objects/test_serializable.py +++ /dev/null @@ -1,151 +0,0 @@ -import tempfile -from pathlib import Path - -import pytest - -from guidellm.objects.pydantic import StandardBaseModel - - -class ExampleModel(StandardBaseModel): - name: str - age: int - - -@pytest.mark.smoke() -def test_serializable_json(): - # to json - example = ExampleModel(name="John Doe", age=30) - json_str = example.to_json() - assert '"name":"John Doe"' in json_str - assert '"age":30' in json_str - - # from json - example = ExampleModel.from_json(json_str) - assert example.name == "John Doe" - assert example.age == 30 - - -@pytest.mark.smoke() -def test_serializable_yaml(): - # to yaml - example = ExampleModel(name="John Doe", age=30) - yaml_str = example.to_yaml() - assert "name: John Doe" in yaml_str - assert "age: 30" in yaml_str - - # from yaml - example = ExampleModel.from_yaml(yaml_str) - assert example.name == "John Doe" - assert example.age == 30 - - -@pytest.mark.smoke() -def test_serializable_file_json(): - example = ExampleModel(name="John Doe", age=30) - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "example.json" - saved_path = example.save_file(file_path, "json") - assert Path(saved_path).exists() - loaded_example = ExampleModel.load_file(saved_path) - assert loaded_example.name == "John Doe" - assert loaded_example.age == 30 - - -@pytest.mark.smoke() -def test_serializable_file_yaml(): - example = ExampleModel(name="John Doe", age=30) - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "example.yaml" - saved_path = example.save_file(file_path, "yaml") - assert Path(saved_path).exists() - loaded_example = ExampleModel.load_file(saved_path) - assert loaded_example.name == "John Doe" - assert loaded_example.age == 30 - - -@pytest.mark.smoke() -def test_serializable_file_without_extension(): - example = ExampleModel(name="John Doe", age=30) - with tempfile.TemporaryDirectory() as temp_dir: - saved_path = example.save_file(temp_dir) - assert Path(saved_path).exists() - assert saved_path.endswith(".yaml") - loaded_example = ExampleModel.load_file(saved_path) - assert loaded_example.name == "John Doe" - assert loaded_example.age == 30 - - -@pytest.mark.sanity() -def test_serializable_file_with_directory_json(): - example = ExampleModel(name="John Doe", age=30) - with tempfile.TemporaryDirectory() as temp_dir: - saved_path = example.save_file(temp_dir, "json") - assert Path(saved_path).exists() - assert saved_path.endswith(".json") - loaded_example = ExampleModel.load_file(saved_path) - assert loaded_example.name == "John Doe" - assert loaded_example.age == 30 - - -@pytest.mark.sanity() -def test_serializable_file_with_directory_yaml(): - example = ExampleModel(name="John Doe", age=30) - with tempfile.TemporaryDirectory() as temp_dir: - saved_path = example.save_file(temp_dir, "yaml") - assert Path(saved_path).exists() - assert saved_path.endswith(".yaml") - loaded_example = ExampleModel.load_file(saved_path) - assert loaded_example.name == "John Doe" - assert loaded_example.age == 30 - - -@pytest.mark.sanity() -def test_serializable_file_infer_extension(): - example = ExampleModel(name="John Doe", age=30) - with tempfile.TemporaryDirectory() as temp_dir: - inferred_path = example.save_file(temp_dir, "json") - assert Path(inferred_path).exists() - assert inferred_path.endswith(".json") - loaded_example = ExampleModel.load_file(inferred_path) - assert loaded_example.name == "John Doe" - assert loaded_example.age == 30 - - -@pytest.mark.regression() -def test_serializable_file_invalid_extension(): - # to file - example = ExampleModel(name="John Doe", age=30) - with tempfile.TemporaryDirectory() as temp_dir: - invalid_file_path = Path(temp_dir) / "example.txt" - with pytest.raises(ValueError, match="Unsupported file extension.*"): - example.save_file(invalid_file_path) - - # to directory - with tempfile.TemporaryDirectory() as temp_dir: - invalid_file_path = Path(temp_dir) - with pytest.raises(ValueError, match="Unsupported file extension.*"): - example.save_file(invalid_file_path, type_="txt") # type: ignore - - # from file - with tempfile.TemporaryDirectory() as temp_dir: - invalid_file_path = Path(temp_dir) / "example.txt" - with invalid_file_path.open("w") as file: - file.write("invalid content") - with pytest.raises(ValueError, match="Unsupported file extension.*"): - ExampleModel.load_file(invalid_file_path) - - -@pytest.mark.regression() -def test_serializable_load_missing_path(): - with tempfile.TemporaryDirectory() as temp_dir: - invalid_file_path = Path(temp_dir) / "example.yaml" - with pytest.raises(FileNotFoundError): - ExampleModel.load_file(invalid_file_path) - - -@pytest.mark.regression() -def test_serializable_load_non_file_path(): - with tempfile.TemporaryDirectory() as temp_dir: - invalid_file_path = Path(temp_dir) - with pytest.raises(ValueError, match="Path is not a file.*"): - ExampleModel.load_file(invalid_file_path) diff --git a/tests/unit/objects/test_statistics.py b/tests/unit/objects/test_statistics.py new file mode 100644 index 00000000..0bd8c083 --- /dev/null +++ b/tests/unit/objects/test_statistics.py @@ -0,0 +1,707 @@ +import math +import time +from typing import List, Literal + +import numpy as np +import pytest + +from guidellm.objects import ( + DistributionSummary, + Percentiles, + RunningStats, + StatusDistributionSummary, + TimeRunningStats, +) + + +def create_default_percentiles() -> Percentiles: + return Percentiles( + p001=0.1, + p01=1.0, + p05=5.0, + p10=10.0, + p25=25.0, + p75=75.0, + p90=90.0, + p95=95.0, + p99=99.0, + p999=99.9, + ) + + +def create_default_distribution_summary() -> DistributionSummary: + return DistributionSummary( + mean=50.0, + median=50.0, + mode=50.0, + variance=835, + std_dev=math.sqrt(835), + min=0.0, + max=100.0, + count=1001, + total_sum=50050.0, + percentiles=create_default_percentiles(), + ) + + +@pytest.mark.smoke() +def test_percentiles_initialization(): + percentiles = create_default_percentiles() + assert percentiles.p001 == 0.1 + assert percentiles.p01 == 1.0 + assert percentiles.p05 == 5.0 + assert percentiles.p10 == 10.0 + assert percentiles.p25 == 25.0 + assert percentiles.p75 == 75.0 + assert percentiles.p90 == 90.0 + assert percentiles.p95 == 95.0 + assert percentiles.p99 == 99.0 + assert percentiles.p999 == 99.9 + + +@pytest.mark.smoke() +def test_percentiles_invalid_initialization(): + test_kwargs = { + "p001": 0.1, + "p01": 1.0, + "p05": 5.0, + "p10": 10.0, + "p25": 25.0, + "p75": 75.0, + "p90": 90.0, + "p95": 95.0, + "p99": 99.0, + "p999": 99.9, + } + test_missing_keys = list(test_kwargs.keys()) + + for missing_key in test_missing_keys: + kwargs = {key: val for key, val in test_kwargs.items() if key != missing_key} + with pytest.raises(ValueError): + Percentiles(**kwargs) + + +@pytest.mark.smoke() +def test_percentiles_marshalling(): + percentiles = create_default_percentiles() + serialized = percentiles.model_dump() + deserialized = Percentiles.model_validate(serialized) + + for key, value in vars(percentiles).items(): + assert getattr(deserialized, key) == value + + +@pytest.mark.smoke() +def test_distribution_summary_initilaization(): + distribution_summary = create_default_distribution_summary() + assert distribution_summary.mean == 50.0 + assert distribution_summary.median == 50.0 + assert distribution_summary.mode == 50.0 + assert distribution_summary.variance == 835 + assert distribution_summary.std_dev == math.sqrt(835) + assert distribution_summary.min == 0.0 + assert distribution_summary.max == 100.0 + assert distribution_summary.count == 1001 + assert distribution_summary.total_sum == 50050.0 + assert distribution_summary.percentiles.p001 == 0.1 + assert distribution_summary.percentiles.p01 == 1.0 + assert distribution_summary.percentiles.p05 == 5.0 + assert distribution_summary.percentiles.p10 == 10.0 + assert distribution_summary.percentiles.p25 == 25.0 + assert distribution_summary.percentiles.p75 == 75.0 + assert distribution_summary.percentiles.p90 == 90.0 + assert distribution_summary.percentiles.p95 == 95.0 + assert distribution_summary.percentiles.p99 == 99.0 + assert distribution_summary.percentiles.p999 == 99.9 + + +@pytest.mark.smoke() +def test_distribution_summary_invalid_initialization(): + test_kwargs = { + "mean": 50.0, + "median": 50.0, + "mode": 50.0, + "variance": 835, + "std_dev": math.sqrt(835), + "min": 0.0, + "max": 100.0, + "count": 1001, + "total_sum": 50050.0, + "percentiles": create_default_percentiles(), + } + test_missing_keys = list(test_kwargs.keys()) + for missing_key in test_missing_keys: + kwargs = {key: val for key, val in test_kwargs.items() if key != missing_key} + with pytest.raises(ValueError): + DistributionSummary(**kwargs) # type: ignore[arg-type] + + +@pytest.mark.smoke() +def test_distribution_summary_marshalling(): + distribution_summary = create_default_distribution_summary() + serialized = distribution_summary.model_dump() + deserialized = DistributionSummary.model_validate(serialized) + + for key, value in vars(distribution_summary).items(): + assert getattr(deserialized, key) == value + + +@pytest.mark.smoke() +def test_distribution_summary_from_distribution_function(): + values = [val / 10.0 for val in range(1001)] + distribution = [(val, 1.0) for val in values] + distribution_summary = DistributionSummary.from_distribution_function(distribution) + assert distribution_summary.mean == pytest.approx(np.mean(values)) + assert distribution_summary.median == pytest.approx(np.median(values)) + assert distribution_summary.mode == 0.0 + assert distribution_summary.variance == pytest.approx(np.var(values, ddof=0)) + assert distribution_summary.std_dev == pytest.approx(np.std(values, ddof=0)) + assert distribution_summary.min == min(values) + assert distribution_summary.max == max(values) + assert distribution_summary.count == len(values) + assert distribution_summary.total_sum == sum(values) + assert distribution_summary.percentiles.p001 == pytest.approx( + np.percentile(values, 0.1) + ) + assert distribution_summary.percentiles.p01 == pytest.approx( + np.percentile(values, 1.0) + ) + assert distribution_summary.percentiles.p05 == pytest.approx( + np.percentile(values, 5.0) + ) + assert distribution_summary.percentiles.p10 == pytest.approx( + np.percentile(values, 10.0) + ) + assert distribution_summary.percentiles.p25 == pytest.approx( + np.percentile(values, 25.0) + ) + assert distribution_summary.percentiles.p75 == pytest.approx( + np.percentile(values, 75.0) + ) + assert distribution_summary.percentiles.p90 == pytest.approx( + np.percentile(values, 90.0) + ) + assert distribution_summary.percentiles.p95 == pytest.approx( + np.percentile(values, 95.0) + ) + assert distribution_summary.percentiles.p99 == pytest.approx( + np.percentile(values, 99.0) + ) + assert distribution_summary.percentiles.p999 == pytest.approx( + np.percentile(values, 99.9) + ) + assert distribution_summary.cumulative_distribution_function is None + + distribution_summary_cdf = DistributionSummary.from_distribution_function( + distribution, include_cdf=True + ) + assert distribution_summary_cdf.cumulative_distribution_function is not None + assert len(distribution_summary_cdf.cumulative_distribution_function) == len(values) + + +def test_distribution_summary_from_values(): + values = [val / 10 for val in range(1001)] + distribution_summary = DistributionSummary.from_values(values) + assert distribution_summary.mean == pytest.approx(np.mean(values)) + assert distribution_summary.median == pytest.approx(np.median(values)) + assert distribution_summary.mode == 0.0 + assert distribution_summary.variance == pytest.approx(np.var(values, ddof=0)) + assert distribution_summary.std_dev == pytest.approx(np.std(values, ddof=0)) + assert distribution_summary.min == min(values) + assert distribution_summary.max == max(values) + assert distribution_summary.count == len(values) + assert distribution_summary.total_sum == sum(values) + assert distribution_summary.percentiles.p001 == pytest.approx( + np.percentile(values, 0.1) + ) + assert distribution_summary.percentiles.p01 == pytest.approx( + np.percentile(values, 1.0) + ) + assert distribution_summary.percentiles.p05 == pytest.approx( + np.percentile(values, 5.0) + ) + assert distribution_summary.percentiles.p10 == pytest.approx( + np.percentile(values, 10.0) + ) + assert distribution_summary.percentiles.p25 == pytest.approx( + np.percentile(values, 25.0) + ) + assert distribution_summary.percentiles.p75 == pytest.approx( + np.percentile(values, 75.0) + ) + assert distribution_summary.percentiles.p90 == pytest.approx( + np.percentile(values, 90.0) + ) + assert distribution_summary.percentiles.p95 == pytest.approx( + np.percentile(values, 95.0) + ) + assert distribution_summary.percentiles.p99 == pytest.approx( + np.percentile(values, 99.0) + ) + assert distribution_summary.percentiles.p999 == pytest.approx( + np.percentile(values, 99.9) + ) + assert distribution_summary.cumulative_distribution_function is None + + distribution_summary_weights = DistributionSummary.from_values( + values, weights=[2] * len(values) + ) + assert distribution_summary_weights.mean == pytest.approx(np.mean(values)) + assert distribution_summary_weights.median == pytest.approx(np.median(values)) + assert distribution_summary_weights.mode == 0.0 + assert distribution_summary_weights.variance == pytest.approx( + np.var(values, ddof=0) + ) + assert distribution_summary_weights.std_dev == pytest.approx(np.std(values, ddof=0)) + assert distribution_summary_weights.min == min(values) + assert distribution_summary_weights.max == max(values) + assert distribution_summary_weights.count == len(values) + assert distribution_summary_weights.total_sum == sum(values) + assert distribution_summary_weights.cumulative_distribution_function is None + + distribution_summary_cdf = DistributionSummary.from_values(values, include_cdf=True) + assert distribution_summary_cdf.cumulative_distribution_function is not None + assert len(distribution_summary_cdf.cumulative_distribution_function) == len(values) + + +def test_distribution_summary_from_request_times_concurrency(): + # create consistent timestamped values matching a rate of 10 per second + requests = [(val / 10, val / 10 + 1) for val in range(10001)] + distribution_summary = DistributionSummary.from_request_times( + requests, distribution_type="concurrency" + ) + assert distribution_summary.mean == pytest.approx(10.0, abs=0.01) + assert distribution_summary.median == pytest.approx(10.0) + assert distribution_summary.mode == 10.0 + assert distribution_summary.variance == pytest.approx(0, abs=0.1) + assert distribution_summary.std_dev == pytest.approx(0, abs=0.3) + assert distribution_summary.min == pytest.approx(1) + assert distribution_summary.max == pytest.approx(10.0) + assert distribution_summary.count == 10 + assert distribution_summary.total_sum == pytest.approx(55.0) + assert distribution_summary.percentiles.p001 == pytest.approx(10, abs=5) + assert distribution_summary.percentiles.p01 == pytest.approx(10) + assert distribution_summary.percentiles.p05 == pytest.approx(10) + assert distribution_summary.percentiles.p10 == pytest.approx(10) + assert distribution_summary.percentiles.p25 == pytest.approx(10) + assert distribution_summary.percentiles.p75 == pytest.approx(10) + assert distribution_summary.percentiles.p90 == pytest.approx(10) + assert distribution_summary.percentiles.p95 == pytest.approx(10) + assert distribution_summary.percentiles.p99 == pytest.approx(10) + assert distribution_summary.percentiles.p999 == pytest.approx(10) + assert distribution_summary.cumulative_distribution_function is None + + distribution_summary_cdf = DistributionSummary.from_request_times( + requests, distribution_type="concurrency", include_cdf=True + ) + assert distribution_summary_cdf.cumulative_distribution_function is not None + assert len(distribution_summary_cdf.cumulative_distribution_function) == 10 + + +def test_distribution_summary_from_request_times_rate(): + # create consistent timestamped values matching a rate of 10 per second + requests = [(val / 10, val / 10 + 1) for val in range(10001)] + distribution_summary = DistributionSummary.from_request_times( + requests, distribution_type="rate" + ) + assert distribution_summary.mean == pytest.approx(10.0, abs=0.01) + assert distribution_summary.median == pytest.approx(10.0) + assert distribution_summary.mode == pytest.approx(10.0) + assert distribution_summary.variance == pytest.approx(0, abs=0.1) + assert distribution_summary.std_dev == pytest.approx(0, abs=0.3) + assert distribution_summary.min == pytest.approx(1.0) + assert distribution_summary.max == pytest.approx(10.0) + assert distribution_summary.count == 12 + assert distribution_summary.total_sum == pytest.approx(111.0) + assert distribution_summary.percentiles.p001 == pytest.approx(10.0, abs=0.5) + assert distribution_summary.percentiles.p01 == pytest.approx(10.0) + assert distribution_summary.percentiles.p05 == pytest.approx(10.0) + assert distribution_summary.percentiles.p10 == pytest.approx(10.0) + assert distribution_summary.percentiles.p25 == pytest.approx(10.0) + assert distribution_summary.percentiles.p75 == pytest.approx(10.0) + assert distribution_summary.percentiles.p90 == pytest.approx(10.0) + assert distribution_summary.percentiles.p95 == pytest.approx(10.0) + assert distribution_summary.percentiles.p99 == pytest.approx(10.0) + assert distribution_summary.percentiles.p999 == pytest.approx(10.0) + assert distribution_summary.cumulative_distribution_function is None + + distribution_summary_cdf = DistributionSummary.from_request_times( + requests, distribution_type="rate", include_cdf=True + ) + assert distribution_summary_cdf.cumulative_distribution_function is not None + assert len(distribution_summary_cdf.cumulative_distribution_function) == 12 + + +def test_distribution_summary_from_iterable_request_times(): + # create consistent timestamped values matching a rate of 10 per second + requests = [(val / 10, val / 10 + 1) for val in range(10001)] + # create 9 iterations for each request with first iter at start + 0.1 + # and spaced at 0.1 seconds apart + first_iter_times = [val / 10 + 0.1 for val in range(10001)] + iter_counts = [9 for _ in range(10001)] + first_iter_counts = [1 for _ in range(10001)] + + distribution_summary = DistributionSummary.from_iterable_request_times( + requests, first_iter_times, iter_counts, first_iter_counts + ) + assert distribution_summary.mean == pytest.approx(90.0, abs=0.1) + assert distribution_summary.median == pytest.approx(80.0) + assert distribution_summary.mode == pytest.approx(80.0) + assert distribution_summary.variance == pytest.approx(704.463, abs=0.001) + assert distribution_summary.std_dev == pytest.approx(26.541, abs=0.001) + assert distribution_summary.min == pytest.approx(0.0) + assert distribution_summary.max == pytest.approx(160.0) + assert distribution_summary.count == 44 + assert distribution_summary.total_sum == pytest.approx(3538.85, abs=0.01) + assert distribution_summary.percentiles.p001 == pytest.approx(80.0) + assert distribution_summary.percentiles.p01 == pytest.approx(80.0) + assert distribution_summary.percentiles.p05 == pytest.approx(80.0) + assert distribution_summary.percentiles.p10 == pytest.approx(80.0) + assert distribution_summary.percentiles.p25 == pytest.approx(80.0) + assert distribution_summary.percentiles.p75 == pytest.approx(80.0) + assert distribution_summary.percentiles.p90 == pytest.approx(160.0) + assert distribution_summary.percentiles.p95 == pytest.approx(160.0) + assert distribution_summary.percentiles.p99 == pytest.approx(160.0) + assert distribution_summary.percentiles.p999 == pytest.approx(160.0) + assert distribution_summary.cumulative_distribution_function is None + + distribution_summary_cdf = DistributionSummary.from_iterable_request_times( + requests, first_iter_times, iter_counts, first_iter_counts, include_cdf=True + ) + assert distribution_summary_cdf.cumulative_distribution_function is not None + assert len(distribution_summary_cdf.cumulative_distribution_function) == 44 + + +def test_status_distribution_summary_initialization(): + status_distribution_summary = StatusDistributionSummary( + total=create_default_distribution_summary(), + successful=create_default_distribution_summary(), + incomplete=create_default_distribution_summary(), + errored=create_default_distribution_summary(), + ) + assert status_distribution_summary.total.mean == 50.0 + assert status_distribution_summary.successful.mean == 50.0 + assert status_distribution_summary.incomplete.mean == 50.0 + assert status_distribution_summary.errored.mean == 50.0 + + +def test_status_distribution_summary_invalid_initialization(): + test_kwargs = { + "total": create_default_distribution_summary(), + "successful": create_default_distribution_summary(), + "incomplete": create_default_distribution_summary(), + "errored": create_default_distribution_summary(), + } + test_missing_keys = list(test_kwargs.keys()) + for missing_key in test_missing_keys: + kwargs = {key: val for key, val in test_kwargs.items() if key != missing_key} + with pytest.raises(ValueError): + StatusDistributionSummary(**kwargs) # type: ignore + + +def test_status_distribution_summary_marshalling(): + status_distribution_summary = StatusDistributionSummary( + total=create_default_distribution_summary(), + successful=create_default_distribution_summary(), + incomplete=create_default_distribution_summary(), + errored=create_default_distribution_summary(), + ) + serialized = status_distribution_summary.model_dump() + deserialized = StatusDistributionSummary.model_validate(serialized) + + for key, value in vars(status_distribution_summary).items(): + for child_key, child_value in vars(value).items(): + assert getattr(getattr(deserialized, key), child_key) == child_value + + +def test_status_distribution_summary_from_values(): + value_types: List[Literal["successful", "incomplete", "error"]] = [ + "successful", + "incomplete", + "error", + ] * 1000 + values = [float(val % 3) for val in range(3000)] + status_distribution_summary = StatusDistributionSummary.from_values( + value_types, values + ) + assert status_distribution_summary.total.count == len(values) + assert status_distribution_summary.total.mean == pytest.approx(np.mean(values)) + assert status_distribution_summary.total.cumulative_distribution_function is None + assert status_distribution_summary.successful.mean == pytest.approx( + np.mean( + [val for ind, val in enumerate(values) if value_types[ind] == "successful"] + ) + ) + assert status_distribution_summary.successful.count == len( + [val for ind, val in enumerate(values) if value_types[ind] == "successful"] + ) + assert ( + status_distribution_summary.successful.cumulative_distribution_function is None + ) + assert status_distribution_summary.incomplete.mean == pytest.approx( + np.mean( + [val for ind, val in enumerate(values) if value_types[ind] == "incomplete"] + ) + ) + assert status_distribution_summary.incomplete.count == len( + [val for ind, val in enumerate(values) if value_types[ind] == "incomplete"] + ) + assert ( + status_distribution_summary.incomplete.cumulative_distribution_function is None + ) + assert status_distribution_summary.errored.mean == pytest.approx( + np.mean([val for ind, val in enumerate(values) if value_types[ind] == "error"]) + ) + assert status_distribution_summary.errored.count == len( + [val for ind, val in enumerate(values) if value_types[ind] == "error"] + ) + assert status_distribution_summary.errored.cumulative_distribution_function is None + + status_distribution_summary_cdf = StatusDistributionSummary.from_values( + value_types, values, include_cdf=True + ) + assert ( + status_distribution_summary_cdf.total.cumulative_distribution_function + is not None + ) + assert ( + status_distribution_summary_cdf.successful.cumulative_distribution_function + is not None + ) + assert ( + status_distribution_summary_cdf.incomplete.cumulative_distribution_function + is not None + ) + assert ( + status_distribution_summary_cdf.errored.cumulative_distribution_function + is not None + ) + + +def test_status_distribution_summary_from_request_times(): + request_types: List[Literal["successful", "incomplete", "error"]] = [ + "successful", + "incomplete", + "error", + ] * 1000 + requests = [((val % 3) / 10, (val % 3) / 10 + 1) for val in range(3000)] + status_distribution_summary = StatusDistributionSummary.from_request_times( + request_types, requests, distribution_type="concurrency" + ) + assert status_distribution_summary.total.mean == pytest.approx(2500.0, abs=0.01) + assert status_distribution_summary.total.cumulative_distribution_function is None + assert status_distribution_summary.successful.mean == pytest.approx( + 1000.0, abs=0.01 + ) + assert ( + status_distribution_summary.successful.cumulative_distribution_function is None + ) + assert status_distribution_summary.incomplete.mean == pytest.approx( + 1000.0, abs=0.01 + ) + assert ( + status_distribution_summary.incomplete.cumulative_distribution_function is None + ) + assert status_distribution_summary.errored.mean == pytest.approx(1000.0, abs=0.01) + assert status_distribution_summary.errored.cumulative_distribution_function is None + + status_distribution_summary_cdf = StatusDistributionSummary.from_request_times( + request_types, requests, distribution_type="concurrency", include_cdf=True + ) + assert ( + status_distribution_summary_cdf.total.cumulative_distribution_function + is not None + ) + assert ( + status_distribution_summary_cdf.successful.cumulative_distribution_function + is not None + ) + assert ( + status_distribution_summary_cdf.incomplete.cumulative_distribution_function + is not None + ) + assert ( + status_distribution_summary_cdf.errored.cumulative_distribution_function + is not None + ) + + +def test_status_distribution_summary_from_iterable_request_times(): + request_types: List[Literal["successful", "incomplete", "error"]] = [ + "successful", + "incomplete", + "error", + ] * 1000 + requests = [(val % 3 / 10, val % 3 / 10 + 1) for val in range(3000)] + first_iter_times = [val % 3 / 10 + 0.1 for val in range(3000)] + iter_counts = [9 for _ in range(3000)] + first_iter_counts = [1 for _ in range(3000)] + status_distribution_summary = StatusDistributionSummary.from_iterable_request_times( + request_types, + requests, + first_iter_times, + iter_counts, + first_iter_counts, + ) + assert status_distribution_summary.total.mean == pytest.approx(21666.66, abs=0.01) + assert status_distribution_summary.total.cumulative_distribution_function is None + assert status_distribution_summary.successful.mean == pytest.approx( + 8000.0, abs=0.01 + ) + assert ( + status_distribution_summary.successful.cumulative_distribution_function is None + ) + assert status_distribution_summary.incomplete.mean == pytest.approx( + 8000.0, abs=0.01 + ) + assert ( + status_distribution_summary.incomplete.cumulative_distribution_function is None + ) + assert status_distribution_summary.errored.mean == pytest.approx(8000.0, abs=0.01) + assert status_distribution_summary.errored.cumulative_distribution_function is None + + status_distribution_summary_cdf = ( + StatusDistributionSummary.from_iterable_request_times( + request_types, + requests, + first_iter_times, + iter_counts, + first_iter_counts, + include_cdf=True, + ) + ) + assert ( + status_distribution_summary_cdf.total.cumulative_distribution_function + is not None + ) + assert ( + status_distribution_summary_cdf.successful.cumulative_distribution_function + is not None + ) + assert ( + status_distribution_summary_cdf.incomplete.cumulative_distribution_function + is not None + ) + assert ( + status_distribution_summary_cdf.errored.cumulative_distribution_function + is not None + ) + + +def test_running_stats_initialization(): + running_stats = RunningStats() + assert running_stats.start_time == pytest.approx(time.time(), abs=0.01) + assert running_stats.count == 0 + assert running_stats.total == 0 + assert running_stats.last == 0 + assert running_stats.mean == 0 + assert running_stats.rate == 0 + + +def test_running_stats_marshalling(): + running_stats = RunningStats() + serialized = running_stats.model_dump() + deserialized = RunningStats.model_validate(serialized) + + for key, value in vars(running_stats).items(): + assert getattr(deserialized, key) == value + + +def test_running_stats_update(): + running_stats = RunningStats() + running_stats.update(1) + assert running_stats.count == 1 + assert running_stats.total == 1 + assert running_stats.last == 1 + assert running_stats.mean == 1 + time.sleep(1.0) + assert running_stats.rate == pytest.approx( + 1.0 / (time.time() - running_stats.start_time), abs=0.1 + ) + + running_stats.update(2) + assert running_stats.count == 2 + assert running_stats.total == 3 + assert running_stats.last == 2 + assert running_stats.mean == 1.5 + time.sleep(1) + assert running_stats.rate == pytest.approx( + 3 / (time.time() - running_stats.start_time), abs=0.1 + ) + + +def test_running_stats_add(): + running_stats = RunningStats() + mean = running_stats + 1 + assert mean == 1 + assert mean == running_stats.mean + assert running_stats.count == 1 + assert running_stats.total == 1 + assert running_stats.last == 1 + + +def test_running_stats_iadd(): + running_stats = RunningStats() + running_stats += 1 + assert running_stats.count == 1 + assert running_stats.total == 1 + assert running_stats.last == 1 + assert running_stats.mean == 1 + + +def test_time_running_stats_initialization(): + time_running_stats = TimeRunningStats() + assert time_running_stats.start_time == pytest.approx(time.time(), abs=0.01) + assert time_running_stats.count == 0 + assert time_running_stats.total == 0 + assert time_running_stats.last == 0 + assert time_running_stats.mean == 0 + assert time_running_stats.rate == 0 + assert time_running_stats.total_ms == 0 + assert time_running_stats.last_ms == 0 + assert time_running_stats.mean_ms == 0 + assert time_running_stats.rate_ms == 0 + + +def test_time_running_stats_marshalling(): + time_running_stats = TimeRunningStats() + serialized = time_running_stats.model_dump() + deserialized = TimeRunningStats.model_validate(serialized) + + for key, value in vars(time_running_stats).items(): + assert getattr(deserialized, key) == value + + +def test_time_running_stats_update(): + time_running_stats = TimeRunningStats() + time_running_stats.update(1) + assert time_running_stats.count == 1 + assert time_running_stats.total == 1 + assert time_running_stats.last == 1 + assert time_running_stats.mean == 1 + assert time_running_stats.total_ms == 1000 + assert time_running_stats.last_ms == 1000 + assert time_running_stats.mean_ms == 1000 + time.sleep(1.0) + assert time_running_stats.rate == pytest.approx( + 1.0 / (time.time() - time_running_stats.start_time), abs=0.1 + ) + assert time_running_stats.rate_ms == pytest.approx( + 1000 / (time.time() - time_running_stats.start_time), abs=0.1 + ) + + time_running_stats.update(2) + assert time_running_stats.count == 2 + assert time_running_stats.total == 3 + assert time_running_stats.last == 2 + assert time_running_stats.mean == 1.5 + assert time_running_stats.total_ms == 3000 + assert time_running_stats.last_ms == 2000 + assert time_running_stats.mean_ms == 1500 + time.sleep(1) + assert time_running_stats.rate == pytest.approx( + 3 / (time.time() - time_running_stats.start_time), abs=0.1 + ) + assert time_running_stats.rate_ms == pytest.approx( + 3000 / (time.time() - time_running_stats.start_time), abs=0.1 + ) diff --git a/tests/unit/request/__init__.py b/tests/unit/request/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/unit/request/test_base.py b/tests/unit/request/test_base.py deleted file mode 100644 index 73cf1b14..00000000 --- a/tests/unit/request/test_base.py +++ /dev/null @@ -1,160 +0,0 @@ -import re -import time -from typing import List -from unittest.mock import MagicMock, Mock, patch - -import pytest - -from guidellm.core import TextGenerationRequest -from tests.dummy.services import TestRequestGenerator - - -@pytest.mark.smoke() -def test_request_generator_sync_constructor(mock_auto_tokenizer): - generator = TestRequestGenerator(mode="sync", tokenizer="mock-tokenizer") - assert generator.mode == "sync" - assert generator.async_queue_size == 50 # Default value - - -@pytest.mark.smoke() -def test_request_generator_async_constructor(mock_auto_tokenizer): - generator = TestRequestGenerator( - mode="async", tokenizer="mock-tokenizer", async_queue_size=10 - ) - assert generator.mode == "async" - assert generator.async_queue_size == 10 - generator.stop() - - -@pytest.mark.smoke() -def test_request_generator_sync_iter(mock_auto_tokenizer): - generator = TestRequestGenerator(mode="sync", tokenizer="mock-tokenizer") - items = [] - for item in generator: - items.append(item) - if len(items) == 5: - break - - assert len(items) == 5 - assert items[0].prompt == "Test prompt" - - -@pytest.mark.smoke() -def test_request_generator_async_iter(mock_auto_tokenizer): - generator = TestRequestGenerator(mode="async", tokenizer="mock-tokenizer") - items = [] - for item in generator: - items.append(item) - if len(items) == 5: - break - - generator.stop() - assert len(items) == 5 - assert items[0].prompt == "Test prompt" - - -@pytest.mark.smoke() -def test_request_generator_iter_calls_create_item(mock_auto_tokenizer): - generator = TestRequestGenerator(mode="sync", tokenizer="mock-tokenizer") - generator.create_item = Mock( # type: ignore - return_value=TextGenerationRequest(prompt="Mock prompt"), - ) - - items = [] - for item in generator: - items.append(item) - if len(items) == 5: - break - - assert len(items) == 5 - generator.create_item.assert_called() - - -@pytest.mark.smoke() -def test_request_generator_async_iter_calls_create_item(mock_auto_tokenizer): - generator = TestRequestGenerator(mode="sync", tokenizer="mock-tokenizer") - generator.create_item = Mock( # type: ignore - return_value=TextGenerationRequest(prompt="Mock prompt"), - ) - - items = [] - for item in generator: - items.append(item) - if len(items) == 5: - break - - generator.stop() - assert len(items) == 5 - generator.create_item.assert_called() - - -@pytest.mark.sanity() -def test_request_generator_repr(mock_auto_tokenizer): - generator = TestRequestGenerator( - mode="sync", tokenizer="mock-tokenizer", async_queue_size=100 - ) - repr_str = repr(generator) - assert repr_str.startswith("RequestGenerator(") - assert "mode=sync" in repr_str - assert "async_queue_size=100" in repr_str - assert "tokenizer= List[int]: - tokens = re.findall(r"\w+|[^\w\s]", text) - return [0] * len(tokens) - - mock_tokenizer = MagicMock() - mock_tokenizer.tokenize = MagicMock(side_effect=_fake_tokenize) - - generator = TestRequestGenerator(tokenizer=mock_tokenizer) - assert generator.tokenizer == mock_tokenizer - - with patch( - "guidellm.request.base.AutoTokenizer", - ) as MockAutoTokenizer: # noqa: N806 - MockAutoTokenizer.from_pretrained.return_value = mock_tokenizer - generator = TestRequestGenerator(tokenizer="mock-tokenizer") - assert generator.tokenizer == mock_tokenizer - MockAutoTokenizer.from_pretrained.assert_called_with("mock-tokenizer") - - -@pytest.mark.regression() -def test_request_generator_populate_queue(mock_auto_tokenizer): - generator = TestRequestGenerator( - mode="async", tokenizer="mock-tokenizer", async_queue_size=2 - ) - generator.create_item = Mock( # type: ignore - return_value=TextGenerationRequest(prompt="Mock prompt") - ) - - time.sleep(0.2) # Allow some time for the queue to populate - generator.stop() - assert generator._queue.qsize() > 0 - - -@pytest.mark.regression() -def test_request_generator_async_stop_during_population(mock_auto_tokenizer): - generator = TestRequestGenerator( - mode="async", tokenizer="mock-tokenizer", async_queue_size=2 - ) - generator.create_item = Mock( # type: ignore - return_value=TextGenerationRequest(prompt="Mock prompt") - ) - - time.sleep(0.1) # Allow some time for the queue to start populating - generator.stop() - - # Ensure the stop event is set and thread is no longer alive - assert generator._stop_event.is_set() - assert not generator._thread.is_alive() diff --git a/tests/unit/request/test_emulated.py b/tests/unit/request/test_emulated.py deleted file mode 100644 index f6af1301..00000000 --- a/tests/unit/request/test_emulated.py +++ /dev/null @@ -1,373 +0,0 @@ -import json -import tempfile -from pathlib import Path -from typing import Tuple, Union - -import numpy as np -import pytest -from transformers import PreTrainedTokenizer # type: ignore - -from guidellm.core.request import TextGenerationRequest -from guidellm.request.emulated import ( - EmulatedConfig, - EmulatedRequestGenerator, - EndlessTokens, -) - - -@pytest.mark.smoke() -def test_emulated_config_construction(): - config = EmulatedConfig( - prompt_tokens=10, - prompt_tokens_variance=2, - prompt_tokens_min=5, - prompt_tokens_max=15, - generated_tokens=20, - generated_tokens_variance=4, - generated_tokens_min=10, - generated_tokens_max=30, - ) - assert config.prompt_tokens == 10 - assert config.prompt_tokens_variance == 2 - assert config.prompt_tokens_min == 5 - assert config.prompt_tokens_max == 15 - assert config.generated_tokens == 20 - assert config.generated_tokens_variance == 4 - assert config.generated_tokens_min == 10 - assert config.generated_tokens_max == 30 - - -@pytest.mark.smoke() -def test_emulated_config_create_dict(): - config_dict = { - "prompt_tokens": 10, - "prompt_tokens_variance": 2, - "prompt_tokens_min": 5, - "prompt_tokens_max": 15, - "generated_tokens": 20, - "generated_tokens_variance": 4, - "generated_tokens_min": 10, - "generated_tokens_max": 30, - } - config = EmulatedConfig.create_config(config_dict) - assert config.prompt_tokens == 10 - assert config.prompt_tokens_variance == 2 - assert config.prompt_tokens_min == 5 - assert config.prompt_tokens_max == 15 - assert config.generated_tokens == 20 - assert config.generated_tokens_variance == 4 - assert config.generated_tokens_min == 10 - assert config.generated_tokens_max == 30 - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("base", "variance", "min_tokens", "max_tokens", "expected_range"), - [ - (10, 2, None, None, (1, 10 + 5 * 2)), - (10, 2, 5, 15, (5, 15)), - (10, None, 5, 15, (5, 15)), - (10, 2, 1, None, (1, 10 + 5 * 2)), - ], -) -def test_emulated_config_token_range( - base: int, - variance: int, - min_tokens: int, - max_tokens: int, - expected_range: Tuple[int, int], -): - assert ( - EmulatedConfig._token_range(base, variance, min_tokens, max_tokens) - == expected_range - ) - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("base", "variance", "min_tokens", "max_tokens", "expected_range"), - [ - (10, None, None, None, (10, 10)), - (10, 5, None, None, (1, 10 + 5 * 2)), - (10, 5, 5, 15, (5, 15)), - (10, None, 5, 15, (5, 15)), - (10, 5, 2, None, (2, 10 + 5 * 2)), - (10, 5, None, 20, (1, 20)), - ], -) -def test_emulated_config_sample_tokens( - base: int, - variance: int, - min_tokens: int, - max_tokens: int, - expected_range: Tuple[int, int], -): - rng = np.random.default_rng() - - for _ in range(100): - token_count = EmulatedConfig._sample_tokens( - base, variance, min_tokens, max_tokens, rng - ) - assert token_count >= expected_range[0] - assert token_count <= expected_range[1] - - -@pytest.mark.sanity() -def test_emulated_config_create(): - test_dict = { - "prompt_tokens": 10, - "prompt_tokens_variance": 2, - "prompt_tokens_min": 5, - "prompt_tokens_max": 15, - "generated_tokens": 20, - "generated_tokens_variance": 4, - "generated_tokens_min": 10, - "generated_tokens_max": 30, - } - compare_config = EmulatedConfig(**test_dict) - - # test dict - test_config = EmulatedConfig.create_config(test_dict) - assert ( - test_config == compare_config - ), f"Dictionary creation failed: {test_config} != {compare_config}" - - # test json str - test_config = EmulatedConfig.create_config(json.dumps(test_dict)) - assert ( - test_config == compare_config - ), f"JSON string creation failed: {test_config} != {compare_config}" - - # test json file str path - with tempfile.TemporaryDirectory() as temp_dir: - test_path = Path(temp_dir) / "test.json" - test_path.write_text(json.dumps(test_dict)) - test_config = EmulatedConfig.create_config(str(test_path)) - assert ( - test_config == compare_config - ), f"JSON file path creation failed: {test_config} != {compare_config}" - - # test json file Path object - with tempfile.TemporaryDirectory() as temp_dir: - test_path = Path(temp_dir) / "test.json" - test_path.write_text(json.dumps(test_dict)) - test_config = EmulatedConfig.create_config(test_path) - assert ( - test_config == compare_config - ), f"JSON file Path object creation failed: {test_config} != {compare_config}" - - # test key value string - test_str = ( - f"prompt_tokens={test_dict['prompt_tokens']}, " - f"prompt_tokens_variance={test_dict['prompt_tokens_variance']}, " - f"prompt_tokens_min={test_dict['prompt_tokens_min']}, " - f"prompt_tokens_max={test_dict['prompt_tokens_max']}, " - f"generated_tokens={test_dict['generated_tokens']}, " - f"generated_tokens_variance={test_dict['generated_tokens_variance']}, " - f"generated_tokens_min={test_dict['generated_tokens_min']}, " - f"generated_tokens_max={test_dict['generated_tokens_max']}" - ) - test_config = EmulatedConfig.create_config(test_str) - assert ( - test_config == compare_config - ), f"Key value string creation failed: {test_config} != {compare_config}" - - -# EndlessTokens - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("data", "expected_words", "expected_indices"), - [ - ( - "word1 word2 word3\nword4 word5", - ["word1", "word2", "word3", "word4", "word5"], - [0, 3], - ), - ( - "word1 word2\n word3 word4\n word5", - ["word1", "word2", "word3", "word4", "word5"], - [0, 2, 4], - ), - ], -) -def test_endless_data_words_construction(data, expected_words, expected_indices): - tokens = EndlessTokens(data) - assert tokens == expected_words - assert tokens.line_indices == expected_indices - - -@pytest.mark.smoke() -def test_endless_data_words_create_from_basic_file(): - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "test.txt" - file_path.write_text("word1 word2 word3\nword4 word5") - - tokens = EndlessTokens(file_path) - assert tokens == ["word1", "word2", "word3", "word4", "word5"] - assert tokens.line_indices == [0, 3] - - tokens = EndlessTokens(str(file_path)) - assert tokens == ["word1", "word2", "word3", "word4", "word5"] - assert tokens.line_indices == [0, 3] - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("data", "start", "length", "expected_text"), - [ - ("word1 word2 word3 word4", 0, 2, "word1 word2"), - ("word1 word2\nword3 word4", 1, 2, "word2\nword3"), - ( - "word1 word2\nword3 word4", - 1, - 6, - "word2\nword3 word4 word1 word2\nword3", - ), - ], -) -def test_endless_data_words_create_text(data, start, length, expected_text): - words = EndlessTokens(data) - text = words.create_text(start, length) - assert text == expected_text - - -# EmulatedRequestGenerator - - -@pytest.mark.smoke() -def test_emulated_request_generator_construction(mocker, mock_auto_tokenizer): - mocker.patch( - "guidellm.request.emulated.EmulatedConfig.create_config", - return_value=EmulatedConfig(prompt_tokens=10), - ) - mocker.patch( - "guidellm.request.emulated.EndlessTokens", - return_value=EndlessTokens("word1 word2"), - ) - generator = EmulatedRequestGenerator( - config="mock_config", tokenizer="mock-tokenizer", mode="sync" - ) - assert isinstance(generator._config, EmulatedConfig) - assert isinstance(generator._tokens, EndlessTokens) - - -@pytest.mark.smoke() -def test_emulated_request_generator_create_item(mocker): - mocker.patch( - "guidellm.request.emulated.EndlessTokens", - return_value=EndlessTokens("word1 word2"), - ) - mock_tokenizer = mocker.Mock(PreTrainedTokenizer) - mock_tokenizer.tokenize.return_value = ["word1", "word2"] - generator = EmulatedRequestGenerator( - config={ - "prompt_tokens": 10, - }, - tokenizer=mock_tokenizer, - mode="sync", - ) - item = generator.create_item() - assert isinstance(item, TextGenerationRequest) - - -@pytest.mark.smoke() -def test_emulated_request_generator_sample_prompt(mocker, mock_auto_tokenizer): - mocker.patch( - "guidellm.request.emulated.EndlessTokens", - return_value=EndlessTokens("word1 word2"), - ) - generator = EmulatedRequestGenerator( - config={"prompt_tokens": 3}, tokenizer="mock-tokenizer", mode="sync" - ) - prompt = generator.sample_prompt(3) - assert prompt == "word1 word2 word1" - - request = generator.create_item() - assert request.prompt_token_count == 3 - - -@pytest.mark.smoke() -def test_emulated_request_generator_random_seed(mocker, mock_auto_tokenizer): - mocker.patch( - "guidellm.request.emulated.EndlessTokens", - return_value=EndlessTokens("word1 word2"), - ) - - rand_gen = EmulatedRequestGenerator( - config={"prompt_tokens": 20, "prompt_tokens_variance": 10}, - tokenizer="mock-tokenizer", - random_seed=42, - mode="sync", - ) - rand_gen_comp_pos = EmulatedRequestGenerator( - config={"prompt_tokens": 20, "prompt_tokens_variance": 10}, - tokenizer="mock-tokenizer", - random_seed=42, - mode="sync", - ) - rand_gen_comp_neg = EmulatedRequestGenerator( - config={"prompt_tokens": 20, "prompt_tokens_variance": 10}, - tokenizer="mock-tokenizer", - random_seed=43, - mode="sync", - ) - - assert rand_gen.create_item().prompt == rand_gen_comp_pos.create_item().prompt - assert rand_gen.create_item().prompt != rand_gen_comp_neg.create_item().prompt - - -@pytest.mark.regression() -@pytest.mark.parametrize( - ("config_type", "config"), - [ - ("dict", {"prompt_tokens": 10, "generated_tokens": 20}), - ("dict", {"prompt_tokens": 10, "prompt_tokens_variance": 2}), - ( - "dict", - { - "prompt_tokens": 10, - "prompt_tokens_min": 5, - "prompt_tokens_max": 15, - "generated_tokens": 20, - }, - ), - ("json_str", json.dumps({"prompt_tokens": 10, "generated_tokens": 20})), - ("key_value_str", "prompt_tokens=10, generated_tokens=20"), - ("file_str", json.dumps({"prompt_tokens": 10, "generated_tokens": 20})), - ("file_path", json.dumps({"prompt_tokens": 10, "generated_tokens": 20})), - ], -) -def test_emulated_request_generator_lifecycle( - mock_requests_pride_and_prejudice, - mock_auto_tokenizer, - config_type: str, - config: Union[str, dict, Path], -): - if config_type in ["dict", "json_str", "key_value_str"]: - generator = EmulatedRequestGenerator(config, tokenizer="mock-tokenizer") - elif config_type in ["file_str", "file_path"]: - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "test.json" - file_path.write_text(config) # type: ignore - generator = EmulatedRequestGenerator( - str(file_path) if config_type == "file_str" else file_path, - tokenizer="mock-tokenizer", - ) - - for _ in range(5): - request = generator.create_item() - prompt_range = generator._config.prompt_tokens_range - outputs_range = generator._config.output_tokens_range - - assert request.prompt_token_count >= prompt_range[0] # type: ignore - assert request.prompt_token_count <= prompt_range[1] # type: ignore - - prompt_tokens = len(generator.tokenizer.tokenize(request.prompt)) - assert request.prompt_token_count == prompt_tokens - - if generator._config.generated_tokens: - assert len(outputs_range) == 2 - assert request.output_token_count >= outputs_range[0] # type: ignore - assert request.output_token_count <= outputs_range[1] # type: ignore diff --git a/tests/unit/request/test_file.py b/tests/unit/request/test_file.py deleted file mode 100644 index 69e538a1..00000000 --- a/tests/unit/request/test_file.py +++ /dev/null @@ -1,161 +0,0 @@ -import tempfile -from pathlib import Path - -import pytest - -from guidellm.core.request import TextGenerationRequest -from guidellm.request.file import FileRequestGenerator - - -@pytest.mark.smoke() -def test_file_request_generator_constructor(mock_auto_tokenizer): - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "example.txt" - file_path.write_text("This is a test.\nThis is another test.") - generator = FileRequestGenerator(file_path, tokenizer="mock-tokenizer") - assert generator._path == file_path - assert generator._data == ["This is a test.", "This is another test."] - assert generator._iterator is not None - - -@pytest.mark.smoke() -def test_file_request_generator_create_item(mock_auto_tokenizer): - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "example.txt" - file_path.write_text("This is a test.\nThis is another test.") - generator = FileRequestGenerator( - file_path, tokenizer="mock-tokenizer", mode="sync" - ) - request = generator.create_item() - assert isinstance(request, TextGenerationRequest) - assert request.prompt == "This is a test." - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("file_extension", "file_content"), - [ - ("txt", "Test content 1.\nTest content 2.\nTest content 3.\n"), - ( - "csv", - "text,label,extra\n" - "Test content 1.,1,extra 1\n" - "Test content 2.,2,extra 2\n" - "Test content 3.,3,extra 3\n", - ), - ( - "jsonl", - '{"text": "Test content 1."}\n' - '{"text": "Test content 2."}\n' - '{"text": "Test content 3."}\n', - ), - ( - "csv", - "prompt,text,extra\n" - "Test content 1., text 1, extra 1\n" - "Test content 2., text 2, extra 2\n" - "Test content 3., text 3, extra 3\n", - ), - ( - "json", - '[{"text": "Test content 1."}, ' - '{"text": "Test content 2."}, ' - '{"text": "Test content 3."}]\n', - ), - ( - "json", - '{"object_1": {"text": "Test content 1."}, ' - '"object_2": {"text": "Test content 2."}, ' - '"object_3": {"text": "Test content 3."}}\n', - ), - ( - "yaml", - "items:\n" - " - text: Test content 1.\n" - " - text: Test content 2.\n" - " - text: Test content 3.\n", - ), - ( - "yaml", - "object_1:\n text: Test content 1.\n" - "object_2:\n text: Test content 2.\n" - "object_3:\n text: Test content 3.\n", - ), - ], -) -def test_file_request_generator_file_types_lifecycle( - mock_auto_tokenizer, file_extension, file_content -): - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / f"example.{file_extension}" - file_path.write_text(file_content) - generator = FileRequestGenerator(file_path, tokenizer="mock-tokenizer") - - for index, request in enumerate(generator): - assert isinstance(request, TextGenerationRequest) - assert request.prompt == f"Test content {index + 1}." - assert request.prompt_token_count == 3 - - if index == 2: - break - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("file_extension", "file_content"), - [ - ("txt", "Test content 1.\nTest content 2.\nTest content 3.\n"), - ( - "csv", - "text,label,extra\n" - "Test content 1.,1,extra 1\n" - "Test content 2.,2,extra 2\n" - "Test content 3.,3,extra 3\n", - ), - ( - "jsonl", - '{"text": "Test content 1."}\n' - '{"text": "Test content 2."}\n' - '{"text": "Test content 3."}\n', - ), - ( - "csv", - "prompt,text,extra\n" - "Test content 1., text 1, extra 1\n" - "Test content 2., text 2, extra 2\n" - "Test content 3., text 3, extra 3\n", - ), - ( - "json", - '[{"text": "Test content 1."}, ' - '{"text": "Test content 2."}, ' - '{"text": "Test content 3."}]\n', - ), - ( - "json", - '{"object_1": {"text": "Test content 1."}, ' - '"object_2": {"text": "Test content 2."}, ' - '"object_3": {"text": "Test content 3."}}\n', - ), - ( - "yaml", - "items:\n" - " - text: Test content 1.\n" - " - text: Test content 2.\n" - " - text: Test content 3.\n", - ), - ( - "yaml", - "object_1:\n text: Test content 1.\n" - "object_2:\n text: Test content 2.\n" - "object_3:\n text: Test content 3.\n", - ), - ], -) -def test_file_request_generator_len(mock_auto_tokenizer, file_extension, file_content): - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / f"example.{file_extension}" - file_path.write_text(file_content) - generator = FileRequestGenerator(file_path, tokenizer="mock-tokenizer") - - assert len(generator) == 3 diff --git a/tests/unit/request/test_transformers.py b/tests/unit/request/test_transformers.py deleted file mode 100644 index d3b45325..00000000 --- a/tests/unit/request/test_transformers.py +++ /dev/null @@ -1,132 +0,0 @@ -from unittest.mock import patch - -import pytest - -from guidellm.core.request import TextGenerationRequest -from guidellm.request.transformers import TransformersDatasetRequestGenerator -from tests.dummy.data.transformers import ( - create_sample_dataset, - create_sample_dataset_dict, - create_sample_iterable_dataset, - create_sample_iterable_dataset_dict, -) - - -@pytest.mark.smoke() -def test_transformers_dataset_request_generator_constructor( - mock_auto_tokenizer, -): - dataset = create_sample_dataset() - with patch( - "guidellm.request.transformers.load_transformers_dataset", - return_value=dataset, - ), patch( - "guidellm.request.transformers.resolve_transformers_dataset_column", - return_value="text", - ): - generator = TransformersDatasetRequestGenerator( - dataset="dummy_dataset", - split="train", - column="text", - tokenizer="mock-tokenizer", - ) - assert generator._dataset == "dummy_dataset" - assert generator._split == "train" - assert generator._column == "text" - assert generator._hf_dataset == dataset - assert generator._hf_column == "text" - assert generator._hf_dataset_iterator is not None - - -@pytest.mark.smoke() -def test_transformers_dataset_request_generator_create_item( - mock_auto_tokenizer, -): - generator = TransformersDatasetRequestGenerator( - dataset=create_sample_dataset_dict(), - split="train", - column="text", - tokenizer="mock-tokenizer", - mode="sync", - ) - request = generator.create_item() - assert isinstance(request, TextGenerationRequest) - assert request.prompt == "sample text 1" - assert request.prompt_token_count == 3 - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("dataset_arg", "dataset"), - [ - ( - "mock/directory/file.csv", - create_sample_dataset_dict(splits=["train"]), - ), - ( - "mock/directory/file.json", - create_sample_dataset(column="prompt"), - ), - ( - "mock/directory/file.py", - create_sample_dataset_dict(splits=["test"], column="output"), - ), - (create_sample_dataset_dict(splits=["val", "train"], column="custom"), None), - (create_sample_dataset(), None), - (create_sample_iterable_dataset_dict(splits=["validation"]), None), - (create_sample_iterable_dataset(), None), - ], -) -def test_transformers_dataset_request_generator_lifecycle( - mock_auto_tokenizer, dataset_arg, dataset -): - with patch( - "guidellm.utils.transformers.load_dataset", - return_value=dataset, - ): - generator = TransformersDatasetRequestGenerator( - dataset=dataset_arg, tokenizer="mock-tokenizer", mode="sync" - ) - - for index, request in enumerate(generator): - assert isinstance(request, TextGenerationRequest) - assert request.prompt == f"sample text {index + 1}" - assert request.prompt_token_count == 3 - - if index == 2: - break - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("dataset_arg", "dataset"), - [ - ( - "mock/directory/file.csv", - create_sample_dataset_dict(splits=["train"]), - ), - ( - "mock/directory/file.json", - create_sample_dataset(column="prompt"), - ), - ( - "mock/directory/file.py", - create_sample_dataset_dict(splits=["test"], column="output"), - ), - (create_sample_dataset_dict(splits=["val", "train"], column="custom"), None), - (create_sample_dataset(), None), - ], -) -def test_transformers_dataset_request_generator_len( - mock_auto_tokenizer, dataset_arg, dataset -): - with patch( - "guidellm.utils.transformers.load_dataset", - return_value=dataset, - ): - generator = TransformersDatasetRequestGenerator( - dataset=dataset_arg, tokenizer="mock-tokenizer", mode="sync" - ) - - # Check if __len__ returns the correct length - assert len(generator) == 3 diff --git a/tests/unit/scheduler/__init__.py b/tests/unit/scheduler/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/unit/scheduler/test_load_generator.py b/tests/unit/scheduler/test_load_generator.py deleted file mode 100644 index 6b84ee01..00000000 --- a/tests/unit/scheduler/test_load_generator.py +++ /dev/null @@ -1,153 +0,0 @@ -import time -from typing import get_args - -import pytest -from scipy.stats import kstest # type: ignore - -from guidellm.scheduler import LoadGenerationMode, LoadGenerator - - -@pytest.mark.smoke() -def test_load_generator_mode(): - assert set(get_args(LoadGenerationMode)) == { - "synchronous", - "constant", - "poisson", - "throughput", - } - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("mode", "rate"), - [ - ("constant", 10), - ("poisson", 5), - ("throughput", None), - ("synchronous", None), - ], -) -def test_load_generator_instantiation(mode, rate): - generator = LoadGenerator(mode=mode, rate=rate) - assert generator.mode == mode - assert generator.rate == rate - - -@pytest.mark.regression() -@pytest.mark.parametrize( - ("mode", "rate", "expected_error"), - [ - ("invalid_mode", None, ValueError), - ("constant", 0, ValueError), - ("poisson", -1, ValueError), - ], -) -def test_load_generator_invalid_instantiation(mode, rate, expected_error): - with pytest.raises(expected_error): - LoadGenerator(mode=mode, rate=rate) - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("mode", "rate"), - [ - ("synchronous", None), - ("throughput", None), - ("constant", 1), - ("poisson", 5), - ], -) -def test_load_generator_times(mode, rate): - # first check that the proper method is called - generator = LoadGenerator(mode=mode, rate=rate) - func_name = f"{mode}_times" - assert hasattr(generator, func_name) - assert callable(getattr(generator, func_name)) - - call_count = 0 - - def _increment_call_count(): - nonlocal call_count - call_count += 1 - yield -1.0 - - setattr(generator, func_name, _increment_call_count) - for time_ in generator.times(): - assert time_ == -1.0 - break - assert call_count == 1 - - # now check that the method generates reasonable timestamps - generator = LoadGenerator(mode=mode, rate=rate) - start_time = time.time() - for index, time_ in enumerate(generator.times()): - if index > 10: - break - - if mode == "synchronous": - assert time_ == -1.0 - else: - assert time_ >= start_time - - -@pytest.mark.smoke() -def test_load_generator_invalid_times(): - generator = LoadGenerator(mode="synchronous") - - for index, time_ in enumerate(generator.synchronous_times()): - if index > 10: - break - - assert time_ == -1.0 - - -@pytest.mark.smoke() -def test_load_generator_throughput_times(): - generator = LoadGenerator(mode="throughput") - - for index, time_ in enumerate(generator.throughput_times()): - if index > 10: - break - - assert time_ <= time.time() - - -@pytest.mark.smoke() -@pytest.mark.parametrize("rate", [1, 10, 42]) -def test_load_generator_constant_times(rate): - generator = LoadGenerator(mode="constant", rate=rate) - start_time = time.time() - - for index, time_ in enumerate(generator.constant_times()): - if index > 10: - break - - assert time_ == pytest.approx(start_time + index / rate, rel=1e-5) - - -@pytest.mark.smoke() -@pytest.mark.flaky(reruns=5) -def test_load_generator_poisson_times(): - rate = 5 - generator = LoadGenerator(mode="poisson", rate=rate) - start_time = time.time() - - times = [] - prev_time = start_time - - for index, current_time in enumerate(generator.poisson_times()): - if index > 100: - break - - times.append(current_time - prev_time) - prev_time = current_time - - mean_inter_arrival_time = 1 / rate - - # Perform Kolmogorov-Smirnov test to compare the sample distribution - # to the expected exponential distribution - ks_statistic, p_value = kstest(times, "expon", args=(0, mean_inter_arrival_time)) - assert p_value > 0.025, ( - f"Poisson-generated inter-arrival times do not " - f"match the expected exponential distribution (p-value: {p_value})" - ) diff --git a/tests/unit/scheduler/test_scheduler.py b/tests/unit/scheduler/test_scheduler.py deleted file mode 100644 index d765280f..00000000 --- a/tests/unit/scheduler/test_scheduler.py +++ /dev/null @@ -1,199 +0,0 @@ -import random -from unittest.mock import create_autospec - -import pytest - -from guidellm.backend import Backend -from guidellm.core import ( - TextGenerationBenchmark, - TextGenerationRequest, - TextGenerationResult, -) -from guidellm.request import RequestGenerator -from guidellm.scheduler import ( - LoadGenerator, - Scheduler, - SchedulerResult, -) - - -@pytest.mark.smoke() -def test_scheduler_result_default_intialization(): - benchmark = create_autospec(TextGenerationBenchmark, instance=True) - scheduler_result = SchedulerResult( - completed=False, - count_total=0, - count_completed=0, - benchmark=benchmark, - ) - - assert scheduler_result.completed is False - assert scheduler_result.count_total == 0 - assert scheduler_result.count_completed == 0 - assert scheduler_result.benchmark == benchmark - assert scheduler_result.current_result is None - - -@pytest.mark.smoke() -def test_scheduler_result_initialization(): - benchmark = create_autospec(TextGenerationBenchmark, instance=True) - result = TextGenerationResult( - request=TextGenerationRequest(prompt="prompt"), output="Test output" - ) - scheduler_result = SchedulerResult( - completed=False, - count_total=10, - count_completed=5, - benchmark=benchmark, - current_result=result, - ) - - assert scheduler_result.completed is False - assert scheduler_result.count_total == 10 - assert scheduler_result.count_completed == 5 - assert scheduler_result.benchmark == benchmark - assert scheduler_result.current_result == result - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("mode", "rate", "max_number", "max_duration"), - [ - ("synchronous", None, 10, None), - ("throughput", 5.0, None, 60.0), - ("poisson", 10.0, 100, None), - ("constant", 1.0, None, 120.0), - ], -) -def test_scheduler_initialization(mode, rate, max_number, max_duration): - generator = create_autospec(RequestGenerator, instance=True) - backend = create_autospec(Backend, instance=True) - scheduler = Scheduler( - generator, - backend, - mode=mode, - rate=rate, - max_number=max_number, - max_duration=max_duration, - ) - - assert scheduler.generator == generator - assert scheduler.backend == backend - assert scheduler.mode == mode - assert scheduler.rate == rate - assert scheduler.max_number == max_number - assert scheduler.max_duration == max_duration - assert isinstance(scheduler.load_generator, LoadGenerator) - assert scheduler.benchmark_mode in {"synchronous", "asynchronous", "throughput"} - - -@pytest.mark.sanity() -@pytest.mark.parametrize( - ("mode", "rate", "max_number", "max_duration"), - [ - # invalid modes - ("invalid_mode", None, 10, None), - # invalid max settings - ("synchronous", None, None, None), - ("synchronous", None, -1, 10), - ("synchronous", None, 10, -1), - # invalid rate settings - ("constant", -1, None, 10), - ("constant", None, None, 10), - ("poisson", -1, None, 10), - ("poisson", None, None, 10), - ], -) -def test_scheduler_invalid_initialization( - mode, - rate, - max_number, - max_duration, -): - generator = create_autospec(RequestGenerator, instance=True) - backend = create_autospec(Backend, instance=True) - - with pytest.raises(ValueError): - Scheduler( - generator, - backend, - mode=mode, - rate=rate, - max_number=max_number, - max_duration=max_duration, - ) - - -@pytest.mark.sanity() -@pytest.mark.asyncio() -@pytest.mark.parametrize( - "mode", - [ - "synchronous", - "throughput", - "poisson", - "constant", - ], -) -async def test_scheduler_run_number(mode, mock_backend): - rate = 10.0 - max_number = 20 - generator = create_autospec(RequestGenerator, instance=True) - - # Mock the request generator and backend submit behavior - generator.__iter__.return_value = iter( - [TextGenerationRequest(prompt="Test", type_=random.choice(["text", "chat"]))] - * (max_number * 2) - ) - - scheduler = Scheduler( - generator, - mock_backend, - mode=mode, - rate=rate, - max_number=max_number, - ) - - run_count = 0 - count_completed = 0 - received_init = False - received_final = False - async for result in scheduler.run(): - run_count += 1 - - assert run_count <= max_number + 2 - assert result.count_total == max_number - assert result.benchmark is not None - assert isinstance(result.benchmark, TextGenerationBenchmark) - - if result.current_result is not None: - count_completed += 1 - - if run_count == 1: - assert not received_init - assert not received_final - assert count_completed == 0 - assert result.count_completed == 0 - assert not result.completed - assert result.current_result is None - received_init = True - elif run_count - 2 == max_number: - assert received_init - assert not received_final - assert count_completed == max_number - assert result.count_completed == max_number - assert result.completed - assert result.current_result is None - received_final = True - else: - assert received_init - assert not received_final - assert count_completed == run_count - 1 - assert result.count_completed == run_count - 1 - assert not result.completed - assert result.current_result is not None - assert isinstance(result.current_result, TextGenerationResult) - - assert received_init - assert received_final - assert count_completed == max_number diff --git a/tests/unit/test_type.py b/tests/unit/test_type.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/unit/utils/__init__.py b/tests/unit/utils/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/unit/utils/test_injector.py b/tests/unit/utils/test_injector.py deleted file mode 100644 index 9a58575e..00000000 --- a/tests/unit/utils/test_injector.py +++ /dev/null @@ -1,70 +0,0 @@ -from pathlib import Path - -import pytest -from pydantic import BaseModel - -from guidellm.config import settings -from guidellm.utils.injector import create_report, inject_data - - -class ExampleModel(BaseModel): - name: str - version: str - - -@pytest.mark.smoke() -def test_inject_data(): - model = ExampleModel(name="Example App", version="1.0.0") - html = "window.report_data = {};" - expected_html = 'window.report_data = {"name":"Example App","version":"1.0.0"};' - - result = inject_data( - model, - html, - settings.report_generation.report_html_match, - settings.report_generation.report_html_placeholder, - ) - assert result == expected_html - - -@pytest.mark.smoke() -def test_create_report_to_file(tmpdir): - model = ExampleModel(name="Example App", version="1.0.0") - html_content = "window.report_data = {};" - expected_html_content = ( - 'window.report_data = {"name":"Example App","version":"1.0.0"};' - ) - - mock_html_path = tmpdir.join("template.html") - mock_html_path.write(html_content) - settings.report_generation.source = str(mock_html_path) - - output_path = tmpdir.join("output.html") - result_path = create_report(model, str(output_path)) - result_content = result_path.read_text() - - assert result_path == output_path - assert result_content == expected_html_content - - -@pytest.mark.smoke() -def test_create_report_to_directory(tmpdir): - model = ExampleModel(name="Example App", version="1.0.0") - html_content = "window.report_data = {};" - expected_html_content = ( - 'window.report_data = {"name":"Example App","version":"1.0.0"};' - ) - - mock_html_path = tmpdir.join("template.html") - mock_html_path.write(html_content) - settings.report_generation.source = str(mock_html_path) - - output_dir = tmpdir.mkdir("output_dir") - output_path = Path(output_dir) / "report.html" - result_path = create_report(model, str(output_dir)) - - with Path(result_path).open("r") as file: - result_content = file.read() - - assert result_path == output_path - assert result_content == expected_html_content diff --git a/tests/unit/utils/test_progress.py b/tests/unit/utils/test_progress.py deleted file mode 100644 index 637b2be2..00000000 --- a/tests/unit/utils/test_progress.py +++ /dev/null @@ -1,116 +0,0 @@ -import pytest - -from guidellm.utils import BenchmarkReportProgress - - -@pytest.fixture() -def benchmark_progress(): - return BenchmarkReportProgress() - - -@pytest.mark.smoke() -def test_initialization(benchmark_progress): - assert benchmark_progress.report_task is None - assert benchmark_progress.benchmark_tasks == [] - assert benchmark_progress.benchmark_tasks_started == [] - assert benchmark_progress.benchmark_tasks_completed == [] - assert benchmark_progress.benchmark_tasks_progress == [] - - -@pytest.mark.smoke() -def test_start_method(benchmark_progress): - descriptions = ["Benchmark 1", "Benchmark 2"] - benchmark_progress.start(descriptions) - - assert len(benchmark_progress.benchmark_tasks) == 2 - assert benchmark_progress.report_task is not None - - benchmark_progress.finish() - - -@pytest.mark.sanity() -def test_update_benchmark(benchmark_progress): - descriptions = ["Benchmark 1"] - benchmark_progress.start(descriptions) - - benchmark_progress.update_benchmark( - index=0, - description="Updating Benchmark 1", - completed=False, - completed_count=50, - completed_total=100, - start_time=0, - req_per_sec=10.5, - ) - assert benchmark_progress.benchmark_tasks_progress[0] == 50.0 - - benchmark_progress.finish() - - -@pytest.mark.sanity() -def test_finish_method(benchmark_progress): - descriptions = ["Benchmark 1", "Benchmark 2"] - benchmark_progress.start(descriptions) - benchmark_progress.finish() - - assert benchmark_progress.report_progress.finished - - -@pytest.mark.regression() -def test_error_on_update_completed_benchmark(benchmark_progress): - descriptions = ["Benchmark 1"] - benchmark_progress.start(descriptions) - benchmark_progress.update_benchmark( - index=0, - description="Benchmark 1", - completed=True, - completed_count=100, - completed_total=100, - start_time=0, - req_per_sec=10.5, - ) - - with pytest.raises(ValueError, match="already completed"): - benchmark_progress.update_benchmark( - index=0, - description="Benchmark 1", - completed=False, - completed_count=50, - completed_total=100, - start_time=0, - req_per_sec=10.5, - ) - - benchmark_progress.finish() - - -@pytest.mark.regression() -def test_multiple_updates(benchmark_progress): - descriptions = ["Benchmark 1", "Benchmark 2"] - benchmark_progress.start(descriptions) - - # First update - benchmark_progress.update_benchmark( - index=0, - description="Updating Benchmark 1", - completed=False, - completed_count=50, - completed_total=100, - start_time=0, - req_per_sec=5.0, - ) - assert benchmark_progress.benchmark_tasks_progress[0] == 50.0 - - # Second update, same task - benchmark_progress.update_benchmark( - index=0, - description="Updating Benchmark 1", - completed=True, - completed_count=100, - completed_total=100, - start_time=0, - req_per_sec=5.0, - ) - assert benchmark_progress.benchmark_tasks_progress[0] == 100.0 - - benchmark_progress.finish() diff --git a/tests/unit/utils/test_text.py b/tests/unit/utils/test_text.py deleted file mode 100644 index 1d89ee31..00000000 --- a/tests/unit/utils/test_text.py +++ /dev/null @@ -1,394 +0,0 @@ -from pathlib import Path -from unittest.mock import patch - -import pytest -import requests - -from guidellm.utils.text import ( - clean_text, - filter_text, - is_path, - is_path_like, - is_url, - load_text, - load_text_lines, - parse_text_objects, - split_lines_by_punctuation, - split_text, -) - - -@pytest.fixture() -def sample_text(): - return "This is a sample text.\nThis is another line!" - - -@pytest.fixture() -def sample_dict_data(): - return [{"text": "line 1"}, {"text": "line 2"}, {"text": "line 3"}] - - -@pytest.fixture() -def sample_csv_data(): - return "text\nline 1\nline 2\nline 3" - - -@pytest.fixture() -def sample_jsonl_data(): - return '{"text": "line 1"}\n{"text": "line 2"}\n{"text": "line 3"}' - - -@pytest.fixture() -def sample_yaml_data(): - return """ - text: - - line 1 - - line 2 - - line 3 - """ - - -@pytest.fixture() -def mock_response(): - response = requests.Response() - response.status_code = 200 - response._content = b"Mock content" - return response - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("text", "start", "end", "expected"), - [ - ("hello world", "hello", "world", "hello "), - ("hello world", "world", None, "world"), - ("hello world", None, "hello", ""), - ("hello world", None, None, "hello world"), - ], -) -def test_filter_text(text, start, end, expected): - assert filter_text(text, start, end) == expected - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ( - "text", - "fix_encoding", - "clean_whitespace", - "remove_empty_lines", - "force_new_line_punctuation", - "expected", - ), - [ - ( - "This is\ta test.\n New line.", - True, - True, - False, - False, - "This is a test.\nNew line.", - ), - ( - "This is\ta test.\n New line.", - True, - True, - True, - False, - "This is a test.\nNew line.", - ), - ( - "This is a test. New line.", - True, - False, - False, - True, - "This is a test.\nNew line.", - ), - ], -) -def test_clean_text( - text, - fix_encoding, - clean_whitespace, - remove_empty_lines, - force_new_line_punctuation, - expected, -): - assert ( - clean_text( - text, - fix_encoding, - clean_whitespace, - remove_empty_lines, - force_new_line_punctuation, - ) - == expected - ) - - -@pytest.mark.smoke() -def test_split_lines_by_punctuation(sample_text): - expected = ["This is a sample text.", "This is another line!"] - assert split_lines_by_punctuation(sample_text) == expected - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("url", "expected"), - [ - ("https://example.com", True), - ("ftp://example.com", True), - ("not a url", False), - ], -) -def test_is_url(url, expected): - assert is_url(url) == expected - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("path", "expected"), - [ - (str(Path(__file__)), True), - ("/non/existent/path", False), - ], -) -def test_is_path(path, expected): - assert is_path(path) == expected - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("path", "enforce_file", "expected"), - [ - (str(Path(__file__)), True, True), - ("/non/existent/path", False, True), - ("https://example.com", False, False), - ], -) -def test_is_path_like(path, enforce_file, expected): - assert is_path_like(path, enforce_file) == expected - - -@pytest.mark.smoke() -def test_split_text(sample_text): - words, separators, new_lines = split_text(sample_text) - assert words == [ - "This", - "is", - "a", - "sample", - "text.", - "This", - "is", - "another", - "line!", - ] - assert separators == [" ", " ", " ", " ", "\n", " ", " ", " ", " "] - assert new_lines == [0, 5] - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("data", "format_", "expected"), - [ - ("text\nline 1\nline 2", "csv", [{"text": "line 1"}, {"text": "line 2"}]), - ( - '{"text": "line 1"}\n{"text": "line 2"}', - "jsonl", - [{"text": "line 1"}, {"text": "line 2"}], - ), - ], -) -def test_parse_text_objects(data, format_, expected): - assert parse_text_objects(data, format_) == expected - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("data", "expected"), - [ - ("https://example.com", "Mock content"), - (str(Path(__file__)), Path(__file__).read_text()), - ], -) -def test_load_text(data, expected, mock_response): - with patch("requests.get", return_value=mock_response): - assert load_text(data) == expected - - -@pytest.mark.regression() -def test_load_text_file_not_found(): - with pytest.raises(FileNotFoundError): - load_text("/non/existent/file.txt") - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("data", "format_", "filters", "expected"), - [ - ("text\nline 1\nline 2", "csv", None, ["line 1", "line 2"]), - ('{"text": "line 1"}\n{"text": "line 2"}', "jsonl", None, ["line 1", "line 2"]), - ("text\nline 1\nline 2", "txt", None, ["text", "line 1", "line 2"]), - ], -) -def test_load_text_lines(data, format_, filters, expected): - assert load_text_lines(data, format_=format_, filters=filters) == expected - - -@pytest.mark.regression() -def test_load_text_lines_invalid_data(): - with pytest.raises(ValueError): - load_text_lines(123) # type: ignore - - -@pytest.mark.regression() -def test_parse_text_objects_invalid_format(): - with pytest.raises(ValueError): - parse_text_objects("text", format_="unsupported") - - -@pytest.mark.regression() -def test_parse_text_objects_invalid_data(): - with pytest.raises(ValueError): - parse_text_objects(123) # type: ignore - - -@pytest.mark.regression() -@pytest.mark.parametrize( - ("data", "format_", "filters", "expected"), - [ - ( - "text\nline 1\nline 2\n", - "csv", - ["text"], - ["line 1", "line 2"], - ), - ], -) -def test_load_text_lines_with_filters(data, format_, filters, expected): - assert load_text_lines(data, format_=format_, filters=filters) == expected - - -@pytest.mark.regression() -def test_is_path_with_symlink(tmp_path): - # Create a symlink to a temporary file - target_file = tmp_path / "target_file.txt" - target_file.write_text("Sample content") - symlink_path = tmp_path / "symlink" - symlink_path.symlink_to(target_file) - - assert is_path(str(symlink_path)) is True - - -@pytest.mark.regression() -def test_is_path_like_with_symlink(tmp_path): - # Create a symlink to a temporary file - target_file = tmp_path / "target_file.txt" - target_file.write_text("Sample content") - symlink_path = tmp_path / "symlink.file" - symlink_path.symlink_to(target_file) - - assert is_path_like(str(symlink_path), enforce_file=True) is True - - -@pytest.mark.regression() -def test_load_text_lines_empty(): - # Test loading text lines from an empty string - assert load_text_lines("") == [] - - -@pytest.mark.regression() -def test_split_text_with_empty_string(): - words, separators, new_lines = split_text("") - assert words == [] - assert separators == [] - assert new_lines == [] - - -@pytest.mark.regression() -def test_split_lines_by_punctuation_with_no_punctuation(): - text = "This is a test without punctuation" - assert split_lines_by_punctuation(text) == [text] - - -@pytest.mark.regression() -def test_is_path_invalid_type(): - assert not is_path(None) - assert not is_path(123) - assert not is_path(["not", "a", "path"]) - - -@pytest.mark.regression() -def test_is_path_like_invalid_type(): - assert not is_path_like(None, enforce_file=False) - assert not is_path_like(123, enforce_file=True) - assert not is_path_like(["not", "a", "path"], enforce_file=False) - - -@pytest.mark.regression() -def test_load_text_invalid_url(): - with pytest.raises(requests.ConnectionError): - load_text("http://invalid.url") - - -@pytest.mark.regression() -def test_parse_text_objects_empty_csv(): - assert parse_text_objects("text\n", "csv") == [] - - -@pytest.mark.regression() -def test_parse_text_objects_empty_jsonl(): - assert parse_text_objects("", "jsonl") == [] - - -@pytest.mark.regression() -def test_parse_text_objects_invalid_jsonl(): - with pytest.raises(ValueError): - parse_text_objects("{invalid_json}", "jsonl") - - -@pytest.mark.regression() -def test_parse_text_objects_empty_yaml(): - assert parse_text_objects("", "yaml") == [] - - -@pytest.mark.regression() -def test_clean_text_with_unicode(): - text = "This is a test with unicode: \u2013 \u2014" - cleaned_text = clean_text(text, fix_encoding=True, clean_whitespace=True) - assert cleaned_text == "This is a test with unicode: – —" - - -@pytest.mark.regression() -def test_split_lines_by_punctuation_with_multiple_punctuations(): - text = "First sentence. Second sentence? Third sentence!" - expected = ["First sentence.", "Second sentence?", "Third sentence!"] - assert split_lines_by_punctuation(text) == expected - - -@pytest.mark.regression() -def test_is_url_empty_string(): - assert not is_url("") - - -@pytest.mark.regression() -def test_load_text_invalid_data(): - with pytest.raises(TypeError): - load_text(123) # type: ignore - - -@pytest.mark.regression() -def test_load_text_lines_empty_format(): - data = "text\nline 1\nline 2" - assert load_text_lines(data, format_="") == ["text", "line 1", "line 2"] - - -@pytest.mark.regression() -def test_split_text_with_mixed_separators(): - text = "This\tis a test\nwith mixed separators." - words, separators, new_lines = split_text(text) - assert words == ["This", "is", "a", "test", "with", "mixed", "separators."] - assert separators == ["\t", " ", " ", "\n", " ", " ", " "] - assert new_lines == [0, 4] diff --git a/tests/unit/utils/test_transformers.py b/tests/unit/utils/test_transformers.py deleted file mode 100644 index 92d1e8b0..00000000 --- a/tests/unit/utils/test_transformers.py +++ /dev/null @@ -1,236 +0,0 @@ -from unittest.mock import patch - -import pytest -from datasets import ( # type: ignore - Dataset, - DatasetDict, - IterableDataset, - IterableDatasetDict, -) - -from guidellm.utils.hf_transformers import ( - load_transformers_dataset, - resolve_transformers_dataset, - resolve_transformers_dataset_column, - resolve_transformers_dataset_split, -) -from tests.dummy.data.transformers import ( - create_sample_dataset, - create_sample_dataset_dict, - create_sample_iterable_dataset, - create_sample_iterable_dataset_dict, -) - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("dataset_arg", "dataset", "split", "preferred_splits", "expected_type"), - [ - ( - "mock/directory/file.csv", - create_sample_dataset_dict(splits=["train"]), - "train", - None, - Dataset, - ), - ( - "mock/directory/file.json", - create_sample_dataset_dict(splits=["test"]), - None, - ("train", "test"), - Dataset, - ), - ( - "mock/directory/file.py", - create_sample_dataset_dict(splits=["test"], column="output"), - None, - None, - Dataset, - ), - ( - create_sample_dataset_dict(splits=["val", "train"], column="custom"), - None, - "val", - None, - Dataset, - ), - ( - create_sample_dataset(), - None, - None, - None, - Dataset, - ), - ( - create_sample_iterable_dataset_dict(splits=["validation"]), - None, - None, - None, - IterableDataset, - ), - ( - create_sample_iterable_dataset(), - None, - "validation", - None, - IterableDataset, - ), - ], -) -def test_load_transformers_dataset( - dataset_arg, dataset, split, preferred_splits, expected_type -): - with patch( - "guidellm.utils.transformers.load_dataset", - return_value=dataset, - ): - loaded_dataset = load_transformers_dataset( - dataset_arg, split=split, preferred_splits=preferred_splits - ) - assert isinstance(loaded_dataset, expected_type) - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("dataset_arg", "dataset", "split", "preferred_splits", "expected_type"), - [ - ( - "mock/directory/file.csv", - create_sample_dataset(), - "train", - None, - Dataset, - ), - ( - "mock/directory/file.json", - create_sample_dataset_dict(splits=["test"]), - None, - ("train", "test"), - DatasetDict, - ), - ( - "mock/directory/file.py", - create_sample_dataset_dict(splits=["test"], column="output"), - None, - None, - DatasetDict, - ), - ( - "mock/directory/file.unk", - create_sample_dataset_dict(splits=["test"], column="output"), - None, - None, - DatasetDict, - ), - ( - create_sample_dataset_dict(splits=["val", "train"], column="custom"), - None, - "val", - None, - DatasetDict, - ), - ( - create_sample_dataset(), - None, - None, - None, - Dataset, - ), - ( - create_sample_iterable_dataset_dict(splits=["validation"]), - None, - None, - None, - IterableDatasetDict, - ), - ( - create_sample_iterable_dataset(), - None, - "validation", - None, - IterableDataset, - ), - ], -) -def test_resolve_transformers_dataset( - dataset_arg, dataset, split, preferred_splits, expected_type -): - with patch( - "guidellm.utils.transformers.load_dataset", - return_value=dataset, - ): - loaded_dataset = resolve_transformers_dataset( - dataset_arg, split=split, preferred_splits=preferred_splits - ) - assert isinstance(loaded_dataset, expected_type) - - -@pytest.mark.sanity() -def test_resolve_transformers_dataset_invalid(): - with pytest.raises(ValueError): - resolve_transformers_dataset(123) - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("dataset", "split", "preferred_splits", "expected_type"), - [ - ( - create_sample_dataset(), - None, - None, - Dataset, - ), - ( - create_sample_iterable_dataset_dict(splits=["validation"]), - None, - None, - IterableDataset, - ), - ( - create_sample_iterable_dataset(), - "validation", - None, - IterableDataset, - ), - ], -) -def test_resolve_transformers_dataset_split( - dataset, split, preferred_splits, expected_type -): - loaded_dataset = resolve_transformers_dataset_split( - dataset, split=split, preferred_splits=preferred_splits - ) - assert isinstance(loaded_dataset, expected_type) - - -def test_resolve_transformers_dataset_split_missing(): - dataset = create_sample_dataset_dict() - with pytest.raises(ValueError): - resolve_transformers_dataset_split(dataset, split="missing") - - -@pytest.mark.smoke() -@pytest.mark.parametrize( - ("dataset", "column", "preferred_columns", "expected_column"), - [ - (create_sample_dataset(), None, None, "text"), - (create_sample_dataset(), "text", None, "text"), - (create_sample_dataset(), None, ["text"], "text"), - (create_sample_dataset(), None, ["data"], "text"), - (create_sample_iterable_dataset(), None, None, "text"), - ], -) -def test_resolve_transformers_dataset_column( - dataset, column, preferred_columns, expected_column -): - resolved_column = resolve_transformers_dataset_column( - dataset, column=column, preferred_columns=preferred_columns - ) - assert resolved_column == expected_column - - -def test_resolve_transformers_dataset_column_missing(): - dataset = create_sample_dataset() - with pytest.raises(ValueError): - resolve_transformers_dataset_column(dataset, column="missing") From 3b821c8326f515019a4c5305b31b4f22496d27d6 Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Thu, 10 Apr 2025 20:16:59 -0400 Subject: [PATCH 26/43] Move metrics to subobject in output --- src/guidellm/benchmark/benchmark.py | 265 +++++++++++++++------------- 1 file changed, 147 insertions(+), 118 deletions(-) diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index f0332c3e..1048b0ab 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -37,8 +37,10 @@ "BenchmarkArgs", "BenchmarkRunStats", "Benchmark", + "BenchmarkMetrics", "GenerativeTextResponseStats", "GenerativeTextErrorStats", + "GenerativeMetrics", "GenerativeBenchmark", ] @@ -234,6 +236,19 @@ def total(self) -> int: return self.total_successful + self.total_incomplete + self.total_errored +class BenchmarkMetrics(StandardBaseModel): + """ + A serializable model representing the metrics for a benchmark run. + """ + + request_per_second: StatusDistributionSummary = Field( + description="The distribution of requests per second for the benchmark.", + ) + request_concurrency: StatusDistributionSummary = Field( + description="The distribution of requests concurrency for the benchmark.", + ) + + class Benchmark(StandardBaseModel): """ The base serializable model representing a benchmark run and its results. @@ -291,11 +306,11 @@ class Benchmark(StandardBaseModel): ) ) - requests_per_second: StatusDistributionSummary = Field( - description="The distribution of requests per second for the benchmark.", - ) - requests_concurrency: StatusDistributionSummary = Field( - description="The distribution of requests concurrency for the benchmark.", + metrics: BenchmarkMetrics = Field( + description=( + "The metrics for the benchmark run represented as a distribution of " + "various per-request statistics." + ), ) @@ -506,6 +521,59 @@ def output_tokens_per_second(self) -> Optional[float]: # type: ignore[override] return super().output_tokens_per_second +class GenerativeMetrics(BenchmarkMetrics): + """ + A serializable model representing the metrics for a generative benchmark run. + """ + + request_latency: StatusDistributionSummary = Field( + description="The distribution of latencies for the completed requests.", + ) + prompt_token_count: StatusDistributionSummary = Field( + description=( + "The distribution of token counts in the prompts for completed, " + "errored, and all requests." + ) + ) + output_token_count: StatusDistributionSummary = Field( + description=( + "The distribution of token counts in the outputs for completed, " + "errored, and all requests." + ) + ) + time_to_first_token_ms: StatusDistributionSummary = Field( + description=( + "The distribution of latencies to receiving the first token in " + "milliseconds for completed, errored, and all requests." + ), + ) + time_per_output_token_ms: StatusDistributionSummary = Field( + description=( + "The distribution of latencies per output token in milliseconds for " + "completed, errored, and all requests. " + "This includes the time to generate the first token and all other tokens." + ), + ) + inter_token_latency_ms: StatusDistributionSummary = Field( + description=( + "The distribution of latencies between tokens in milliseconds for " + "completed, errored, and all requests." + ), + ) + output_tokens_per_second: StatusDistributionSummary = Field( + description=( + "The distribution of output tokens per second for completed, " + "errored, and all requests." + ), + ) + tokens_per_second: StatusDistributionSummary = Field( + description=( + "The distribution of tokens per second, including prompt and output tokens " + "for completed, errored, and all requests." + ), + ) + + class GenerativeBenchmark(Benchmark): """ A serializable model representing a benchmark run and its results for generative @@ -568,51 +636,10 @@ class GenerativeBenchmark(Benchmark): end_time: float = Field( description="The end time of the last request for the benchmark.", ) - - request_latency: StatusDistributionSummary = Field( - description="The distribution of latencies for the completed requests.", - ) - prompt_token_count: StatusDistributionSummary = Field( - description=( - "The distribution of token counts in the prompts for completed, " - "errored, and all requests." - ) - ) - output_token_count: StatusDistributionSummary = Field( - description=( - "The distribution of token counts in the outputs for completed, " - "errored, and all requests." - ) - ) - time_to_first_token_ms: StatusDistributionSummary = Field( + metrics: GenerativeMetrics = Field( description=( - "The distribution of latencies to receiving the first token in " - "milliseconds for completed, errored, and all requests." - ), - ) - time_per_output_token_ms: StatusDistributionSummary = Field( - description=( - "The distribution of latencies per output token in milliseconds for " - "completed, errored, and all requests. " - "This includes the time to generate the first token and all other tokens." - ), - ) - inter_token_latency_ms: StatusDistributionSummary = Field( - description=( - "The distribution of latencies between tokens in milliseconds for " - "completed, errored, and all requests." - ), - ) - output_tokens_per_second: StatusDistributionSummary = Field( - description=( - "The distribution of output tokens per second for completed, " - "errored, and all requests." - ), - ) - tokens_per_second: StatusDistributionSummary = Field( - description=( - "The distribution of tokens per second, including prompt and output tokens " - "for completed, errored, and all requests." + "The metrics for the benchmark run represented as a distribution of " + "various per-request statistics." ), ) @@ -793,74 +820,76 @@ def from_stats( errored_requests=errored, start_time=start_time, end_time=end_time, - requests_per_second=StatusDistributionSummary.from_request_times( - request_types=total_types, - requests=[(req.start_time, req.end_time) for req in total], - distribution_type="rate", - ), - requests_concurrency=StatusDistributionSummary.from_request_times( - request_types=total_types, - requests=[(req.start_time, req.end_time) for req in total], - distribution_type="concurrency", - ), - request_latency=StatusDistributionSummary.from_values( - value_types=total_types, - values=[req.request_latency for req in total], - ), - prompt_token_count=StatusDistributionSummary.from_values( - value_types=list(total_types_with_prompt), - values=[req.prompt_tokens for req in total_with_prompt], - ), - output_token_count=StatusDistributionSummary.from_values( - value_types=list(total_types_with_output_first), - values=[req.output_tokens for req in total_with_output_first], - ), - time_to_first_token_ms=StatusDistributionSummary.from_values( - value_types=list(total_types_with_output_first), - values=[ - req.time_to_first_token_ms or 0 for req in total_with_output_first - ], - ), - time_per_output_token_ms=StatusDistributionSummary.from_values( - value_types=list(total_types_with_output_first), - values=[ - req.time_per_output_token_ms or 0 for req in total_with_output_first - ], - weights=[req.output_tokens for req in total_with_output_first], - ), - inter_token_latency_ms=StatusDistributionSummary.from_values( - value_types=list(total_types_with_output_multi), - values=[ - req.inter_token_latency_ms or 0 for req in total_with_output_multi - ], - weights=[req.output_tokens - 1 for req in total_with_output_multi], - ), - output_tokens_per_second=StatusDistributionSummary.from_iterable_request_times( - request_types=list(total_types_with_output_first), - requests=[ - (req.start_time, req.end_time) for req in total_with_output_first - ], - first_iter_times=[ - req.first_token_time or req.start_time - for req in total_with_output_first - ], - iter_counts=[req.output_tokens for req in total_with_output_first], - ), - tokens_per_second=StatusDistributionSummary.from_iterable_request_times( - request_types=list(total_types_with_output_first), - requests=[ - (req.start_time, req.end_time) for req in total_with_output_first - ], - first_iter_times=[ - req.first_token_time or req.start_time - for req in total_with_output_first - ], - iter_counts=[ - req.prompt_tokens + req.output_tokens - for req in total_with_output_first - ], - first_iter_counts=[ - req.prompt_tokens for req in total_with_output_first - ], + metrics=GenerativeMetrics( + request_per_second=StatusDistributionSummary.from_request_times( + request_types=total_types, + requests=[(req.start_time, req.end_time) for req in total], + distribution_type="rate", + ), + request_concurrency=StatusDistributionSummary.from_request_times( + request_types=total_types, + requests=[(req.start_time, req.end_time) for req in total], + distribution_type="concurrency", + ), + request_latency=StatusDistributionSummary.from_values( + value_types=total_types, + values=[req.request_latency for req in total], + ), + prompt_token_count=StatusDistributionSummary.from_values( + value_types=list(total_types_with_prompt), + values=[req.prompt_tokens for req in total_with_prompt], + ), + output_token_count=StatusDistributionSummary.from_values( + value_types=list(total_types_with_output_first), + values=[req.output_tokens for req in total_with_output_first], + ), + time_to_first_token_ms=StatusDistributionSummary.from_values( + value_types=list(total_types_with_output_first), + values=[ + req.time_to_first_token_ms or 0 for req in total_with_output_first + ], + ), + time_per_output_token_ms=StatusDistributionSummary.from_values( + value_types=list(total_types_with_output_first), + values=[ + req.time_per_output_token_ms or 0 for req in total_with_output_first + ], + weights=[req.output_tokens for req in total_with_output_first], + ), + inter_token_latency_ms=StatusDistributionSummary.from_values( + value_types=list(total_types_with_output_multi), + values=[ + req.inter_token_latency_ms or 0 for req in total_with_output_multi + ], + weights=[req.output_tokens - 1 for req in total_with_output_multi], + ), + output_tokens_per_second=StatusDistributionSummary.from_iterable_request_times( + request_types=list(total_types_with_output_first), + requests=[ + (req.start_time, req.end_time) for req in total_with_output_first + ], + first_iter_times=[ + req.first_token_time or req.start_time + for req in total_with_output_first + ], + iter_counts=[req.output_tokens for req in total_with_output_first], + ), + tokens_per_second=StatusDistributionSummary.from_iterable_request_times( + request_types=list(total_types_with_output_first), + requests=[ + (req.start_time, req.end_time) for req in total_with_output_first + ], + first_iter_times=[ + req.first_token_time or req.start_time + for req in total_with_output_first + ], + iter_counts=[ + req.prompt_tokens + req.output_tokens + for req in total_with_output_first + ], + first_iter_counts=[ + req.prompt_tokens for req in total_with_output_first + ], + ), ), ) From 5e63061087781ba050f5cc42f3f5609f54a5d1c6 Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Thu, 10 Apr 2025 20:41:03 -0400 Subject: [PATCH 27/43] Move requests to subobject in output --- src/guidellm/benchmark/benchmark.py | 60 +++++++++++++++++++---------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index 1048b0ab..8ae23d14 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -41,6 +41,7 @@ "GenerativeTextResponseStats", "GenerativeTextErrorStats", "GenerativeMetrics", + "GenerativeRequestsBreakdown", "GenerativeBenchmark", ] @@ -574,6 +575,23 @@ class GenerativeMetrics(BenchmarkMetrics): ) +class GenerativeRequestsBreakdown(StandardBaseModel): + """ + A serializable model representing the breakdown of requests for a generative + benchmark run. + """ + + successful: List[GenerativeTextResponseStats] = Field( + description="The list of completed requests.", + ) + incomplete: List[GenerativeTextErrorStats] = Field( + description="The list of incomplete requests.", + ) + errored: List[GenerativeTextErrorStats] = Field( + description="The list of errored requests.", + ) + + class GenerativeBenchmark(Benchmark): """ A serializable model representing a benchmark run and its results for generative @@ -595,9 +613,6 @@ class GenerativeBenchmark(Benchmark): "the benchmark. None if no sampling was applied." ), ) - successful_requests: List[GenerativeTextResponseStats] = Field( - description="The list of completed requests.", - ) incomplete_total: int = Field( description=( "The total number of incomplete requests in the benchmark, " @@ -611,9 +626,6 @@ class GenerativeBenchmark(Benchmark): "the benchmark. None if no sampling was applied." ), ) - incomplete_requests: List[GenerativeTextErrorStats] = Field( - description="The list of incomplete requests.", - ) errored_total: int = Field( description=( "The total number of errored requests in the benchmark, " @@ -627,9 +639,6 @@ class GenerativeBenchmark(Benchmark): "the benchmark. None if no sampling was applied." ), ) - errored_requests: List[GenerativeTextErrorStats] = Field( - description="The list of errored requests.", - ) start_time: float = Field( description="The start time of the first request for the benchmark.", ) @@ -642,6 +651,13 @@ class GenerativeBenchmark(Benchmark): "various per-request statistics." ), ) + # Output is ordered so keep this at the end + requests: GenerativeRequestsBreakdown = Field( + description=( + "The breakdown of requests for the benchmark run including completed, " + "incomplete, and errored requests." + ), + ) @computed_field # type: ignore[misc] @property @@ -707,22 +723,22 @@ def create_sampled( f"a larger size, given: {error_sample_size}" ) - sample_size = min(sample_size, len(self.successful_requests)) - incomplete_sample_size = min(sample_size, len(self.incomplete_requests)) - error_sample_size = min(error_sample_size, len(self.errored_requests)) + sample_size = min(sample_size, len(self.requests.successful)) + incomplete_sample_size = min(sample_size, len(self.requests.incomplete)) + error_sample_size = min(error_sample_size, len(self.requests.errored)) sampled_instance = self.model_copy() sampled_instance.successful_sampled_size = sample_size - sampled_instance.successful_requests = random.sample( - self.successful_requests, sample_size + sampled_instance.requests.successful = random.sample( + self.requests.successful, sample_size ) sampled_instance.incomplete_sampled_size = incomplete_sample_size - sampled_instance.incomplete_requests = random.sample( - self.incomplete_requests, incomplete_sample_size + sampled_instance.requests.incomplete = random.sample( + self.requests.incomplete, incomplete_sample_size ) sampled_instance.errored_sampled_size = error_sample_size - sampled_instance.errored_requests = random.sample( - self.errored_requests, error_sample_size + sampled_instance.requests.errored = random.sample( + self.requests.errored, error_sample_size ) return sampled_instance @@ -813,11 +829,8 @@ def from_stats( request_loader=requests_loader, extras=extras or {}, successful_total=len(successful), - successful_requests=successful, incomplete_total=len(incomplete), - incomplete_requests=incomplete, errored_total=len(errored), - errored_requests=errored, start_time=start_time, end_time=end_time, metrics=GenerativeMetrics( @@ -892,4 +905,9 @@ def from_stats( ], ), ), + requests=GenerativeRequestsBreakdown( + successful=successful, + incomplete=incomplete, + errored=errored, + ), ) From 82a381f3d1540f1700fee6f937908f445ed6b9c1 Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Thu, 10 Apr 2025 21:36:36 -0400 Subject: [PATCH 28/43] Replace Request breakdown with generic class --- src/guidellm/benchmark/benchmark.py | 49 ++++++++++++++++------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index 8ae23d14..5b1b5a7a 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -1,6 +1,6 @@ import random import uuid -from typing import Any, Dict, List, Literal, Optional, TypeVar, Union +from typing import Any, Dict, Generic, List, Literal, Optional, TypeVar, Union from pydantic import Field, computed_field @@ -34,6 +34,7 @@ __all__ = [ "BENCH", + "StatusBreakdown", "BenchmarkArgs", "BenchmarkRunStats", "Benchmark", @@ -41,11 +42,30 @@ "GenerativeTextResponseStats", "GenerativeTextErrorStats", "GenerativeMetrics", - "GenerativeRequestsBreakdown", "GenerativeBenchmark", ] +SuccessfulT = TypeVar("SuccessfulT") +IncompleteT = TypeVar("IncompleteT") +ErroredT = TypeVar("ErroredT") +class StatusBreakdown(StandardBaseModel, Generic[SuccessfulT, IncompleteT, ErroredT]): + """ + A serializable model representing the breakdown of statistics for a benchmark run + split into successful, incomplete, and errored. + """ + + successful: SuccessfulT = Field( + description="Successful", + ) + incomplete: IncompleteT = Field( + description="Incomplete", + ) + errored: ErroredT = Field( + description="Errored", + ) + + class BenchmarkArgs(StandardBaseModel): """ A serializable model representing the arguments used to specify a benchmark run @@ -575,23 +595,6 @@ class GenerativeMetrics(BenchmarkMetrics): ) -class GenerativeRequestsBreakdown(StandardBaseModel): - """ - A serializable model representing the breakdown of requests for a generative - benchmark run. - """ - - successful: List[GenerativeTextResponseStats] = Field( - description="The list of completed requests.", - ) - incomplete: List[GenerativeTextErrorStats] = Field( - description="The list of incomplete requests.", - ) - errored: List[GenerativeTextErrorStats] = Field( - description="The list of errored requests.", - ) - - class GenerativeBenchmark(Benchmark): """ A serializable model representing a benchmark run and its results for generative @@ -652,7 +655,11 @@ class GenerativeBenchmark(Benchmark): ), ) # Output is ordered so keep this at the end - requests: GenerativeRequestsBreakdown = Field( + requests: StatusBreakdown[ + List[GenerativeTextResponseStats], + List[GenerativeTextErrorStats], + List[GenerativeTextErrorStats] + ] = Field( description=( "The breakdown of requests for the benchmark run including completed, " "incomplete, and errored requests." @@ -905,7 +912,7 @@ def from_stats( ], ), ), - requests=GenerativeRequestsBreakdown( + requests=StatusBreakdown( successful=successful, incomplete=incomplete, errored=errored, From 31adea445694dd6530fe04f8153a94c6ec3dc4cd Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Thu, 10 Apr 2025 22:06:19 -0400 Subject: [PATCH 29/43] Define sampling sizes and counts as StatusBreakdowns --- src/guidellm/benchmark/benchmark.py | 96 ++++++++++++----------------- 1 file changed, 38 insertions(+), 58 deletions(-) diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index 5b1b5a7a..9e897fe5 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -47,9 +47,9 @@ SuccessfulT = TypeVar("SuccessfulT") -IncompleteT = TypeVar("IncompleteT") -ErroredT = TypeVar("ErroredT") -class StatusBreakdown(StandardBaseModel, Generic[SuccessfulT, IncompleteT, ErroredT]): +ErroredT = TypeVar("ErroredT", default=SuccessfulT) +IncompleteT = TypeVar("IncompleteT", default=ErroredT) +class StatusBreakdown(StandardBaseModel, Generic[SuccessfulT, ErroredT, IncompleteT]): """ A serializable model representing the breakdown of statistics for a benchmark run split into successful, incomplete, and errored. @@ -603,42 +603,16 @@ class GenerativeBenchmark(Benchmark): """ type_: Literal["generative_benchmark"] = "generative_benchmark" # type: ignore[assignment] - successful_total: int = Field( + total_count: StatusBreakdown[int] = Field( description=( - "The total number of completed requests in the benchmark, " + "The total number of requests in the benchmark, " "excluding warmup and cooldown." ) ) - successful_sampled_size: Optional[int] = Field( + sampled_size: Optional[StatusBreakdown[int]] = Field( default=None, description=( - "The number of completed requests that were randomly sampled for " - "the benchmark. None if no sampling was applied." - ), - ) - incomplete_total: int = Field( - description=( - "The total number of incomplete requests in the benchmark, " - "excluding warmup and cooldown." - ) - ) - incomplete_sampled_size: Optional[int] = Field( - default=None, - description=( - "The number of incomplete requests that were randomly sampled for " - "the benchmark. None if no sampling was applied." - ), - ) - errored_total: int = Field( - description=( - "The total number of errored requests in the benchmark, " - "excluding warmup and cooldown." - ) - ) - errored_sampled_size: Optional[int] = Field( - default=None, - description=( - "The number of errored requests that were randomly sampled for " + "The number of requests that were randomly sampled for " "the benchmark. None if no sampling was applied." ), ) @@ -648,6 +622,15 @@ class GenerativeBenchmark(Benchmark): end_time: float = Field( description="The end time of the last request for the benchmark.", ) + @computed_field # type: ignore[misc] + @property + def duration(self) -> float: + """ + :return: The duration of the benchmark in seconds from the start of the + first request to the end of the last request. + """ + return self.end_time - self.start_time + metrics: GenerativeMetrics = Field( description=( "The metrics for the benchmark run represented as a distribution of " @@ -658,7 +641,6 @@ class GenerativeBenchmark(Benchmark): requests: StatusBreakdown[ List[GenerativeTextResponseStats], List[GenerativeTextErrorStats], - List[GenerativeTextErrorStats] ] = Field( description=( "The breakdown of requests for the benchmark run including completed, " @@ -666,15 +648,6 @@ class GenerativeBenchmark(Benchmark): ), ) - @computed_field # type: ignore[misc] - @property - def duration(self) -> float: - """ - :return: The duration of the benchmark in seconds from the start of the - first request to the end of the last request. - """ - return self.end_time - self.start_time - def create_sampled( self, sample_size: int, error_sample_size: Optional[int] = None ) -> "GenerativeBenchmark": @@ -703,30 +676,30 @@ def create_sampled( ) if ( - self.successful_sampled_size is not None - and sample_size > self.successful_sampled_size + self.sampled_size is not None + and sample_size > self.sampled_size.successful ): raise ValueError( "The benchmark's completed response have already been sampled with " - f"size {self.successful_sampled_size} and cannot be resampled with " + f"size {self.sampled_size.successful} and cannot be resampled with " f"a larger size, given: {sample_size}" ) if ( - self.incomplete_sampled_size is not None - and sample_size > self.incomplete_sampled_size + self.sampled_size is not None + and sample_size > self.sampled_size.incomplete ): raise ValueError( "The benchmark's incomplete response have already been sampled with " - f"size {self.incomplete_sampled_size} and cannot be resampled with " + f"size {self.sampled_size.incomplete} and cannot be resampled with " f"a larger size, given: {sample_size}" ) if ( - self.errored_sampled_size is not None - and error_sample_size > self.errored_sampled_size + self.sampled_size is not None + and error_sample_size > self.sampled_size.errored ): raise ValueError( "The benchmark's errored response have already been sampled with " - f"size {self.errored_sampled_size} and cannot be resampled with " + f"size {self.sampled_size.errored} and cannot be resampled with " f"a larger size, given: {error_sample_size}" ) @@ -735,15 +708,20 @@ def create_sampled( error_sample_size = min(error_sample_size, len(self.requests.errored)) sampled_instance = self.model_copy() - sampled_instance.successful_sampled_size = sample_size + sampled_instance.sampled_size = StatusBreakdown( + successful=0, + incomplete=0, + errored=0, + ) + sampled_instance.sampled_size.successful = sample_size sampled_instance.requests.successful = random.sample( self.requests.successful, sample_size ) - sampled_instance.incomplete_sampled_size = incomplete_sample_size + sampled_instance.sampled_size.incomplete = incomplete_sample_size sampled_instance.requests.incomplete = random.sample( self.requests.incomplete, incomplete_sample_size ) - sampled_instance.errored_sampled_size = error_sample_size + sampled_instance.sampled_size.errored = error_sample_size sampled_instance.requests.errored = random.sample( self.requests.errored, error_sample_size ) @@ -835,9 +813,11 @@ def from_stats( worker=worker, request_loader=requests_loader, extras=extras or {}, - successful_total=len(successful), - incomplete_total=len(incomplete), - errored_total=len(errored), + total_count=StatusBreakdown( + successful=len(successful), + incomplete=len(incomplete), + errored=len(errored), + ), start_time=start_time, end_time=end_time, metrics=GenerativeMetrics( From f8c5e7ab544e882c88e7b7cf677e6813a43b51ae Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Thu, 10 Apr 2025 22:08:45 -0400 Subject: [PATCH 30/43] Set a default type for SuccessfulT Mypy gets unhappy when we instantiate a generic class without declaring its type (aka infer from the inital value). As a workaround just declare our default type as Any. --- src/guidellm/benchmark/benchmark.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index 9e897fe5..8711696d 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -1,8 +1,9 @@ import random import uuid -from typing import Any, Dict, Generic, List, Literal, Optional, TypeVar, Union +from typing import Any, Dict, Generic, List, Literal, Optional, Union from pydantic import Field, computed_field +from typing_extensions import TypeVar from guidellm.benchmark.profile import ( AsyncProfile, @@ -46,7 +47,7 @@ ] -SuccessfulT = TypeVar("SuccessfulT") +SuccessfulT = TypeVar("SuccessfulT", default=Any) ErroredT = TypeVar("ErroredT", default=SuccessfulT) IncompleteT = TypeVar("IncompleteT", default=ErroredT) class StatusBreakdown(StandardBaseModel, Generic[SuccessfulT, ErroredT, IncompleteT]): From 93f0fd1c942a5a9506c7ef08ef635912d321db8b Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Thu, 10 Apr 2025 22:32:37 -0400 Subject: [PATCH 31/43] Fix case chnage on requests_per_second --- src/guidellm/benchmark/benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index 8711696d..5200f82a 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -263,7 +263,7 @@ class BenchmarkMetrics(StandardBaseModel): A serializable model representing the metrics for a benchmark run. """ - request_per_second: StatusDistributionSummary = Field( + requests_per_second: StatusDistributionSummary = Field( description="The distribution of requests per second for the benchmark.", ) request_concurrency: StatusDistributionSummary = Field( From 00a210d03a14720612ffb7f40bd14da867428a34 Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Thu, 10 Apr 2025 22:54:18 -0400 Subject: [PATCH 32/43] Plumb output changes though progress and summary --- src/guidellm/benchmark/benchmark.py | 2 +- src/guidellm/benchmark/benchmarker.py | 4 +- src/guidellm/benchmark/output.py | 62 +++++++++++++-------------- src/guidellm/benchmark/progress.py | 16 +++---- 4 files changed, 42 insertions(+), 42 deletions(-) diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index 5200f82a..dc28860a 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -277,7 +277,7 @@ class Benchmark(StandardBaseModel): Specific benchmarker implementations should extend this model to include additional information or metadata as needed. - Note, requests_per_second and requests_concurrency are kept at this level + Note, requests_per_second and request_concurrency are kept at this level and are expected to be populated by the subclass implementation to ensure the logic for Profiles can include more complicated logic for determining what rates and concurrency values to use for subsequent strategies. diff --git a/src/guidellm/benchmark/benchmarker.py b/src/guidellm/benchmark/benchmarker.py index 332d2ad1..9e53130b 100644 --- a/src/guidellm/benchmark/benchmarker.py +++ b/src/guidellm/benchmark/benchmarker.py @@ -235,8 +235,8 @@ async def run( benchmark: BENCH = aggregator.compile() profile.completed_strategy( - average_rate=benchmark.requests_per_second.successful.mean, - average_concurrency=benchmark.requests_concurrency.successful.mean, + average_rate=benchmark.metrics.requests_per_second.successful.mean, + average_concurrency=benchmark.metrics.request_concurrency.successful.mean, ) yield BenchmarkerResult( diff --git a/src/guidellm/benchmark/output.py b/src/guidellm/benchmark/output.py index de34d626..8bcdc097 100644 --- a/src/guidellm/benchmark/output.py +++ b/src/guidellm/benchmark/output.py @@ -258,29 +258,29 @@ def print_benchmarks_info(self): f"{datetime.fromtimestamp(benchmark.end_time).strftime("%H:%M:%S")}", f"{(benchmark.end_time - benchmark.start_time):.1f}", ( - f"{benchmark.successful_total:>5} / " - f"{benchmark.incomplete_total} / " - f"{benchmark.errored_total}" + f"{benchmark.total_count.successful:>5} / " + f"{benchmark.total_count.incomplete} / " + f"{benchmark.total_count.errored}" ), ( - f"{benchmark.prompt_token_count.successful.mean:>5.1f} / " - f"{benchmark.prompt_token_count.incomplete.mean:.1f} / " - f"{benchmark.prompt_token_count.errored.mean:.1f}" + f"{benchmark.metrics.prompt_token_count.successful.mean:>5.1f} / " + f"{benchmark.metrics.prompt_token_count.incomplete.mean:.1f} / " + f"{benchmark.metrics.prompt_token_count.errored.mean:.1f}" ), ( - f"{benchmark.output_token_count.successful.mean:>5.1f} / " - f"{benchmark.output_token_count.incomplete.mean:.1f} / " - f"{benchmark.output_token_count.errored.mean:.1f}" + f"{benchmark.metrics.output_token_count.successful.mean:>5.1f} / " + f"{benchmark.metrics.output_token_count.incomplete.mean:.1f} / " + f"{benchmark.metrics.output_token_count.errored.mean:.1f}" ), ( - f"{benchmark.prompt_token_count.successful.total_sum:>6.0f} / " - f"{benchmark.prompt_token_count.incomplete.total_sum:.0f} / " - f"{benchmark.prompt_token_count.errored.total_sum:.0f}" + f"{benchmark.metrics.prompt_token_count.successful.total_sum:>6.0f} / " + f"{benchmark.metrics.prompt_token_count.incomplete.total_sum:.0f} / " + f"{benchmark.metrics.prompt_token_count.errored.total_sum:.0f}" ), ( - f"{benchmark.output_token_count.successful.total_sum:>6.0f} / " - f"{benchmark.output_token_count.incomplete.total_sum:.0f} / " - f"{benchmark.output_token_count.errored.total_sum:.0f}" + f"{benchmark.metrics.output_token_count.successful.total_sum:>6.0f} / " + f"{benchmark.metrics.output_token_count.incomplete.total_sum:.0f} / " + f"{benchmark.metrics.output_token_count.errored.total_sum:.0f}" ), ] ) @@ -313,29 +313,29 @@ def print_benchmarks_stats(self): rows.append( [ strategy_display_str(benchmark.args.strategy), - f"{benchmark.requests_per_second.successful.mean:.2f}", - f"{benchmark.requests_concurrency.successful.mean:.2f}", - f"{benchmark.output_tokens_per_second.total.mean:.1f}", - f"{benchmark.tokens_per_second.total.mean:.1f}", + f"{benchmark.metrics.requests_per_second.successful.mean:.2f}", + f"{benchmark.metrics.request_concurrency.successful.mean:.2f}", + f"{benchmark.metrics.output_tokens_per_second.total.mean:.1f}", + f"{benchmark.metrics.tokens_per_second.total.mean:.1f}", ( - f"{benchmark.request_latency.successful.mean:.2f} / " - f"{benchmark.request_latency.successful.median:.2f} / " - f"{benchmark.request_latency.successful.percentiles.p99:.2f}" + f"{benchmark.metrics.request_latency.successful.mean:.2f} / " + f"{benchmark.metrics.request_latency.successful.median:.2f} / " + f"{benchmark.metrics.request_latency.successful.percentiles.p99:.2f}" ), ( - f"{benchmark.time_to_first_token_ms.successful.mean:.1f} / " - f"{benchmark.time_to_first_token_ms.successful.median:.1f} / " - f"{benchmark.time_to_first_token_ms.successful.percentiles.p99:.1f}" + f"{benchmark.metrics.time_to_first_token_ms.successful.mean:.1f} / " + f"{benchmark.metrics.time_to_first_token_ms.successful.median:.1f} / " + f"{benchmark.metrics.time_to_first_token_ms.successful.percentiles.p99:.1f}" ), ( - f"{benchmark.inter_token_latency_ms.successful.mean:.1f} / " - f"{benchmark.inter_token_latency_ms.successful.median:.1f} / " - f"{benchmark.inter_token_latency_ms.successful.percentiles.p99:.1f}" + f"{benchmark.metrics.inter_token_latency_ms.successful.mean:.1f} / " + f"{benchmark.metrics.inter_token_latency_ms.successful.median:.1f} / " + f"{benchmark.metrics.inter_token_latency_ms.successful.percentiles.p99:.1f}" ), ( - f"{benchmark.time_per_output_token_ms.successful.mean:.1f} / " - f"{benchmark.time_per_output_token_ms.successful.median:.1f} / " - f"{benchmark.time_per_output_token_ms.successful.percentiles.p99:.1f}" + f"{benchmark.metrics.time_per_output_token_ms.successful.mean:.1f} / " + f"{benchmark.metrics.time_per_output_token_ms.successful.median:.1f} / " + f"{benchmark.metrics.time_per_output_token_ms.successful.percentiles.p99:.1f}" ), ] ) diff --git a/src/guidellm/benchmark/progress.py b/src/guidellm/benchmark/progress.py index b7b4042f..7ba1a9cb 100644 --- a/src/guidellm/benchmark/progress.py +++ b/src/guidellm/benchmark/progress.py @@ -553,10 +553,10 @@ def handle_update_benchmark_compiled( progress_state.compiling = False progress_state.ended = True progress_state.requests_rate = ( - current_benchmark.requests_per_second.successful.mean + current_benchmark.metric.requests_per_second.successful.mean ) progress_state.requests_processing = ( - current_benchmark.requests_concurrency.successful.mean + current_benchmark.metric.requests_concurrency.successful.mean ) def handle_end(self, result: BenchmarkerResult): # noqa: ARG002 @@ -647,22 +647,22 @@ def handle_update_benchmark_compiled( progress_state.requests_successful = current_benchmark.successful_total progress_state.requests_errored = current_benchmark.errored_total progress_state.output_tokens = ( - current_benchmark.output_token_count.successful.mean + current_benchmark.metric.output_token_count.successful.mean ) progress_state.prompt_tokens = ( - current_benchmark.prompt_token_count.successful.mean + current_benchmark.metric.prompt_token_count.successful.mean ) progress_state.output_tokens_rate = ( - current_benchmark.output_tokens_per_second.successful.mean + current_benchmark.metric.output_tokens_per_second.successful.mean ) progress_state.total_tokens_rate = ( - current_benchmark.tokens_per_second.successful.mean + current_benchmark.metric.tokens_per_second.successful.mean ) progress_state.tokens_ttft = ( - current_benchmark.time_to_first_token_ms.successful.mean + current_benchmark.metric.time_to_first_token_ms.successful.mean ) progress_state.tokens_itl = ( - current_benchmark.inter_token_latency_ms.successful.mean + current_benchmark.metric.inter_token_latency_ms.successful.mean ) def create_task_progress_state( From cf160b60cc6d653edb542644b2745ae036c72844 Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Thu, 10 Apr 2025 22:58:00 -0400 Subject: [PATCH 33/43] Pluralization is hard --- src/guidellm/benchmark/benchmark.py | 2 +- src/guidellm/benchmark/progress.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index dc28860a..9563b42a 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -822,7 +822,7 @@ def from_stats( start_time=start_time, end_time=end_time, metrics=GenerativeMetrics( - request_per_second=StatusDistributionSummary.from_request_times( + requests_per_second=StatusDistributionSummary.from_request_times( request_types=total_types, requests=[(req.start_time, req.end_time) for req in total], distribution_type="rate", diff --git a/src/guidellm/benchmark/progress.py b/src/guidellm/benchmark/progress.py index 7ba1a9cb..4b3abb02 100644 --- a/src/guidellm/benchmark/progress.py +++ b/src/guidellm/benchmark/progress.py @@ -553,10 +553,10 @@ def handle_update_benchmark_compiled( progress_state.compiling = False progress_state.ended = True progress_state.requests_rate = ( - current_benchmark.metric.requests_per_second.successful.mean + current_benchmark.metrics.requests_per_second.successful.mean ) progress_state.requests_processing = ( - current_benchmark.metric.requests_concurrency.successful.mean + current_benchmark.metrics.request_concurrency.successful.mean ) def handle_end(self, result: BenchmarkerResult): # noqa: ARG002 @@ -647,22 +647,22 @@ def handle_update_benchmark_compiled( progress_state.requests_successful = current_benchmark.successful_total progress_state.requests_errored = current_benchmark.errored_total progress_state.output_tokens = ( - current_benchmark.metric.output_token_count.successful.mean + current_benchmark.metrics.output_token_count.successful.mean ) progress_state.prompt_tokens = ( - current_benchmark.metric.prompt_token_count.successful.mean + current_benchmark.metrics.prompt_token_count.successful.mean ) progress_state.output_tokens_rate = ( - current_benchmark.metric.output_tokens_per_second.successful.mean + current_benchmark.metrics.output_tokens_per_second.successful.mean ) progress_state.total_tokens_rate = ( - current_benchmark.metric.tokens_per_second.successful.mean + current_benchmark.metrics.tokens_per_second.successful.mean ) progress_state.tokens_ttft = ( - current_benchmark.metric.time_to_first_token_ms.successful.mean + current_benchmark.metrics.time_to_first_token_ms.successful.mean ) progress_state.tokens_itl = ( - current_benchmark.metric.inter_token_latency_ms.successful.mean + current_benchmark.metrics.inter_token_latency_ms.successful.mean ) def create_task_progress_state( From 2bedc6d8e4a6628cc883b461c7f7477d9f1fbe6f Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Thu, 10 Apr 2025 23:29:39 -0400 Subject: [PATCH 34/43] Fix changes after rebase --- src/guidellm/benchmark/progress.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/guidellm/benchmark/progress.py b/src/guidellm/benchmark/progress.py index 4b3abb02..35e49a0e 100644 --- a/src/guidellm/benchmark/progress.py +++ b/src/guidellm/benchmark/progress.py @@ -639,13 +639,13 @@ def handle_update_benchmark_compiled( current_benchmark: GenerativeBenchmark = result.current_benchmark # type: ignore[assignment] progress_state.request_latency = ( - current_benchmark.request_latency.successful.mean + current_benchmark.metrics.request_latency.successful.mean ) progress_state.requests_processing = ( - current_benchmark.requests_concurrency.successful.mean + current_benchmark.metrics.request_concurrency.successful.mean ) - progress_state.requests_successful = current_benchmark.successful_total - progress_state.requests_errored = current_benchmark.errored_total + progress_state.requests_successful = current_benchmark.total_count.successful + progress_state.requests_errored = current_benchmark.total_count.errored progress_state.output_tokens = ( current_benchmark.metrics.output_token_count.successful.mean ) From 331978b704c287dd69b862d5270f45699e983ca9 Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Thu, 10 Apr 2025 23:44:01 -0400 Subject: [PATCH 35/43] Fix/ignore linting errors due to line length changes --- src/guidellm/benchmark/benchmark.py | 15 ++++++++++----- src/guidellm/benchmark/output.py | 24 ++++++++++++------------ 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index 9563b42a..388ddd6a 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -847,27 +847,31 @@ def from_stats( time_to_first_token_ms=StatusDistributionSummary.from_values( value_types=list(total_types_with_output_first), values=[ - req.time_to_first_token_ms or 0 for req in total_with_output_first + req.time_to_first_token_ms or 0 + for req in total_with_output_first ], ), time_per_output_token_ms=StatusDistributionSummary.from_values( value_types=list(total_types_with_output_first), values=[ - req.time_per_output_token_ms or 0 for req in total_with_output_first + req.time_per_output_token_ms or 0 + for req in total_with_output_first ], weights=[req.output_tokens for req in total_with_output_first], ), inter_token_latency_ms=StatusDistributionSummary.from_values( value_types=list(total_types_with_output_multi), values=[ - req.inter_token_latency_ms or 0 for req in total_with_output_multi + req.inter_token_latency_ms or 0 + for req in total_with_output_multi ], weights=[req.output_tokens - 1 for req in total_with_output_multi], ), output_tokens_per_second=StatusDistributionSummary.from_iterable_request_times( request_types=list(total_types_with_output_first), requests=[ - (req.start_time, req.end_time) for req in total_with_output_first + (req.start_time, req.end_time) + for req in total_with_output_first ], first_iter_times=[ req.first_token_time or req.start_time @@ -878,7 +882,8 @@ def from_stats( tokens_per_second=StatusDistributionSummary.from_iterable_request_times( request_types=list(total_types_with_output_first), requests=[ - (req.start_time, req.end_time) for req in total_with_output_first + (req.start_time, req.end_time) + for req in total_with_output_first ], first_iter_times=[ req.first_token_time or req.start_time diff --git a/src/guidellm/benchmark/output.py b/src/guidellm/benchmark/output.py index 8bcdc097..a75cbb38 100644 --- a/src/guidellm/benchmark/output.py +++ b/src/guidellm/benchmark/output.py @@ -263,23 +263,23 @@ def print_benchmarks_info(self): f"{benchmark.total_count.errored}" ), ( - f"{benchmark.metrics.prompt_token_count.successful.mean:>5.1f} / " + f"{benchmark.metrics.prompt_token_count.successful.mean:>5.1f} / " # noqa: E501 f"{benchmark.metrics.prompt_token_count.incomplete.mean:.1f} / " f"{benchmark.metrics.prompt_token_count.errored.mean:.1f}" ), ( - f"{benchmark.metrics.output_token_count.successful.mean:>5.1f} / " + f"{benchmark.metrics.output_token_count.successful.mean:>5.1f} / " # noqa: E501 f"{benchmark.metrics.output_token_count.incomplete.mean:.1f} / " f"{benchmark.metrics.output_token_count.errored.mean:.1f}" ), ( - f"{benchmark.metrics.prompt_token_count.successful.total_sum:>6.0f} / " - f"{benchmark.metrics.prompt_token_count.incomplete.total_sum:.0f} / " + f"{benchmark.metrics.prompt_token_count.successful.total_sum:>6.0f} / " # noqa: E501 + f"{benchmark.metrics.prompt_token_count.incomplete.total_sum:.0f} / " # noqa: E501 f"{benchmark.metrics.prompt_token_count.errored.total_sum:.0f}" ), ( - f"{benchmark.metrics.output_token_count.successful.total_sum:>6.0f} / " - f"{benchmark.metrics.output_token_count.incomplete.total_sum:.0f} / " + f"{benchmark.metrics.output_token_count.successful.total_sum:>6.0f} / " # noqa: E501 + f"{benchmark.metrics.output_token_count.incomplete.total_sum:.0f} / " # noqa: E501 f"{benchmark.metrics.output_token_count.errored.total_sum:.0f}" ), ] @@ -323,18 +323,18 @@ def print_benchmarks_stats(self): f"{benchmark.metrics.request_latency.successful.percentiles.p99:.2f}" ), ( - f"{benchmark.metrics.time_to_first_token_ms.successful.mean:.1f} / " - f"{benchmark.metrics.time_to_first_token_ms.successful.median:.1f} / " + f"{benchmark.metrics.time_to_first_token_ms.successful.mean:.1f} / " # noqa: E501 + f"{benchmark.metrics.time_to_first_token_ms.successful.median:.1f} / " # noqa: E501 f"{benchmark.metrics.time_to_first_token_ms.successful.percentiles.p99:.1f}" ), ( - f"{benchmark.metrics.inter_token_latency_ms.successful.mean:.1f} / " - f"{benchmark.metrics.inter_token_latency_ms.successful.median:.1f} / " + f"{benchmark.metrics.inter_token_latency_ms.successful.mean:.1f} / " # noqa: E501 + f"{benchmark.metrics.inter_token_latency_ms.successful.median:.1f} / " # noqa: E501 f"{benchmark.metrics.inter_token_latency_ms.successful.percentiles.p99:.1f}" ), ( - f"{benchmark.metrics.time_per_output_token_ms.successful.mean:.1f} / " - f"{benchmark.metrics.time_per_output_token_ms.successful.median:.1f} / " + f"{benchmark.metrics.time_per_output_token_ms.successful.mean:.1f} / " # noqa: E501 + f"{benchmark.metrics.time_per_output_token_ms.successful.median:.1f} / " # noqa: E501 f"{benchmark.metrics.time_per_output_token_ms.successful.percentiles.p99:.1f}" ), ] From c449bdebb423eec4450cfc3d5c2d27cf7c959192 Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Fri, 11 Apr 2025 00:25:24 -0400 Subject: [PATCH 36/43] Fix double quotes inside a double qoute f-string f"{"test"}" is valid in 3.12 but not in older pythons --- src/guidellm/benchmark/output.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/guidellm/benchmark/output.py b/src/guidellm/benchmark/output.py index a75cbb38..327275f7 100644 --- a/src/guidellm/benchmark/output.py +++ b/src/guidellm/benchmark/output.py @@ -254,8 +254,8 @@ def print_benchmarks_info(self): rows.append( [ strategy_display_str(benchmark.args.strategy), - f"{datetime.fromtimestamp(benchmark.start_time).strftime("%H:%M:%S")}", - f"{datetime.fromtimestamp(benchmark.end_time).strftime("%H:%M:%S")}", + f"{datetime.fromtimestamp(benchmark.start_time).strftime('%H:%M:%S')}", + f"{datetime.fromtimestamp(benchmark.end_time).strftime('%H:%M:%S')}", f"{(benchmark.end_time - benchmark.start_time):.1f}", ( f"{benchmark.total_count.successful:>5} / " From 4cd904d7b8adb5905871e15bdec00b786109f1c5 Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Fri, 11 Apr 2025 00:28:24 -0400 Subject: [PATCH 37/43] importlib.resources.files requires valid module In python < 3.12 importlib.resources.files will fail if the module contains no python. --- src/guidellm/data/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 src/guidellm/data/__init__.py diff --git a/src/guidellm/data/__init__.py b/src/guidellm/data/__init__.py new file mode 100644 index 00000000..8a48204e --- /dev/null +++ b/src/guidellm/data/__init__.py @@ -0,0 +1,4 @@ +""" +Required for python < 3.12 +https://docs.python.org/3/library/importlib.resources.html#importlib.resources.files +""" From 48098fc2e274bff8433823459b67c56917936dd2 Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Fri, 11 Apr 2025 15:50:03 +0000 Subject: [PATCH 38/43] Fix for restructuring of output and general simplification based on feedback --- .github/workflows/development.yml | 8 +- .github/workflows/nightly.yml | 6 +- .github/workflows/quality.yml | 10 +- .github/workflows/release.yml | 6 +- src/guidellm/benchmark/__init__.py | 8 +- src/guidellm/benchmark/aggregator.py | 724 +++++++++++++------------- src/guidellm/benchmark/benchmark.py | 206 +++----- src/guidellm/benchmark/benchmarker.py | 83 ++- src/guidellm/benchmark/output.py | 30 +- src/guidellm/benchmark/progress.py | 67 ++- src/guidellm/config.py | 8 +- src/guidellm/objects/__init__.py | 9 +- src/guidellm/objects/pydantic.py | 38 +- src/guidellm/objects/statistics.py | 34 +- src/guidellm/scheduler/__init__.py | 6 +- src/guidellm/scheduler/result.py | 8 +- src/guidellm/scheduler/scheduler.py | 18 +- src/guidellm/scheduler/types.py | 6 +- src/guidellm/scheduler/worker.py | 26 +- tests/unit/objects/test_statistics.py | 14 - 20 files changed, 635 insertions(+), 680 deletions(-) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index f8604d0f..7ee36c49 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -11,8 +11,8 @@ jobs: strategy: matrix: python: - - "3.12" - - "3.8" + - "3.13" + - "3.9" steps: - uses: actions/checkout@v4 - name: Set up Python @@ -29,8 +29,8 @@ jobs: strategy: matrix: python: - - "3.12" - - "3.8" + - "3.13" + - "3.9" steps: - uses: actions/checkout@v4 - name: Set up Python diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index bf7f34cf..634ab52c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -10,11 +10,11 @@ jobs: strategy: matrix: python: + - "3.13" - "3.12" - "3.11" - "3.10" - "3.9" - - "3.8" steps: - uses: actions/checkout@v4 - name: Set up Python @@ -31,11 +31,11 @@ jobs: strategy: matrix: python: + - "3.13" - "3.12" - "3.11" - "3.10" - "3.9" - - "3.8" steps: - uses: actions/checkout@v4 - name: Set up Python @@ -52,11 +52,11 @@ jobs: strategy: matrix: python: + - "3.13" - "3.12" - "3.11" - "3.10" - "3.9" - - "3.8" steps: - uses: actions/checkout@v4 - name: Set up Python diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 95d44af4..5060149a 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -14,8 +14,8 @@ jobs: strategy: matrix: python: - - "3.12" - - "3.8" + - "3.13" + - "3.9" steps: - uses: actions/checkout@v4 - name: Set up Python @@ -32,8 +32,8 @@ jobs: strategy: matrix: python: - - "3.12" - - "3.8" + - "3.13" + - "3.9" steps: - uses: actions/checkout@v4 - name: Set up Python @@ -52,7 +52,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.9" - name: Install pre-commit run: pip install pre-commit - name: Run pre-commit checks diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d4fe2494..c7c7b8f7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,11 +11,11 @@ jobs: strategy: matrix: python: + - "3.13" - "3.12" - "3.11" - "3.10" - "3.9" - - "3.8" steps: - uses: actions/checkout@v4 - name: Set up Python @@ -32,11 +32,11 @@ jobs: strategy: matrix: python: + - "3.13" - "3.12" - "3.11" - "3.10" - "3.9" - - "3.8" steps: - uses: actions/checkout@v4 - name: Set up Python @@ -53,11 +53,11 @@ jobs: strategy: matrix: python: + - "3.13" - "3.12" - "3.11" - "3.10" - "3.9" - - "3.8" steps: - uses: actions/checkout@v4 - name: Set up Python diff --git a/src/guidellm/benchmark/__init__.py b/src/guidellm/benchmark/__init__.py index a9da9e80..dc100596 100644 --- a/src/guidellm/benchmark/__init__.py +++ b/src/guidellm/benchmark/__init__.py @@ -1,5 +1,5 @@ -from .aggregator import AGG, BenchmarkAggregator, GenerativeBenchmarkAggregator -from .benchmark import BENCH, Benchmark, GenerativeBenchmark +from .aggregator import AggregatorT, BenchmarkAggregator, GenerativeBenchmarkAggregator +from .benchmark import Benchmark, BenchmarkT, GenerativeBenchmark from .benchmarker import Benchmarker, BenchmarkerResult, GenerativeBenchmarker from .entrypoints import benchmark_generative_text from .profile import ( @@ -14,8 +14,8 @@ ) __all__ = [ - "AGG", - "BENCH", + "AggregatorT", + "BenchmarkT", "Benchmark", "BenchmarkAggregator", "GenerativeBenchmark", diff --git a/src/guidellm/benchmark/aggregator.py b/src/guidellm/benchmark/aggregator.py index d7273a02..3d050a87 100644 --- a/src/guidellm/benchmark/aggregator.py +++ b/src/guidellm/benchmark/aggregator.py @@ -17,177 +17,48 @@ from guidellm.backend import ResponseSummary from guidellm.benchmark.benchmark import ( - BENCH, BenchmarkArgs, BenchmarkRunStats, + BenchmarkT, GenerativeBenchmark, GenerativeTextErrorStats, GenerativeTextResponseStats, ) -from guidellm.benchmark.profile import ( - AsyncProfile, - ConcurrentProfile, - Profile, - SweepProfile, - SynchronousProfile, - ThroughputProfile, -) from guidellm.config import settings -from guidellm.objects import RunningStats, StandardBaseModel, TimeRunningStats +from guidellm.objects import ( + RunningStats, + StandardBaseModel, + StatusBreakdown, + TimeRunningStats, +) from guidellm.request import ( GenerationRequest, GenerativeRequestLoaderDescription, RequestLoaderDescription, ) from guidellm.scheduler import ( - REQ, - RES, - AsyncConstantStrategy, - AsyncPoissonStrategy, - ConcurrentStrategy, GenerativeRequestsWorkerDescription, + RequestT, + ResponseT, SchedulerRequestResult, - SchedulingStrategy, - SynchronousStrategy, - ThroughputStrategy, WorkerDescription, ) from guidellm.utils import check_load_processor __all__ = [ - "AGG", + "AggregatorT", "BenchmarkAggregator", "GenerativeBenchmarkAggregator", ] -class BenchmarkAggregator(ABC, StandardBaseModel, Generic[BENCH, REQ, RES]): +class SchedulerRunningStats(StandardBaseModel): """ - A pydantic base class representing the base class for aggregating benchmark results. - The purpose is to receive and process results from a Benchmarker as it iterates - through a Scheduler for an individual benchmark run. - As results are added, lightweight statistics are updated and stored for immediate - progress and informational updates to the caller. - Once the benchmark run is complete, the `compile` method is called to finalize - the benchmark and return a Benchmark object with all the results and statistics - fully calculated. + The metrics for the scheduler stored as running statistics for easy calculations + of rates, averages, totals, etc. """ - type_: Literal["benchmark_aggregator"] = "benchmark_aggregator" - run_id: str = Field( - description=( - "The unique identifier for the encompasing benchmark run that this " - "benchmark was a part of." - ) - ) - profile: Union[ - AsyncProfile, - SweepProfile, - ConcurrentProfile, - ThroughputProfile, - SynchronousProfile, - Profile, - ] = Field( - description=( - "The profile used for the entire benchamrk run that the strategy for " - "the active benchmark was pulled from." - ), - discriminator="type_", - ) - strategy_index: int = Field( - description=( - "The index of the strategy in the profile that was used for this benchmark." - ) - ) - strategy: Union[ - ConcurrentStrategy, - SchedulingStrategy, - ThroughputStrategy, - SynchronousStrategy, - AsyncPoissonStrategy, - AsyncConstantStrategy, - SchedulingStrategy, - ] = Field( - description="The scheduling strategy used to run this benchmark. ", - discriminator="type_", - ) - max_number: Optional[int] = Field( - description="The maximum number of requests to run for this benchmark, if any." - ) - max_duration: Optional[float] = Field( - description="The maximum duration in seconds to run this benchmark, if any." - ) - warmup_number: Optional[int] = Field( - description=( - "The number of requests to run for the warmup phase of this benchmark, " - "if any. These are requests that were not included in the final results." - ) - ) - warmup_duration: Optional[float] = Field( - description=( - "The duration in seconds to run for the warmup phase of this benchmark, " - "if any. These are requests that were not included in the final results." - ) - ) - cooldown_number: Optional[int] = Field( - description=( - "The number of requests to run for the cooldown phase of this benchmark, " - "if any. These are requests that were not included in the final results." - ) - ) - cooldown_duration: Optional[float] = Field( - description=( - "The duration in seconds to run for the cooldown phase of this benchmark, " - "if any. These are requests that were not included in the final results." - ) - ) - worker_description: Union[ - GenerativeRequestsWorkerDescription, WorkerDescription - ] = Field( - description=( - "The description and specifics for the worker used to resolve requests " - "for this benchmark." - ), - discriminator="type_", - ) - request_loader_description: Union[ - GenerativeRequestLoaderDescription, RequestLoaderDescription - ] = Field( - description=( - "The description and specifics for the request loader used to create " - "requests for this benchmark." - ), - discriminator="type_", - ) - extras: Dict[str, Any] = Field( - description=( - "Any additional information or metadata that was passed for this benchmark." - ) - ) - - results: List[SchedulerRequestResult[REQ, RES]] = Field( - default_factory=list, - description=( - "The list of all results from the benchmark (complete, incomplete, error), " - "that were not within the warmup or cooldown periods." - ), - ) - in_warmup: bool = Field( - description=( - "A flag to indicate if the benchmark is currently in the warmup phase." - ), - default=False, - exclude=True, - ) - in_cooldown: bool = Field( - description=( - "A flag to indicate if the benchmark is currently in the cooldown phase." - ), - default=False, - exclude=True, - ) - - scheduler_created_requests: RunningStats = Field( + created_requests: RunningStats = Field( description=( "The running statistics for the number of requests created for this " "benchmark run. This includes all requests created, regardless of " @@ -195,7 +66,7 @@ class BenchmarkAggregator(ABC, StandardBaseModel, Generic[BENCH, REQ, RES]): ), default_factory=RunningStats, ) - scheduler_queued_requests: RunningStats = Field( + queued_requests: RunningStats = Field( description=( "The running statistics for the number of requests pending in queue " "for this benchmark run. This includes requests that are waiting to " @@ -203,21 +74,21 @@ class BenchmarkAggregator(ABC, StandardBaseModel, Generic[BENCH, REQ, RES]): ), default_factory=RunningStats, ) - scheduler_scheduled_requests: RunningStats = Field( + scheduled_requests: RunningStats = Field( description=( "The running statistics for the number of requests scheduled (actively " "running but waiting for the desired start time) for this benchmark run." ), default_factory=RunningStats, ) - scheduler_processing_requests: RunningStats = Field( + processing_requests: RunningStats = Field( description=( "The running statistics for the number of requests actively being " "processed by the worker for this benchmark run." ), default_factory=RunningStats, ) - scheduler_completed_requests: RunningStats = Field( + completed_requests: RunningStats = Field( description=( "The running statistics for the number of requests completed for this " "benchmark run. This includes requests within the warmup and cooldown " @@ -226,34 +97,27 @@ class BenchmarkAggregator(ABC, StandardBaseModel, Generic[BENCH, REQ, RES]): default_factory=RunningStats, ) - successful_requests: RunningStats = Field( - description=( - "The running statistics for the number of requests that completed " - "successfully without error. This is a subset of the completed requests " - "for any that did not error. This includes requests within the warmup " - "and cooldown period, if any, along with the final results." - ), - default_factory=RunningStats, - ) - incomplete_requests: RunningStats = Field( - description=( - "The running statistics for the number of requests that were incomplete " - "or preempted during processing. This includes requests " - "within the warmup and cooldown period, if any, along with the final " - "results." - ), - default_factory=RunningStats, - ) - errored_requests: RunningStats = Field( - description=( - "The running statistics for the number of requests that errored during " - "processing. This is a subset of the completed requests for any that " - "errored. This includes requests within the warmup and cooldown period, " - "if any, along with the final results." - ), - default_factory=RunningStats, - ) +class RequestsRunningStats(StandardBaseModel): + """ + The metrics for requests that have succeeded, been canceled, or errored stored + as running statistics for easy calculations of rates, averages, totals, etc. + """ + + totals: StatusBreakdown[RunningStats, RunningStats, RunningStats, RunningStats] = ( + Field( + description=( + "The running statistics for the total number of requests that " + "completed within the benchmark run." + ), + default_factory=lambda: StatusBreakdown( + successful=RunningStats(), + errored=RunningStats(), + incomplete=RunningStats(), + total=RunningStats(), + ), + ) + ) queued_time: TimeRunningStats = Field( description=( "The running statistics for the time spent in queue for all requests that " @@ -348,9 +212,106 @@ class BenchmarkAggregator(ABC, StandardBaseModel, Generic[BENCH, REQ, RES]): default_factory=TimeRunningStats, ) + +class BenchmarkAggregator( + ABC, StandardBaseModel, Generic[BenchmarkT, RequestT, ResponseT] +): + """ + A pydantic base class representing the base class for aggregating benchmark results. + The purpose is to receive and process results from a Benchmarker as it iterates + through a Scheduler for an individual benchmark run. + As results are added, lightweight statistics are updated and stored for immediate + progress and informational updates to the caller. + Once the benchmark run is complete, the `compile` method is called to finalize + the benchmark and return a Benchmark object with all the results and statistics + fully calculated. + """ + + type_: Literal["benchmark_aggregator"] = "benchmark_aggregator" + run_id: str = Field( + description=( + "The unique identifier for the encompasing benchmark run that this " + "benchmark was a part of." + ) + ) + args: BenchmarkArgs = Field( + description=( + "The arguments used to create the benchmark run that this benchmark was " + "a part of." + ) + ) + worker_description: Union[ + GenerativeRequestsWorkerDescription, WorkerDescription + ] = Field( + description=( + "The description and specifics for the worker used to resolve requests " + "for this benchmark." + ), + discriminator="type_", + ) + request_loader_description: Union[ + GenerativeRequestLoaderDescription, RequestLoaderDescription + ] = Field( + description=( + "The description and specifics for the request loader used to create " + "requests for this benchmark." + ), + discriminator="type_", + ) + extras: Dict[str, Any] = Field( + description=( + "Any additional information or metadata that was passed for this benchmark." + ) + ) + in_warmup: bool = Field( + description=( + "A flag to indicate if the benchmark is currently in the warmup phase." + ), + default=False, + exclude=True, + ) + in_cooldown: bool = Field( + description=( + "A flag to indicate if the benchmark is currently in the cooldown phase." + ), + default=False, + exclude=True, + ) + scheduler_stats: SchedulerRunningStats = Field( + description=( + "The running statistics for the scheduler for this benchmark run. " + "This includes all requests created, regardless of their status." + ), + default_factory=SchedulerRunningStats, + ) + requests_stats: RequestsRunningStats = Field( + description=( + "The running statistics for the requests for this benchmark run. " + "This includes all requests created, regardless of their status." + ), + default_factory=RequestsRunningStats, + ) + results: StatusBreakdown[ + List[SchedulerRequestResult[RequestT, ResponseT]], + List[SchedulerRequestResult[RequestT, ResponseT]], + List[SchedulerRequestResult[RequestT, ResponseT]], + None, + ] = Field( + description=( + "The completed requests for this benchmark run broken down by status" + "and excluding warmup and cooldown requests." + ), + default_factory=lambda: StatusBreakdown( + successful=[], + errored=[], + incomplete=[], + total=None, + ), + ) + def add_result( self, - result: SchedulerRequestResult[REQ, RES], + result: SchedulerRequestResult[RequestT, ResponseT], ) -> bool: """ Add a result to the aggregator. This will update the internal statistics @@ -362,14 +323,20 @@ def add_result( did not fit within the warmup or cooldown period, was not requested, or is not finished """ - # Add base scheduler statistics to the aggregator - self.scheduler_created_requests += max(0, result.run_info.created_requests) - self.scheduler_queued_requests += max(0, result.run_info.queued_requests) - self.scheduler_scheduled_requests += max(0, result.run_info.scheduled_requests) - self.scheduler_processing_requests += max( + # Add scheduler statistics + self.scheduler_stats.created_requests += max( + 0, result.run_info.created_requests + ) + self.scheduler_stats.queued_requests += max(0, result.run_info.queued_requests) + self.scheduler_stats.scheduled_requests += max( + 0, result.run_info.scheduled_requests + ) + self.scheduler_stats.processing_requests += max( 0, result.run_info.processing_requests ) - self.scheduler_completed_requests += max(0, result.run_info.completed_requests) + self.scheduler_stats.completed_requests += max( + 0, result.run_info.completed_requests + ) if result.type_ != "request_complete" or ( result.request_info.canceled and not result.request_info.requested @@ -378,23 +345,25 @@ def add_result( # If the result was canceled and not started, ignore it return False - # add base result statistics given this was not preempted and it's completed - if result.request_info.completed: - self.successful_requests += 1 - elif result.request_info.canceled: - self.incomplete_requests += 1 + # Add request statistics + self.requests_stats.totals.total += 1 + if result.request_info.canceled: + self.requests_stats.totals.incomplete += 1 elif result.request_info.errored: - self.errored_requests += 1 + self.requests_stats.totals.errored += 1 + elif result.request_info.completed: + self.requests_stats.totals.successful += 1 else: raise ValueError( "Unexpected state: request_info must be either " - "completed, canceled, or errored." + "completed, canceled, or errored. " + f"Got {result.request_info}" ) - self.queued_time += ( # type: ignore[misc] + self.requests_stats.queued_time += ( result.request_info.dequeued_time - result.request_info.queued_time ) - self.scheduled_time_delay += ( # type: ignore[misc] + self.requests_stats.scheduled_time_delay += ( result.request_info.scheduled_time - result.request_info.dequeued_time ) sleep_time = max( @@ -402,57 +371,83 @@ def add_result( result.request_info.targeted_start_time - result.request_info.scheduled_time, ) - self.scheduled_time_sleep += sleep_time # type: ignore[misc] - time_to_worker_start = ( # type: ignore[misc] + self.requests_stats.scheduled_time_sleep += sleep_time + time_to_worker_start = ( result.request_info.worker_start - result.request_info.scheduled_time ) - self.worker_start_delay += time_to_worker_start - sleep_time # type: ignore[misc] - self.worker_time += ( # type: ignore[misc] + self.requests_stats.worker_start_delay += time_to_worker_start - sleep_time + self.requests_stats.worker_time += ( result.request_info.worker_end - result.request_info.worker_start ) - self.worker_start_time_targeted_delay += ( # type: ignore[misc] + self.requests_stats.worker_start_time_targeted_delay += ( + result.request_info.worker_start - result.request_info.targeted_start_time + ) + self.requests_stats.request_start_time_delay += ( + result.request_info.worker_start - result.request_info.targeted_start_time + ) + self.requests_stats.request_start_time_targeted_delay += ( result.request_info.worker_start - result.request_info.targeted_start_time ) + self.requests_stats.request_time_delay += ( + result.request_info.worker_end - result.request_info.worker_start + ) - (result.request_info.worker_end - result.request_info.worker_start) + self.requests_stats.request_time += ( + result.request_info.worker_end - result.request_info.worker_start + ) # Add result to the list of results provided we are not in warmup or cooldown - total_completed = ( - self.successful_requests.total - + self.incomplete_requests.total - + self.errored_requests.total - ) - global_start_time = self.scheduler_created_requests.start_time + total_completed = self.requests_stats.totals.total + global_start_time = self.requests_stats.totals.total.start_time - if (self.warmup_number and total_completed <= self.warmup_number) or ( - self.warmup_duration + in_warmup_number = ( + self.args.warmup_number and total_completed <= self.args.warmup_number + ) + in_warmup_duration = ( + self.args.warmup_duration and result.request_info.worker_start - <= (global_start_time + self.warmup_duration) - ): - # within warmup period + <= (global_start_time - self.args.warmup_duration) + ) + + if in_warmup_number or in_warmup_duration: self.in_warmup = True return True - if ( - self.cooldown_number - and self.max_number - and total_completed > self.max_number - self.cooldown_number - ) or ( - self.cooldown_duration - and self.max_duration + self.in_warmup = False + in_cooldown_number = ( + self.args.cooldown_number + and self.args.max_number + and total_completed > self.args.max_number - self.args.cooldown_number + ) + in_cooldown_duration = ( + self.args.cooldown_duration + and self.args.max_duration and result.request_info.worker_start - >= global_start_time + self.max_duration - self.cooldown_duration - ): - # within cooldown period + > global_start_time + self.args.max_duration - self.args.cooldown_duration + ) + + if in_cooldown_number or in_cooldown_duration: self.in_cooldown = True return True - self.in_warmup = False self.in_cooldown = False - self.results.append(result) + + if result.request_info.canceled: + self.results.incomplete.append(result) + elif result.request_info.errored: + self.results.errored.append(result) + elif result.request_info.completed: + self.results.successful.append(result) + else: + raise ValueError( + "Unexpected state: request_info must be either " + "completed, canceled, or errored. " + f"Got {result.request_info}" + ) return True @abstractmethod - def compile(self) -> BENCH: + def compile(self) -> BenchmarkT: """ Compile the benchmark results and statistics into a Benchmark object. This is required to be implemented by subclasses to finalize the benchmark @@ -461,27 +456,14 @@ def compile(self) -> BENCH: ... -AGG = TypeVar("AGG", bound=BenchmarkAggregator) +AggregatorT = TypeVar("AggregatorT", bound=BenchmarkAggregator) -class GenerativeBenchmarkAggregator( - BenchmarkAggregator[GenerativeBenchmark, GenerationRequest, ResponseSummary] -): - type_: Literal["generative_benchmark_aggregator"] = ( - "generative_benchmark_aggregator" # type: ignore[assignment] - ) - processor: Optional[Union[str, Path, Any]] = Field( - description=( - "The tokenizer to use for calculating token counts when none are " - "avaiable that match the preferred source." - ) - ) - processor_args: Optional[Dict[str, Any]] = Field( - description=( - "Additional arguments to pass to the tokenizer if it requires " - "any specific configuration for loading or processing." - ), - ) +class GenerativeRequestsRunningStats(RequestsRunningStats): + """ + The metrics for generative requests that have succeeded, been canceled, or errored + stored as running statistics for easy calculations of rates, averages, totals, etc. + """ time_to_first_token: TimeRunningStats = Field( description=( @@ -520,6 +502,33 @@ class GenerativeBenchmarkAggregator( default_factory=RunningStats, ) + +class GenerativeBenchmarkAggregator( + BenchmarkAggregator[GenerativeBenchmark, GenerationRequest, ResponseSummary] +): + type_: Literal["generative_benchmark_aggregator"] = ( + "generative_benchmark_aggregator" # type: ignore[assignment] + ) + processor: Optional[Union[str, Path, Any]] = Field( + description=( + "The tokenizer to use for calculating token counts when none are " + "avaiable that match the preferred source." + ) + ) + processor_args: Optional[Dict[str, Any]] = Field( + description=( + "Additional arguments to pass to the tokenizer if it requires " + "any specific configuration for loading or processing." + ), + ) + requests_stats: GenerativeRequestsRunningStats = Field( + description=( + "The running statistics for the requests for this benchmark run. " + "This includes all requests created, regardless of their status." + ), + default_factory=GenerativeRequestsRunningStats, + ) + def add_result( self, result: SchedulerRequestResult[GenerationRequest, ResponseSummary] ) -> bool: @@ -539,35 +548,35 @@ def add_result( if result.response is None: raise ValueError("Response is None, cannot add result.") - self.request_start_time_delay += ( # type: ignore[misc] + self.requests_stats.request_start_time_delay += ( result.response.start_time - result.request_info.worker_start ) - self.request_start_time_targeted_delay += ( # type: ignore[misc] + self.requests_stats.request_start_time_targeted_delay += ( result.response.start_time - result.request_info.targeted_start_time ) - self.request_time_delay += ( # type: ignore[misc] + self.requests_stats.request_time_delay += ( (result.response.start_time - result.request_info.worker_start) + result.request_info.worker_end - result.response.end_time ) - self.request_time += result.response.end_time - result.response.start_time # type: ignore[misc] - - self.time_to_first_token += ( # type: ignore[misc] - (result.response.first_iter_time - result.response.start_time) * 1000.0 - if result.response.first_iter_time - else 0.0 - ) - self.inter_token_latency.update( - (result.response.last_iter_time - result.response.first_iter_time) * 1000.0 - if result.response.last_iter_time and result.response.first_iter_time - else 0.0, - count=(result.response.output_tokens or 1) - 1, + self.requests_stats.request_time += ( + result.response.end_time - result.response.start_time ) - self.prompt_tokens += result.response.prompt_tokens or 0 - self.output_tokens += result.response.output_tokens or 0 - self.total_tokens += (result.response.prompt_tokens or 0) + ( - result.response.output_tokens or 0 + if result.response.first_iter_time: + self.requests_stats.time_to_first_token += ( + result.response.first_iter_time - result.response.start_time + ) + if result.response.last_iter_time and result.response.first_iter_time: + self.requests_stats.inter_token_latency.update( + result.response.last_iter_time - result.response.first_iter_time, + count=(result.response.output_tokens or 1) - 1, + ) + self.requests_stats.prompt_tokens += result.response.request_prompt_tokens or 0 + self.requests_stats.output_tokens += result.response.request_output_tokens or 0 + total_tokens = (result.response.request_prompt_tokens or 0) + ( + result.response.request_output_tokens or 0 ) + self.requests_stats.total_tokens += total_tokens return True @@ -584,33 +593,26 @@ def compile(self) -> GenerativeBenchmark: successful=successful, incomplete=incomplete, errored=errored, - args=BenchmarkArgs( - profile=self.profile, - strategy_index=self.strategy_index, - strategy=self.strategy, - max_number=self.max_number, - max_duration=self.max_duration, - warmup_number=self.warmup_number, - warmup_duration=self.warmup_duration, - cooldown_number=self.cooldown_number, - cooldown_duration=self.cooldown_duration, - ), + args=self.args, run_stats=BenchmarkRunStats( - start_time=self.scheduler_created_requests.start_time, + start_time=self.requests_stats.totals.total.start_time, end_time=time.time(), - total_successful=int(self.successful_requests.total), - total_incomplete=int(self.incomplete_requests.total), - total_errored=int(self.errored_requests.total), - queued_time_avg=self.queued_time.mean, - scheduled_time_delay_avg=self.scheduled_time_delay.mean, - scheduled_time_sleep_avg=self.scheduled_time_sleep.mean, - worker_start_delay_avg=self.worker_start_delay.mean, - worker_time_avg=self.worker_time.mean, - worker_start_time_targeted_delay_avg=self.worker_start_time_targeted_delay.mean, - request_start_time_delay_avg=self.request_start_time_delay.mean, - request_start_time_targeted_delay_avg=self.request_start_time_targeted_delay.mean, - request_time_delay_avg=self.request_time_delay.mean, - request_time_avg=self.request_time.mean, + requests_made=StatusBreakdown( + successful=self.requests_stats.totals.successful.total, + errored=self.requests_stats.totals.errored.total, + incomplete=self.requests_stats.totals.incomplete.total, + total=self.requests_stats.totals.total.total, + ), + queued_time_avg=self.requests_stats.queued_time.mean, + scheduled_time_delay_avg=self.requests_stats.scheduled_time_delay.mean, + scheduled_time_sleep_avg=self.requests_stats.scheduled_time_sleep.mean, + worker_start_delay_avg=self.requests_stats.worker_start_delay.mean, + worker_time_avg=self.requests_stats.worker_time.mean, + worker_start_time_targeted_delay_avg=self.requests_stats.worker_start_time_targeted_delay.mean, + request_start_time_delay_avg=self.requests_stats.request_start_time_delay.mean, + request_start_time_targeted_delay_avg=self.requests_stats.request_start_time_targeted_delay.mean, + request_time_delay_avg=self.requests_stats.request_time_delay.mean, + request_time_avg=self.requests_stats.request_time.mean, ), worker=self.worker_description, requests_loader=self.request_loader_description, @@ -624,82 +626,92 @@ def _compile_results( List[GenerativeTextErrorStats], List[GenerativeTextErrorStats], ]: - successful: List[GenerativeTextResponseStats] = [] - incomplete: List[GenerativeTextErrorStats] = [] - error: List[GenerativeTextErrorStats] = [] - - for result in self.results: - if result.request is None: - raise ValueError("Request is None, cannot compile results.") - - if result.response is None: - raise ValueError("Response is None, cannot compile results.") - - prompt_tokens = self._compile_tokens_count( - value=str(result.request.content), - requests_tokens=result.response.request_prompt_tokens, - response_tokens=result.response.response_prompt_tokens, - preferred_tokens_source=settings.preferred_prompt_tokens_source, - errored=result.request_info.errored, + successful: List[GenerativeTextResponseStats] = [ + GenerativeTextResponseStats( + request_id=result.request.request_id, + request_type=result.request.request_type, + scheduler_info=result.request_info, + prompt=str(result.request.content), + prompt_tokens=self._compile_tokens_count( + value=str(result.request.content), + requests_tokens=result.response.request_prompt_tokens, + response_tokens=result.response.response_prompt_tokens, + preferred_tokens_source=settings.preferred_prompt_tokens_source, + errored=False, + ), + output=result.response.value, + output_tokens=self._compile_tokens_count( + value=result.response.value, + requests_tokens=result.response.request_output_tokens, + response_tokens=result.response.response_output_tokens, + preferred_tokens_source=settings.preferred_output_tokens_source, + errored=False, + ), + start_time=result.response.start_time, + end_time=result.response.end_time, + first_token_time=result.response.first_iter_time or -1.0, + last_token_time=result.response.last_iter_time or -1.0, ) - output_tokens = self._compile_tokens_count( - value=result.response.value, - requests_tokens=result.response.request_output_tokens, - response_tokens=result.response.response_output_tokens, - preferred_tokens_source=settings.preferred_output_tokens_source, - errored=result.request_info.errored, + for result in self.results.successful + ] + incomplete: List[GenerativeTextErrorStats] = [ + GenerativeTextErrorStats( + error=result.response.error or "", + request_id=result.request.request_id, + request_type=result.request.request_type, + scheduler_info=result.request_info, + prompt=str(result.request.content), + prompt_tokens=self._compile_tokens_count( + value=str(result.request.content), + requests_tokens=result.response.request_prompt_tokens, + response_tokens=result.response.response_prompt_tokens, + preferred_tokens_source=settings.preferred_prompt_tokens_source, + errored=True, + ), + output=result.response.value, + output_tokens=self._compile_tokens_count( + value=result.response.value, + requests_tokens=result.response.request_output_tokens, + response_tokens=result.response.response_output_tokens, + preferred_tokens_source=settings.preferred_output_tokens_source, + errored=True, + ), + start_time=result.response.start_time, + end_time=result.response.end_time, + first_token_time=result.response.first_iter_time, + last_token_time=result.response.last_iter_time, ) - - if result.request_info.canceled: - incomplete.append( - GenerativeTextErrorStats( - error=result.response.error or "", - request_id=result.request.request_id, - request_type=result.request.request_type, - scheduler_info=result.request_info, - prompt=str(result.request.content), - prompt_tokens=prompt_tokens, - output=result.response.value, - output_tokens=output_tokens, - start_time=result.response.start_time, - end_time=result.response.end_time, - first_token_time=result.response.first_iter_time, - last_token_time=result.response.last_iter_time, - ) - ) - elif result.request_info.errored: - error.append( - GenerativeTextErrorStats( - error=result.response.error or "", - request_id=result.request.request_id, - request_type=result.request.request_type, - scheduler_info=result.request_info, - prompt=str(result.request.content), - prompt_tokens=prompt_tokens, - output=result.response.value, - output_tokens=output_tokens, - start_time=result.response.start_time, - end_time=result.response.end_time, - first_token_time=result.response.first_iter_time, - last_token_time=result.response.last_iter_time, - ) - ) - else: - successful.append( - GenerativeTextResponseStats( - request_id=result.request.request_id, - request_type=result.request.request_type, - scheduler_info=result.request_info, - prompt=str(result.request.content), - prompt_tokens=prompt_tokens, - output=result.response.value, - output_tokens=output_tokens, - start_time=result.response.start_time, - end_time=result.response.end_time, - first_token_time=result.response.first_iter_time or -1, - last_token_time=result.response.last_iter_time or -1, - ) - ) + for result in self.results.incomplete + ] + error: List[GenerativeTextErrorStats] = [ + GenerativeTextErrorStats( + error=result.response.error or "", + request_id=result.request.request_id, + request_type=result.request.request_type, + scheduler_info=result.request_info, + prompt=str(result.request.content), + prompt_tokens=self._compile_tokens_count( + value=str(result.request.content), + requests_tokens=result.response.request_prompt_tokens, + response_tokens=result.response.response_prompt_tokens, + preferred_tokens_source=settings.preferred_prompt_tokens_source, + errored=True, + ), + output=result.response.value, + output_tokens=self._compile_tokens_count( + value=result.response.value, + requests_tokens=result.response.request_output_tokens, + response_tokens=result.response.response_output_tokens, + preferred_tokens_source=settings.preferred_output_tokens_source, + errored=True, + ), + start_time=result.response.start_time, + end_time=result.response.end_time, + first_token_time=result.response.first_iter_time, + last_token_time=result.response.last_iter_time, + ) + for result in self.results.errored + ] return successful, incomplete, error diff --git a/src/guidellm/benchmark/benchmark.py b/src/guidellm/benchmark/benchmark.py index 388ddd6a..f1f9187c 100644 --- a/src/guidellm/benchmark/benchmark.py +++ b/src/guidellm/benchmark/benchmark.py @@ -1,9 +1,8 @@ import random import uuid -from typing import Any, Dict, Generic, List, Literal, Optional, Union +from typing import Any, Dict, List, Literal, Optional, TypeVar, Union from pydantic import Field, computed_field -from typing_extensions import TypeVar from guidellm.benchmark.profile import ( AsyncProfile, @@ -15,6 +14,7 @@ ) from guidellm.objects import ( StandardBaseModel, + StatusBreakdown, StatusDistributionSummary, ) from guidellm.request import ( @@ -34,7 +34,7 @@ ) __all__ = [ - "BENCH", + "BenchmarkT", "StatusBreakdown", "BenchmarkArgs", "BenchmarkRunStats", @@ -47,26 +47,6 @@ ] -SuccessfulT = TypeVar("SuccessfulT", default=Any) -ErroredT = TypeVar("ErroredT", default=SuccessfulT) -IncompleteT = TypeVar("IncompleteT", default=ErroredT) -class StatusBreakdown(StandardBaseModel, Generic[SuccessfulT, ErroredT, IncompleteT]): - """ - A serializable model representing the breakdown of statistics for a benchmark run - split into successful, incomplete, and errored. - """ - - successful: SuccessfulT = Field( - description="Successful", - ) - incomplete: IncompleteT = Field( - description="Incomplete", - ) - errored: ErroredT = Field( - description="Errored", - ) - - class BenchmarkArgs(StandardBaseModel): """ A serializable model representing the arguments used to specify a benchmark run @@ -148,26 +128,12 @@ class BenchmarkRunStats(StandardBaseModel): end_time: float = Field( description="The end time of the benchmark run.", ) - - total_successful: int = Field( - description=( - "The total number of successful requests in the benchmark run, " - "including warmup and cooldown." - ), - ) - total_incomplete: int = Field( - description=( - "The total number of incomplete requests in the benchmark run, " - "including warmup and cooldown." - ) - ) - total_errored: int = Field( + requests_made: StatusBreakdown[int, int, int, int] = Field( description=( - "The total number of errored requests in the benchmark run, " - "including warmup and cooldown." + "The number of requests made for the benchmark run broken down by " + "status including successful, incomplete, errored, and the sum of all three" ) ) - queued_time_avg: float = Field( description=( "The average time spent in the queue for each request in the benchmark " @@ -248,15 +214,6 @@ class BenchmarkRunStats(StandardBaseModel): ) ) - @computed_field # type: ignore[misc] - @property - def total(self) -> int: - """ - :return: The total number of requests in the benchmark run, including - warmup and cooldown. - """ - return self.total_successful + self.total_incomplete + self.total_errored - class BenchmarkMetrics(StandardBaseModel): """ @@ -304,30 +261,23 @@ class Benchmark(StandardBaseModel): "The process statistics for the entire benchmark run across all requests." ) ) - worker: Optional[Union[GenerativeRequestsWorkerDescription, WorkerDescription]] = ( - Field( - description=( - "The description and specifics for the worker used to resolve requests " - "for this benchmark." - ), - discriminator="type_", - ) + worker: Union[WorkerDescription] = Field( + description=( + "The description and specifics for the worker used to resolve requests " + "for this benchmark." + ), ) - request_loader: Optional[ - Union[GenerativeRequestLoaderDescription, RequestLoaderDescription] - ] = Field( + request_loader: Union[RequestLoaderDescription] = Field( description=( "The description and specifics for the request loader used to create " "requests for this benchmark." ), - discriminator="type_", ) extras: Dict[str, Any] = Field( description=( "Any additional information or metadata that was passed for this benchmark." ) ) - metrics: BenchmarkMetrics = Field( description=( "The metrics for the benchmark run represented as a distribution of " @@ -336,7 +286,7 @@ class Benchmark(StandardBaseModel): ) -BENCH = TypeVar("BENCH", bound=Benchmark) +BenchmarkT = TypeVar("BenchmarkT", bound=Benchmark) class GenerativeTextResponseStats(StandardBaseModel): @@ -604,25 +554,13 @@ class GenerativeBenchmark(Benchmark): """ type_: Literal["generative_benchmark"] = "generative_benchmark" # type: ignore[assignment] - total_count: StatusBreakdown[int] = Field( - description=( - "The total number of requests in the benchmark, " - "excluding warmup and cooldown." - ) - ) - sampled_size: Optional[StatusBreakdown[int]] = Field( - default=None, - description=( - "The number of requests that were randomly sampled for " - "the benchmark. None if no sampling was applied." - ), - ) start_time: float = Field( description="The start time of the first request for the benchmark.", ) end_time: float = Field( description="The end time of the last request for the benchmark.", ) + @computed_field # type: ignore[misc] @property def duration(self) -> float: @@ -632,99 +570,82 @@ def duration(self) -> float: """ return self.end_time - self.start_time + worker: GenerativeRequestsWorkerDescription = Field( + description=( + "The description and specifics for the worker used to resolve requests " + "for this benchmark." + ), + ) + request_loader: GenerativeRequestLoaderDescription = Field( + description=( + "The description and specifics for the request loader used to create " + "requests for this benchmark." + ), + ) metrics: GenerativeMetrics = Field( description=( "The metrics for the benchmark run represented as a distribution of " "various per-request statistics." ), ) - # Output is ordered so keep this at the end + # Output is ordered so keep the requests at the end for better readability in files + request_totals: StatusBreakdown[int, int, int, int] = Field( + description=( + "The number of requests made for the benchmark broken down by status " + "including successful, incomplete, errored, and the sum of all three" + ) + ) + request_samples: Optional[StatusBreakdown[int, int, int, None]] = Field( + description=( + "The number of requests that were randomly sampled for " + "the benchmark. None if no sampling was applied." + ), + default=None, + ) requests: StatusBreakdown[ List[GenerativeTextResponseStats], List[GenerativeTextErrorStats], + List[GenerativeTextErrorStats], + None, ] = Field( description=( - "The breakdown of requests for the benchmark run including completed, " + "The breakdown of requests for the benchmark run including successful, " "incomplete, and errored requests." ), ) - def create_sampled( - self, sample_size: int, error_sample_size: Optional[int] = None - ) -> "GenerativeBenchmark": + def create_sampled(self, sample_size: int) -> "GenerativeBenchmark": """ Create a new benchmark instance with a random sample of the completed and errored requests based on the given sample sizes. If the sample sizes are larger than the total number of requests, the sample sizes are capped at the total number of requests. - :param sample_size: The number of completed requests to sample. - :param error_sample_size: The number of errored requests to sample. - If None, defaults to the sample_size. + :param sample_size: The number of requests to sample for each status type. :return: A new benchmark instance with the sampled requests. - :raises ValueError: If the sample sizes are negative or if the - GenerativeBenchmark has already been sampled and the requested sample - sizes are larger than the previously sampled sizes. + :raises ValueError: If the sample sizes are negative. """ - if error_sample_size is None: - error_sample_size = sample_size - if sample_size < 0: raise ValueError(f"Sample size must be non-negative, given {sample_size}") - if error_sample_size < 0: - raise ValueError( - f"Error sample size must be non-negative, given {error_sample_size}" - ) - - if ( - self.sampled_size is not None - and sample_size > self.sampled_size.successful - ): - raise ValueError( - "The benchmark's completed response have already been sampled with " - f"size {self.sampled_size.successful} and cannot be resampled with " - f"a larger size, given: {sample_size}" - ) - if ( - self.sampled_size is not None - and sample_size > self.sampled_size.incomplete - ): - raise ValueError( - "The benchmark's incomplete response have already been sampled with " - f"size {self.sampled_size.incomplete} and cannot be resampled with " - f"a larger size, given: {sample_size}" - ) - if ( - self.sampled_size is not None - and error_sample_size > self.sampled_size.errored - ): - raise ValueError( - "The benchmark's errored response have already been sampled with " - f"size {self.sampled_size.errored} and cannot be resampled with " - f"a larger size, given: {error_sample_size}" - ) sample_size = min(sample_size, len(self.requests.successful)) + error_sample_size = min(sample_size, len(self.requests.errored)) incomplete_sample_size = min(sample_size, len(self.requests.incomplete)) - error_sample_size = min(error_sample_size, len(self.requests.errored)) sampled_instance = self.model_copy() - sampled_instance.sampled_size = StatusBreakdown( - successful=0, - incomplete=0, - errored=0, - ) - sampled_instance.sampled_size.successful = sample_size sampled_instance.requests.successful = random.sample( self.requests.successful, sample_size ) - sampled_instance.sampled_size.incomplete = incomplete_sample_size + sampled_instance.requests.errored = random.sample( + self.requests.errored, error_sample_size + ) sampled_instance.requests.incomplete = random.sample( self.requests.incomplete, incomplete_sample_size ) - sampled_instance.sampled_size.errored = error_sample_size - sampled_instance.requests.errored = random.sample( - self.requests.errored, error_sample_size + sampled_instance.request_samples = StatusBreakdown( + successful=len(sampled_instance.requests.successful), + incomplete=len(sampled_instance.requests.incomplete), + errored=len(sampled_instance.requests.errored), ) return sampled_instance @@ -737,8 +658,8 @@ def from_stats( errored: List[GenerativeTextErrorStats], args: BenchmarkArgs, run_stats: BenchmarkRunStats, - worker: WorkerDescription, - requests_loader: RequestLoaderDescription, + worker: GenerativeRequestsWorkerDescription, + requests_loader: GenerativeRequestLoaderDescription, extras: Optional[Dict[str, Any]], ) -> "GenerativeBenchmark": """ @@ -811,16 +732,11 @@ def from_stats( run_id=run_id, args=args, run_stats=run_stats, - worker=worker, - request_loader=requests_loader, extras=extras or {}, - total_count=StatusBreakdown( - successful=len(successful), - incomplete=len(incomplete), - errored=len(errored), - ), start_time=start_time, end_time=end_time, + worker=worker, + request_loader=requests_loader, metrics=GenerativeMetrics( requests_per_second=StatusDistributionSummary.from_request_times( request_types=total_types, @@ -898,6 +814,12 @@ def from_stats( ], ), ), + request_totals=StatusBreakdown( + successful=len(successful), + incomplete=len(incomplete), + errored=len(errored), + total=len(total), + ), requests=StatusBreakdown( successful=successful, incomplete=incomplete, diff --git a/src/guidellm/benchmark/benchmarker.py b/src/guidellm/benchmark/benchmarker.py index 9e53130b..71d7c89a 100644 --- a/src/guidellm/benchmark/benchmarker.py +++ b/src/guidellm/benchmark/benchmarker.py @@ -17,16 +17,20 @@ from transformers import PreTrainedTokenizer # type: ignore # noqa: PGH003 from guidellm.backend import Backend, ResponseSummary -from guidellm.benchmark.aggregator import AGG, BENCH, GenerativeBenchmarkAggregator -from guidellm.benchmark.benchmark import GenerativeBenchmark +from guidellm.benchmark.aggregator import ( + AggregatorT, + BenchmarkT, + GenerativeBenchmarkAggregator, +) +from guidellm.benchmark.benchmark import BenchmarkArgs, GenerativeBenchmark from guidellm.benchmark.profile import Profile from guidellm.objects import StandardBaseModel from guidellm.request import GenerationRequest, RequestLoaderDescription from guidellm.scheduler import ( - REQ, - RES, GenerativeRequestsWorker, RequestsWorker, + RequestT, + ResponseT, Scheduler, SchedulerRequestResult, SchedulingStrategy, @@ -35,7 +39,9 @@ __all__ = ["Benchmarker", "BenchmarkerResult", "GenerativeBenchmarker"] -class BenchmarkerResult(StandardBaseModel, Generic[AGG, BENCH, REQ, RES]): +class BenchmarkerResult( + StandardBaseModel, Generic[AggregatorT, BenchmarkT, RequestT, ResponseT] +): type_: Literal[ "run_start", "run_complete", @@ -49,9 +55,9 @@ class BenchmarkerResult(StandardBaseModel, Generic[AGG, BENCH, REQ, RES]): profile: Profile current_index: int current_strategy: Optional[SchedulingStrategy] = None - current_aggregator: Optional[AGG] = None - current_benchmark: Optional[BENCH] = None - current_result: Optional[SchedulerRequestResult[REQ, RES]] = None + current_aggregator: Optional[AggregatorT] = None + current_benchmark: Optional[BenchmarkT] = None + current_result: Optional[SchedulerRequestResult[RequestT, ResponseT]] = None class BenchmarkerStrategyLimits(StandardBaseModel): @@ -120,16 +126,16 @@ def cooldown_duration(self) -> Optional[float]: return self.cooldown_percent_per_strategy * self.max_duration -class Benchmarker(Generic[AGG, BENCH, REQ, RES], ABC): +class Benchmarker(Generic[AggregatorT, BenchmarkT, RequestT, ResponseT], ABC): def __init__( self, - worker: RequestsWorker[REQ, RES], - request_loader: Iterable[REQ], + worker: RequestsWorker[RequestT, ResponseT], + request_loader: Iterable[RequestT], requests_loader_description: RequestLoaderDescription, benchmark_save_extras: Optional[Dict[str, Any]] = None, ): self.worker = worker - self.scheduler: Scheduler[REQ, RES] = Scheduler( + self.scheduler: Scheduler[RequestT, ResponseT] = Scheduler( worker=worker, request_loader=request_loader ) self.requests_loader_description = requests_loader_description @@ -142,7 +148,9 @@ async def run( max_duration_per_strategy: Optional[float], warmup_percent_per_strategy: Optional[float], cooldown_percent_per_strategy: Optional[float], - ) -> AsyncGenerator[BenchmarkerResult[AGG, BENCH, REQ, RES], None]: + ) -> AsyncGenerator[ + BenchmarkerResult[AggregatorT, BenchmarkT, RequestT, ResponseT], None + ]: try: requests_loader_size = len(self.scheduler.request_loader) # type: ignore[arg-type] except Exception: # noqa: BLE001 @@ -179,12 +187,7 @@ async def run( profile=profile, strategy_index=current_index, strategy=scheduling_strategy, - max_number=strategy_limits.max_number, - max_duration=strategy_limits.max_duration, - warmup_number=strategy_limits.warmup_number, - warmup_duration=strategy_limits.warmup_duration, - cooldown_number=strategy_limits.cooldown_number, - cooldown_duration=strategy_limits.cooldown_duration, + limits=strategy_limits, ) async for result in self.scheduler.run( @@ -233,7 +236,7 @@ async def run( else: raise ValueError(f"Unexpected result type: {type(result)}") - benchmark: BENCH = aggregator.compile() + benchmark: BenchmarkT = aggregator.compile() profile.completed_strategy( average_rate=benchmark.metrics.requests_per_second.successful.mean, average_concurrency=benchmark.metrics.request_concurrency.successful.mean, @@ -270,13 +273,8 @@ def create_benchmark_aggregator( profile: Profile, strategy_index: int, strategy: SchedulingStrategy, - max_number: Optional[int], - max_duration: Optional[float], - warmup_number: Optional[int], - warmup_duration: Optional[float], - cooldown_number: Optional[int], - cooldown_duration: Optional[float], - ) -> AGG: ... + limits: BenchmarkerStrategyLimits, + ) -> AggregatorT: ... class GenerativeBenchmarker( @@ -311,27 +309,24 @@ def create_benchmark_aggregator( profile: Profile, strategy_index: int, strategy: SchedulingStrategy, - max_number: Optional[int], - max_duration: Optional[float], - warmup_number: Optional[int], - warmup_duration: Optional[float], - cooldown_number: Optional[int], - cooldown_duration: Optional[float], + limits: BenchmarkerStrategyLimits, ) -> GenerativeBenchmarkAggregator: return GenerativeBenchmarkAggregator( - processor=self.processor, - processor_args=self.processor_args, run_id=run_id, - profile=profile, - strategy_index=strategy_index, - strategy=strategy, - max_number=max_number, - max_duration=max_duration, - warmup_number=warmup_number, - warmup_duration=warmup_duration, - cooldown_number=cooldown_number, - cooldown_duration=cooldown_duration, + args=BenchmarkArgs( + profile=profile, + strategy_index=strategy_index, + strategy=strategy, + max_number=limits.max_number, + max_duration=limits.max_duration, + warmup_number=limits.warmup_number, + warmup_duration=limits.warmup_duration, + cooldown_number=limits.cooldown_number, + cooldown_duration=limits.cooldown_duration, + ), worker_description=self.worker.description, request_loader_description=self.requests_loader_description, extras=self.benchmark_save_extras or {}, + processor=self.processor, + processor_args=self.processor_args, ) diff --git a/src/guidellm/benchmark/output.py b/src/guidellm/benchmark/output.py index 327275f7..d0bdc103 100644 --- a/src/guidellm/benchmark/output.py +++ b/src/guidellm/benchmark/output.py @@ -258,28 +258,28 @@ def print_benchmarks_info(self): f"{datetime.fromtimestamp(benchmark.end_time).strftime('%H:%M:%S')}", f"{(benchmark.end_time - benchmark.start_time):.1f}", ( - f"{benchmark.total_count.successful:>5} / " - f"{benchmark.total_count.incomplete} / " - f"{benchmark.total_count.errored}" + f"{benchmark.request_totals.successful:>5} / " + f"{benchmark.request_totals.incomplete} / " + f"{benchmark.request_totals.errored}" ), ( - f"{benchmark.metrics.prompt_token_count.successful.mean:>5.1f} / " # noqa: E501 + f"{benchmark.metrics.prompt_token_count.successful.mean:>5.1f} / " # noqa: E501 f"{benchmark.metrics.prompt_token_count.incomplete.mean:.1f} / " f"{benchmark.metrics.prompt_token_count.errored.mean:.1f}" ), ( - f"{benchmark.metrics.output_token_count.successful.mean:>5.1f} / " # noqa: E501 + f"{benchmark.metrics.output_token_count.successful.mean:>5.1f} / " # noqa: E501 f"{benchmark.metrics.output_token_count.incomplete.mean:.1f} / " f"{benchmark.metrics.output_token_count.errored.mean:.1f}" ), ( - f"{benchmark.metrics.prompt_token_count.successful.total_sum:>6.0f} / " # noqa: E501 - f"{benchmark.metrics.prompt_token_count.incomplete.total_sum:.0f} / " # noqa: E501 + f"{benchmark.metrics.prompt_token_count.successful.total_sum:>6.0f} / " # noqa: E501 + f"{benchmark.metrics.prompt_token_count.incomplete.total_sum:.0f} / " # noqa: E501 f"{benchmark.metrics.prompt_token_count.errored.total_sum:.0f}" ), ( - f"{benchmark.metrics.output_token_count.successful.total_sum:>6.0f} / " # noqa: E501 - f"{benchmark.metrics.output_token_count.incomplete.total_sum:.0f} / " # noqa: E501 + f"{benchmark.metrics.output_token_count.successful.total_sum:>6.0f} / " # noqa: E501 + f"{benchmark.metrics.output_token_count.incomplete.total_sum:.0f} / " # noqa: E501 f"{benchmark.metrics.output_token_count.errored.total_sum:.0f}" ), ] @@ -323,18 +323,18 @@ def print_benchmarks_stats(self): f"{benchmark.metrics.request_latency.successful.percentiles.p99:.2f}" ), ( - f"{benchmark.metrics.time_to_first_token_ms.successful.mean:.1f} / " # noqa: E501 - f"{benchmark.metrics.time_to_first_token_ms.successful.median:.1f} / " # noqa: E501 + f"{benchmark.metrics.time_to_first_token_ms.successful.mean:.1f} / " # noqa: E501 + f"{benchmark.metrics.time_to_first_token_ms.successful.median:.1f} / " # noqa: E501 f"{benchmark.metrics.time_to_first_token_ms.successful.percentiles.p99:.1f}" ), ( - f"{benchmark.metrics.inter_token_latency_ms.successful.mean:.1f} / " # noqa: E501 - f"{benchmark.metrics.inter_token_latency_ms.successful.median:.1f} / " # noqa: E501 + f"{benchmark.metrics.inter_token_latency_ms.successful.mean:.1f} / " # noqa: E501 + f"{benchmark.metrics.inter_token_latency_ms.successful.median:.1f} / " # noqa: E501 f"{benchmark.metrics.inter_token_latency_ms.successful.percentiles.p99:.1f}" ), ( - f"{benchmark.metrics.time_per_output_token_ms.successful.mean:.1f} / " # noqa: E501 - f"{benchmark.metrics.time_per_output_token_ms.successful.median:.1f} / " # noqa: E501 + f"{benchmark.metrics.time_per_output_token_ms.successful.mean:.1f} / " # noqa: E501 + f"{benchmark.metrics.time_per_output_token_ms.successful.median:.1f} / " # noqa: E501 f"{benchmark.metrics.time_per_output_token_ms.successful.percentiles.p99:.1f}" ), ] diff --git a/src/guidellm/benchmark/progress.py b/src/guidellm/benchmark/progress.py index 35e49a0e..059c4b06 100644 --- a/src/guidellm/benchmark/progress.py +++ b/src/guidellm/benchmark/progress.py @@ -481,10 +481,10 @@ def handle_update_scheduler_start( progress_state.started = True current_aggregator: BenchmarkAggregator = result.current_aggregator # type: ignore[assignment] progress_state.start_time = ( - current_aggregator.scheduler_created_requests.start_time + current_aggregator.requests_stats.totals.total.start_time ) - progress_state.max_number = current_aggregator.max_number - progress_state.max_duration = current_aggregator.max_duration + progress_state.max_number = current_aggregator.args.max_number + progress_state.max_duration = current_aggregator.args.max_duration def handle_update_scheduler_update( self, progress_state: BTPS, result: BenchmarkerResult @@ -498,31 +498,36 @@ def handle_update_scheduler_update( current_aggregator: BenchmarkAggregator = result.current_aggregator # type: ignore[assignment] progress_state.in_warmup = current_aggregator.in_warmup progress_state.in_cooldown = current_aggregator.in_cooldown - progress_state.requests_rate = current_aggregator.successful_requests.rate - progress_state.request_latency = current_aggregator.request_time.mean + progress_state.requests_rate = ( + current_aggregator.requests_stats.totals.successful.rate + ) + progress_state.request_latency = ( + current_aggregator.requests_stats.request_time.mean + ) progress_state.requests_processing = ( - current_aggregator.scheduler_processing_requests.last + current_aggregator.scheduler_stats.processing_requests.last ) progress_state.requests_successful = ( - current_aggregator.successful_requests.total + current_aggregator.requests_stats.totals.successful.total ) progress_state.requests_incomplete = ( - current_aggregator.incomplete_requests.total + current_aggregator.requests_stats.totals.incomplete.total + ) + progress_state.requests_errored = ( + current_aggregator.requests_stats.totals.errored.total ) - progress_state.requests_errored = current_aggregator.errored_requests.total - progress_state.worker_overheads_time_ms = ( - current_aggregator.scheduled_time_delay.mean_ms - + current_aggregator.worker_start_delay.mean_ms + current_aggregator.requests_stats.scheduled_time_delay.mean_ms + + current_aggregator.requests_stats.worker_start_delay.mean_ms ) progress_state.backend_overheads_time_ms = ( - current_aggregator.request_time_delay.mean_ms + current_aggregator.requests_stats.request_time_delay.mean_ms ) progress_state.requests_sleep_time_ms = ( - current_aggregator.scheduled_time_sleep.mean_ms + current_aggregator.requests_stats.scheduled_time_sleep.mean_ms ) progress_state.requests_targeted_start_time_delay_ms = ( - current_aggregator.request_start_time_targeted_delay.mean_ms + current_aggregator.requests_stats.request_start_time_targeted_delay.mean_ms ) def handle_update_scheduler_complete( @@ -623,12 +628,24 @@ def handle_update_scheduler_update( ): super().handle_update_scheduler_update(progress_state, result) current_aggregator: GenerativeBenchmarkAggregator = result.current_aggregator # type: ignore[assignment] - progress_state.output_tokens = current_aggregator.output_tokens.mean - progress_state.prompt_tokens = current_aggregator.prompt_tokens.mean - progress_state.output_tokens_rate = current_aggregator.output_tokens.rate - progress_state.total_tokens_rate = current_aggregator.total_tokens.rate - progress_state.tokens_ttft = current_aggregator.time_to_first_token.mean - progress_state.tokens_itl = current_aggregator.inter_token_latency.mean + progress_state.output_tokens = ( + current_aggregator.requests_stats.output_tokens.mean + ) + progress_state.prompt_tokens = ( + current_aggregator.requests_stats.prompt_tokens.mean + ) + progress_state.output_tokens_rate = ( + current_aggregator.requests_stats.output_tokens.rate + ) + progress_state.total_tokens_rate = ( + current_aggregator.requests_stats.total_tokens.rate + ) + progress_state.tokens_ttft = ( + current_aggregator.requests_stats.time_to_first_token.mean_ms + ) + progress_state.tokens_itl = ( + current_aggregator.requests_stats.inter_token_latency.mean_ms + ) def handle_update_benchmark_compiled( self, @@ -641,11 +658,9 @@ def handle_update_benchmark_compiled( progress_state.request_latency = ( current_benchmark.metrics.request_latency.successful.mean ) - progress_state.requests_processing = ( - current_benchmark.metrics.request_concurrency.successful.mean - ) - progress_state.requests_successful = current_benchmark.total_count.successful - progress_state.requests_errored = current_benchmark.total_count.errored + progress_state.requests_successful = current_benchmark.request_totals.successful + progress_state.requests_errored = current_benchmark.request_totals.errored + progress_state.requests_incomplete = current_benchmark.request_totals.incomplete progress_state.output_tokens = ( current_benchmark.metrics.output_token_count.successful.mean ) diff --git a/src/guidellm/config.py b/src/guidellm/config.py index f3d0e09b..ece9d63f 100644 --- a/src/guidellm/config.py +++ b/src/guidellm/config.py @@ -137,8 +137,12 @@ class Settings(BaseSettings): dataset: DatasetSettings = DatasetSettings() # Request/stats settings - preferred_prompt_tokens_source: Optional[Literal["request", "response"]] = None - preferred_output_tokens_source: Optional[Literal["request", "response"]] = None + preferred_prompt_tokens_source: Optional[ + Literal["request", "response", "local"] + ] = None + preferred_output_tokens_source: Optional[ + Literal["request", "response", "local"] + ] = None preferred_backend: Literal["openai"] = "openai" openai: OpenAISettings = OpenAISettings() diff --git a/src/guidellm/objects/__init__.py b/src/guidellm/objects/__init__.py index 9329a70f..168570dd 100644 --- a/src/guidellm/objects/__init__.py +++ b/src/guidellm/objects/__init__.py @@ -1,4 +1,4 @@ -from .pydantic import StandardBaseModel +from .pydantic import StandardBaseModel, StatusBreakdown from .statistics import ( DistributionSummary, Percentiles, @@ -8,10 +8,11 @@ ) __all__ = [ - "Percentiles", - "DistributionSummary", - "StatusDistributionSummary", "StandardBaseModel", + "StatusBreakdown", + "DistributionSummary", + "Percentiles", "RunningStats", + "StatusDistributionSummary", "TimeRunningStats", ] diff --git a/src/guidellm/objects/pydantic.py b/src/guidellm/objects/pydantic.py index 68b87b97..87e2cb99 100644 --- a/src/guidellm/objects/pydantic.py +++ b/src/guidellm/objects/pydantic.py @@ -1,9 +1,9 @@ -from typing import Any +from typing import Any, Generic, TypeVar from loguru import logger -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, Field -__all__ = ["StandardBaseModel"] +__all__ = ["StandardBaseModel", "StatusBreakdown"] class StandardBaseModel(BaseModel): @@ -26,3 +26,35 @@ def __init__(self, /, **data: Any) -> None: self.__class__.__name__, data, ) + + +SuccessfulT = TypeVar("SuccessfulT") +ErroredT = TypeVar("ErroredT") +IncompleteT = TypeVar("IncompleteT") +TotalT = TypeVar("TotalT") + + +class StatusBreakdown(BaseModel, Generic[SuccessfulT, ErroredT, IncompleteT, TotalT]): + """ + A base class for Pydantic models that are separated by statuses including + successful, incomplete, and errored. It additionally enables the inclusion + of total, which is intended as the combination of all statuses. + Total may or may not be used depending on if it duplicates information. + """ + + successful: SuccessfulT = Field( + description="The results with a successful status.", + default=None, + ) + errored: ErroredT = Field( + description="The results with an errored status.", + default=None, + ) + incomplete: IncompleteT = Field( + description="The results with an incomplete status.", + default=None, + ) + total: TotalT = Field( + description="The combination of all statuses.", + default=None, + ) diff --git a/src/guidellm/objects/statistics.py b/src/guidellm/objects/statistics.py index e192dee6..722f3d92 100644 --- a/src/guidellm/objects/statistics.py +++ b/src/guidellm/objects/statistics.py @@ -6,7 +6,7 @@ import numpy as np from pydantic import Field, computed_field -from guidellm.objects import StandardBaseModel +from guidellm.objects.pydantic import StandardBaseModel, StatusBreakdown __all__ = [ "Percentiles", @@ -401,7 +401,14 @@ def from_iterable_request_times( ) -class StatusDistributionSummary(StandardBaseModel): +class StatusDistributionSummary( + StatusBreakdown[ + DistributionSummary, + DistributionSummary, + DistributionSummary, + DistributionSummary, + ] +): """ A pydantic model representing a statistical summary for a given distribution of numerical values grouped by status. @@ -409,29 +416,6 @@ class StatusDistributionSummary(StandardBaseModel): and errored values for a benchmark or other statistical summary. """ - total: DistributionSummary = Field( - description=( - "The dist summary for all statuses (successful, incomplete, error)." - ), - ) - successful: DistributionSummary = Field( - description=( - "The distribution summary for successful statuses " - "(e.g., successful requests)." - ) - ) - incomplete: DistributionSummary = Field( - description=( - "The distribution summary for incomplete statuses " - "(e.g., requests that hit a timeout error and were unable to complete)." - ), - ) - errored: DistributionSummary = Field( - description=( - "The distribution summary for errored statuses (e.g., failed requests)." - ) - ) - @staticmethod def from_values( value_types: List[Literal["successful", "incomplete", "error"]], diff --git a/src/guidellm/scheduler/__init__.py b/src/guidellm/scheduler/__init__.py index 73256a60..e26f3bb3 100644 --- a/src/guidellm/scheduler/__init__.py +++ b/src/guidellm/scheduler/__init__.py @@ -15,7 +15,7 @@ ThroughputStrategy, strategy_display_str, ) -from .types import REQ, RES +from .types import RequestT, ResponseT from .worker import ( GenerativeRequestsWorker, GenerativeRequestsWorkerDescription, @@ -40,8 +40,8 @@ "SynchronousStrategy", "ThroughputStrategy", "strategy_display_str", - "REQ", - "RES", + "RequestT", + "ResponseT", "WorkerProcessRequest", "WorkerProcessResult", "ResolveStatus", diff --git a/src/guidellm/scheduler/result.py b/src/guidellm/scheduler/result.py index 8a60b077..ab1094ad 100644 --- a/src/guidellm/scheduler/result.py +++ b/src/guidellm/scheduler/result.py @@ -6,7 +6,7 @@ from guidellm.objects import StandardBaseModel from guidellm.scheduler.strategy import SchedulingStrategy -from guidellm.scheduler.types import REQ, RES +from guidellm.scheduler.types import RequestT, ResponseT __all__ = [ "SchedulerResult", @@ -124,7 +124,7 @@ class SchedulerResult(StandardBaseModel): class SchedulerRequestResult( SchedulerResult, - Generic[REQ, RES], + Generic[RequestT, ResponseT], ): pydantic_type: Literal["scheduler_request_result"] = "scheduler_request_result" # type: ignore[assignment] type_: Literal[ @@ -132,6 +132,6 @@ class SchedulerRequestResult( "request_start", "request_complete", ] - request: REQ + request: RequestT request_info: SchedulerRequestInfo - response: Optional[RES] = None + response: Optional[ResponseT] = None diff --git a/src/guidellm/scheduler/scheduler.py b/src/guidellm/scheduler/scheduler.py index 9c801d15..0be0ebb7 100644 --- a/src/guidellm/scheduler/scheduler.py +++ b/src/guidellm/scheduler/scheduler.py @@ -25,7 +25,7 @@ SchedulerRunInfo, ) from guidellm.scheduler.strategy import SchedulingStrategy -from guidellm.scheduler.types import REQ, RES +from guidellm.scheduler.types import RequestT, ResponseT from guidellm.scheduler.worker import ( RequestsWorker, WorkerProcessRequest, @@ -35,7 +35,7 @@ __all__ = ["Scheduler"] -class Scheduler(Generic[REQ, RES]): +class Scheduler(Generic[RequestT, ResponseT]): """ A class that handles the scheduling of requests to a worker. This class is responsible for managing the lifecycle of the requests, @@ -57,8 +57,8 @@ class Scheduler(Generic[REQ, RES]): def __init__( self, - worker: RequestsWorker[REQ, RES], - request_loader: Iterable[REQ], + worker: RequestsWorker[RequestT, ResponseT], + request_loader: Iterable[RequestT], ): if not isinstance(worker, RequestsWorker): raise ValueError(f"Invalid worker: {worker}") @@ -74,7 +74,9 @@ async def run( scheduling_strategy: SchedulingStrategy, max_number: Optional[int] = None, max_duration: Optional[float] = None, - ) -> AsyncGenerator[Union[SchedulerResult, SchedulerRequestResult[REQ, RES]], None]: + ) -> AsyncGenerator[ + Union[SchedulerResult, SchedulerRequestResult[RequestT, ResponseT]], None + ]: """ The main method that runs the scheduler. This method is a generator that yields SchedulerResult objects @@ -286,7 +288,7 @@ def _add_requests( raise StopIteration request = next(requests_iter) - work_req: WorkerProcessRequest[REQ] = WorkerProcessRequest( + work_req: WorkerProcessRequest[RequestT] = WorkerProcessRequest( request=request, start_time=request_time, timeout_time=run_info.end_time, @@ -308,9 +310,9 @@ def _check_result_ready( self, responses_queue: multiprocessing.Queue, run_info: SchedulerRunInfo, - ) -> Optional[SchedulerRequestResult[REQ, RES]]: + ) -> Optional[SchedulerRequestResult[RequestT, ResponseT]]: try: - process_response: WorkerProcessResult[REQ, RES] = ( + process_response: WorkerProcessResult[RequestT, ResponseT] = ( responses_queue.get_nowait() ) except multiprocessing.queues.Empty: # type: ignore[attr-defined] diff --git a/src/guidellm/scheduler/types.py b/src/guidellm/scheduler/types.py index 46ff4b9b..42535d71 100644 --- a/src/guidellm/scheduler/types.py +++ b/src/guidellm/scheduler/types.py @@ -1,7 +1,7 @@ from typing import TypeVar -__all__ = ["REQ", "RES"] +__all__ = ["RequestT", "ResponseT"] -REQ = TypeVar("REQ") -RES = TypeVar("RES") +RequestT = TypeVar("RequestT") +ResponseT = TypeVar("ResponseT") diff --git a/src/guidellm/scheduler/worker.py b/src/guidellm/scheduler/worker.py index 36c75fc8..44444c51 100644 --- a/src/guidellm/scheduler/worker.py +++ b/src/guidellm/scheduler/worker.py @@ -29,7 +29,7 @@ from guidellm.objects import StandardBaseModel from guidellm.request import GenerationRequest from guidellm.scheduler.result import SchedulerRequestInfo -from guidellm.scheduler.types import REQ, RES +from guidellm.scheduler.types import RequestT, ResponseT __all__ = [ "WorkerProcessRequest", @@ -43,18 +43,18 @@ @dataclass -class WorkerProcessRequest(Generic[REQ]): - request: REQ +class WorkerProcessRequest(Generic[RequestT]): + request: RequestT start_time: float timeout_time: float queued_time: float @dataclass -class WorkerProcessResult(Generic[REQ, RES]): +class WorkerProcessResult(Generic[RequestT, ResponseT]): type_: Literal["request_scheduled", "request_start", "request_complete"] - request: REQ - response: Optional[RES] + request: RequestT + response: Optional[ResponseT] info: SchedulerRequestInfo @@ -73,7 +73,7 @@ class WorkerDescription(StandardBaseModel): type_: Literal["worker"] = "worker" -class RequestsWorker(ABC, Generic[REQ, RES]): +class RequestsWorker(ABC, Generic[RequestT, ResponseT]): """ An abstract base class for a worker that processes requests. This class defines the interface for a worker that can resolve requests @@ -107,9 +107,9 @@ async def prepare_multiprocessing(self): @abstractmethod async def resolve( self, - request: REQ, + request: RequestT, timeout_time: float, - ) -> Tuple[ResolveStatus, RES]: + ) -> Tuple[ResolveStatus, ResponseT]: """ An abstract method that must be implemented by subclasses. This method should handle the resolution of a request through asyncio, @@ -124,13 +124,13 @@ async def resolve( async def get_request( self, requests_queue: multiprocessing.Queue - ) -> Optional[WorkerProcessRequest[REQ]]: + ) -> Optional[WorkerProcessRequest[RequestT]]: return await asyncio.to_thread(requests_queue.get) # type: ignore[attr-defined] async def send_result( self, results_queue: multiprocessing.Queue, - result: WorkerProcessResult[REQ, RES], + result: WorkerProcessResult[RequestT, ResponseT], ): await asyncio.to_thread(results_queue.put, result) # type: ignore[attr-defined] @@ -151,7 +151,7 @@ async def resolve_scheduler_request( scheduled_time=time.time(), process_id=process_id, ) - result: WorkerProcessResult[REQ, RES] = WorkerProcessResult( + result: WorkerProcessResult[RequestT, ResponseT] = WorkerProcessResult( type_="request_scheduled", request=request, response=None, @@ -177,6 +177,8 @@ async def resolve_scheduler_request( info.completed = status.completed info.errored = status.errored info.canceled = status.canceled + info.request_start = status.request_start + info.request_end = status.request_end result = WorkerProcessResult( type_="request_complete", request=request, diff --git a/tests/unit/objects/test_statistics.py b/tests/unit/objects/test_statistics.py index 0bd8c083..692db4b6 100644 --- a/tests/unit/objects/test_statistics.py +++ b/tests/unit/objects/test_statistics.py @@ -385,20 +385,6 @@ def test_status_distribution_summary_initialization(): assert status_distribution_summary.errored.mean == 50.0 -def test_status_distribution_summary_invalid_initialization(): - test_kwargs = { - "total": create_default_distribution_summary(), - "successful": create_default_distribution_summary(), - "incomplete": create_default_distribution_summary(), - "errored": create_default_distribution_summary(), - } - test_missing_keys = list(test_kwargs.keys()) - for missing_key in test_missing_keys: - kwargs = {key: val for key, val in test_kwargs.items() if key != missing_key} - with pytest.raises(ValueError): - StatusDistributionSummary(**kwargs) # type: ignore - - def test_status_distribution_summary_marshalling(): status_distribution_summary = StatusDistributionSummary( total=create_default_distribution_summary(), From 02633619faf0663dd198c1198049adc37ccfd196 Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Fri, 11 Apr 2025 16:16:59 +0000 Subject: [PATCH 39/43] Fix quality, unit, integration, and e2e tests --- src/guidellm/benchmark/aggregator.py | 64 +++++++++++++++++---------- src/guidellm/benchmark/benchmarker.py | 12 +++-- src/guidellm/objects/pydantic.py | 8 ++-- tests/e2e/test_placeholder.py | 2 + tests/integration/test_placeholder.py | 2 + tests/unit/backend/test_backend.py | 11 +++-- tests/unit/conftest.py | 17 ------- tests/unit/mock_backend.py | 2 +- 8 files changed, 65 insertions(+), 53 deletions(-) create mode 100644 tests/e2e/test_placeholder.py create mode 100644 tests/integration/test_placeholder.py diff --git a/src/guidellm/benchmark/aggregator.py b/src/guidellm/benchmark/aggregator.py index 3d050a87..6bd69d28 100644 --- a/src/guidellm/benchmark/aggregator.py +++ b/src/guidellm/benchmark/aggregator.py @@ -301,7 +301,7 @@ class BenchmarkAggregator( "The completed requests for this benchmark run broken down by status" "and excluding warmup and cooldown requests." ), - default_factory=lambda: StatusBreakdown( + default_factory=lambda: StatusBreakdown( # type: ignore[arg-type] successful=[], errored=[], incomplete=[], @@ -360,10 +360,10 @@ def add_result( f"Got {result.request_info}" ) - self.requests_stats.queued_time += ( + self.requests_stats.queued_time.update( result.request_info.dequeued_time - result.request_info.queued_time ) - self.requests_stats.scheduled_time_delay += ( + self.requests_stats.scheduled_time_delay.update( result.request_info.scheduled_time - result.request_info.dequeued_time ) sleep_time = max( @@ -371,32 +371,33 @@ def add_result( result.request_info.targeted_start_time - result.request_info.scheduled_time, ) - self.requests_stats.scheduled_time_sleep += sleep_time + self.requests_stats.scheduled_time_sleep.update(sleep_time) time_to_worker_start = ( result.request_info.worker_start - result.request_info.scheduled_time ) - self.requests_stats.worker_start_delay += time_to_worker_start - sleep_time - self.requests_stats.worker_time += ( + self.requests_stats.worker_start_delay.update(time_to_worker_start - sleep_time) + self.requests_stats.worker_time.update( result.request_info.worker_end - result.request_info.worker_start ) - self.requests_stats.worker_start_time_targeted_delay += ( + self.requests_stats.worker_start_time_targeted_delay.update( result.request_info.worker_start - result.request_info.targeted_start_time ) - self.requests_stats.request_start_time_delay += ( + self.requests_stats.request_start_time_delay.update( result.request_info.worker_start - result.request_info.targeted_start_time ) - self.requests_stats.request_start_time_targeted_delay += ( + self.requests_stats.request_start_time_targeted_delay.update( result.request_info.worker_start - result.request_info.targeted_start_time ) - self.requests_stats.request_time_delay += ( - result.request_info.worker_end - result.request_info.worker_start - ) - (result.request_info.worker_end - result.request_info.worker_start) - self.requests_stats.request_time += ( + self.requests_stats.request_time_delay.update( + (result.request_info.worker_end - result.request_info.worker_start) + - (result.request_info.worker_end - result.request_info.worker_start) + ) + self.requests_stats.request_time.update( result.request_info.worker_end - result.request_info.worker_start ) # Add result to the list of results provided we are not in warmup or cooldown - total_completed = self.requests_stats.totals.total + total_completed = self.requests_stats.totals.total.total global_start_time = self.requests_stats.totals.total.start_time in_warmup_number = ( @@ -521,6 +522,20 @@ class GenerativeBenchmarkAggregator( "any specific configuration for loading or processing." ), ) + worker_description: GenerativeRequestsWorkerDescription = Field( + description=( + "The description and specifics for the worker used to resolve requests " + "for this benchmark." + ), + discriminator="type_", + ) + request_loader_description: GenerativeRequestLoaderDescription = Field( + description=( + "The description and specifics for the request loader used to create " + "requests for this benchmark." + ), + discriminator="type_", + ) requests_stats: GenerativeRequestsRunningStats = Field( description=( "The running statistics for the requests for this benchmark run. " @@ -548,22 +563,22 @@ def add_result( if result.response is None: raise ValueError("Response is None, cannot add result.") - self.requests_stats.request_start_time_delay += ( + self.requests_stats.request_start_time_delay.update( result.response.start_time - result.request_info.worker_start ) - self.requests_stats.request_start_time_targeted_delay += ( + self.requests_stats.request_start_time_targeted_delay.update( result.response.start_time - result.request_info.targeted_start_time ) - self.requests_stats.request_time_delay += ( + self.requests_stats.request_time_delay.update( (result.response.start_time - result.request_info.worker_start) + result.request_info.worker_end - result.response.end_time ) - self.requests_stats.request_time += ( + self.requests_stats.request_time.update( result.response.end_time - result.response.start_time ) if result.response.first_iter_time: - self.requests_stats.time_to_first_token += ( + self.requests_stats.time_to_first_token.update( result.response.first_iter_time - result.response.start_time ) if result.response.last_iter_time and result.response.first_iter_time: @@ -598,10 +613,10 @@ def compile(self) -> GenerativeBenchmark: start_time=self.requests_stats.totals.total.start_time, end_time=time.time(), requests_made=StatusBreakdown( - successful=self.requests_stats.totals.successful.total, - errored=self.requests_stats.totals.errored.total, - incomplete=self.requests_stats.totals.incomplete.total, - total=self.requests_stats.totals.total.total, + successful=int(self.requests_stats.totals.successful.total), + errored=int(self.requests_stats.totals.errored.total), + incomplete=int(self.requests_stats.totals.incomplete.total), + total=int(self.requests_stats.totals.total.total), ), queued_time_avg=self.requests_stats.queued_time.mean, scheduled_time_delay_avg=self.requests_stats.scheduled_time_delay.mean, @@ -653,6 +668,7 @@ def _compile_results( last_token_time=result.response.last_iter_time or -1.0, ) for result in self.results.successful + if result.request and result.response ] incomplete: List[GenerativeTextErrorStats] = [ GenerativeTextErrorStats( @@ -682,6 +698,7 @@ def _compile_results( last_token_time=result.response.last_iter_time, ) for result in self.results.incomplete + if result.request and result.response ] error: List[GenerativeTextErrorStats] = [ GenerativeTextErrorStats( @@ -711,6 +728,7 @@ def _compile_results( last_token_time=result.response.last_iter_time, ) for result in self.results.errored + if result.request and result.response ] return successful, incomplete, error diff --git a/src/guidellm/benchmark/benchmarker.py b/src/guidellm/benchmark/benchmarker.py index 71d7c89a..9f54c8aa 100644 --- a/src/guidellm/benchmark/benchmarker.py +++ b/src/guidellm/benchmark/benchmarker.py @@ -25,7 +25,11 @@ from guidellm.benchmark.benchmark import BenchmarkArgs, GenerativeBenchmark from guidellm.benchmark.profile import Profile from guidellm.objects import StandardBaseModel -from guidellm.request import GenerationRequest, RequestLoaderDescription +from guidellm.request import ( + GenerationRequest, + GenerativeRequestLoaderDescription, + RequestLoaderDescription, +) from guidellm.scheduler import ( GenerativeRequestsWorker, RequestsWorker, @@ -289,7 +293,7 @@ def __init__( self, backend: Backend, request_loader: Iterable[GenerationRequest], - request_loader_description: RequestLoaderDescription, + request_loader_description: GenerativeRequestLoaderDescription, benchmark_save_extras: Optional[Dict[str, Any]] = None, processor: Optional[Union[str, Path, PreTrainedTokenizer]] = None, processor_args: Optional[Dict[str, Any]] = None, @@ -324,8 +328,8 @@ def create_benchmark_aggregator( cooldown_number=limits.cooldown_number, cooldown_duration=limits.cooldown_duration, ), - worker_description=self.worker.description, - request_loader_description=self.requests_loader_description, + worker_description=self.worker.description, # type: ignore[arg-type] + request_loader_description=self.requests_loader_description, # type: ignore[arg-type] extras=self.benchmark_save_extras or {}, processor=self.processor, processor_args=self.processor_args, diff --git a/src/guidellm/objects/pydantic.py b/src/guidellm/objects/pydantic.py index 87e2cb99..b6e998fa 100644 --- a/src/guidellm/objects/pydantic.py +++ b/src/guidellm/objects/pydantic.py @@ -44,17 +44,17 @@ class StatusBreakdown(BaseModel, Generic[SuccessfulT, ErroredT, IncompleteT, Tot successful: SuccessfulT = Field( description="The results with a successful status.", - default=None, + default=None, # type: ignore[assignment] ) errored: ErroredT = Field( description="The results with an errored status.", - default=None, + default=None, # type: ignore[assignment] ) incomplete: IncompleteT = Field( description="The results with an incomplete status.", - default=None, + default=None, # type: ignore[assignment] ) total: TotalT = Field( description="The combination of all statuses.", - default=None, + default=None, # type: ignore[assignment] ) diff --git a/tests/e2e/test_placeholder.py b/tests/e2e/test_placeholder.py new file mode 100644 index 00000000..3ada1ee4 --- /dev/null +++ b/tests/e2e/test_placeholder.py @@ -0,0 +1,2 @@ +def test_placeholder(): + assert True diff --git a/tests/integration/test_placeholder.py b/tests/integration/test_placeholder.py new file mode 100644 index 00000000..3ada1ee4 --- /dev/null +++ b/tests/integration/test_placeholder.py @@ -0,0 +1,2 @@ +def test_placeholder(): + assert True diff --git a/tests/unit/backend/test_backend.py b/tests/unit/backend/test_backend.py index 29a008e1..1c16d397 100644 --- a/tests/unit/backend/test_backend.py +++ b/tests/unit/backend/test_backend.py @@ -124,10 +124,13 @@ async def test_backend_chat_completions(mock_backend): @pytest.mark.smoke() -def test_backend_models(mock_backend): - assert mock_backend.available_models() == ["mock-model"] +@pytest.mark.asyncio() +async def test_backend_models(mock_backend): + models = await mock_backend.available_models() + assert models == ["mock-model"] @pytest.mark.smoke() -def test_backend_validate(mock_backend): - mock_backend.validate() +@pytest.mark.asyncio() +async def test_backend_validate(mock_backend): + await mock_backend.validate() diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 2a31df5d..41c0fbf5 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,11 +1,9 @@ import json -from pathlib import Path from typing import Any, AsyncIterable, Dict, List, Literal, Optional from unittest.mock import MagicMock, patch import httpx import pytest -import requests_mock import respx from guidellm.backend import ResponseSummary, StreamingTextResponse @@ -27,21 +25,6 @@ def _fake_tokenize(text: str) -> List[int]: yield mock_tokenizer -@pytest.fixture() -def mock_requests_pride_and_prejudice(): - text_path = ( - Path(__file__).parent.parent / "dummy" / "data" / "pride_and_prejudice.txt" - ) - text_content = text_path.read_text() - - with requests_mock.Mocker() as mock: - mock.get( - "https://www.gutenberg.org/files/1342/1342-0.txt", - text=text_content, - ) - yield mock - - @pytest.fixture() def mock_backend(request): params = request.param if hasattr(request, "param") else {} diff --git a/tests/unit/mock_backend.py b/tests/unit/mock_backend.py index e7335f61..0e59e93e 100644 --- a/tests/unit/mock_backend.py +++ b/tests/unit/mock_backend.py @@ -43,7 +43,7 @@ def info(self) -> Dict[str, Any]: async def prepare_multiprocessing(self): pass - def check_setup(self): + async def check_setup(self): pass async def available_models(self) -> List[str]: From b164b4bcb3b05c4b9b91e5f40e0c433487a394ef Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Fri, 11 Apr 2025 16:28:39 +0000 Subject: [PATCH 40/43] Just kidding, let's try that again and hopefully fix quality and tests --- src/guidellm/benchmark/entrypoints.py | 8 +++++--- src/guidellm/dataset/entrypoints.py | 3 ++- src/guidellm/objects/statistics.py | 6 +++--- tests/e2e/test_placeholder.py | 4 ++++ tests/integration/test_placeholder.py | 4 ++++ 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/guidellm/benchmark/entrypoints.py b/src/guidellm/benchmark/entrypoints.py index 9b5a85a3..fc98219e 100644 --- a/src/guidellm/benchmark/entrypoints.py +++ b/src/guidellm/benchmark/entrypoints.py @@ -1,8 +1,10 @@ from pathlib import Path -from typing import Any, Callable, Dict, Iterable, List, Literal, Optional, Union +from typing import Any, Dict, Iterable, List, Literal, Optional, Union from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict -from transformers import PreTrainedTokenizer # type: ignore[import] +from transformers import ( # type: ignore[import] + PreTrainedTokenizerBase, +) from guidellm.backend import Backend, BackendType from guidellm.benchmark.benchmark import GenerativeBenchmark @@ -22,7 +24,7 @@ async def benchmark_generative_text( backend_type: BackendType, backend_args: Optional[Dict[str, Any]], model: Optional[str], - processor: Optional[Union[str, Path, PreTrainedTokenizer, Callable]], + processor: Optional[Optional[Union[str, Path, PreTrainedTokenizerBase]]], processor_args: Optional[Dict[str, Any]], data: Union[ str, diff --git a/src/guidellm/dataset/entrypoints.py b/src/guidellm/dataset/entrypoints.py index 643ea0f5..5abf0112 100644 --- a/src/guidellm/dataset/entrypoints.py +++ b/src/guidellm/dataset/entrypoints.py @@ -1,3 +1,4 @@ +from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Union from datasets import Dataset, IterableDataset @@ -15,7 +16,7 @@ def load_dataset( data: Any, data_args: Optional[Dict[str, Any]], - processor: PreTrainedTokenizerBase, + processor: Optional[Union[str, Path, PreTrainedTokenizerBase]], processor_args: Optional[Dict[str, Any]], random_seed: int = 42, split_pref_order: Optional[List[str]] = None, diff --git a/src/guidellm/objects/statistics.py b/src/guidellm/objects/statistics.py index 722f3d92..0e43cdbd 100644 --- a/src/guidellm/objects/statistics.py +++ b/src/guidellm/objects/statistics.py @@ -117,14 +117,14 @@ def from_distribution_function( :return: An instance of DistributionSummary with calculated values. """ values, weights = zip(*distribution) if distribution else ([], []) - values = np.array(values) - weights = np.array(weights) + values = np.array(values) # type: ignore[assignment] + weights = np.array(weights) # type: ignore[assignment] # create the PDF probabilities = weights / np.sum(weights) # type: ignore[operator] pdf = np.column_stack((values, probabilities)) pdf = pdf[np.argsort(pdf[:, 0])] - values = pdf[:, 0] + values = pdf[:, 0] # type: ignore[assignment] probabilities = pdf[:, 1] # calculate the CDF diff --git a/tests/e2e/test_placeholder.py b/tests/e2e/test_placeholder.py index 3ada1ee4..d028e3f9 100644 --- a/tests/e2e/test_placeholder.py +++ b/tests/e2e/test_placeholder.py @@ -1,2 +1,6 @@ +import pytest + + +@pytest.mark.smoke() def test_placeholder(): assert True diff --git a/tests/integration/test_placeholder.py b/tests/integration/test_placeholder.py index 3ada1ee4..d028e3f9 100644 --- a/tests/integration/test_placeholder.py +++ b/tests/integration/test_placeholder.py @@ -1,2 +1,6 @@ +import pytest + + +@pytest.mark.smoke() def test_placeholder(): assert True From b411db383d7f950295a2b1ee7b9705f4a42b9465 Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Fri, 11 Apr 2025 16:39:06 +0000 Subject: [PATCH 41/43] Trying one more time for quality --- src/guidellm/benchmark/benchmarker.py | 4 ++-- src/guidellm/request/loader.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/guidellm/benchmark/benchmarker.py b/src/guidellm/benchmark/benchmarker.py index 9f54c8aa..985d9c4f 100644 --- a/src/guidellm/benchmark/benchmarker.py +++ b/src/guidellm/benchmark/benchmarker.py @@ -14,7 +14,7 @@ ) from pydantic import Field -from transformers import PreTrainedTokenizer # type: ignore # noqa: PGH003 +from transformers import PreTrainedTokenizerBase # type: ignore # noqa: PGH003 from guidellm.backend import Backend, ResponseSummary from guidellm.benchmark.aggregator import ( @@ -295,7 +295,7 @@ def __init__( request_loader: Iterable[GenerationRequest], request_loader_description: GenerativeRequestLoaderDescription, benchmark_save_extras: Optional[Dict[str, Any]] = None, - processor: Optional[Union[str, Path, PreTrainedTokenizer]] = None, + processor: Optional[Union[str, Path, PreTrainedTokenizerBase]] = None, processor_args: Optional[Dict[str, Any]] = None, ): super().__init__( diff --git a/src/guidellm/request/loader.py b/src/guidellm/request/loader.py index ac0704a6..de11e9c3 100644 --- a/src/guidellm/request/loader.py +++ b/src/guidellm/request/loader.py @@ -2,7 +2,6 @@ from pathlib import Path from typing import ( Any, - Callable, Dict, Iterable, Iterator, @@ -13,7 +12,7 @@ ) from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict -from transformers import PreTrainedTokenizer # type: ignore[import] +from transformers import PreTrainedTokenizerBase # type: ignore[import] from guidellm.dataset import ColumnInputTypes, load_dataset from guidellm.objects import StandardBaseModel @@ -80,7 +79,7 @@ def __init__( IterableDatasetDict, ], data_args: Optional[Dict[str, Any]], - processor: Optional[Union[str, Path, PreTrainedTokenizer, Callable]], + processor: Optional[Union[str, Path, PreTrainedTokenizerBase]], processor_args: Optional[Dict[str, Any]], shuffle: bool = True, iter_type: Literal["finite", "infinite"] = "finite", From d0d31d39ddad340c0a1ca54b565484cbd14d4cbd Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Fri, 11 Apr 2025 15:59:42 -0400 Subject: [PATCH 42/43] Bump min python to 3.9 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4eb171f6..9a5ac42a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ name = "guidellm" version = "0.1.0" description = "Guidance platform for deploying and managing large language models." readme = { file = "README.md", content-type = "text/markdown" } -requires-python = ">=3.8.0,<4.0" +requires-python = ">=3.9.0,<4.0" license = { file = "LICENSE" } authors = [ { name = "Neuralmagic, Inc." } ] urls = { homepage = "https://github.com/neuralmagic/guidellm" } From 70aa669ff7ed7e5cbc3ee0bb76652c6291c3a6c7 Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Fri, 11 Apr 2025 16:05:36 -0400 Subject: [PATCH 43/43] Revert "Bump min python to 3.9" Since python 3.9 adds support for using list and dict as types this has made ruff unhappy with using the typing constructs of List and Dict. Reverting my change for now; we can revisit in separate PR. This reverts commit d0d31d39ddad340c0a1ca54b565484cbd14d4cbd. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9a5ac42a..4eb171f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ name = "guidellm" version = "0.1.0" description = "Guidance platform for deploying and managing large language models." readme = { file = "README.md", content-type = "text/markdown" } -requires-python = ">=3.9.0,<4.0" +requires-python = ">=3.8.0,<4.0" license = { file = "LICENSE" } authors = [ { name = "Neuralmagic, Inc." } ] urls = { homepage = "https://github.com/neuralmagic/guidellm" }