diff --git a/guardrails/classes/validation/validation_summary.py b/guardrails/classes/validation/validation_summary.py new file mode 100644 index 000000000..a1c9b13d0 --- /dev/null +++ b/guardrails/classes/validation/validation_summary.py @@ -0,0 +1,54 @@ +# TODO Temp to update once generated class is in +from typing import Iterator, List + +from guardrails.classes.generic.arbitrary_model import ArbitraryModel +from guardrails.classes.validation.validation_result import FailResult +from guardrails.classes.validation.validator_logs import ValidatorLogs +from guardrails_api_client import ValidationSummary as IValidationSummary + + +class ValidationSummary(IValidationSummary, ArbitraryModel): + @staticmethod + def _generate_summaries_from_validator_logs( + validator_logs: List[ValidatorLogs], + ) -> Iterator["ValidationSummary"]: + """ + Generate a list of ValidationSummary objects from a list of + ValidatorLogs objects. Using an iterator to allow serializing + the summaries to other formats. + """ + for log in validator_logs: + validation_result = log.validation_result + is_fail_result = isinstance(validation_result, FailResult) + failure_reason = validation_result.error_message if is_fail_result else None + error_spans = validation_result.error_spans if is_fail_result else [] + yield ValidationSummary( + validatorName=log.validator_name, + validatorStatus=log.validation_result.outcome, # type: ignore + propertyPath=log.property_path, + failureReason=failure_reason, + errorSpans=error_spans, # type: ignore + ) + + @staticmethod + def from_validator_logs( + validator_logs: List[ValidatorLogs], + ) -> List["ValidationSummary"]: + summaries = [] + for summary in ValidationSummary._generate_summaries_from_validator_logs( + validator_logs + ): + summaries.append(summary) + return summaries + + @staticmethod + def from_validator_logs_only_fails( + validator_logs: List[ValidatorLogs], + ) -> List["ValidationSummary"]: + summaries = [] + for summary in ValidationSummary._generate_summaries_from_validator_logs( + validator_logs + ): + if summary.failure_reason: + summaries.append(summary) + return summaries diff --git a/guardrails/classes/validation_outcome.py b/guardrails/classes/validation_outcome.py index 39f26a363..1182fe6b2 100644 --- a/guardrails/classes/validation_outcome.py +++ b/guardrails/classes/validation_outcome.py @@ -1,4 +1,4 @@ -from typing import Generic, Iterator, Optional, Tuple, Union, cast +from typing import Generic, Iterator, List, Optional, Tuple, Union, cast from pydantic import Field from rich.pretty import pretty_repr @@ -11,6 +11,7 @@ from guardrails.classes.history import Call, Iteration from guardrails.classes.output_type import OT from guardrails.classes.generic.arbitrary_model import ArbitraryModel +from guardrails.classes.validation.validation_summary import ValidationSummary from guardrails.constants import pass_status from guardrails.utils.safe_get import safe_get @@ -31,6 +32,11 @@ class ValidationOutcome(IValidationOutcome, ArbitraryModel, Generic[OT]): error: If the validation failed, this field will contain the error message """ + validation_summaries: Optional[List["ValidationSummary"]] = Field( + description="The summaries of the validation results.", default=[] + ) + """The summaries of the validation results.""" + raw_llm_output: Optional[str] = Field( description="The raw, unchanged output from the LLM call.", default=None ) @@ -75,6 +81,10 @@ def from_guard_history(cls, call: Call): list(last_iteration.reasks), 0 ) validation_passed = call.status == pass_status + validator_logs = last_iteration.validator_logs or [] + validation_summaries = ValidationSummary.from_validator_logs_only_fails( + validator_logs + ) reask = last_output if isinstance(last_output, ReAsk) else None error = call.error output = cast(OT, call.guarded_output) @@ -84,6 +94,7 @@ def from_guard_history(cls, call: Call): validated_output=output, reask=reask, validation_passed=validation_passed, + validation_summaries=validation_summaries, error=error, ) diff --git a/guardrails/guard.py b/guardrails/guard.py index 383c9f230..880e0f0bc 100644 --- a/guardrails/guard.py +++ b/guardrails/guard.py @@ -35,6 +35,7 @@ from guardrails.classes.output_type import OT from guardrails.classes.rc import RC from guardrails.classes.validation.validation_result import ErrorSpan +from guardrails.classes.validation.validation_summary import ValidationSummary from guardrails.classes.validation_outcome import ValidationOutcome from guardrails.classes.execution import GuardExecutionOptions from guardrails.classes.generic import Stack @@ -1217,6 +1218,13 @@ def _single_server_call(self, *, payload: Dict[str, Any]) -> ValidationOutcome[O ) self.history.extend([Call.from_interface(call) for call in guard_history]) + validation_summaries = [] + if self.history.last and self.history.last.iterations.last: + validator_logs = self.history.last.iterations.last.validator_logs + validation_summaries = ValidationSummary.from_validator_logs_only_fails( + validator_logs + ) + # TODO: See if the below statement is still true # Our interfaces are too different for this to work right now. # Once we move towards shared interfaces for both the open source @@ -1232,6 +1240,7 @@ def _single_server_call(self, *, payload: Dict[str, Any]) -> ValidationOutcome[O raw_llm_output=validation_output.raw_llm_output, validated_output=validated_output, validation_passed=(validation_output.validation_passed is True), + validation_summaries=validation_summaries, ) else: raise ValueError("Guard does not have an api client!") diff --git a/poetry.lock b/poetry.lock index 794698da3..2bfd08f55 100644 --- a/poetry.lock +++ b/poetry.lock @@ -512,17 +512,17 @@ files = [ [[package]] name = "boto3" -version = "1.35.31" +version = "1.35.32" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.35.31-py3-none-any.whl", hash = "sha256:2e9af74d10d8af7610a8d8468d2914961f116912a024fce17351825260385a52"}, - {file = "boto3-1.35.31.tar.gz", hash = "sha256:8c593af260c4ea3eb6f079c09908f94494ca2222aa4e40a7ff490fab1cee8b39"}, + {file = "boto3-1.35.32-py3-none-any.whl", hash = "sha256:786a243f4b4827c6ae149442bf544c2ae449570cf23616a5d386f7a2633e0e08"}, + {file = "boto3-1.35.32.tar.gz", hash = "sha256:a7652962897340d34bc930ffc9311dcc441da975dd1b904d0172b06adbea3601"}, ] [package.dependencies] -botocore = ">=1.35.31,<1.36.0" +botocore = ">=1.35.32,<1.36.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -531,13 +531,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.35.31" +version = "1.35.32" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.35.31-py3-none-any.whl", hash = "sha256:4cee814875bc78656aef4011d3d6b2231e96f53ea3661ee428201afb579d5c31"}, - {file = "botocore-1.35.31.tar.gz", hash = "sha256:f7bfa910cf2cbcc8c2307c1cf7b93495d614c2d699883417893e0a337fe4eb63"}, + {file = "botocore-1.35.32-py3-none-any.whl", hash = "sha256:2c0c2b62dd156daf904525f3f523ae22bf34ac109d727acf0bbfbca291440fc3"}, + {file = "botocore-1.35.32.tar.gz", hash = "sha256:5ecbcd2f112e991b1f95c88ebb0f8df1a7c4ad9681aff80ec77e67cc4836eaa9"}, ] [package.dependencies] @@ -2192,20 +2192,6 @@ Werkzeug = ">=3.0.3,<4" [package.extras] dev = ["coverage", "gunicorn (>=22.0.0,<23)", "pytest", "pytest-mock", "ruff"] -[[package]] -name = "guardrails-api-client" -version = "0.3.9" -description = "Guardrails API Client." -optional = false -python-versions = "<4,>=3.8" -files = [ - {file = "guardrails_api_client-0.3.9-py3-none-any.whl", hash = "sha256:f24110690669007ca276ddfea928770c9e1ba360e4a70b06cb6e63aceffd0d4f"}, - {file = "guardrails_api_client-0.3.9.tar.gz", hash = "sha256:dac110182381cd779579c217028534db20168dd659f2484c5c2d21aa9560d2fc"}, -] - -[package.extras] -dev = ["pyright", "pytest", "pytest-cov", "ruff"] - [[package]] name = "guardrails-api-client" version = "0.3.13" @@ -7971,18 +7957,18 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "urllib3" -version = "2.2.3" +version = "2.0.7" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=3.8" +python-versions = ">=3.7" files = [ - {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"}, - {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"}, + {file = "urllib3-2.0.7-py3-none-any.whl", hash = "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e"}, + {file = "urllib3-2.0.7.tar.gz", hash = "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84"}, ] [package.extras] brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -h2 = ["h2 (>=4,<5)"] +secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] @@ -8553,4 +8539,4 @@ vectordb = ["faiss-cpu", "numpy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "96293d271b0c1be36b7544ce23261cab6b8a0546e1530f5044e6daace3c335e2" +content-hash = "75778b3799a2c211eac627d23d0d9dd78a5ee578eabc8ade3a713f6a70a81314" diff --git a/pyproject.toml b/pyproject.toml index bb0b00e7c..f9fe4507f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ opentelemetry-sdk = "^1.24.0" opentelemetry-exporter-otlp-proto-grpc = "^1.24.0" opentelemetry-exporter-otlp-proto-http = "^1.24.0" guardrails-hub-types = "^0.0.4" -guardrails-api-client = ">=0.3.8" +guardrails-api-client = ">=0.3.13" diff-match-patch = "^20230430" guardrails-api = ">=0.0.1" mlflow = {version = ">=2.0.1", optional = true}