From dcd99288f48d31b0c49d59d6dbb01f0dbde7e7d7 Mon Sep 17 00:00:00 2001
From: Alejandro <ae@alejandro.ltd>
Date: Mon, 23 Sep 2024 15:21:48 -0700
Subject: [PATCH 01/13] wip: validation summary integration

---
 .../classes/validation/validation_summary.py  | 35 +++++++++++++++++++
 guardrails/classes/validation_outcome.py      | 11 +++++-
 guardrails/guard.py                           |  9 +++++
 guardrails/run/async_stream_runner.py         | 12 +++++++
 guardrails/run/stream_runner.py               | 18 +++++++++-
 5 files changed, 83 insertions(+), 2 deletions(-)
 create mode 100644 guardrails/classes/validation/validation_summary.py

diff --git a/guardrails/classes/validation/validation_summary.py b/guardrails/classes/validation/validation_summary.py
new file mode 100644
index 000000000..59d4fc496
--- /dev/null
+++ b/guardrails/classes/validation/validation_summary.py
@@ -0,0 +1,35 @@
+# TODO Temp to update once generated class is in
+from typing import List, Optional
+
+from guardrails.classes.generic.arbitrary_model import ArbitraryModel
+from guardrails.classes.validation.validation_result import ErrorSpan, FailResult
+from guardrails.classes.validation.validator_logs import ValidatorLogs
+
+
+class ValidationSummary(ArbitraryModel):
+    validator_name: str
+    validator_status: str
+    failure_reason: Optional[str]
+    error_spans: Optional[List["ErrorSpan"]] = []
+    property_path: Optional[str]
+
+    @staticmethod
+    def from_validator_logs(
+        validator_logs: List[ValidatorLogs],
+    ) -> List["ValidationSummary"]:
+        summaries = []
+        for log in validator_logs:
+            validation_result = log.validation_result
+            is_fail_result = isinstance(validation_result, FailResult)
+            failure_reason = validation_result.error_message if is_fail_result else None
+            error_spans = validation_result.error_spans if is_fail_result else []
+            summaries.append(
+                ValidationSummary(
+                    validator_name=log.validator_name,
+                    validator_status=log.validation_result.outcome,
+                    property_path=log.property_path,
+                    failure_reason=failure_reason,
+                    error_spans=error_spans,
+                )
+            )
+        return summaries
diff --git a/guardrails/classes/validation_outcome.py b/guardrails/classes/validation_outcome.py
index 39f26a363..e7899b124 100644
--- a/guardrails/classes/validation_outcome.py
+++ b/guardrails/classes/validation_outcome.py
@@ -1,4 +1,4 @@
-from typing import Generic, Iterator, Optional, Tuple, Union, cast
+from typing import Generic, Iterator, List, Optional, Tuple, Union, cast
 
 from pydantic import Field
 from rich.pretty import pretty_repr
@@ -11,6 +11,7 @@
 from guardrails.classes.history import Call, Iteration
 from guardrails.classes.output_type import OT
 from guardrails.classes.generic.arbitrary_model import ArbitraryModel
+from guardrails.classes.validation.validation_summary import ValidationSummary
 from guardrails.constants import pass_status
 from guardrails.utils.safe_get import safe_get
 
@@ -31,6 +32,11 @@ class ValidationOutcome(IValidationOutcome, ArbitraryModel, Generic[OT]):
         error: If the validation failed, this field will contain the error message
     """
 
+    validation_summaries: Optional[List["ValidationSummary"]] = Field(
+        description="The summaries of the validation results.", default=[]
+    )
+    """The summaries of the validation results."""
+
     raw_llm_output: Optional[str] = Field(
         description="The raw, unchanged output from the LLM call.", default=None
     )
@@ -75,6 +81,8 @@ def from_guard_history(cls, call: Call):
             list(last_iteration.reasks), 0
         )
         validation_passed = call.status == pass_status
+        validator_logs = last_iteration.validator_logs or []
+        validation_summaries = ValidationSummary.from_validator_logs(validator_logs)
         reask = last_output if isinstance(last_output, ReAsk) else None
         error = call.error
         output = cast(OT, call.guarded_output)
@@ -84,6 +92,7 @@ def from_guard_history(cls, call: Call):
             validated_output=output,
             reask=reask,
             validation_passed=validation_passed,
+            validation_summaries=validation_summaries,
             error=error,
         )
 
diff --git a/guardrails/guard.py b/guardrails/guard.py
index ac8198e5e..2cc11b30a 100644
--- a/guardrails/guard.py
+++ b/guardrails/guard.py
@@ -1193,6 +1193,11 @@ def _single_server_call(self, *, payload: Dict[str, Any]) -> ValidationOutcome[O
             )
             self.history.extend([Call.from_interface(call) for call in guard_history])
 
+            # TODO Validation Summary
+            # validator_logs = self.history.last.iterations.last.validator_logs
+            # validation_summaries = ValidationSummary.
+            # from_validator_logs(validator_logs)
+
             # TODO: See if the below statement is still true
             # Our interfaces are too different for this to work right now.
             # Once we move towards shared interfaces for both the open source
@@ -1203,6 +1208,7 @@ def _single_server_call(self, *, payload: Dict[str, Any]) -> ValidationOutcome[O
                 if validation_output.validated_output
                 else None
             )
+            # TODO: Validation Summary
             return ValidationOutcome[OT](
                 call_id=validation_output.call_id,  # type: ignore
                 raw_llm_output=validation_output.raw_llm_output,
@@ -1224,9 +1230,11 @@ def _stream_server_call(
                 payload=ValidatePayload.from_dict(payload),  # type: ignore
                 openai_api_key=get_call_kwarg("api_key"),
             )
+            print("Server response:", response)
             for fragment in response:
                 validation_output = fragment
                 if validation_output is None:
+                    # TODO Validation Summary
                     yield ValidationOutcome[OT](
                         call_id="0",  # type: ignore
                         raw_llm_output=None,
@@ -1240,6 +1248,7 @@ def _stream_server_call(
                         if validation_output.validated_output
                         else None
                     )
+                    # TODO Validation Summary
                     yield ValidationOutcome[OT](
                         call_id=validation_output.call_id,  # type: ignore
                         raw_llm_output=validation_output.raw_llm_output,
diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index 5285a9634..7042a3f57 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -13,6 +13,7 @@
 from guardrails.classes import ValidationOutcome
 from guardrails.classes.history import Call, Inputs, Iteration, Outputs
 from guardrails.classes.output_type import OutputTypes
+from guardrails.classes.validation.validation_summary import ValidationSummary
 from guardrails.constants import pass_status
 from guardrails.llm_providers import (
     AsyncLiteLLMCallable,
@@ -164,11 +165,16 @@ async def async_step(
                     )
                 validation_response += cast(str, validated_fragment)
                 passed = call_log.status == pass_status
+                validator_logs = iteration.validator_logs
+                validation_summaries = ValidationSummary.from_validator_logs(
+                    validator_logs
+                )
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     raw_llm_output=chunk_text,
                     validated_output=validated_fragment,
                     validation_passed=passed,
+                    validation_summaries=validation_summaries,
                 )
         else:
             async for chunk in stream_output:
@@ -204,11 +210,17 @@ async def async_step(
                     validation_response = cast(list, validated_fragment)
                 else:
                     validation_response = cast(dict, validated_fragment)
+
+                validator_logs = iteration.validator_logs
+                validation_summaries = ValidationSummary.from_validator_logs(
+                    validator_logs
+                )
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     raw_llm_output=fragment,
                     validated_output=chunk_text,
                     validation_passed=validated_fragment is not None,
+                    validation_summaries=validation_summaries,
                 )
 
         iteration.outputs.raw_output = fragment
diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index 2e041abf4..2a188d46c 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -1,8 +1,10 @@
 from typing import Any, Dict, Generator, Iterable, List, Optional, Tuple, Union, cast
 
+
 from guardrails import validator_service
 from guardrails.classes.history import Call, Inputs, Iteration, Outputs
 from guardrails.classes.output_type import OT, OutputTypes
+from guardrails.classes.validation.validation_summary import ValidationSummary
 from guardrails.classes.validation_outcome import ValidationOutcome
 from guardrails.llm_providers import (
     LiteLLMCallable,
@@ -176,7 +178,9 @@ def prepare_chunk_generator(stream) -> Iterable[Tuple[Any, bool]]:
                 "$",
                 validate_subschema=True,
             )
-
+            # Not sure I like adding all this info to every chunk
+            # maybe move last chunk?
+            validator_logs = iteration.validator_logs
             for res in gen:
                 chunk = res.chunk
                 original_text = res.original_text
@@ -195,6 +199,11 @@ def prepare_chunk_generator(stream) -> Iterable[Tuple[Any, bool]]:
                     )
                 # 5. Convert validated fragment to a pretty JSON string
                 validation_response += cast(str, chunk)
+                validator_logs = call_log.iterations.last.validator_logs
+
+                validation_summaries = ValidationSummary.from_validator_logs(
+                    validator_logs
+                )
                 passed = call_log.status == pass_status
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
@@ -202,6 +211,7 @@ def prepare_chunk_generator(stream) -> Iterable[Tuple[Any, bool]]:
                     raw_llm_output=original_text,
                     validated_output=chunk,
                     validation_passed=passed,
+                    validation_summaries=validation_summaries,
                 )
 
         # handle non string schema
@@ -246,11 +256,17 @@ def prepare_chunk_generator(stream) -> Iterable[Tuple[Any, bool]]:
                 else:
                     validation_response = cast(dict, validated_fragment)
                 # 5. Convert validated fragment to a pretty JSON string
+
+                validator_logs = iteration.validator_logs
+                validation_summaries = ValidationSummary.from_validator_logs(
+                    validator_logs
+                )
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     raw_llm_output=fragment,
                     validated_output=validated_fragment,
                     validation_passed=validated_fragment is not None,
+                    validation_summaries=validation_summaries,
                 )
 
         # # Finally, add to logs

From 505628cca8700af56a8e150ff4c449518c3d538e Mon Sep 17 00:00:00 2001
From: Alejandro <ae@alejandro.ltd>
Date: Tue, 24 Sep 2024 12:17:11 -0700
Subject: [PATCH 02/13] removed summaries from streaming due to peformance hit

---
 guardrails/run/async_stream_runner.py | 12 ------------
 guardrails/run/stream_runner.py       | 18 +-----------------
 2 files changed, 1 insertion(+), 29 deletions(-)

diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index 7042a3f57..5285a9634 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -13,7 +13,6 @@
 from guardrails.classes import ValidationOutcome
 from guardrails.classes.history import Call, Inputs, Iteration, Outputs
 from guardrails.classes.output_type import OutputTypes
-from guardrails.classes.validation.validation_summary import ValidationSummary
 from guardrails.constants import pass_status
 from guardrails.llm_providers import (
     AsyncLiteLLMCallable,
@@ -165,16 +164,11 @@ async def async_step(
                     )
                 validation_response += cast(str, validated_fragment)
                 passed = call_log.status == pass_status
-                validator_logs = iteration.validator_logs
-                validation_summaries = ValidationSummary.from_validator_logs(
-                    validator_logs
-                )
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     raw_llm_output=chunk_text,
                     validated_output=validated_fragment,
                     validation_passed=passed,
-                    validation_summaries=validation_summaries,
                 )
         else:
             async for chunk in stream_output:
@@ -210,17 +204,11 @@ async def async_step(
                     validation_response = cast(list, validated_fragment)
                 else:
                     validation_response = cast(dict, validated_fragment)
-
-                validator_logs = iteration.validator_logs
-                validation_summaries = ValidationSummary.from_validator_logs(
-                    validator_logs
-                )
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     raw_llm_output=fragment,
                     validated_output=chunk_text,
                     validation_passed=validated_fragment is not None,
-                    validation_summaries=validation_summaries,
                 )
 
         iteration.outputs.raw_output = fragment
diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index 2a188d46c..2e041abf4 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -1,10 +1,8 @@
 from typing import Any, Dict, Generator, Iterable, List, Optional, Tuple, Union, cast
 
-
 from guardrails import validator_service
 from guardrails.classes.history import Call, Inputs, Iteration, Outputs
 from guardrails.classes.output_type import OT, OutputTypes
-from guardrails.classes.validation.validation_summary import ValidationSummary
 from guardrails.classes.validation_outcome import ValidationOutcome
 from guardrails.llm_providers import (
     LiteLLMCallable,
@@ -178,9 +176,7 @@ def prepare_chunk_generator(stream) -> Iterable[Tuple[Any, bool]]:
                 "$",
                 validate_subschema=True,
             )
-            # Not sure I like adding all this info to every chunk
-            # maybe move last chunk?
-            validator_logs = iteration.validator_logs
+
             for res in gen:
                 chunk = res.chunk
                 original_text = res.original_text
@@ -199,11 +195,6 @@ def prepare_chunk_generator(stream) -> Iterable[Tuple[Any, bool]]:
                     )
                 # 5. Convert validated fragment to a pretty JSON string
                 validation_response += cast(str, chunk)
-                validator_logs = call_log.iterations.last.validator_logs
-
-                validation_summaries = ValidationSummary.from_validator_logs(
-                    validator_logs
-                )
                 passed = call_log.status == pass_status
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
@@ -211,7 +202,6 @@ def prepare_chunk_generator(stream) -> Iterable[Tuple[Any, bool]]:
                     raw_llm_output=original_text,
                     validated_output=chunk,
                     validation_passed=passed,
-                    validation_summaries=validation_summaries,
                 )
 
         # handle non string schema
@@ -256,17 +246,11 @@ def prepare_chunk_generator(stream) -> Iterable[Tuple[Any, bool]]:
                 else:
                     validation_response = cast(dict, validated_fragment)
                 # 5. Convert validated fragment to a pretty JSON string
-
-                validator_logs = iteration.validator_logs
-                validation_summaries = ValidationSummary.from_validator_logs(
-                    validator_logs
-                )
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     raw_llm_output=fragment,
                     validated_output=validated_fragment,
                     validation_passed=validated_fragment is not None,
-                    validation_summaries=validation_summaries,
                 )
 
         # # Finally, add to logs

From 5620dc3af136e766fb453a8f35b1f2b874068d78 Mon Sep 17 00:00:00 2001
From: Alejandro <ae@alejandro.ltd>
Date: Tue, 24 Sep 2024 12:19:08 -0700
Subject: [PATCH 03/13] added generator for generate summaries from logs

---
 .../classes/validation/validation_summary.py  | 47 +++++++++++--------
 1 file changed, 27 insertions(+), 20 deletions(-)

diff --git a/guardrails/classes/validation/validation_summary.py b/guardrails/classes/validation/validation_summary.py
index 59d4fc496..3c8d96097 100644
--- a/guardrails/classes/validation/validation_summary.py
+++ b/guardrails/classes/validation/validation_summary.py
@@ -1,35 +1,42 @@
 # TODO Temp to update once generated class is in
-from typing import List, Optional
+from typing import Iterator, List
 
 from guardrails.classes.generic.arbitrary_model import ArbitraryModel
-from guardrails.classes.validation.validation_result import ErrorSpan, FailResult
+from guardrails.classes.validation.validation_result import FailResult
 from guardrails.classes.validation.validator_logs import ValidatorLogs
+from guardrails_api_client import ValidationSummary as IValidationSummary
 
 
-class ValidationSummary(ArbitraryModel):
-    validator_name: str
-    validator_status: str
-    failure_reason: Optional[str]
-    error_spans: Optional[List["ErrorSpan"]] = []
-    property_path: Optional[str]
-
+class ValidationSummary(IValidationSummary, ArbitraryModel):
     @staticmethod
-    def from_validator_logs(
+    def _generate_summaries_from_validator_logs(
         validator_logs: List[ValidatorLogs],
-    ) -> List["ValidationSummary"]:
-        summaries = []
+    ) -> Iterator["ValidationSummary"]:
+        """
+        Generate a list of ValidationSummary objects from a list of
+        ValidatorLogs objects. Using an iterator to allow serializing
+        the summaries to other formats.
+        """
         for log in validator_logs:
             validation_result = log.validation_result
             is_fail_result = isinstance(validation_result, FailResult)
             failure_reason = validation_result.error_message if is_fail_result else None
             error_spans = validation_result.error_spans if is_fail_result else []
-            summaries.append(
-                ValidationSummary(
-                    validator_name=log.validator_name,
-                    validator_status=log.validation_result.outcome,
-                    property_path=log.property_path,
-                    failure_reason=failure_reason,
-                    error_spans=error_spans,
-                )
+            yield ValidationSummary(
+                validatorName=log.validator_name,
+                validatorStatus=log.validation_result.outcome,  # type: ignore
+                propertyPath=log.property_path,
+                failureReason=failure_reason,
+                errorSpans=error_spans,  # type: ignore
             )
+
+    @staticmethod
+    def from_validator_logs(
+        validator_logs: List[ValidatorLogs],
+    ) -> List["ValidationSummary"]:
+        summaries = []
+        for summary in ValidationSummary._generate_summaries_from_validator_logs(
+            validator_logs
+        ):
+            summaries.append(summary)
         return summaries

From dfe0910f3ca423dbf15c917150d91062df71994d Mon Sep 17 00:00:00 2001
From: Alejandro <ae@alejandro.ltd>
Date: Tue, 24 Sep 2024 12:20:40 -0700
Subject: [PATCH 04/13] added to guard.py

---
 guardrails/guard.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/guardrails/guard.py b/guardrails/guard.py
index 2cc11b30a..108a12d4a 100644
--- a/guardrails/guard.py
+++ b/guardrails/guard.py
@@ -33,6 +33,7 @@
 from guardrails.api_client import GuardrailsApiClient
 from guardrails.classes.output_type import OT
 from guardrails.classes.validation.validation_result import ErrorSpan
+from guardrails.classes.validation.validation_summary import ValidationSummary
 from guardrails.classes.validation_outcome import ValidationOutcome
 from guardrails.classes.credentials import Credentials
 from guardrails.classes.execution import GuardExecutionOptions
@@ -1193,10 +1194,8 @@ def _single_server_call(self, *, payload: Dict[str, Any]) -> ValidationOutcome[O
             )
             self.history.extend([Call.from_interface(call) for call in guard_history])
 
-            # TODO Validation Summary
-            # validator_logs = self.history.last.iterations.last.validator_logs
-            # validation_summaries = ValidationSummary.
-            # from_validator_logs(validator_logs)
+            validator_logs = self.history.last.iterations.last.validator_logs
+            validation_summaries = ValidationSummary.from_validator_logs(validator_logs)
 
             # TODO: See if the below statement is still true
             # Our interfaces are too different for this to work right now.
@@ -1208,12 +1207,12 @@ def _single_server_call(self, *, payload: Dict[str, Any]) -> ValidationOutcome[O
                 if validation_output.validated_output
                 else None
             )
-            # TODO: Validation Summary
             return ValidationOutcome[OT](
                 call_id=validation_output.call_id,  # type: ignore
                 raw_llm_output=validation_output.raw_llm_output,
                 validated_output=validated_output,
                 validation_passed=(validation_output.validation_passed is True),
+                validation_summaries=validation_summaries,
             )
         else:
             raise ValueError("Guard does not have an api client!")

From 8a49b5ee2fbdce9fb00cbc16012223b6465692ef Mon Sep 17 00:00:00 2001
From: Alejandro <ae@alejandro.ltd>
Date: Tue, 24 Sep 2024 12:22:06 -0700
Subject: [PATCH 05/13] removed print and comments

---
 guardrails/guard.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/guardrails/guard.py b/guardrails/guard.py
index 108a12d4a..118fe67d8 100644
--- a/guardrails/guard.py
+++ b/guardrails/guard.py
@@ -1229,11 +1229,9 @@ def _stream_server_call(
                 payload=ValidatePayload.from_dict(payload),  # type: ignore
                 openai_api_key=get_call_kwarg("api_key"),
             )
-            print("Server response:", response)
             for fragment in response:
                 validation_output = fragment
                 if validation_output is None:
-                    # TODO Validation Summary
                     yield ValidationOutcome[OT](
                         call_id="0",  # type: ignore
                         raw_llm_output=None,
@@ -1247,7 +1245,6 @@ def _stream_server_call(
                         if validation_output.validated_output
                         else None
                     )
-                    # TODO Validation Summary
                     yield ValidationOutcome[OT](
                         call_id=validation_output.call_id,  # type: ignore
                         raw_llm_output=validation_output.raw_llm_output,

From adcdd01691605279f4e85373a2b7d82a2f0515d0 Mon Sep 17 00:00:00 2001
From: Alejandro <ae@alejandro.ltd>
Date: Tue, 24 Sep 2024 12:28:33 -0700
Subject: [PATCH 06/13] bump minimum api client

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 7d0c1eb91..2ab9c7e9d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -56,7 +56,7 @@ opentelemetry-sdk = "^1.24.0"
 opentelemetry-exporter-otlp-proto-grpc = "^1.24.0"
 opentelemetry-exporter-otlp-proto-http = "^1.24.0"
 guardrails-hub-types = "^0.0.4"
-guardrails-api-client = ">=0.3.8"
+guardrails-api-client = ">=0.3.12"
 diff-match-patch = "^20230430"
 guardrails-api = ">=0.0.1"
 mlflow = {version = ">=2.0.1", optional = true}

From f43db05bf27a69ff4829cc5b46c4d1d2994911c8 Mon Sep 17 00:00:00 2001
From: Alejandro <ae@alejandro.ltd>
Date: Tue, 24 Sep 2024 12:57:18 -0700
Subject: [PATCH 07/13] updated poetry lock

---
 poetry.lock | 32 +++++++++++---------------------
 1 file changed, 11 insertions(+), 21 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index 24b48dec6..d64a1bd98 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -2123,15 +2123,22 @@ dev = ["coverage", "gunicorn (>=22.0.0,<23)", "pytest", "pytest-mock", "ruff"]
 
 [[package]]
 name = "guardrails-api-client"
-version = "0.3.8"
+version = "0.3.12"
 description = "Guardrails API Client."
 optional = false
 python-versions = "<4,>=3.8"
 files = [
-    {file = "guardrails_api_client-0.3.8-py3-none-any.whl", hash = "sha256:2becd5ac9c720879a997e50e2a813d9e77a6fb1a7068a96b1b9712dd6dd9efb8"},
-    {file = "guardrails_api_client-0.3.8.tar.gz", hash = "sha256:2e1e45cddf727534a378bc99f7f98da0800960918c75bda010b8bc493b946d16"},
+    {file = "guardrails_api_client-0.3.12-py3-none-any.whl", hash = "sha256:ba9c2a8cf12e5f78486890e19739fb94e7aaf6955ef4453f096cc730df84794e"},
+    {file = "guardrails_api_client-0.3.12.tar.gz", hash = "sha256:3c0a40549c2857744b059baceae63e0646a9178305700e72320b1b53dbaeb33e"},
 ]
 
+[package.dependencies]
+pydantic = ">=2"
+python-dateutil = ">=2.5.3"
+setuptools = ">=21.0.0"
+typing-extensions = ">=4.7.1"
+urllib3 = ">=1.25.3,<2.1.0"
+
 [package.extras]
 dev = ["pyright", "pytest", "pytest-cov", "ruff"]
 
@@ -7854,23 +7861,6 @@ brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotl
 secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"]
 socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
 
-[[package]]
-name = "urllib3"
-version = "2.2.1"
-description = "HTTP library with thread-safe connection pooling, file post, and more."
-optional = false
-python-versions = ">=3.8"
-files = [
-    {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"},
-    {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"},
-]
-
-[package.extras]
-brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"]
-h2 = ["h2 (>=4,<5)"]
-socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"]
-zstd = ["zstandard (>=0.18.0)"]
-
 [[package]]
 name = "uvloop"
 version = "0.20.0"
@@ -8419,4 +8409,4 @@ vectordb = ["faiss-cpu", "numpy"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.9"
-content-hash = "6253610141bb5686330057ae658550f9257aabe83ee7b279b783a7f4418a26a6"
+content-hash = "8cbf51c8006599fb64f6810841dc8da70f11c9f889934fce77b2766da6a46c79"

From a010d014cd7420c6608357705c7e34a6b7943f76 Mon Sep 17 00:00:00 2001
From: Alejandro <ae@alejandro.ltd>
Date: Tue, 24 Sep 2024 13:51:59 -0700
Subject: [PATCH 08/13] changed to only output validation summaries with
 failures

---
 .../classes/validation/validation_summary.py  | 12 ++++++++++++
 guardrails/classes/validation_outcome.py      |  4 +++-
 guardrails/run/async_stream_runner.py         | 11 +++++++++++
 guardrails/run/stream_runner.py               | 19 +++++++++++++++++--
 4 files changed, 43 insertions(+), 3 deletions(-)

diff --git a/guardrails/classes/validation/validation_summary.py b/guardrails/classes/validation/validation_summary.py
index 3c8d96097..a1c9b13d0 100644
--- a/guardrails/classes/validation/validation_summary.py
+++ b/guardrails/classes/validation/validation_summary.py
@@ -40,3 +40,15 @@ def from_validator_logs(
         ):
             summaries.append(summary)
         return summaries
+
+    @staticmethod
+    def from_validator_logs_only_fails(
+        validator_logs: List[ValidatorLogs],
+    ) -> List["ValidationSummary"]:
+        summaries = []
+        for summary in ValidationSummary._generate_summaries_from_validator_logs(
+            validator_logs
+        ):
+            if summary.failure_reason:
+                summaries.append(summary)
+        return summaries
diff --git a/guardrails/classes/validation_outcome.py b/guardrails/classes/validation_outcome.py
index e7899b124..1182fe6b2 100644
--- a/guardrails/classes/validation_outcome.py
+++ b/guardrails/classes/validation_outcome.py
@@ -82,7 +82,9 @@ def from_guard_history(cls, call: Call):
         )
         validation_passed = call.status == pass_status
         validator_logs = last_iteration.validator_logs or []
-        validation_summaries = ValidationSummary.from_validator_logs(validator_logs)
+        validation_summaries = ValidationSummary.from_validator_logs_only_fails(
+            validator_logs
+        )
         reask = last_output if isinstance(last_output, ReAsk) else None
         error = call.error
         output = cast(OT, call.guarded_output)
diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index 5285a9634..a38709be0 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -13,6 +13,7 @@
 from guardrails.classes import ValidationOutcome
 from guardrails.classes.history import Call, Inputs, Iteration, Outputs
 from guardrails.classes.output_type import OutputTypes
+from guardrails.classes.validation.validation_summary import ValidationSummary
 from guardrails.constants import pass_status
 from guardrails.llm_providers import (
     AsyncLiteLLMCallable,
@@ -164,11 +165,16 @@ async def async_step(
                     )
                 validation_response += cast(str, validated_fragment)
                 passed = call_log.status == pass_status
+                validator_logs = iteration.validator_logs
+                validation_summaries = ValidationSummary.from_validator_logs_only_fails(
+                    validator_logs
+                )
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     raw_llm_output=chunk_text,
                     validated_output=validated_fragment,
                     validation_passed=passed,
+                    validation_summaries=validation_summaries,
                 )
         else:
             async for chunk in stream_output:
@@ -204,11 +210,16 @@ async def async_step(
                     validation_response = cast(list, validated_fragment)
                 else:
                     validation_response = cast(dict, validated_fragment)
+                validator_logs = iteration.validator_logs
+                validation_summaries = ValidationSummary.from_validator_logs_only_fails(
+                    validator_logs
+                )
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     raw_llm_output=fragment,
                     validated_output=chunk_text,
                     validation_passed=validated_fragment is not None,
+                    validation_summaries=validation_summaries,
                 )
 
         iteration.outputs.raw_output = fragment
diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index 2e041abf4..39584d711 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -1,8 +1,10 @@
 from typing import Any, Dict, Generator, Iterable, List, Optional, Tuple, Union, cast
 
+
 from guardrails import validator_service
 from guardrails.classes.history import Call, Inputs, Iteration, Outputs
 from guardrails.classes.output_type import OT, OutputTypes
+from guardrails.classes.validation.validation_summary import ValidationSummary
 from guardrails.classes.validation_outcome import ValidationOutcome
 from guardrails.llm_providers import (
     LiteLLMCallable,
@@ -176,8 +178,7 @@ def prepare_chunk_generator(stream) -> Iterable[Tuple[Any, bool]]:
                 "$",
                 validate_subschema=True,
             )
-
-            for res in gen:
+            for chunk_index, res in enumerate(gen):
                 chunk = res.chunk
                 original_text = res.original_text
                 if isinstance(chunk, SkeletonReAsk):
@@ -196,12 +197,20 @@ def prepare_chunk_generator(stream) -> Iterable[Tuple[Any, bool]]:
                 # 5. Convert validated fragment to a pretty JSON string
                 validation_response += cast(str, chunk)
                 passed = call_log.status == pass_status
+
+                validator_logs = call_log.iterations.last.validator_logs
+
+                validation_summaries = ValidationSummary.from_validator_logs_only_fails(
+                    validator_logs
+                )
+
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     #  The chunk or the whole output?
                     raw_llm_output=original_text,
                     validated_output=chunk,
                     validation_passed=passed,
+                    validation_summaries=validation_summaries,
                 )
 
         # handle non string schema
@@ -246,11 +255,17 @@ def prepare_chunk_generator(stream) -> Iterable[Tuple[Any, bool]]:
                 else:
                     validation_response = cast(dict, validated_fragment)
                 # 5. Convert validated fragment to a pretty JSON string
+
+                validator_logs = iteration.validator_logs
+                validation_summaries = ValidationSummary.from_validator_logs_only_fails(
+                    validator_logs
+                )
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     raw_llm_output=fragment,
                     validated_output=validated_fragment,
                     validation_passed=validated_fragment is not None,
+                    validation_summaries=validation_summaries,
                 )
 
         # # Finally, add to logs

From 938b4479a6b74a34b3e3118cf758669d97bc2af0 Mon Sep 17 00:00:00 2001
From: Alejandro <ae@alejandro.ltd>
Date: Tue, 24 Sep 2024 14:07:36 -0700
Subject: [PATCH 09/13] only report failures in summaries in non streaming
 server calls

---
 guardrails/guard.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/guardrails/guard.py b/guardrails/guard.py
index 118fe67d8..00b1f916e 100644
--- a/guardrails/guard.py
+++ b/guardrails/guard.py
@@ -1195,7 +1195,9 @@ def _single_server_call(self, *, payload: Dict[str, Any]) -> ValidationOutcome[O
             self.history.extend([Call.from_interface(call) for call in guard_history])
 
             validator_logs = self.history.last.iterations.last.validator_logs
-            validation_summaries = ValidationSummary.from_validator_logs(validator_logs)
+            validation_summaries = ValidationSummary.from_validator_logs_only_fails(
+                validator_logs
+            )
 
             # TODO: See if the below statement is still true
             # Our interfaces are too different for this to work right now.

From b05dc6aa7d2f8fd8d15c0f544f400938ecfa4906 Mon Sep 17 00:00:00 2001
From: Alejandro <ae@alejandro.ltd>
Date: Tue, 24 Sep 2024 14:58:54 -0700
Subject: [PATCH 10/13] bumpted min api client version

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index d35bc08af..07d11cd50 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -56,7 +56,7 @@ opentelemetry-sdk = "^1.24.0"
 opentelemetry-exporter-otlp-proto-grpc = "^1.24.0"
 opentelemetry-exporter-otlp-proto-http = "^1.24.0"
 guardrails-hub-types = "^0.0.4"
-guardrails-api-client = ">=0.3.12"
+guardrails-api-client = ">=0.3.13"
 diff-match-patch = "^20230430"
 guardrails-api = ">=0.0.1"
 mlflow = {version = ">=2.0.1", optional = true}

From 7d9080ffd74dbb27431e9981cd66366e0b31489c Mon Sep 17 00:00:00 2001
From: Alejandro <ae@alejandro.ltd>
Date: Tue, 24 Sep 2024 19:47:25 -0700
Subject: [PATCH 11/13] undo stream implementation

---
 guardrails/run/async_stream_runner.py | 11 -----------
 guardrails/run/stream_runner.py       | 19 ++-----------------
 2 files changed, 2 insertions(+), 28 deletions(-)

diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index eaabe6e23..8f39c21f2 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -13,7 +13,6 @@
 from guardrails.classes import ValidationOutcome
 from guardrails.classes.history import Call, Inputs, Iteration, Outputs
 from guardrails.classes.output_type import OutputTypes
-from guardrails.classes.validation.validation_summary import ValidationSummary
 from guardrails.constants import pass_status
 from guardrails.llm_providers import (
     AsyncLiteLLMCallable,
@@ -168,16 +167,11 @@ async def async_step(
                     )
                 validation_response += cast(str, validated_fragment)
                 passed = call_log.status == pass_status
-                validator_logs = iteration.validator_logs
-                validation_summaries = ValidationSummary.from_validator_logs_only_fails(
-                    validator_logs
-                )
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     raw_llm_output=chunk_text,
                     validated_output=validated_fragment,
                     validation_passed=passed,
-                    validation_summaries=validation_summaries,
                 )
         else:
             async for chunk in stream_output:
@@ -213,16 +207,11 @@ async def async_step(
                     validation_response = cast(list, validated_fragment)
                 else:
                     validation_response = cast(dict, validated_fragment)
-                validator_logs = iteration.validator_logs
-                validation_summaries = ValidationSummary.from_validator_logs_only_fails(
-                    validator_logs
-                )
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     raw_llm_output=fragment,
                     validated_output=chunk_text,
                     validation_passed=validated_fragment is not None,
-                    validation_summaries=validation_summaries,
                 )
 
         iteration.outputs.raw_output = fragment
diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index 173c1ef95..2f1272c6b 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -1,10 +1,8 @@
 from typing import Any, Dict, Iterator, List, Optional, Tuple, Union, cast
 
-
 from guardrails import validator_service
 from guardrails.classes.history import Call, Inputs, Iteration, Outputs
 from guardrails.classes.output_type import OT, OutputTypes
-from guardrails.classes.validation.validation_summary import ValidationSummary
 from guardrails.classes.validation_outcome import ValidationOutcome
 from guardrails.llm_providers import (
     LiteLLMCallable,
@@ -181,7 +179,8 @@ def prepare_chunk_generator(stream) -> Iterator[Tuple[Any, bool]]:
                 "$",
                 validate_subschema=True,
             )
-            for chunk_index, res in enumerate(gen):
+
+            for res in gen:
                 chunk = res.chunk
                 original_text = res.original_text
                 if isinstance(chunk, SkeletonReAsk):
@@ -200,20 +199,12 @@ def prepare_chunk_generator(stream) -> Iterator[Tuple[Any, bool]]:
                 # 5. Convert validated fragment to a pretty JSON string
                 validation_response += cast(str, chunk)
                 passed = call_log.status == pass_status
-
-                validator_logs = call_log.iterations.last.validator_logs
-
-                validation_summaries = ValidationSummary.from_validator_logs_only_fails(
-                    validator_logs
-                )
-
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     #  The chunk or the whole output?
                     raw_llm_output=original_text,
                     validated_output=chunk,
                     validation_passed=passed,
-                    validation_summaries=validation_summaries,
                 )
 
         # handle non string schema
@@ -258,17 +249,11 @@ def prepare_chunk_generator(stream) -> Iterator[Tuple[Any, bool]]:
                 else:
                     validation_response = cast(dict, validated_fragment)
                 # 5. Convert validated fragment to a pretty JSON string
-
-                validator_logs = iteration.validator_logs
-                validation_summaries = ValidationSummary.from_validator_logs_only_fails(
-                    validator_logs
-                )
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     raw_llm_output=fragment,
                     validated_output=validated_fragment,
                     validation_passed=validated_fragment is not None,
-                    validation_summaries=validation_summaries,
                 )
 
         # # Finally, add to logs

From c16ca9eef2b3a4347bcc4d0605cb71ae4119304d Mon Sep 17 00:00:00 2001
From: Alejandro <ae@alejandro.ltd>
Date: Wed, 25 Sep 2024 15:01:28 -0700
Subject: [PATCH 12/13] update poetry.lock

---
 poetry.lock | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index 2ddb19df9..949f30720 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -2123,13 +2123,13 @@ dev = ["coverage", "gunicorn (>=22.0.0,<23)", "pytest", "pytest-mock", "ruff"]
 
 [[package]]
 name = "guardrails-api-client"
-version = "0.3.12"
+version = "0.3.13"
 description = "Guardrails API Client."
 optional = false
 python-versions = "<4,>=3.8"
 files = [
-    {file = "guardrails_api_client-0.3.12-py3-none-any.whl", hash = "sha256:ba9c2a8cf12e5f78486890e19739fb94e7aaf6955ef4453f096cc730df84794e"},
-    {file = "guardrails_api_client-0.3.12.tar.gz", hash = "sha256:3c0a40549c2857744b059baceae63e0646a9178305700e72320b1b53dbaeb33e"},
+    {file = "guardrails_api_client-0.3.13-py3-none-any.whl", hash = "sha256:c9c5297355d022428a573e6e845d4876cdfbda81635d9c65fb107db1f3bb0ac2"},
+    {file = "guardrails_api_client-0.3.13.tar.gz", hash = "sha256:4f0f1a7e7ef100138fb99634d75e3330e9abba9d4e8363f27af8bc7e722e2d88"},
 ]
 
 [package.dependencies]
@@ -8420,4 +8420,4 @@ vectordb = ["faiss-cpu", "numpy"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.9"
-content-hash = "8cbf51c8006599fb64f6810841dc8da70f11c9f889934fce77b2766da6a46c79"
+content-hash = "e2e1fa42497acd1313f1a4fb4529e235c3e8f958474e06cefc96c05b27e093a0"

From 9f5790a48ac1814374d1f223af37ca0a0c770edc Mon Sep 17 00:00:00 2001
From: Alejandro <ae@alejandro.ltd>
Date: Wed, 25 Sep 2024 16:46:11 -0700
Subject: [PATCH 13/13] fix typing issue

---
 guardrails/guard.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/guardrails/guard.py b/guardrails/guard.py
index 75113089e..2d35f8eb9 100644
--- a/guardrails/guard.py
+++ b/guardrails/guard.py
@@ -1167,10 +1167,12 @@ def _single_server_call(self, *, payload: Dict[str, Any]) -> ValidationOutcome[O
             )
             self.history.extend([Call.from_interface(call) for call in guard_history])
 
-            validator_logs = self.history.last.iterations.last.validator_logs
-            validation_summaries = ValidationSummary.from_validator_logs_only_fails(
-                validator_logs
-            )
+            validation_summaries = []
+            if self.history.last and self.history.last.iterations.last:
+                validator_logs = self.history.last.iterations.last.validator_logs
+                validation_summaries = ValidationSummary.from_validator_logs_only_fails(
+                    validator_logs
+                )
 
             # TODO: See if the below statement is still true
             # Our interfaces are too different for this to work right now.