diff --git a/.circleci/config.yml b/.circleci/config.yml index 1356cd43..79d895ce 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,10 +17,11 @@ jobs: - run: name: Install Environment Dependencies command: | # install dependencies - apt-get -y install curl - pip install --upgrade pip + apt-get update + apt-get -y install curl libgeos-dev + pip install --upgrade pip pip install poetry - poetry install + poetry install -E shapely - run: name: Black Formatting Check # Only validation, without re-formatting diff --git a/CHANGELOG.md b/CHANGELOG.md index 0228853e..1321a680 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.9.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.9.0) - 2022-04-07 + +### Added + +- Validate metrics support metadata and field filtering on input annotation and predictions +- 3D/Cuboid metrics: Recall, Precision, 3D IOU and birds eye 2D IOU``` +- Shapely can be used for metric development if the optional scale-nucleus[shapely] is installed +- Full support for passing parameters to evaluation configurations + ## [0.8.4](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.8.4) - 2022-04-06 - Changing `camera_params` of dataset items can now be done through the dataset method `update_items_metadata` diff --git a/README.md b/README.md index 1c9ea3b4..52417fe6 100644 --- a/README.md +++ b/README.md @@ -179,3 +179,24 @@ cd docs sphinx-autobuild . ./_build/html --watch ../nucleus ``` `sphinx-autobuild` will spin up a server on localhost (port 8000 by default) that will watch for and automatically rebuild a version of the API reference based on your local docstring changes. + + +## Custom Metrics using Shapely in scale-validate + +Certain metrics use `shapely` which is added as an optional dependency. +```bash +pip install scale-nucleus[metrics] +``` + +Note that you might need to install a local GEOS package since Shapely doesn't provide binaries bundled with GEOS for every platform. + +```bash +#Mac OS +brew install geos +# Ubuntu/Debian flavors +apt-get install libgeos-dev +``` + +To develop it locally use + +`poetry install --extra shapely` diff --git a/nucleus/annotation.py b/nucleus/annotation.py index 6019fac1..95a3ea6c 100644 --- a/nucleus/annotation.py +++ b/nucleus/annotation.py @@ -557,6 +557,7 @@ class SegmentationAnnotation(Annotation): annotations: List[Segment] reference_id: str annotation_id: Optional[str] = None + # metadata: Optional[dict] = None # TODO(sc: 422637) def __post_init__(self): if not self.mask_url: @@ -574,6 +575,7 @@ def from_json(cls, payload: dict): ], reference_id=payload[REFERENCE_ID_KEY], annotation_id=payload.get(ANNOTATION_ID_KEY, None), + # metadata=payload.get(METADATA_KEY, None), # TODO(sc: 422637) ) def to_payload(self) -> dict: @@ -582,6 +584,7 @@ def to_payload(self) -> dict: MASK_URL_KEY: self.mask_url, ANNOTATIONS_KEY: [ann.to_payload() for ann in self.annotations], ANNOTATION_ID_KEY: self.annotation_id, + # METADATA_KEY: self.metadata, # TODO(sc: 422637) } payload[REFERENCE_ID_KEY] = self.reference_id diff --git a/nucleus/metrics/__init__.py b/nucleus/metrics/__init__.py index 1fd038a2..460561f7 100644 --- a/nucleus/metrics/__init__.py +++ b/nucleus/metrics/__init__.py @@ -1,5 +1,12 @@ from .base import Metric, ScalarResult from .categorization_metrics import CategorizationF1 +from .cuboid_metrics import CuboidIOU, CuboidPrecision, CuboidRecall +from .filtering import ( + FieldFilter, + ListOfOrAndFilters, + MetadataFilter, + apply_filters, +) from .polygon_metrics import ( PolygonAveragePrecision, PolygonIOU, diff --git a/nucleus/metrics/base.py b/nucleus/metrics/base.py index 75d916e9..b316e1be 100644 --- a/nucleus/metrics/base.py +++ b/nucleus/metrics/base.py @@ -1,9 +1,14 @@ import sys from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import Iterable, List +from typing import Iterable, List, Optional, Union from nucleus.annotation import AnnotationList +from nucleus.metrics.filtering import ( + ListOfAndFilters, + ListOfOrAndFilters, + apply_filters, +) from nucleus.prediction import PredictionList @@ -86,12 +91,107 @@ def __call__( metric(annotations, predictions) """ + def __init__( + self, + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + ): + """ + Args: + annotation_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + prediction_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + """ + self.annotation_filters = annotation_filters + self.prediction_filters = prediction_filters + @abstractmethod - def __call__( + def call_metric( self, annotations: AnnotationList, predictions: PredictionList ) -> MetricResult: """A metric must override this method and return a metric result, given annotations and predictions.""" + def __call__( + self, annotations: AnnotationList, predictions: PredictionList + ) -> MetricResult: + annotations = self._filter_annotations(annotations) + predictions = self._filter_predictions(predictions) + return self.call_metric(annotations, predictions) + + def _filter_annotations(self, annotations: AnnotationList): + if ( + self.annotation_filters is None + or len(self.annotation_filters) == 0 + ): + return annotations + annotations.box_annotations = apply_filters( + annotations.box_annotations, self.annotation_filters + ) + annotations.line_annotations = apply_filters( + annotations.line_annotations, self.annotation_filters + ) + annotations.polygon_annotations = apply_filters( + annotations.polygon_annotations, self.annotation_filters + ) + annotations.cuboid_annotations = apply_filters( + annotations.cuboid_annotations, self.annotation_filters + ) + annotations.category_annotations = apply_filters( + annotations.category_annotations, self.annotation_filters + ) + annotations.multi_category_annotations = apply_filters( + annotations.multi_category_annotations, self.annotation_filters + ) + annotations.segmentation_annotations = apply_filters( + annotations.segmentation_annotations, self.annotation_filters + ) + return annotations + + def _filter_predictions(self, predictions: PredictionList): + if ( + self.prediction_filters is None + or len(self.prediction_filters) == 0 + ): + return predictions + predictions.box_predictions = apply_filters( + predictions.box_predictions, self.prediction_filters + ) + predictions.line_predictions = apply_filters( + predictions.line_predictions, self.prediction_filters + ) + predictions.polygon_predictions = apply_filters( + predictions.polygon_predictions, self.prediction_filters + ) + predictions.cuboid_predictions = apply_filters( + predictions.cuboid_predictions, self.prediction_filters + ) + predictions.category_predictions = apply_filters( + predictions.category_predictions, self.prediction_filters + ) + predictions.segmentation_predictions = apply_filters( + predictions.segmentation_predictions, self.prediction_filters + ) + return predictions + @abstractmethod def aggregate_score(self, results: List[MetricResult]) -> ScalarResult: """A metric must define how to aggregate results from single items to a single ScalarResult. diff --git a/nucleus/metrics/categorization_metrics.py b/nucleus/metrics/categorization_metrics.py index 80979c8e..6c6e1354 100644 --- a/nucleus/metrics/categorization_metrics.py +++ b/nucleus/metrics/categorization_metrics.py @@ -1,11 +1,12 @@ from abc import abstractmethod from dataclasses import dataclass -from typing import List, Set, Tuple, Union +from typing import List, Optional, Set, Tuple, Union from sklearn.metrics import f1_score from nucleus.annotation import AnnotationList, CategoryAnnotation from nucleus.metrics.base import Metric, MetricResult, ScalarResult +from nucleus.metrics.filtering import ListOfAndFilters, ListOfOrAndFilters from nucleus.metrics.filters import confidence_filter from nucleus.prediction import CategoryPrediction, PredictionList @@ -56,12 +57,37 @@ class CategorizationMetric(Metric): def __init__( self, confidence_threshold: float = 0.0, + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, ): """Initializes CategorizationMetric abstract object. Args: confidence_threshold: minimum confidence threshold for predictions to be taken into account for evaluation. Must be in [0, 1]. Default 0.0 + annotation_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + prediction_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). """ + super().__init__(annotation_filters, prediction_filters) assert 0 <= confidence_threshold <= 1 self.confidence_threshold = confidence_threshold @@ -83,7 +109,7 @@ def eval( def aggregate_score(self, results: List[CategorizationResult]) -> ScalarResult: # type: ignore[override] pass - def __call__( + def call_metric( self, annotations: AnnotationList, predictions: PredictionList ) -> CategorizationResult: if self.confidence_threshold > 0: @@ -139,7 +165,15 @@ class CategorizationF1(CategorizationMetric): """Evaluation method that matches categories and returns a CategorizationF1Result that aggregates to the F1 score""" def __init__( - self, confidence_threshold: float = 0.0, f1_method: str = "macro" + self, + confidence_threshold: float = 0.0, + f1_method: str = "macro", + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, ): """ Args: @@ -169,8 +203,28 @@ def __init__( Calculate metrics for each instance, and find their average (only meaningful for multilabel classification where this differs from :func:`accuracy_score`). + annotation_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + prediction_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). """ - super().__init__(confidence_threshold) + super().__init__( + confidence_threshold, annotation_filters, prediction_filters + ) assert ( f1_method in F1_METHODS ), f"Invalid f1_method {f1_method}, expected one of {F1_METHODS}" diff --git a/nucleus/metrics/cuboid_metrics.py b/nucleus/metrics/cuboid_metrics.py new file mode 100644 index 00000000..a6a11d03 --- /dev/null +++ b/nucleus/metrics/cuboid_metrics.py @@ -0,0 +1,267 @@ +import sys +from abc import abstractmethod +from typing import List, Optional, Union + +from nucleus.annotation import AnnotationList, CuboidAnnotation +from nucleus.prediction import CuboidPrediction, PredictionList + +from .base import Metric, ScalarResult +from .cuboid_utils import detection_iou, label_match_wrapper, recall_precision +from .filtering import ListOfAndFilters, ListOfOrAndFilters +from .filters import confidence_filter + + +class CuboidMetric(Metric): + """Abstract class for metrics of cuboids. + + The CuboidMetric class automatically filters incoming annotations and + predictions for only cuboid annotations. It also filters + predictions whose confidence is less than the provided confidence_threshold. + Finally, it provides support for enforcing matching labels. If + `enforce_label_match` is set to True, then annotations and predictions will + only be matched if they have the same label. + + To create a new concrete CuboidMetric, override the `eval` function + with logic to define a metric between cuboid annotations and predictions. + """ + + def __init__( + self, + enforce_label_match: bool = False, + confidence_threshold: float = 0.0, + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + ): + """Initializes CuboidMetric abstract object. + + Args: + enforce_label_match: whether to enforce that annotation and prediction labels must match. Default False + confidence_threshold: minimum confidence threshold for predictions. Must be in [0, 1]. Default 0.0 + annotation_filters: MetadataFilter predicates. Predicates are expressed in disjunctive normal form (DNF), + like [[MetadataFilter('x', '==', 0), FieldFilter('label', '==', 'pedestrian')], ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single field predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective and multiple column predicate. Finally, the most outer list combines + these filters as a disjunction (OR). + prediction_filters: MetadataFilter predicates. Predicates are expressed in disjunctive normal form (DNF), + like [[MetadataFilter('x', '==', 0), FieldFilter('label', '==', 'pedestrian')], ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single field predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective and multiple column predicate. Finally, the most outer list combines + these filters as a disjunction (OR). + """ + self.enforce_label_match = enforce_label_match + assert 0 <= confidence_threshold <= 1 + self.confidence_threshold = confidence_threshold + super().__init__(annotation_filters, prediction_filters) + + @abstractmethod + def eval( + self, + annotations: List[CuboidAnnotation], + predictions: List[CuboidPrediction], + ) -> ScalarResult: + # Main evaluation function that subclasses must override. + pass + + def aggregate_score(self, results: List[ScalarResult]) -> ScalarResult: # type: ignore[override] + return ScalarResult.aggregate(results) + + def call_metric( + self, annotations: AnnotationList, predictions: PredictionList + ) -> ScalarResult: + if self.confidence_threshold > 0: + predictions = confidence_filter( + predictions, self.confidence_threshold + ) + cuboid_annotations: List[CuboidAnnotation] = [] + cuboid_annotations.extend(annotations.cuboid_annotations) + cuboid_predictions: List[CuboidPrediction] = [] + cuboid_predictions.extend(predictions.cuboid_predictions) + + eval_fn = label_match_wrapper(self.eval) + result = eval_fn( + cuboid_annotations, + cuboid_predictions, + enforce_label_match=self.enforce_label_match, + ) + return result + + +class CuboidIOU(CuboidMetric): + """Calculates the average IOU between cuboid annotations and predictions.""" + + # TODO: Remove defaults once these are surfaced more cleanly to users. + def __init__( + self, + enforce_label_match: bool = True, + iou_threshold: float = 0.0, + confidence_threshold: float = 0.0, + iou_2d: bool = False, + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + ): + """Initializes CuboidIOU object. + + Args: + enforce_label_match: whether to enforce that annotation and prediction labels must match. Defaults to True + iou_threshold: IOU threshold to consider detection as valid. Must be in [0, 1]. Default 0.0 + iou_2d: whether to return the BEV 2D IOU if true, or the 3D IOU if false. + confidence_threshold: minimum confidence threshold for predictions. Must be in [0, 1]. Default 0.0 + annotation_filters: MetadataFilter predicates. Predicates are expressed in disjunctive normal form (DNF), like + [[MetadataFilter('x', '=', 0), ...], ...]. DNF allows arbitrary boolean logical combinations of single field + predicates. The innermost structures each describe a single column predicate. The list of inner predicates is + interpreted as a conjunction (AND), forming a more selective and multiple column predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + prediction_filters: MetadataFilter predicates. Predicates are expressed in disjunctive normal form (DNF), like + [[MetadataFilter('x', '=', 0), ...], ...]. DNF allows arbitrary boolean logical combinations of single field + predicates. The innermost structures each describe a single column predicate. The list of inner predicates is + interpreted as a conjunction (AND), forming a more selective and multiple column predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + """ + assert ( + 0 <= iou_threshold <= 1 + ), "IoU threshold must be between 0 and 1." + self.iou_threshold = iou_threshold + self.iou_2d = iou_2d + super().__init__( + enforce_label_match=enforce_label_match, + confidence_threshold=confidence_threshold, + annotation_filters=annotation_filters, + prediction_filters=prediction_filters, + ) + + def eval( + self, + annotations: List[CuboidAnnotation], + predictions: List[CuboidPrediction], + ) -> ScalarResult: + iou_3d_metric, iou_2d_metric = detection_iou( + predictions, + annotations, + threshold_in_overlap_ratio=self.iou_threshold, + ) + + weight = max(len(annotations), len(predictions)) + if self.iou_2d: + avg_iou = iou_2d_metric.sum() / max(weight, sys.float_info.epsilon) + else: + avg_iou = iou_3d_metric.sum() / max(weight, sys.float_info.epsilon) + + return ScalarResult(avg_iou, weight) + + +class CuboidPrecision(CuboidMetric): + """Calculates the average precision between cuboid annotations and predictions.""" + + # TODO: Remove defaults once these are surfaced more cleanly to users. + def __init__( + self, + enforce_label_match: bool = True, + iou_threshold: float = 0.0, + confidence_threshold: float = 0.0, + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + ): + """Initializes CuboidIOU object. + + Args: + enforce_label_match: whether to enforce that annotation and prediction labels must match. Defaults to True + iou_threshold: IOU threshold to consider detection as valid. Must be in [0, 1]. Default 0.0 + confidence_threshold: minimum confidence threshold for predictions. Must be in [0, 1]. Default 0.0 + annotation_filters: MetadataFilter predicates. Predicates are expressed in disjunctive normal form (DNF), like + [[MetadataFilter('x', '==', 0), ...], ...]. DNF allows arbitrary boolean logical combinations of single field + predicates. The innermost structures each describe a single column predicate. The list of inner predicates is + interpreted as a conjunction (AND), forming a more selective and multiple column predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + prediction_filters: MetadataFilter predicates. Predicates are expressed in disjunctive normal form (DNF), like + [[MetadataFilter('x', '==', 0), ...], ...]. DNF allows arbitrary boolean logical combinations of single field + predicates. The innermost structures each describe a single column predicate. The list of inner predicates is + interpreted as a conjunction (AND), forming a more selective and multiple column predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + """ + assert ( + 0 <= iou_threshold <= 1 + ), "IoU threshold must be between 0 and 1." + self.iou_threshold = iou_threshold + super().__init__( + enforce_label_match=enforce_label_match, + confidence_threshold=confidence_threshold, + annotation_filters=annotation_filters, + prediction_filters=prediction_filters, + ) + + def eval( + self, + annotations: List[CuboidAnnotation], + predictions: List[CuboidPrediction], + ) -> ScalarResult: + stats = recall_precision( + predictions, + annotations, + threshold_in_overlap_ratio=self.iou_threshold, + ) + weight = stats["tp_sum"] + stats["fp_sum"] + precision = stats["tp_sum"] / max(weight, sys.float_info.epsilon) + return ScalarResult(precision, weight) + + +class CuboidRecall(CuboidMetric): + """Calculates the average recall between cuboid annotations and predictions.""" + + # TODO: Remove defaults once these are surfaced more cleanly to users. + def __init__( + self, + enforce_label_match: bool = True, + iou_threshold: float = 0.0, + confidence_threshold: float = 0.0, + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + ): + """Initializes CuboidIOU object. + + Args: + enforce_label_match: whether to enforce that annotation and prediction labels must match. Defaults to True + iou_threshold: IOU threshold to consider detection as valid. Must be in [0, 1]. Default 0.0 + confidence_threshold: minimum confidence threshold for predictions. Must be in [0, 1]. Default 0.0 + """ + assert ( + 0 <= iou_threshold <= 1 + ), "IoU threshold must be between 0 and 1." + self.iou_threshold = iou_threshold + super().__init__( + enforce_label_match=enforce_label_match, + confidence_threshold=confidence_threshold, + annotation_filters=annotation_filters, + prediction_filters=prediction_filters, + ) + + def eval( + self, + annotations: List[CuboidAnnotation], + predictions: List[CuboidPrediction], + ) -> ScalarResult: + stats = recall_precision( + predictions, + annotations, + threshold_in_overlap_ratio=self.iou_threshold, + ) + weight = stats["tp_sum"] + stats["fn_sum"] + recall = stats["tp_sum"] / max(weight, sys.float_info.epsilon) + return ScalarResult(recall, weight) diff --git a/nucleus/metrics/cuboid_utils.py b/nucleus/metrics/cuboid_utils.py new file mode 100644 index 00000000..e23821de --- /dev/null +++ b/nucleus/metrics/cuboid_utils.py @@ -0,0 +1,380 @@ +from functools import wraps +from typing import Dict, List, Tuple + +import numpy as np + +try: + from shapely.geometry import Polygon +except ModuleNotFoundError: + import sys + + class Polygon: # type: ignore + def __init__(self, *args, **kwargs): + """Object to make sure we only raise errors if actually trying to use shapely""" + if sys.platform.startswith("darwin"): + platform_specific_msg = ( + "Depending on Python environment used GEOS might need to be installed via " + "`brew install geos`." + ) + elif sys.platform.startswith("linux"): + platform_specific_msg = ( + "Depending on Python environment used GEOS might need to be installed via " + "system package `libgeos-dev`." + ) + else: + platform_specific_msg = "GEOS package will need to be installed see (https://trac.osgeo.org/geos/)" + raise ModuleNotFoundError( + f"Module 'shapely' not found. Install optionally with `scale-nucleus[shapely]` or when developing " + f"`poetry install -E shapely`. {platform_specific_msg}" + ) + + +from nucleus.annotation import CuboidAnnotation +from nucleus.prediction import CuboidPrediction + +from .base import ScalarResult + + +def group_cuboids_by_label( + annotations: List[CuboidAnnotation], + predictions: List[CuboidPrediction], +) -> Dict[str, Tuple[List[CuboidAnnotation], List[CuboidPrediction]]]: + """Groups input annotations and predictions by label. + + Args: + annotations: list of input cuboid annotations + predictions: list of input cuboid predictions + + Returns: + Mapping from each label to (annotations, predictions) tuple + """ + labels = set(annotation.label for annotation in annotations) + labels |= set(prediction.label for prediction in predictions) + grouped: Dict[ + str, Tuple[List[CuboidAnnotation], List[CuboidPrediction]] + ] = {label: ([], []) for label in labels} + for annotation in annotations: + grouped[annotation.label][0].append(annotation) + for prediction in predictions: + grouped[prediction.label][1].append(prediction) + return grouped + + +def label_match_wrapper(metric_fn): + """Decorator to add the ability to only apply metric to annotations and + predictions with matching labels. + + Args: + metric_fn: Metric function that takes a list of annotations, a list + of predictions, and optional args and kwargs. + + Returns: + Metric function which can optionally enforce matching labels. + """ + + @wraps(metric_fn) + def wrapper( + annotations: List[CuboidAnnotation], + predictions: List[CuboidPrediction], + *args, + enforce_label_match: bool = False, + **kwargs, + ) -> ScalarResult: + # Simply return the metric if we are not enforcing label matches. + if not enforce_label_match: + return metric_fn(annotations, predictions, *args, **kwargs) + + # For each bin of annotations/predictions, compute the metric applied + # only to that bin. Then aggregate results across all bins. + grouped_inputs = group_cuboids_by_label(annotations, predictions) + metric_results = [] + for binned_annotations, binned_predictions in grouped_inputs.values(): + metric_result = metric_fn( + binned_annotations, binned_predictions, *args, **kwargs + ) + metric_results.append(metric_result) + assert all( + isinstance(r, ScalarResult) for r in metric_results + ), "Expected every result to be a ScalarResult" + return ScalarResult.aggregate(metric_results) + + return wrapper + + +def process_dataitem(dataitem): + processed_item = {} + processed_item["xyz"] = np.array( + [[ann.position.x, ann.position.y, ann.position.z] for ann in dataitem] + ) + processed_item["wlh"] = np.array( + [ + [ann.dimensions.x, ann.dimensions.y, ann.dimensions.z] + for ann in dataitem + ] + ) + processed_item["yaw"] = np.array([ann.yaw for ann in dataitem]) + return processed_item + + +def compute_outer_iou( + xyz_0: np.ndarray, + wlh_0: np.ndarray, + yaw_0: np.ndarray, + xyz_1: np.ndarray, + wlh_1: np.ndarray, + yaw_1: np.ndarray, + scale_convention: bool = True, + distance_threshold=25, +) -> Tuple[np.ndarray, np.ndarray]: + """ + Computes outer 3D and 2D IoU + :param xyz_0: (n, 3) + :param wlh_0: (n, 3) + :param yaw_0: (n,) + :param xyz_1: (m, 3) + :param wlh_1: (m, 3) + :param yaw_1: (m,) + :param scale_convention: flag whether the internal Scale convention is used (have to be adjusted by pi/2) + :param distance_threshold: computes iou only within this distance (~3x speedup) + :return: (n, m) 3D IoU, (n, m) 2D IoU + """ + + bottom_z = np.maximum.outer( + xyz_0[:, 2] - (wlh_0[:, 2] / 2), xyz_1[:, 2] - (wlh_1[:, 2] / 2) + ) + top_z = np.minimum.outer( + xyz_0[:, 2] + (wlh_0[:, 2] / 2), xyz_1[:, 2] + (wlh_1[:, 2] / 2) + ) + height_intersection = np.maximum(0, top_z - bottom_z) + + cuboid_corners_0 = get_batch_cuboid_corners( + xyz_0, wlh_0, yaw_0, scale_convention=scale_convention + ) + cuboid_corners_1 = get_batch_cuboid_corners( + xyz_1, wlh_1, yaw_1, scale_convention=scale_convention + ) + polygons_1 = [ + Polygon(corners_1[[1, 0, 4, 5, 1], :2]) + for corners_1 in cuboid_corners_1 + ] + area_intersection = np.zeros( + (cuboid_corners_0.shape[0], cuboid_corners_1.shape[0]), + dtype=np.float32, + ) + + if cuboid_corners_0.shape[0] != 0 and cuboid_corners_1.shape[0] != 0: + distance_mask = ( + np.linalg.norm( + xyz_0[:, np.newaxis, :] - xyz_1[np.newaxis, :, :], axis=2 + ) + < distance_threshold + ) + + for i, corners_0 in enumerate(cuboid_corners_0): + for j, polygon_1 in enumerate(polygons_1): + if distance_mask[i, j]: + area_intersection[i, j] = ( + Polygon(corners_0[[1, 0, 4, 5, 1], :2]) + .intersection(polygon_1) + .area + ) + + intersection = height_intersection * area_intersection + area_0 = wlh_0[:, 0] * wlh_0[:, 1] + area_1 = wlh_1[:, 0] * wlh_1[:, 1] + union_2d = np.add.outer(area_0, area_1) - area_intersection + + volume_0 = area_0 * wlh_0[:, 2] + volume_1 = area_1 * wlh_1[:, 2] + union = np.add.outer(volume_0, volume_1) - intersection + return intersection / union, area_intersection / union_2d + + +def get_batch_cuboid_corners( + xyz: np.ndarray, + wlh: np.ndarray, + yaw: np.ndarray, + pitch: np.ndarray = None, + roll: np.ndarray = None, + scale_convention: bool = True, +) -> np.ndarray: + """ + Vectorized batch version of get_cuboid_corners + :param xyz: (n, 3) + :param wlh: (n, 3) + :param yaw: (n,) + :param pitch: (n,) + :param roll: (n,) + :param scale_convention: flag whether the internal Scale convention is used (have to be adjusted by pi/2) + :return: (n, 8, 3) + """ + if scale_convention: + yaw = yaw.copy() + np.pi / 2 + + w, l, h = wlh[:, 0, None], wlh[:, 1, None], wlh[:, 2, None] + + x_corners = l / 2 * np.array([1, 1, 1, 1, -1, -1, -1, -1]) + y_corners = w / 2 * np.array([1, -1, -1, 1, 1, -1, -1, 1]) + z_corners = h / 2 * np.array([1, 1, -1, -1, 1, 1, -1, -1]) + corners = np.stack((x_corners, y_corners, z_corners), axis=1) + + rot_mats = get_batch_rotation_matrices(yaw, pitch, roll) + corners = np.matmul(rot_mats, corners) + + x, y, z = xyz[:, 0, None], xyz[:, 1, None], xyz[:, 2, None] + corners[:, 0, :] = corners[:, 0, :] + x + corners[:, 1, :] = corners[:, 1, :] + y + corners[:, 2, :] = corners[:, 2, :] + z + return corners.swapaxes(1, 2) + + +def get_batch_rotation_matrices( + yaw: np.ndarray, pitch: np.ndarray = None, roll: np.ndarray = None +) -> np.ndarray: + if pitch is None: + pitch = np.zeros_like(yaw) + if roll is None: + roll = np.zeros_like(yaw) + cy = np.cos(yaw) + sy = np.sin(yaw) + cp = np.cos(pitch) + sp = np.sin(pitch) + cr = np.cos(roll) + sr = np.sin(roll) + return np.stack( + ( + np.stack( + (cy * cp, cy * sp * sr - sy * cr, cy * sp * cr + sy * sr), 1 + ), + np.stack( + (sy * cp, sy * sp * sr + cy * cr, sy * sp * cr - cy * sr), 1 + ), + np.stack((-sp, cp * sr, cp * cr), 1), + ), + 1, + ) + + +def associate_cuboids_on_iou( + xyz_0: np.ndarray, + wlh_0: np.ndarray, + yaw_0: np.ndarray, + xyz_1: np.ndarray, + wlh_1: np.ndarray, + yaw_1: np.ndarray, + threshold_in_overlap_ratio: float = 0.1, +) -> List[Tuple[int, int]]: + if xyz_0.shape[0] < 1 or xyz_1.shape[0] < 1: + return [] + iou_matrix, _ = compute_outer_iou(xyz_0, wlh_0, yaw_0, xyz_1, wlh_1, yaw_1) + mapping = [] + for i, m in enumerate(iou_matrix.max(axis=1)): + if m >= threshold_in_overlap_ratio: + mapping.append((i, iou_matrix[i].argmax())) + return mapping + + +def recall_precision( + prediction: List[CuboidPrediction], + groundtruth: List[CuboidAnnotation], + threshold_in_overlap_ratio: float, +) -> Dict[str, float]: + """ + Calculates the precision and recall of each lidar frame. + + Args: + :param predictions: list of cuboid annotation predictions. + :param ground_truth: list of cuboid annotation groundtruths. + :param threshold: IOU threshold to consider detection as valid. Must be in [0, 1]. + """ + + tp_sum = 0 + fp_sum = 0 + fn_sum = 0 + num_predicted = 0 + num_instances = 0 + + gt_items = process_dataitem(groundtruth) + pred_items = process_dataitem(prediction) + + num_predicted += pred_items["xyz"].shape[0] + num_instances += gt_items["xyz"].shape[0] + + tp = np.zeros(pred_items["xyz"].shape[0]) + fp = np.ones(pred_items["xyz"].shape[0]) + fn = np.ones(gt_items["xyz"].shape[0]) + + mapping = associate_cuboids_on_iou( + pred_items["xyz"], + pred_items["wlh"], + pred_items["yaw"] + np.pi / 2, + gt_items["xyz"], + gt_items["wlh"], + gt_items["yaw"] + np.pi / 2, + threshold_in_overlap_ratio=threshold_in_overlap_ratio, + ) + + for pred_id, gt_id in mapping: + if fn[gt_id] == 0: + continue + tp[pred_id] = 1 + fp[pred_id] = 0 + fn[gt_id] = 0 + + tp_sum += tp.sum() + fp_sum += fp.sum() + fn_sum += fn.sum() + + return { + "tp_sum": tp_sum, + "fp_sum": fp_sum, + "fn_sum": fn_sum, + "precision": tp_sum / (tp_sum + fp_sum), + "recall": tp_sum / (tp_sum + fn_sum), + "num_predicted": num_predicted, + "num_instances": num_instances, + } + + +def detection_iou( + prediction: List[CuboidPrediction], + groundtruth: List[CuboidAnnotation], + threshold_in_overlap_ratio: float, +) -> Tuple[np.ndarray, np.ndarray]: + """ + Calculates the 2D IOU and 3D IOU overlap between predictions and groundtruth. + Uses linear sum assignment to associate cuboids. + + Args: + :param predictions: list of cuboid annotation predictions. + :param ground_truth: list of cuboid annotation groundtruths. + :param threshold: IOU threshold to consider detection as valid. Must be in [0, 1]. + """ + + gt_items = process_dataitem(groundtruth) + pred_items = process_dataitem(prediction) + + meter_2d = [] + meter_3d = [] + + if gt_items["xyz"].shape[0] == 0 or pred_items["xyz"].shape[0] == 0: + return np.array([0.0]), np.array([0.0]) + + iou_3d, iou_2d = compute_outer_iou( + gt_items["xyz"], + gt_items["wlh"], + gt_items["yaw"], + pred_items["xyz"], + pred_items["wlh"], + pred_items["yaw"], + ) + + for i, m in enumerate(iou_3d.max(axis=1)): + if m >= threshold_in_overlap_ratio: + j = iou_3d[i].argmax() + meter_3d.append(iou_3d[i, j]) + meter_2d.append(iou_2d[i, j]) + + meter_3d = np.array(meter_3d) + meter_2d = np.array(meter_2d) + return meter_3d, meter_2d diff --git a/nucleus/metrics/filtering.py b/nucleus/metrics/filtering.py new file mode 100644 index 00000000..2ef517f0 --- /dev/null +++ b/nucleus/metrics/filtering.py @@ -0,0 +1,371 @@ +import enum +import functools +import logging +from enum import Enum +from typing import Callable, Iterable, List, NamedTuple, Sequence, Set, Union + +from nucleus.annotation import ( + BoxAnnotation, + CategoryAnnotation, + CuboidAnnotation, + LineAnnotation, + MultiCategoryAnnotation, + PolygonAnnotation, + SegmentationAnnotation, +) +from nucleus.prediction import ( + BoxPrediction, + CategoryPrediction, + CuboidPrediction, + LinePrediction, + PolygonPrediction, + SegmentationPrediction, +) + + +class FilterOp(str, Enum): + GT = ">" + GTE = ">=" + LT = "<" + LTE = "<=" + EQ = "=" + EQEQ = "==" + NEQ = "!=" + IN = "in" + NOT_IN = "not in" + + +class FilterType(str, enum.Enum): + """The type of the filter decides the getter used for the comparison. + Attributes: + FIELD: Access the attribute field of an object + METADATA: Access the metadata dictionary of an object + """ + + FIELD = "field" + METADATA = "metadata" + + +FilterableBaseVals = Union[str, float, int, bool] +FilterableTypes = Union[ + FilterableBaseVals, + Sequence[FilterableBaseVals], + Set[FilterableBaseVals], + Iterable[FilterableBaseVals], +] + +AnnotationTypes = Union[ + BoxAnnotation, + CategoryAnnotation, + CuboidAnnotation, + LineAnnotation, + MultiCategoryAnnotation, + PolygonAnnotation, + SegmentationAnnotation, +] +PredictionTypes = Union[ + BoxPrediction, + CategoryPrediction, + CuboidPrediction, + LinePrediction, + PolygonPrediction, + SegmentationPrediction, +] + + +class AnnotationOrPredictionFilter(NamedTuple): + """Internal type for reconstruction of JSON encoded payload. Type field decides if filter behaves like FieldFilter + or MetadataFilter + + Attributes: + key: key to compare with value + op: :class:`FilterOp` or one of [">", ">=", "<", "<=", "=", "==", "!=", "in", "not in"] to define comparison + with value field + value: bool, str, float or int to compare the field with key or list of the same values for 'in' and 'not in' + ops + allow_missing: Allow missing field values. Will REMOVE the object with the missing field from the selection + type: DO NOT USE. Internal type for serialization over the wire. Changing this will change the `NamedTuple` + type as well. + """ + + key: str + op: Union[FilterOp, str] + value: FilterableTypes + allow_missing: bool + type: FilterType + + +class FieldFilter(NamedTuple): + """Filter on standard field of AnnotationTypes or PredictionTypes + + Examples: + FieldFilter("x", ">", 10) would pass every :class:`BoxAnnotation` with `x` attribute larger than 10 + FieldFilter("label", "in", [) would pass every :class:`BoxAnnotation` with `x` attribute larger than 10 + + Attributes: + key: key to compare with value + op: :class:`FilterOp` or one of [">", ">=", "<", "<=", "=", "==", "!=", "in", "not in"] to define comparison + with value field + value: bool, str, float or int to compare the field with key or list of the same values for 'in' and 'not in' + ops + allow_missing: Allow missing field values. Will REMOVE the object with the missing field from the selection + type: DO NOT USE. Internal type for serialization over the wire. Changing this will change the `NamedTuple` + type as well. + """ + + key: str + op: Union[FilterOp, str] + value: FilterableTypes + allow_missing: bool = False + type: FilterType = FilterType.FIELD + + +class MetadataFilter(NamedTuple): + """Filter on customer provided metadata associated with AnnotationTypes or PredictionTypes + + Attributes: + key: key to compare with value + op: :class:`FilterOp` or one of [">", ">=", "<", "<=", "=", "==", "!=", "in", "not in"] to define comparison + with value field + value: bool, str, float or int to compare the field with key or list of the same values for 'in' and 'not in' + ops + allow_missing: Allow missing metada values. Will REMOVE the object with the missing field from the selection + type: DO NOT USE. Internal type for serialization over the wire. Changing this will change the `NamedTuple` + type as well. + """ + + key: str + op: Union[FilterOp, str] + value: FilterableTypes + allow_missing: bool = False + type: FilterType = FilterType.METADATA + + +Filter = Union[FieldFilter, MetadataFilter, AnnotationOrPredictionFilter] +OrAndDNFFilters = List[List[Filter]] +OrAndDNFFilters.__doc__ = """\ +Disjunctive normal form (DNF) filters. +DNF allows arbitrary boolean logical combinations of single field predicates. +The innermost structures each describe a single field predicate. + +The list of inner predicates is interpreted as a conjunction (AND), forming a more selective and multiple column +predicate. + +Finally, the most outer list combines these filters as a disjunction (OR). +""" +ListOfOrAndJSONSerialized = List[List[List]] +ListOfOrAndJSONSerialized.__doc__ = """\ +JSON serialized form of DNFFilters. The innermost list has to be trivially expandable (*list) to a +:class:`AnnotationOrPredictionFilter`. + +Disjunctive normal form (DNF) filters. +DNF allows arbitrary boolean logical combinations of single field predicates. +The innermost structures each describe a single field predicate. + -The list of inner predicates is interpreted as a conjunction (AND), forming a more selective and multiple column + predicate. + -Finally, the most outer list combines these filters as a disjunction (OR). + + +""" +ListOfOrAndFilters = Union[OrAndDNFFilters, ListOfOrAndJSONSerialized] +ListOfAndJSONSerialized = List[List] +ListOfAndFilterTuple = List[Filter] +ListOfAndFilterTuple.__doc__ = """\ +List of AND filters. +The list of predicates is interpreted as a conjunction (AND), forming a multiple field predicate. + +If providing a doubly nested list the innermost list has to be trivially expandable (*list) to a +:class:`AnnotationOrPredictionFilter` +""" +ListOfAndFilters = Union[ + ListOfAndFilterTuple, + ListOfAndJSONSerialized, +] + + +def _attribute_getter( + field_name: str, + allow_missing: bool, + ann_or_pred: Union[AnnotationTypes, PredictionTypes], +): + """Create a function to get object fields""" + if allow_missing: + return ( + getattr(ann_or_pred, field_name) + if hasattr(ann_or_pred, field_name) + else AlwaysFalseComparison() + ) + else: + return getattr(ann_or_pred, field_name) + + +class AlwaysFalseComparison: + """Helper class to make sure that allow filtering out missing fields (by always returning a false comparison)""" + + def __gt__(self, other): + return False + + def __ge__(self, other): + return False + + def __lt__(self, other): + return False + + def __le__(self, other): + return False + + def __eq__(self, other): + return False + + def __ne__(self, other): + return False + + +def _metadata_field_getter( + field_name: str, + allow_missing: bool, + ann_or_pred: Union[AnnotationTypes, PredictionTypes], +): + """Create a function to get a metadata field""" + if isinstance( + ann_or_pred, (SegmentationAnnotation, SegmentationPrediction) + ): + if allow_missing: + logging.warning( + "Trying to filter metadata on SegmentationAnnotation or Prediction. " + "This will never work until metadata is supported for this format." + ) + return AlwaysFalseComparison() + else: + raise RuntimeError( + f"{type(ann_or_pred)} doesn't support metadata filtering" + ) + + if allow_missing: + return ( + ann_or_pred.metadata.get(field_name, AlwaysFalseComparison()) + if ann_or_pred.metadata + else AlwaysFalseComparison() + ) + else: + return ( + ann_or_pred.metadata[field_name] + if ann_or_pred.metadata + else RuntimeError( + f"No metadata on {ann_or_pred}, trying to access {field_name}" + ) + ) + + +def _filter_to_comparison_function( # pylint: disable=too-many-return-statements + filter_def: Filter, +) -> Callable[[Union[AnnotationTypes, PredictionTypes]], bool]: + """Creates a comparison function from a filter configuration to apply to annotations or predictions + + Parameters: + filter_def: Definition of a filter conditions + + Returns: + + """ + if FilterType(filter_def.type) == FilterType.FIELD: + getter = functools.partial( + _attribute_getter, filter_def.key, filter_def.allow_missing + ) + elif FilterType(filter_def.type) == FilterType.METADATA: + getter = functools.partial( + _metadata_field_getter, filter_def.key, filter_def.allow_missing + ) + op = FilterOp(filter_def.op) + if op is FilterOp.GT: + return lambda ann_or_pred: getter(ann_or_pred) > filter_def.value + elif op is FilterOp.GTE: + return lambda ann_or_pred: getter(ann_or_pred) >= filter_def.value + elif op is FilterOp.LT: + return lambda ann_or_pred: getter(ann_or_pred) < filter_def.value + elif op is FilterOp.LTE: + return lambda ann_or_pred: getter(ann_or_pred) <= filter_def.value + elif op is FilterOp.EQ or op is FilterOp.EQEQ: + return lambda ann_or_pred: getter(ann_or_pred) == filter_def.value + elif op is FilterOp.NEQ: + return lambda ann_or_pred: getter(ann_or_pred) != filter_def.value + elif op is FilterOp.IN: + return lambda ann_or_pred: getter(ann_or_pred) in set( + filter_def.value # type: ignore + ) + elif op is FilterOp.NOT_IN: + return lambda ann_or_pred: getter(ann_or_pred) not in set( + filter_def.value # type:ignore + ) + else: + raise RuntimeError( + f"Fell through all op cases, no match for: '{op}' - MetadataFilter: {filter_def}," + ) + + +def apply_filters( + ann_or_pred: Union[Sequence[AnnotationTypes], Sequence[PredictionTypes]], + filters: Union[ListOfOrAndFilters, ListOfAndFilters], +): + """Apply filters to list of annotations or list of predictions + Attributes: + ann_or_pred: Prediction or Annotation + filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field + predicates. The innermost structures each describe a single column predicate. The list of inner predicates + is interpreted as a conjunction (AND), forming a more selective `and` multiple column predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + """ + if filters is None or len(filters) == 0: + return ann_or_pred + + filters = ensureDNFFilters(filters) + + dnf_condition_functions = [] + for or_branch in filters: + and_conditions = [ + _filter_to_comparison_function(cond) for cond in or_branch + ] + dnf_condition_functions.append(and_conditions) + + filtered = [] + for item in ann_or_pred: + for or_conditions in dnf_condition_functions: + if all(c(item) for c in or_conditions): + filtered.append(item) + break + return filtered + + +def ensureDNFFilters(filters) -> OrAndDNFFilters: + """JSON encoding creates a triple nested lists from the doubly nested tuples. This function creates the + tuple form again.""" + if isinstance(filters[0], (MetadataFilter, FieldFilter)): + # Normalize into DNF + filters: ListOfOrAndFilters = [filters] # type: ignore + + # NOTE: We have to handle JSON transformed tuples which become two or three layers of lists + if ( + isinstance(filters, list) + and isinstance(filters[0], list) + and isinstance(filters[0][0], str) + ): + filters = [filters] + if ( + isinstance(filters, list) + and isinstance(filters[0], list) + and isinstance(filters[0][0], list) + ): + formatted_filter = [] + for or_branch in filters: + and_chain = [ + AnnotationOrPredictionFilter(*condition) + for condition in or_branch + ] + formatted_filter.append(and_chain) + filters = formatted_filter + return filters diff --git a/nucleus/metrics/polygon_metrics.py b/nucleus/metrics/polygon_metrics.py index 7ebbf20d..82a67020 100644 --- a/nucleus/metrics/polygon_metrics.py +++ b/nucleus/metrics/polygon_metrics.py @@ -1,6 +1,6 @@ import sys from abc import abstractmethod -from typing import List, Union +from typing import List, Optional, Union import numpy as np @@ -8,6 +8,7 @@ from nucleus.prediction import BoxPrediction, PolygonPrediction, PredictionList from .base import Metric, ScalarResult +from .filtering import ListOfAndFilters, ListOfOrAndFilters from .filters import confidence_filter, polygon_label_filter from .metric_utils import compute_average_precision from .polygon_utils import ( @@ -82,13 +83,38 @@ def __init__( self, enforce_label_match: bool = False, confidence_threshold: float = 0.0, + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, ): """Initializes PolygonMetric abstract object. Args: enforce_label_match: whether to enforce that annotation and prediction labels must match. Default False confidence_threshold: minimum confidence threshold for predictions. Must be in [0, 1]. Default 0.0 + annotation_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + prediction_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). """ + super().__init__(annotation_filters, prediction_filters) self.enforce_label_match = enforce_label_match assert 0 <= confidence_threshold <= 1 self.confidence_threshold = confidence_threshold @@ -105,7 +131,7 @@ def eval( def aggregate_score(self, results: List[ScalarResult]) -> ScalarResult: # type: ignore[override] return ScalarResult.aggregate(results) - def __call__( + def call_metric( self, annotations: AnnotationList, predictions: PredictionList ) -> ScalarResult: if self.confidence_threshold > 0: @@ -169,6 +195,12 @@ def __init__( enforce_label_match: bool = False, iou_threshold: float = 0.0, confidence_threshold: float = 0.0, + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, ): """Initializes PolygonIOU object. @@ -176,12 +208,35 @@ def __init__( enforce_label_match: whether to enforce that annotation and prediction labels must match. Defaults to False iou_threshold: IOU threshold to consider detection as valid. Must be in [0, 1]. Default 0.0 confidence_threshold: minimum confidence threshold for predictions. Must be in [0, 1]. Default 0.0 + annotation_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + prediction_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). """ assert ( 0 <= iou_threshold <= 1 ), "IoU threshold must be between 0 and 1." self.iou_threshold = iou_threshold - super().__init__(enforce_label_match, confidence_threshold) + super().__init__( + enforce_label_match, + confidence_threshold, + annotation_filters, + prediction_filters, + ) def eval( self, @@ -237,6 +292,12 @@ def __init__( enforce_label_match: bool = False, iou_threshold: float = 0.5, confidence_threshold: float = 0.0, + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, ): """Initializes PolygonPrecision object. @@ -244,12 +305,35 @@ def __init__( enforce_label_match: whether to enforce that annotation and prediction labels must match. Defaults to False iou_threshold: IOU threshold to consider detection as valid. Must be in [0, 1]. Default 0.5 confidence_threshold: minimum confidence threshold for predictions. Must be in [0, 1]. Default 0.0 + annotation_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + prediction_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). """ assert ( 0 <= iou_threshold <= 1 ), "IoU threshold must be between 0 and 1." self.iou_threshold = iou_threshold - super().__init__(enforce_label_match, confidence_threshold) + super().__init__( + enforce_label_match, + confidence_threshold, + annotation_filters, + prediction_filters, + ) def eval( self, @@ -306,6 +390,12 @@ def __init__( enforce_label_match: bool = False, iou_threshold: float = 0.5, confidence_threshold: float = 0.0, + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, ): """Initializes PolygonRecall object. @@ -313,12 +403,35 @@ def __init__( enforce_label_match: whether to enforce that annotation and prediction labels must match. Defaults to False iou_threshold: IOU threshold to consider detection as valid. Must be in [0, 1]. Default 0.5 confidence_threshold: minimum confidence threshold for predictions. Must be in [0, 1]. Default 0.0 + annotation_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + prediction_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). """ assert ( 0 <= iou_threshold <= 1 ), "IoU threshold must be between 0 and 1." self.iou_threshold = iou_threshold - super().__init__(enforce_label_match, confidence_threshold) + super().__init__( + enforce_label_match, + confidence_threshold, + annotation_filters=annotation_filters, + prediction_filters=prediction_filters, + ) def eval( self, @@ -374,18 +487,47 @@ def __init__( self, label, iou_threshold: float = 0.5, + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, ): """Initializes PolygonRecall object. Args: iou_threshold: IOU threshold to consider detection as valid. Must be in [0, 1]. Default 0.5 + annotation_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + prediction_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). """ assert ( 0 <= iou_threshold <= 1 ), "IoU threshold must be between 0 and 1." self.iou_threshold = iou_threshold self.label = label - super().__init__(enforce_label_match=False, confidence_threshold=0) + super().__init__( + enforce_label_match=False, + confidence_threshold=0, + annotation_filters=annotation_filters, + prediction_filters=prediction_filters, + ) def eval( self, @@ -450,17 +592,46 @@ class PolygonMAP(PolygonMetric): def __init__( self, iou_threshold: float = 0.5, + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, ): """Initializes PolygonRecall object. Args: iou_threshold: IOU threshold to consider detection as valid. Must be in [0, 1]. Default 0.5 + annotation_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + prediction_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). """ assert ( 0 <= iou_threshold <= 1 ), "IoU threshold must be between 0 and 1." self.iou_threshold = iou_threshold - super().__init__(enforce_label_match=False, confidence_threshold=0) + super().__init__( + enforce_label_match=False, + confidence_threshold=0, + annotation_filters=annotation_filters, + prediction_filters=prediction_filters, + ) def eval( self, diff --git a/nucleus/prediction.py b/nucleus/prediction.py index 31176d84..a73d8bee 100644 --- a/nucleus/prediction.py +++ b/nucleus/prediction.py @@ -120,6 +120,7 @@ def from_json(cls, payload: dict): ], reference_id=payload[REFERENCE_ID_KEY], annotation_id=payload.get(ANNOTATION_ID_KEY, None), + # metadata=payload.get(METADATA_KEY, None), # TODO(sc: 422637) ) diff --git a/nucleus/validate/client.py b/nucleus/validate/client.py index cfc5fbb5..43e2f7f1 100644 --- a/nucleus/validate/client.py +++ b/nucleus/validate/client.py @@ -5,7 +5,10 @@ from .constants import SCENARIO_TEST_ID_KEY from .data_transfer_objects.eval_function import GetEvalFunctions -from .data_transfer_objects.scenario_test import CreateScenarioTestRequest +from .data_transfer_objects.scenario_test import ( + CreateScenarioTestRequest, + EvalFunctionListEntry, +) from .errors import CreateScenarioTestError from .eval_functions.available_eval_functions import AvailableEvalFunctions from .eval_functions.base_eval_function import EvalFunctionConfig @@ -83,7 +86,10 @@ def create_scenario_test( name=name, slice_id=slice_id, evaluation_functions=[ - ef.to_entry() for ef in evaluation_functions # type:ignore + EvalFunctionListEntry( + id=ef.id, eval_func_arguments=ef.eval_func_arguments + ) + for ef in evaluation_functions ], ).dict(), "validate/scenario_test", diff --git a/nucleus/validate/data_transfer_objects/scenario_test.py b/nucleus/validate/data_transfer_objects/scenario_test.py index 174571ff..4e029867 100644 --- a/nucleus/validate/data_transfer_objects/scenario_test.py +++ b/nucleus/validate/data_transfer_objects/scenario_test.py @@ -4,13 +4,16 @@ from nucleus.pydantic_base import ImmutableModel -from .eval_function import EvalFunctionEntry + +class EvalFunctionListEntry(ImmutableModel): + id: str + eval_func_arguments: dict class CreateScenarioTestRequest(ImmutableModel): name: str slice_id: str - evaluation_functions: List[EvalFunctionEntry] + evaluation_functions: List[EvalFunctionListEntry] @validator("slice_id") def startswith_slice_indicator(cls, v): # pylint: disable=no-self-argument diff --git a/nucleus/validate/eval_functions/available_eval_functions.py b/nucleus/validate/eval_functions/available_eval_functions.py index a2ba14cd..80cb2dd7 100644 --- a/nucleus/validate/eval_functions/available_eval_functions.py +++ b/nucleus/validate/eval_functions/available_eval_functions.py @@ -6,11 +6,10 @@ EvalFunctionConfig, ) +from ...metrics.filtering import ListOfAndFilters, ListOfOrAndFilters from ..data_transfer_objects.eval_function import EvalFunctionEntry from ..errors import EvalFunctionNotAvailableError -MEAN_AVG_PRECISION_NAME = "mean_average_precision_boxes" - class PolygonIOUConfig(EvalFunctionConfig): def __call__( @@ -18,6 +17,12 @@ def __call__( enforce_label_match: bool = False, iou_threshold: float = 0.0, confidence_threshold: float = 0.0, + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, **kwargs, ): """Configures a call to :class:`PolygonIOU` object. @@ -38,6 +43,24 @@ def __call__( enforce_label_match: whether to enforce that annotation and prediction labels must match. Defaults to False iou_threshold: IOU threshold to consider detection as valid. Must be in [0, 1]. Default 0.0 confidence_threshold: minimum confidence threshold for predictions. Must be in [0, 1]. Default 0.0 + annotation_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + prediction_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). """ return super().__call__( enforce_label_match=enforce_label_match, @@ -55,6 +78,12 @@ class PolygonMAPConfig(EvalFunctionConfig): def __call__( self, iou_threshold: float = 0.5, + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, **kwargs, ): """Configures a call to :class:`PolygonMAP` object. @@ -73,6 +102,24 @@ def __call__( Args: iou_threshold: IOU threshold to consider detection as valid. Must be in [0, 1]. Default 0.0 + annotation_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + prediction_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). """ return super().__call__( iou_threshold=iou_threshold, @@ -90,6 +137,12 @@ def __call__( enforce_label_match: bool = False, iou_threshold: float = 0.5, confidence_threshold: float = 0.0, + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, **kwargs, ): """Configures a call to :class:`PolygonRecall` object. @@ -110,6 +163,24 @@ def __call__( enforce_label_match: whether to enforce that annotation and prediction labels must match. Defaults to False iou_threshold: IOU threshold to consider detection as valid. Must be in [0, 1]. Default 0.0 confidence_threshold: minimum confidence threshold for predictions. Must be in [0, 1]. Default 0.0 + annotation_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + prediction_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). """ return super().__call__( enforce_label_match=enforce_label_match, @@ -129,6 +200,12 @@ def __call__( enforce_label_match: bool = False, iou_threshold: float = 0.5, confidence_threshold: float = 0.0, + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, **kwargs, ): """Configures a call to :class:`PolygonPrecision` object. @@ -149,6 +226,24 @@ def __call__( enforce_label_match: whether to enforce that annotation and prediction labels must match. Defaults to False iou_threshold: IOU threshold to consider detection as valid. Must be in [0, 1]. Default 0.0 confidence_threshold: minimum confidence threshold for predictions. Must be in [0, 1]. Default 0.0 + annotation_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + prediction_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). """ return super().__call__( enforce_label_match=enforce_label_match, @@ -162,11 +257,234 @@ def expected_name(cls) -> str: return "bbox_precision" +class CuboidIOU2DConfig(EvalFunctionConfig): + def __call__( + self, + enforce_label_match: bool = True, + iou_threshold: float = 0.0, + confidence_threshold: float = 0.0, + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + **kwargs, + ): + """Configure a call to CuboidIOU object. + + Args: + enforce_label_match: whether to enforce that annotation and prediction labels must match. Defaults to True + iou_threshold: IOU threshold to consider detection as valid. Must be in [0, 1]. Default 0.0 + confidence_threshold: minimum confidence threshold for predictions. Must be in [0, 1]. Default 0.0 + annotation_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + prediction_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + """ + return super().__call__( + enforce_label_match=enforce_label_match, + iou_threshold=iou_threshold, + confidence_threshold=confidence_threshold, + iou_2d=True, + annotation_filters=annotation_filters, + prediction_filters=prediction_filters, + **kwargs, + ) + + @classmethod + def expected_name(cls) -> str: + return "cuboid_iou_2d" + + +class CuboidIOU3DConfig(EvalFunctionConfig): + def __call__( + self, + enforce_label_match: bool = True, + iou_threshold: float = 0.0, + confidence_threshold: float = 0.0, + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + **kwargs, + ): + """Configure a call to CuboidIOU object. + + Args: + enforce_label_match: whether to enforce that annotation and prediction labels must match. Defaults to True + iou_threshold: IOU threshold to consider detection as valid. Must be in [0, 1]. Default 0.0 + iou_2d: whether to return the BEV 2D IOU if true, or the 3D IOU if false. + confidence_threshold: minimum confidence threshold for predictions. Must be in [0, 1]. Default 0.0 + annotation_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + prediction_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + """ + return super().__call__( + enforce_label_match=enforce_label_match, + iou_threshold=iou_threshold, + confidence_threshold=confidence_threshold, + iou_2d=False, + annotation_filters=annotation_filters, + prediction_filters=prediction_filters, + **kwargs, + ) + + @classmethod + def expected_name(cls) -> str: + return "cuboid_iou_3d" + + +class CuboidPrecisionConfig(EvalFunctionConfig): + def __call__( + self, + enforce_label_match: bool = True, + iou_threshold: float = 0.0, + confidence_threshold: float = 0.0, + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + **kwargs, + ): + """Configure a call to CuboidPrecision object. + + Args: + enforce_label_match: whether to enforce that annotation and prediction labels must match. Defaults to True + iou_threshold: IOU threshold to consider detection as valid. Must be in [0, 1]. Default 0.0 + iou_2d: whether to return the BEV 2D IOU if true, or the 3D IOU if false. + confidence_threshold: minimum confidence threshold for predictions. Must be in [0, 1]. Default 0.0 + annotation_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + prediction_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + """ + return super().__call__( + enforce_label_match=enforce_label_match, + iou_threshold=iou_threshold, + confidence_threshold=confidence_threshold, + annotation_filters=annotation_filters, + prediction_filters=prediction_filters, + **kwargs, + ) + + @classmethod + def expected_name(cls) -> str: + return "cuboid_precision" + + +class CuboidRecallConfig(EvalFunctionConfig): + def __call__( + self, + enforce_label_match: bool = True, + iou_threshold: float = 0.0, + confidence_threshold: float = 0.0, + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + **kwargs, + ): + """Configure a call to a CuboidRecall object. + + Args: + enforce_label_match: whether to enforce that annotation and prediction labels must match. Defaults to True + iou_threshold: IOU threshold to consider detection as valid. Must be in [0, 1]. Default 0.0 + iou_2d: whether to return the BEV 2D IOU if true, or the 3D IOU if false. + confidence_threshold: minimum confidence threshold for predictions. Must be in [0, 1]. Default 0.0 + annotation_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + prediction_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + """ + return super().__call__( + enforce_label_match=enforce_label_match, + iou_threshold=iou_threshold, + confidence_threshold=confidence_threshold, + annotation_filters=annotation_filters, + prediction_filters=prediction_filters, + **kwargs, + ) + + @classmethod + def expected_name(cls) -> str: + return "cuboid_recall" + + class CategorizationF1Config(EvalFunctionConfig): def __call__( self, confidence_threshold: Optional[float] = None, f1_method: Optional[str] = None, + annotation_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, + prediction_filters: Optional[ + Union[ListOfOrAndFilters, ListOfAndFilters] + ] = None, **kwargs, ): """ Configure an evaluation of :class:`CategorizationF1`. @@ -210,6 +528,24 @@ def __call__( Calculate metrics for each instance, and find their average (only meaningful for multilabel classification where this differs from :func:`accuracy_score`). + annotation_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). + prediction_filters: Filter predicates. Allowed formats are: + ListOfAndFilters where each Filter forms a chain of AND predicates. + or + ListOfOrAndFilters where Filters are expressed in disjunctive normal form (DNF), like + [[MetadataFilter("short_haired", "==", True), FieldFilter("label", "in", ["cat", "dog"]), ...]. + DNF allows arbitrary boolean logical combinations of single field predicates. The innermost structures + each describe a single column predicate. The list of inner predicates is interpreted as a conjunction + (AND), forming a more selective `and` multiple field predicate. + Finally, the most outer list combines these filters as a disjunction (OR). """ return super().__call__( confidence_threshold=confidence_threshold, f1_method=f1_method @@ -273,6 +609,10 @@ def expected_name(cls) -> str: PolygonMAPConfig, PolygonPrecisionConfig, PolygonRecallConfig, + CuboidRecallConfig, + CuboidIOU2DConfig, + CuboidIOU3DConfig, + CuboidPrecisionConfig, CategorizationF1Config, CustomEvalFunction, EvalFunctionNotAvailable, @@ -324,6 +664,10 @@ def __init__(self, available_functions: List[EvalFunctionEntry]): self.cat_f1: CategorizationF1Config = self._assign_eval_function_if_defined( CategorizationF1Config # type: ignore ) + self.cuboid_iou_2d: CuboidIOU2DConfig = self._assign_eval_function_if_defined(CuboidIOU2DConfig) # type: ignore + self.cuboid_iou_3d: CuboidIOU3DConfig = self._assign_eval_function_if_defined(CuboidIOU3DConfig) # type: ignore + self.cuboid_precision: CuboidPrecisionConfig = self._assign_eval_function_if_defined(CuboidPrecisionConfig) # type: ignore + self.cuboid_recall: CuboidRecallConfig = self._assign_eval_function_if_defined(CuboidRecallConfig) # type: ignore # Add public entries that have not been implemented as an attribute on this class for func_entry in self._public_func_entries.values(): diff --git a/nucleus/validate/eval_functions/base_eval_function.py b/nucleus/validate/eval_functions/base_eval_function.py index af823f70..fa59ec31 100644 --- a/nucleus/validate/eval_functions/base_eval_function.py +++ b/nucleus/validate/eval_functions/base_eval_function.py @@ -62,6 +62,3 @@ def _op_to_test_metric(self, comparison: ThresholdComparison, value): threshold=value, eval_func_arguments=self.eval_func_arguments, ) - - def to_entry(self): - return self.eval_func_entry diff --git a/poetry.lock b/poetry.lock index 55dc71a2..85b3ba09 100644 --- a/poetry.lock +++ b/poetry.lock @@ -81,7 +81,7 @@ python-versions = "*" [[package]] name = "appnope" -version = "0.1.2" +version = "0.1.3" description = "Disable App Nap on macOS >= 10.9" category = "dev" optional = false @@ -122,7 +122,7 @@ tests = ["pytest"] [[package]] name = "astroid" -version = "2.9.3" +version = "2.11.2" description = "An abstract syntax tree for Python with inference support." category = "dev" optional = false @@ -132,7 +132,7 @@ python-versions = ">=3.6.2" lazy-object-proxy = ">=1.4.0" typed-ast = {version = ">=1.4.0,<2.0", markers = "implementation_name == \"cpython\" and python_version < \"3.8\""} typing-extensions = {version = ">=3.10", markers = "python_version < \"3.10\""} -wrapt = ">=1.11,<1.14" +wrapt = ">=1.11,<2" [[package]] name = "async-generator" @@ -408,7 +408,7 @@ python-versions = ">=3.6,<4.0" [[package]] name = "cryptography" -version = "36.0.1" +version = "36.0.2" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "dev" optional = false @@ -449,6 +449,17 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "dill" +version = "0.3.4" +description = "serialize all of python" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*" + +[package.extras] +graph = ["objgraph (>=1.7.2)"] + [[package]] name = "distlib" version = "0.3.4" @@ -593,7 +604,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "immutables" -version = "0.16" +version = "0.17" description = "Immutable Collections" category = "dev" optional = false @@ -1102,7 +1113,7 @@ test = ["check-manifest", "fastjsonschema", "testpath", "pytest", "pytest-cov"] [[package]] name = "nest-asyncio" -version = "1.5.4" +version = "1.5.5" description = "Patch asyncio to allow nested event loops" category = "main" optional = false @@ -1118,7 +1129,7 @@ python-versions = "*" [[package]] name = "notebook" -version = "6.4.8" +version = "6.4.10" description = "A web-based notebook environment for interactive computing" category = "dev" optional = false @@ -1131,7 +1142,7 @@ ipython-genutils = "*" jinja2 = "*" jupyter-client = ">=5.3.4" jupyter-core = ">=4.6.1" -nbconvert = "*" +nbconvert = ">=5" nbformat = "*" nest-asyncio = ">=1.5" prometheus-client = "*" @@ -1326,7 +1337,7 @@ twisted = ["twisted"] [[package]] name = "prompt-toolkit" -version = "3.0.28" +version = "3.0.29" description = "Library for building powerful interactive command lines in Python" category = "dev" optional = false @@ -1409,21 +1420,25 @@ python-versions = "*" [[package]] name = "pylint" -version = "2.12.2" +version = "2.13.4" description = "python code static checker" category = "dev" optional = false python-versions = ">=3.6.2" [package.dependencies] -astroid = ">=2.9.0,<2.10" +astroid = ">=2.11.2,<=2.12.0-dev0" colorama = {version = "*", markers = "sys_platform == \"win32\""} +dill = ">=0.2" isort = ">=4.2.5,<6" -mccabe = ">=0.6,<0.7" +mccabe = ">=0.6,<0.8" platformdirs = ">=2.2.0" -toml = ">=0.9.2" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""} +[package.extras] +testutil = ["gitpython (>3)"] + [[package]] name = "pyparsing" version = "3.0.7" @@ -1508,7 +1523,7 @@ six = ">=1.5" [[package]] name = "pytz" -version = "2021.3" +version = "2022.1" description = "World timezone definitions, modern and historical" category = "dev" optional = false @@ -1560,7 +1575,7 @@ py = {version = "*", markers = "implementation_name == \"pypy\""} [[package]] name = "regex" -version = "2022.3.2" +version = "2022.3.15" description = "Alternative regular expression module, to replace re." category = "dev" optional = false @@ -1669,6 +1684,19 @@ nativelib = ["pyobjc-framework-cocoa", "pywin32"] objc = ["pyobjc-framework-cocoa"] win32 = ["pywin32"] +[[package]] +name = "shapely" +version = "1.8.1.post1" +description = "Geometric objects, predicates, and operations" +category = "main" +optional = true +python-versions = ">=3.6" + +[package.extras] +all = ["pytest", "pytest-cov", "numpy"] +test = ["pytest", "pytest-cov"] +vectorized = ["numpy"] + [[package]] name = "shellingham" version = "1.4.0" @@ -1893,9 +1921,17 @@ category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "tomli" +version = "1.2.3" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.6" + [[package]] name = "tomlkit" -version = "0.10.0" +version = "0.10.1" description = "Style preserving TOML library" category = "dev" optional = false @@ -1911,7 +1947,7 @@ python-versions = ">= 3.5" [[package]] name = "tqdm" -version = "4.63.0" +version = "4.64.0" description = "Fast, Extensible Progress Meter" category = "main" optional = false @@ -1924,6 +1960,7 @@ importlib-resources = {version = "*", markers = "python_version < \"3.7\""} [package.extras] dev = ["py-make (>=0.1.0)", "twine", "wheel"] notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] telegram = ["requests"] [[package]] @@ -1960,7 +1997,7 @@ python-versions = ">=3.6" [[package]] name = "unidecode" -version = "1.3.3" +version = "1.3.4" description = "ASCII transliterations of Unicode text" category = "dev" optional = false @@ -1968,20 +2005,20 @@ python-versions = ">=3.5" [[package]] name = "urllib3" -version = "1.26.8" +version = "1.26.9" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" [package.extras] -brotli = ["brotlipy (>=0.6.0)"] +brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.13.3" +version = "20.14.0" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -2030,7 +2067,7 @@ test = ["websockets"] [[package]] name = "wrapt" -version = "1.13.3" +version = "1.14.0" description = "Module for decorators, wrappers and monkey patching." category = "dev" optional = false @@ -2061,10 +2098,13 @@ python-versions = ">=3.6" docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] +[extras] +shapely = ["Shapely"] + [metadata] lock-version = "1.1" python-versions = "^3.6.2" -content-hash = "8d6269a2d5d30685e0d1e9b38187b6105826042e868acfb5bdb5a3e7188560f8" +content-hash = "9ec43abb1b42a874d8504909bd50eb85a1b19a32fb27e61f43ed4dbe2f09614c" [metadata.files] absl-py = [ @@ -2162,8 +2202,8 @@ appdirs = [ {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, ] appnope = [ - {file = "appnope-0.1.2-py2.py3-none-any.whl", hash = "sha256:93aa393e9d6c54c5cd570ccadd8edad61ea0c4b9ea7a01409020c9aa019eb442"}, - {file = "appnope-0.1.2.tar.gz", hash = "sha256:dd83cd4b5b460958838f6eb3000c660b1f9caf2a5b1de4264e941512f603258a"}, + {file = "appnope-0.1.3-py2.py3-none-any.whl", hash = "sha256:265a455292d0bd8a72453494fa24df5a11eb18373a60c7c0430889f22548605e"}, + {file = "appnope-0.1.3.tar.gz", hash = "sha256:02bd91c4de869fbb1e1c50aafc4098827a7a54ab2f39d9dcba6c9547ed920e24"}, ] argon2-cffi = [ {file = "argon2-cffi-21.3.0.tar.gz", hash = "sha256:d384164d944190a7dd7ef22c6aa3ff197da12962bd04b17f64d4e93d934dba5b"}, @@ -2193,8 +2233,8 @@ argon2-cffi-bindings = [ {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5e00316dabdaea0b2dd82d141cc66889ced0cdcbfa599e8b471cf22c620c329a"}, ] astroid = [ - {file = "astroid-2.9.3-py3-none-any.whl", hash = "sha256:506daabe5edffb7e696ad82483ad0228245a9742ed7d2d8c9cdb31537decf9f6"}, - {file = "astroid-2.9.3.tar.gz", hash = "sha256:1efdf4e867d4d8ba4a9f6cf9ce07cd182c4c41de77f23814feb27ca93ca9d877"}, + {file = "astroid-2.11.2-py3-none-any.whl", hash = "sha256:cc8cc0d2d916c42d0a7c476c57550a4557a083081976bf42a73414322a6411d9"}, + {file = "astroid-2.11.2.tar.gz", hash = "sha256:8d0a30fe6481ce919f56690076eafbb2fb649142a89dc874f1ec0e7a011492d0"}, ] async-generator = [ {file = "async_generator-1.10-py3-none-any.whl", hash = "sha256:01c7bf666359b4967d2cda0000cc2e4af16a0ae098cbffcb8472fb9e8ad6585b"}, @@ -2389,26 +2429,26 @@ crashtest = [ {file = "crashtest-0.3.1.tar.gz", hash = "sha256:42ca7b6ce88b6c7433e2ce47ea884e91ec93104a4b754998be498a8e6c3d37dd"}, ] cryptography = [ - {file = "cryptography-36.0.1-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:73bc2d3f2444bcfeac67dd130ff2ea598ea5f20b40e36d19821b4df8c9c5037b"}, - {file = "cryptography-36.0.1-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:2d87cdcb378d3cfed944dac30596da1968f88fb96d7fc34fdae30a99054b2e31"}, - {file = "cryptography-36.0.1-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:74d6c7e80609c0f4c2434b97b80c7f8fdfaa072ca4baab7e239a15d6d70ed73a"}, - {file = "cryptography-36.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:6c0c021f35b421ebf5976abf2daacc47e235f8b6082d3396a2fe3ccd537ab173"}, - {file = "cryptography-36.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d59a9d55027a8b88fd9fd2826c4392bd487d74bf628bb9d39beecc62a644c12"}, - {file = "cryptography-36.0.1-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a817b961b46894c5ca8a66b599c745b9a3d9f822725221f0e0fe49dc043a3a3"}, - {file = "cryptography-36.0.1-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:94ae132f0e40fe48f310bba63f477f14a43116f05ddb69d6fa31e93f05848ae2"}, - {file = "cryptography-36.0.1-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:7be0eec337359c155df191d6ae00a5e8bbb63933883f4f5dffc439dac5348c3f"}, - {file = "cryptography-36.0.1-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:e0344c14c9cb89e76eb6a060e67980c9e35b3f36691e15e1b7a9e58a0a6c6dc3"}, - {file = "cryptography-36.0.1-cp36-abi3-win32.whl", hash = "sha256:4caa4b893d8fad33cf1964d3e51842cd78ba87401ab1d2e44556826df849a8ca"}, - {file = "cryptography-36.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:391432971a66cfaf94b21c24ab465a4cc3e8bf4a939c1ca5c3e3a6e0abebdbcf"}, - {file = "cryptography-36.0.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:bb5829d027ff82aa872d76158919045a7c1e91fbf241aec32cb07956e9ebd3c9"}, - {file = "cryptography-36.0.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebc15b1c22e55c4d5566e3ca4db8689470a0ca2babef8e3a9ee057a8b82ce4b1"}, - {file = "cryptography-36.0.1-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:596f3cd67e1b950bc372c33f1a28a0692080625592ea6392987dba7f09f17a94"}, - {file = "cryptography-36.0.1-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:30ee1eb3ebe1644d1c3f183d115a8c04e4e603ed6ce8e394ed39eea4a98469ac"}, - {file = "cryptography-36.0.1-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ec63da4e7e4a5f924b90af42eddf20b698a70e58d86a72d943857c4c6045b3ee"}, - {file = "cryptography-36.0.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca238ceb7ba0bdf6ce88c1b74a87bffcee5afbfa1e41e173b1ceb095b39add46"}, - {file = "cryptography-36.0.1-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:ca28641954f767f9822c24e927ad894d45d5a1e501767599647259cbf030b903"}, - {file = "cryptography-36.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:39bdf8e70eee6b1c7b289ec6e5d84d49a6bfa11f8b8646b5b3dfe41219153316"}, - {file = "cryptography-36.0.1.tar.gz", hash = "sha256:53e5c1dc3d7a953de055d77bef2ff607ceef7a2aac0353b5d630ab67f7423638"}, + {file = "cryptography-36.0.2-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:4e2dddd38a5ba733be6a025a1475a9f45e4e41139d1321f412c6b360b19070b6"}, + {file = "cryptography-36.0.2-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:4881d09298cd0b669bb15b9cfe6166f16fc1277b4ed0d04a22f3d6430cb30f1d"}, + {file = "cryptography-36.0.2-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ea634401ca02367c1567f012317502ef3437522e2fc44a3ea1844de028fa4b84"}, + {file = "cryptography-36.0.2-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:7be666cc4599b415f320839e36367b273db8501127b38316f3b9f22f17a0b815"}, + {file = "cryptography-36.0.2-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8241cac0aae90b82d6b5c443b853723bcc66963970c67e56e71a2609dc4b5eaf"}, + {file = "cryptography-36.0.2-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b2d54e787a884ffc6e187262823b6feb06c338084bbe80d45166a1cb1c6c5bf"}, + {file = "cryptography-36.0.2-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:c2c5250ff0d36fd58550252f54915776940e4e866f38f3a7866d92b32a654b86"}, + {file = "cryptography-36.0.2-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:ec6597aa85ce03f3e507566b8bcdf9da2227ec86c4266bd5e6ab4d9e0cc8dab2"}, + {file = "cryptography-36.0.2-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ca9f686517ec2c4a4ce930207f75c00bf03d94e5063cbc00a1dc42531511b7eb"}, + {file = "cryptography-36.0.2-cp36-abi3-win32.whl", hash = "sha256:f64b232348ee82f13aac22856515ce0195837f6968aeaa94a3d0353ea2ec06a6"}, + {file = "cryptography-36.0.2-cp36-abi3-win_amd64.whl", hash = "sha256:53e0285b49fd0ab6e604f4c5d9c5ddd98de77018542e88366923f152dbeb3c29"}, + {file = "cryptography-36.0.2-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:32db5cc49c73f39aac27574522cecd0a4bb7384e71198bc65a0d23f901e89bb7"}, + {file = "cryptography-36.0.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b3d199647468d410994dbeb8cec5816fb74feb9368aedf300af709ef507e3e"}, + {file = "cryptography-36.0.2-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:da73d095f8590ad437cd5e9faf6628a218aa7c387e1fdf67b888b47ba56a17f0"}, + {file = "cryptography-36.0.2-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:0a3bf09bb0b7a2c93ce7b98cb107e9170a90c51a0162a20af1c61c765b90e60b"}, + {file = "cryptography-36.0.2-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8897b7b7ec077c819187a123174b645eb680c13df68354ed99f9b40a50898f77"}, + {file = "cryptography-36.0.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82740818f2f240a5da8dfb8943b360e4f24022b093207160c77cadade47d7c85"}, + {file = "cryptography-36.0.2-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:1f64a62b3b75e4005df19d3b5235abd43fa6358d5516cfc43d87aeba8d08dd51"}, + {file = "cryptography-36.0.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e167b6b710c7f7bc54e67ef593f8731e1f45aa35f8a8a7b72d6e42ec76afd4b3"}, + {file = "cryptography-36.0.2.tar.gz", hash = "sha256:70f8f4f7bb2ac9f340655cbac89d68c527af5bb4387522a8413e841e3e6628c9"}, ] dataclasses = [ {file = "dataclasses-0.7-py3-none-any.whl", hash = "sha256:3459118f7ede7c8bea0fe795bff7c6c2ce287d01dd226202f7c9ebc0610a7836"}, @@ -2422,6 +2462,10 @@ defusedxml = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, ] +dill = [ + {file = "dill-0.3.4-py2.py3-none-any.whl", hash = "sha256:7e40e4a70304fd9ceab3535d36e58791d9c4a776b38ec7f7ec9afc8d3dca4d4f"}, + {file = "dill-0.3.4.zip", hash = "sha256:9f9734205146b2b353ab3fec9af0070237b6ddae78452af83d2fca84d739e675"}, +] distlib = [ {file = "distlib-0.3.4-py2.py3-none-any.whl", hash = "sha256:6564fe0a8f51e734df6333d08b8b94d4ea8ee6b99b5ed50613f731fd4089f34b"}, {file = "distlib-0.3.4.zip", hash = "sha256:e4b58818180336dc9c529bfb9a0b58728ffc09ad92027a3f30b7cd91e3458579"}, @@ -2544,33 +2588,55 @@ imagesize = [ {file = "imagesize-1.3.0.tar.gz", hash = "sha256:cd1750d452385ca327479d45b64d9c7729ecf0b3969a58148298c77092261f9d"}, ] immutables = [ - {file = "immutables-0.16-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:acbfa79d44228d96296279068441f980dc63dbed52522d9227ff9f4d96c6627e"}, - {file = "immutables-0.16-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c9ed003eacb92e630ef200e31f47236c2139b39476894f7963b32bd39bafa3"}, - {file = "immutables-0.16-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0a396314b9024fa55bf83a27813fd76cf9f27dce51f53b0f19b51de035146251"}, - {file = "immutables-0.16-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:4a2a71678348fb95b13ca108d447f559a754c41b47bd1e7e4fb23974e735682d"}, - {file = "immutables-0.16-cp36-cp36m-win32.whl", hash = "sha256:064001638ab5d36f6aa05b6101446f4a5793fb71e522bc81b8fc65a1894266ff"}, - {file = "immutables-0.16-cp36-cp36m-win_amd64.whl", hash = "sha256:1de393f1b188740ca7b38f946f2bbc7edf3910d2048f03bbb8d01f17a038d67c"}, - {file = "immutables-0.16-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fcf678a3074613119385a02a07c469ec5130559f5ea843c85a0840c80b5b71c6"}, - {file = "immutables-0.16-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a307eb0984eb43e815dcacea3ac50c11d00a936ecf694c46991cd5a23bcb0ec0"}, - {file = "immutables-0.16-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7a58825ff2254e2612c5a932174398a4ea8fbddd8a64a02c880cc32ee28b8820"}, - {file = "immutables-0.16-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:798b095381eb42cf40db6876339e7bed84093e5868018a9e73d8e1f7ab4bb21e"}, - {file = "immutables-0.16-cp37-cp37m-win32.whl", hash = "sha256:19bdede174847c2ef1292df0f23868ab3918b560febb09fcac6eec621bd4812b"}, - {file = "immutables-0.16-cp37-cp37m-win_amd64.whl", hash = "sha256:9ccf4c0e3e2e3237012b516c74c49de8872ccdf9129739f7a0b9d7444a8c4862"}, - {file = "immutables-0.16-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:d59beef203a3765db72b1d0943547425c8318ecf7d64c451fd1e130b653c2fbb"}, - {file = "immutables-0.16-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0020aaa4010b136056c20a46ce53204e1407a9e4464246cb2cf95b90808d9161"}, - {file = "immutables-0.16-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edd9f67671555af1eb99ad3c7550238487dd7ac0ac5205b40204ed61c9a922ac"}, - {file = "immutables-0.16-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:298a301f85f307b4c056a0825eb30f060e64d73605e783289f3df37dd762bab8"}, - {file = "immutables-0.16-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b779617f5b94486bfd0f22162cd72eb5f2beb0214a14b75fdafb7b2c908ed0cb"}, - {file = "immutables-0.16-cp38-cp38-win32.whl", hash = "sha256:511c93d8b1bbbf103ff3f1f120c5a68a9866ce03dea6ac406537f93ca9b19139"}, - {file = "immutables-0.16-cp38-cp38-win_amd64.whl", hash = "sha256:b651b61c1af6cda2ee201450f2ffe048a5959bc88e43e6c312f4c93e69c9e929"}, - {file = "immutables-0.16-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:aa7bf572ae1e006104c584be70dc634849cf0dc62f42f4ee194774f97e7fd17d"}, - {file = "immutables-0.16-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:50793a44ba0d228ed8cad4d0925e00dfd62ea32f44ddee8854f8066447272d05"}, - {file = "immutables-0.16-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:799621dcdcdcbb2516546a40123b87bf88de75fe7459f7bd8144f079ace6ec3e"}, - {file = "immutables-0.16-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7bcf52aeb983bd803b7c6106eae1b2d9a0c7ab1241bc6b45e2174ba2b7283031"}, - {file = "immutables-0.16-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:734c269e82e5f307fb6e17945953b67659d1731e65309787b8f7ba267d1468f2"}, - {file = "immutables-0.16-cp39-cp39-win32.whl", hash = "sha256:a454d5d3fee4b7cc627345791eb2ca4b27fa3bbb062ccf362ecaaa51679a07ed"}, - {file = "immutables-0.16-cp39-cp39-win_amd64.whl", hash = "sha256:2505d93395d3f8ae4223e21465994c3bc6952015a38dc4f03cb3e07a2b8d8325"}, - {file = "immutables-0.16.tar.gz", hash = "sha256:d67e86859598eed0d926562da33325dac7767b7b1eff84e232c22abea19f4360"}, + {file = "immutables-0.17-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cab10d65a29b2019fffd7a3924f6965a8f785e7bd409641ce36ab2d3335f88c4"}, + {file = "immutables-0.17-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f73088c9b8595ddfd45a5658f8cce0cb3ae6e5890458381fccba3ed3035081d4"}, + {file = "immutables-0.17-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef632832fa1acae6861d83572b866126f9e35706ab6e581ce6b175b3e0b7a3c4"}, + {file = "immutables-0.17-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0efdcec7b63859b41f794ffa0cd0d6dc87e77d1be4ff0ec23471a3a1e719235f"}, + {file = "immutables-0.17-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3eca96f12bc1535657d24eae2c69816d0b22c4a4bc7f4753115e028a137e8dad"}, + {file = "immutables-0.17-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:01a25b1056754aa486afea5471ca348410d77f458477ccb6fa3baf2d3e3ff3d5"}, + {file = "immutables-0.17-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c41a6648f7355f1241da677c418edae56fdc45af19ad3540ca8a1e7a81606a7a"}, + {file = "immutables-0.17-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0b578bba11bd8ae55dee9536edf8d82be18463d15d4b4c9827e27eeeb73826bf"}, + {file = "immutables-0.17-cp310-cp310-win32.whl", hash = "sha256:a28682e115191e909673aedb9ccea3377da3a6a929f8bd86982a2a76bdfa89db"}, + {file = "immutables-0.17-cp310-cp310-win_amd64.whl", hash = "sha256:293ddb681502945f29b3065e688a962e191e752320040892316b9dd1e3b9c8c9"}, + {file = "immutables-0.17-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:ec04fc7d9f76f26d82a5d9d1715df0409d0096309828fc46cd1a2067c7fbab95"}, + {file = "immutables-0.17-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f024f25e9fda42251a2b2167668ca70678c19fb3ab6ed509cef0b4b431d0ff73"}, + {file = "immutables-0.17-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b02083b2052cc201ac5cbd38f34a5da21fcd51016cb4ddd1fb43d7dc113eac17"}, + {file = "immutables-0.17-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea32db31afb82d8369e98f85c5b815ff81610a12fbc837830a34388f1b56f080"}, + {file = "immutables-0.17-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:898a9472d1dd3d17f291114395a1be65be035355fc65af0b2c88238f8fbeaa62"}, + {file = "immutables-0.17-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:736dd3d88d44da0ee48804792bd095c01a344c5d1b0f10beeb9ccb3a00b9c19d"}, + {file = "immutables-0.17-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:15ff4139720f79b902f435a25e3c00f9c8adcc41d79bed64b7e51ae36cfe9620"}, + {file = "immutables-0.17-cp36-cp36m-win32.whl", hash = "sha256:4f018a6c4c3689b82f763ad4f84dec6aa91c83981db7f6bafef963f036e5e815"}, + {file = "immutables-0.17-cp36-cp36m-win_amd64.whl", hash = "sha256:d7400a6753b292ac80102ed026efa8da2c3fedd50c443924cbe9b6448d3b19e4"}, + {file = "immutables-0.17-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f7a6e0380bddb99c46bb3f12ae5eee9a23d6a66d99bbf0fb10fa552f935c2e8d"}, + {file = "immutables-0.17-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7696c42d1f9a16ecda0ee46229848df8706973690b45e8a090d995d647a5ec57"}, + {file = "immutables-0.17-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:892b6a1619cd8c398fa70302c4cfa9768a694377639330e7a58cc7be111ab23e"}, + {file = "immutables-0.17-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89093d5a85357250b1d5ae218fdcfdbac4097cbb2d8b55004aa7a2ca2a00a09f"}, + {file = "immutables-0.17-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:99a8bc6d0623300eb46beea74f7a5061968fb3efc4e072f23f6c0b21c588238d"}, + {file = "immutables-0.17-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:00380474f8e3b4a2eeb06ce694e0e3cb85a144919140a2b3116defb6c1587471"}, + {file = "immutables-0.17-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:078e3ed63be0ac36523b80bbabbfb1bb57e55009f4efb5650b0e3b3ed569c3f1"}, + {file = "immutables-0.17-cp37-cp37m-win32.whl", hash = "sha256:14905aecc62b318d86045dcf8d35ef2063803d9d331aeccd88958f03caadc7b0"}, + {file = "immutables-0.17-cp37-cp37m-win_amd64.whl", hash = "sha256:3774d403d1570105a1da2e00c38ce3f04065fd1deff04cf998f8d8e946d0ae13"}, + {file = "immutables-0.17-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e5a9caee1b99eccf1447056ae6bda77edd15c357421293e81fa1a4f28e83448a"}, + {file = "immutables-0.17-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:fed1e1baf1de1bc94a0310da29814892064928d7d40ff5a3b86bcd11d5e7cfff"}, + {file = "immutables-0.17-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d7daa340d76747ba5a8f64816b48def74bd4be45a9508073b34fa954d099fba"}, + {file = "immutables-0.17-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a4644c29fe07fb92ba84b26659708e1799fecaaf781214adf13edd8a4d7495a9"}, + {file = "immutables-0.17-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1e9ea0e2a31db44fb01617ff875d4c26f962696e1c5ff11ed7767c2d8dedac4"}, + {file = "immutables-0.17-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:64100dfdb29fae2bc84748fff5d66dd6b3997806c717eeb75f7099aeee9b1878"}, + {file = "immutables-0.17-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5f933e5bf6f2c1afb24bc2fc8bea8b132096a4a6ba54f36be59787981f3e50ff"}, + {file = "immutables-0.17-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:9508a087a47f9f9506adf2fa8383ab14c46a222b57eea8612bc4c2aa9a9550fe"}, + {file = "immutables-0.17-cp38-cp38-win32.whl", hash = "sha256:dfd2c63f15d1e5ea1ed2a05b7c602b5f61a64337415d299df20e103a57ae4906"}, + {file = "immutables-0.17-cp38-cp38-win_amd64.whl", hash = "sha256:301c539660c988c5b24051ccad1e36c040a916f1e58fa3e245e3122fc50dd28d"}, + {file = "immutables-0.17-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:563bc2ddbe75c10faa3b4b0206870653b44a231b97ed23cff8ab8aff503d922d"}, + {file = "immutables-0.17-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f621ea6130393cd14d0fbd35b306d4dc70bcd0fda550a8cd313db8015e34ca60"}, + {file = "immutables-0.17-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57c2d1b16b716bca70345db334dd6a861bf45c46cb11bb1801277f8a9012e864"}, + {file = "immutables-0.17-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a08e1a80bd8c5df72c2bf0af24a37ceec17e8ffdb850ed5a62d0bba1d4d86018"}, + {file = "immutables-0.17-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b99155ad112149d43208c611c6c42f19e16716526dacc0fcc16736d2f5d2e20"}, + {file = "immutables-0.17-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ed71e736f8fb82545d00c8969dbc167547c15e85729058edbed3c03b94fca86c"}, + {file = "immutables-0.17-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:19e4b8e5810dd7cab63fa700373f787a369d992166eabc23f4b962e5704d33c5"}, + {file = "immutables-0.17-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:305062012497d4c4a70fe35e20cef2c6f65744e721b04671092a63354799988d"}, + {file = "immutables-0.17-cp39-cp39-win32.whl", hash = "sha256:f5c6bd012384a8d6af7bb25675719214d76640fe6c336e2b5fba9eef1407ae6a"}, + {file = "immutables-0.17-cp39-cp39-win_amd64.whl", hash = "sha256:615ab26873a794559ccaf4e0e9afdb5aefad0867c15262ba64a55a12a5a41573"}, + {file = "immutables-0.17.tar.gz", hash = "sha256:ad894446355b6f5289a9c84fb46f7c47c6ef2b1bfbdd2be6cb177dbb7f1587ad"}, ] importlib-metadata = [ {file = "importlib_metadata-1.7.0-py2.py3-none-any.whl", hash = "sha256:dc15b2969b4ce36305c51eebe62d418ac7791e9a157911d58bfb1f9ccd8e2070"}, @@ -2704,6 +2770,9 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad"}, {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d"}, {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4dc8f9fb58f7364b63fd9f85013b780ef83c11857ae79f2feda41e270468dd9b"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:20dca64a3ef2d6e4d5d615a3fd418ad3bde77a47ec8a23d984a12b5b4c74491a"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cdfba22ea2f0029c9261a4bd07e830a8da012291fbe44dc794e488b6c9bb353a"}, {file = "MarkupSafe-2.0.1-cp310-cp310-win32.whl", hash = "sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28"}, {file = "MarkupSafe-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51"}, @@ -2715,6 +2784,9 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:deb993cacb280823246a026e3b2d81c493c53de6acfd5e6bfe31ab3402bb37dd"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:63f3268ba69ace99cab4e3e3b5840b03340efed0948ab8f78d2fd87ee5442a4f"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:8d206346619592c6200148b01a2142798c989edcb9c896f9ac9722a99d4e77e6"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-win32.whl", hash = "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567"}, @@ -2726,6 +2798,9 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d6c7ebd4e944c85e2c3421e612a7057a2f48d478d79e61800d81468a8d842207"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f0567c4dc99f264f49fe27da5f735f414c4e7e7dd850cfd8e69f0862d7c74ea9"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:89c687013cb1cd489a0f0ac24febe8c7a666e6e221b783e53ac50ebf68e45d86"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-win32.whl", hash = "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914"}, {file = "MarkupSafe-2.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9"}, @@ -2738,6 +2813,9 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:aca6377c0cb8a8253e493c6b451565ac77e98c2951c45f913e0b52facdcff83f"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:04635854b943835a6ea959e948d19dcd311762c5c0c6e1f0e16ee57022669194"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6300b8454aa6930a24b9618fbb54b5a68135092bc666f7b06901f897fa5c2fee"}, {file = "MarkupSafe-2.0.1-cp38-cp38-win32.whl", hash = "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64"}, {file = "MarkupSafe-2.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833"}, {file = "MarkupSafe-2.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26"}, @@ -2750,6 +2828,9 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1"}, {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac"}, {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4296f2b1ce8c86a6aea78613c34bb1a672ea0e3de9c6ba08a960efe0b0a09047"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f02365d4e99430a12647f09b6cc8bab61a6564363f313126f775eb4f6ef798e"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5b6d930f030f8ed98e3e6c98ffa0652bdb82601e7a016ec2ab5d7ff23baa78d1"}, {file = "MarkupSafe-2.0.1-cp39-cp39-win32.whl", hash = "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74"}, {file = "MarkupSafe-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8"}, {file = "MarkupSafe-2.0.1.tar.gz", hash = "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a"}, @@ -2917,16 +2998,16 @@ nbformat = [ {file = "nbformat-5.1.3.tar.gz", hash = "sha256:b516788ad70771c6250977c1374fcca6edebe6126fd2adb5a69aa5c2356fd1c8"}, ] nest-asyncio = [ - {file = "nest_asyncio-1.5.4-py3-none-any.whl", hash = "sha256:3fdd0d6061a2bb16f21fe8a9c6a7945be83521d81a0d15cff52e9edee50101d6"}, - {file = "nest_asyncio-1.5.4.tar.gz", hash = "sha256:f969f6013a16fadb4adcf09d11a68a4f617c6049d7af7ac2c676110169a63abd"}, + {file = "nest_asyncio-1.5.5-py3-none-any.whl", hash = "sha256:b98e3ec1b246135e4642eceffa5a6c23a3ab12c82ff816a92c612d68205813b2"}, + {file = "nest_asyncio-1.5.5.tar.gz", hash = "sha256:e442291cd942698be619823a17a86a5759eabe1f8613084790de189fe9e16d65"}, ] nodeenv = [ {file = "nodeenv-1.6.0-py2.py3-none-any.whl", hash = "sha256:621e6b7076565ddcacd2db0294c0381e01fd28945ab36bcf00f41c5daf63bef7"}, {file = "nodeenv-1.6.0.tar.gz", hash = "sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b"}, ] notebook = [ - {file = "notebook-6.4.8-py3-none-any.whl", hash = "sha256:3e702fcc54b8ae597533c3864793b7a1e971dec9e112f67235828d8a798fd654"}, - {file = "notebook-6.4.8.tar.gz", hash = "sha256:1e985c9dc6f678bdfffb9dc657306b5469bfa62d73e03f74e8defbf76d284312"}, + {file = "notebook-6.4.10-py3-none-any.whl", hash = "sha256:49cead814bff0945fcb2ee07579259418672ac175d3dc3d8102a4b0a656ed4df"}, + {file = "notebook-6.4.10.tar.gz", hash = "sha256:2408a76bc6289283a8eecfca67e298ec83c67db51a4c2e1b713dd180bb39e90e"}, ] numpy = [ {file = "numpy-1.19.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cc6bd4fd593cb261332568485e20a0712883cf631f6f5e8e86a52caa8b2b50ff"}, @@ -3021,8 +3102,8 @@ prometheus-client = [ {file = "prometheus_client-0.13.1.tar.gz", hash = "sha256:ada41b891b79fca5638bd5cfe149efa86512eaa55987893becd2c6d8d0a5dfc5"}, ] prompt-toolkit = [ - {file = "prompt_toolkit-3.0.28-py3-none-any.whl", hash = "sha256:30129d870dcb0b3b6a53efdc9d0a83ea96162ffd28ffe077e94215b233dc670c"}, - {file = "prompt_toolkit-3.0.28.tar.gz", hash = "sha256:9f1cd16b1e86c2968f2519d7fb31dd9d669916f515612c269d14e9ed52b51650"}, + {file = "prompt_toolkit-3.0.29-py3-none-any.whl", hash = "sha256:62291dad495e665fca0bda814e342c69952086afb0f4094d0893d357e5c78752"}, + {file = "prompt_toolkit-3.0.29.tar.gz", hash = "sha256:bd640f60e8cecd74f0dc249713d433ace2ddc62b65ee07f96d358e0b152b6ea7"}, ] ptyprocess = [ {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"}, @@ -3090,8 +3171,8 @@ pylev = [ {file = "pylev-1.4.0.tar.gz", hash = "sha256:9e77e941042ad3a4cc305dcdf2b2dec1aec2fbe3dd9015d2698ad02b173006d1"}, ] pylint = [ - {file = "pylint-2.12.2-py3-none-any.whl", hash = "sha256:daabda3f7ed9d1c60f52d563b1b854632fd90035bcf01443e234d3dc794e3b74"}, - {file = "pylint-2.12.2.tar.gz", hash = "sha256:9d945a73640e1fec07ee34b42f5669b770c759acd536ec7b16d7e4b87a9c9ff9"}, + {file = "pylint-2.13.4-py3-none-any.whl", hash = "sha256:8672cf7441b81410f5de7defdf56e2d559c956fd0579652f2e0a0a35bea2d546"}, + {file = "pylint-2.13.4.tar.gz", hash = "sha256:7cc6d0c4f61dff440f9ed8b657f4ecd615dcfe35345953eb7b1dc74afe901d7a"}, ] pyparsing = [ {file = "pyparsing-3.0.7-py3-none-any.whl", hash = "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484"}, @@ -3137,8 +3218,8 @@ python-dateutil = [ {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, ] pytz = [ - {file = "pytz-2021.3-py2.py3-none-any.whl", hash = "sha256:3672058bc3453457b622aab7a1c3bfd5ab0bdae451512f6cf25f64ed37f5b87c"}, - {file = "pytz-2021.3.tar.gz", hash = "sha256:acad2d8b20a1af07d4e4c9d2e9285c5ed9104354062f275f3fcd88dcef4f1326"}, + {file = "pytz-2022.1-py2.py3-none-any.whl", hash = "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c"}, + {file = "pytz-2022.1.tar.gz", hash = "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7"}, ] pywin32 = [ {file = "pywin32-303-cp310-cp310-win32.whl", hash = "sha256:6fed4af057039f309263fd3285d7b8042d41507343cd5fa781d98fcc5b90e8bb"}, @@ -3250,80 +3331,80 @@ pyzmq = [ {file = "pyzmq-22.3.0.tar.gz", hash = "sha256:8eddc033e716f8c91c6a2112f0a8ebc5e00532b4a6ae1eb0ccc48e027f9c671c"}, ] regex = [ - {file = "regex-2022.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ab69b4fe09e296261377d209068d52402fb85ef89dc78a9ac4a29a895f4e24a7"}, - {file = "regex-2022.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5bc5f921be39ccb65fdda741e04b2555917a4bced24b4df14eddc7569be3b493"}, - {file = "regex-2022.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43eba5c46208deedec833663201752e865feddc840433285fbadee07b84b464d"}, - {file = "regex-2022.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c68d2c04f7701a418ec2e5631b7f3552efc32f6bcc1739369c6eeb1af55f62e0"}, - {file = "regex-2022.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:caa2734ada16a44ae57b229d45091f06e30a9a52ace76d7574546ab23008c635"}, - {file = "regex-2022.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef806f684f17dbd6263d72a54ad4073af42b42effa3eb42b877e750c24c76f86"}, - {file = "regex-2022.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:be319f4eb400ee567b722e9ea63d5b2bb31464e3cf1b016502e3ee2de4f86f5c"}, - {file = "regex-2022.3.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:42bb37e2b2d25d958c25903f6125a41aaaa1ed49ca62c103331f24b8a459142f"}, - {file = "regex-2022.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:fbc88d3ba402b5d041d204ec2449c4078898f89c4a6e6f0ed1c1a510ef1e221d"}, - {file = "regex-2022.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:91e0f7e7be77250b808a5f46d90bf0032527d3c032b2131b63dee54753a4d729"}, - {file = "regex-2022.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:cb3652bbe6720786b9137862205986f3ae54a09dec8499a995ed58292bdf77c2"}, - {file = "regex-2022.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:878c626cbca3b649e14e972c14539a01191d79e58934e3f3ef4a9e17f90277f8"}, - {file = "regex-2022.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6df070a986fc064d865c381aecf0aaff914178fdf6874da2f2387e82d93cc5bd"}, - {file = "regex-2022.3.2-cp310-cp310-win32.whl", hash = "sha256:b549d851f91a4efb3e65498bd4249b1447ab6035a9972f7fc215eb1f59328834"}, - {file = "regex-2022.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:8babb2b5751105dc0aef2a2e539f4ba391e738c62038d8cb331c710f6b0f3da7"}, - {file = "regex-2022.3.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:1977bb64264815d3ef016625adc9df90e6d0e27e76260280c63eca993e3f455f"}, - {file = "regex-2022.3.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e73652057473ad3e6934944af090852a02590c349357b79182c1b681da2c772"}, - {file = "regex-2022.3.2-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b22ff939a8856a44f4822da38ef4868bd3a9ade22bb6d9062b36957c850e404f"}, - {file = "regex-2022.3.2-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:878f5d649ba1db9f52cc4ef491f7dba2d061cdc48dd444c54260eebc0b1729b9"}, - {file = "regex-2022.3.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0008650041531d0eadecc96a73d37c2dc4821cf51b0766e374cb4f1ddc4e1c14"}, - {file = "regex-2022.3.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:06b1df01cf2aef3a9790858af524ae2588762c8a90e784ba00d003f045306204"}, - {file = "regex-2022.3.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57484d39447f94967e83e56db1b1108c68918c44ab519b8ecfc34b790ca52bf7"}, - {file = "regex-2022.3.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:74d86e8924835f863c34e646392ef39039405f6ce52956d8af16497af4064a30"}, - {file = "regex-2022.3.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:ae17fc8103f3b63345709d3e9654a274eee1c6072592aec32b026efd401931d0"}, - {file = "regex-2022.3.2-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:5f92a7cdc6a0ae2abd184e8dfd6ef2279989d24c85d2c85d0423206284103ede"}, - {file = "regex-2022.3.2-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:5dcc4168536c8f68654f014a3db49b6b4a26b226f735708be2054314ed4964f4"}, - {file = "regex-2022.3.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:1e30762ddddb22f7f14c4f59c34d3addabc789216d813b0f3e2788d7bcf0cf29"}, - {file = "regex-2022.3.2-cp36-cp36m-win32.whl", hash = "sha256:286ff9ec2709d56ae7517040be0d6c502642517ce9937ab6d89b1e7d0904f863"}, - {file = "regex-2022.3.2-cp36-cp36m-win_amd64.whl", hash = "sha256:d326ff80ed531bf2507cba93011c30fff2dd51454c85f55df0f59f2030b1687b"}, - {file = "regex-2022.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:9d828c5987d543d052b53c579a01a52d96b86f937b1777bbfe11ef2728929357"}, - {file = "regex-2022.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c87ac58b9baaf50b6c1b81a18d20eda7e2883aa9a4fb4f1ca70f2e443bfcdc57"}, - {file = "regex-2022.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d6c2441538e4fadd4291c8420853431a229fcbefc1bf521810fbc2629d8ae8c2"}, - {file = "regex-2022.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f3356afbb301ec34a500b8ba8b47cba0b44ed4641c306e1dd981a08b416170b5"}, - {file = "regex-2022.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d96eec8550fd2fd26f8e675f6d8b61b159482ad8ffa26991b894ed5ee19038b"}, - {file = "regex-2022.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf668f26604e9f7aee9f8eaae4ca07a948168af90b96be97a4b7fa902a6d2ac1"}, - {file = "regex-2022.3.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0eb0e2845e81bdea92b8281a3969632686502565abf4a0b9e4ab1471c863d8f3"}, - {file = "regex-2022.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:87bc01226cd288f0bd9a4f9f07bf6827134dc97a96c22e2d28628e824c8de231"}, - {file = "regex-2022.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:09b4b6ccc61d4119342b26246ddd5a04accdeebe36bdfe865ad87a0784efd77f"}, - {file = "regex-2022.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:9557545c10d52c845f270b665b52a6a972884725aa5cf12777374e18f2ea8960"}, - {file = "regex-2022.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:0be0c34a39e5d04a62fd5342f0886d0e57592a4f4993b3f9d257c1f688b19737"}, - {file = "regex-2022.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:7b103dffb9f6a47ed7ffdf352b78cfe058b1777617371226c1894e1be443afec"}, - {file = "regex-2022.3.2-cp37-cp37m-win32.whl", hash = "sha256:f8169ec628880bdbca67082a9196e2106060a4a5cbd486ac51881a4df805a36f"}, - {file = "regex-2022.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:4b9c16a807b17b17c4fa3a1d8c242467237be67ba92ad24ff51425329e7ae3d0"}, - {file = "regex-2022.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:67250b36edfa714ba62dc62d3f238e86db1065fccb538278804790f578253640"}, - {file = "regex-2022.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5510932596a0f33399b7fff1bd61c59c977f2b8ee987b36539ba97eb3513584a"}, - {file = "regex-2022.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6f7ee2289176cb1d2c59a24f50900f8b9580259fa9f1a739432242e7d254f93"}, - {file = "regex-2022.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86d7a68fa53688e1f612c3246044157117403c7ce19ebab7d02daf45bd63913e"}, - {file = "regex-2022.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aaf5317c961d93c1a200b9370fb1c6b6836cc7144fef3e5a951326912bf1f5a3"}, - {file = "regex-2022.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad397bc7d51d69cb07ef89e44243f971a04ce1dca9bf24c992c362406c0c6573"}, - {file = "regex-2022.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:297c42ede2c81f0cb6f34ea60b5cf6dc965d97fa6936c11fc3286019231f0d66"}, - {file = "regex-2022.3.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:af4d8cc28e4c7a2f6a9fed544228c567340f8258b6d7ea815b62a72817bbd178"}, - {file = "regex-2022.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:452519bc4c973e961b1620c815ea6dd8944a12d68e71002be5a7aff0a8361571"}, - {file = "regex-2022.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:cb34c2d66355fb70ae47b5595aafd7218e59bb9c00ad8cc3abd1406ca5874f07"}, - {file = "regex-2022.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:3d146e5591cb67c5e836229a04723a30af795ef9b70a0bbd913572e14b7b940f"}, - {file = "regex-2022.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:03299b0bcaa7824eb7c0ebd7ef1e3663302d1b533653bfe9dc7e595d453e2ae9"}, - {file = "regex-2022.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:9ccb0a4ab926016867260c24c192d9df9586e834f5db83dfa2c8fffb3a6e5056"}, - {file = "regex-2022.3.2-cp38-cp38-win32.whl", hash = "sha256:f7e8f1ee28e0a05831c92dc1c0c1c94af5289963b7cf09eca5b5e3ce4f8c91b0"}, - {file = "regex-2022.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:35ed2f3c918a00b109157428abfc4e8d1ffabc37c8f9abc5939ebd1e95dabc47"}, - {file = "regex-2022.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:55820bc631684172b9b56a991d217ec7c2e580d956591dc2144985113980f5a3"}, - {file = "regex-2022.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:83f03f0bd88c12e63ca2d024adeee75234d69808b341e88343b0232329e1f1a1"}, - {file = "regex-2022.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42d6007722d46bd2c95cce700181570b56edc0dcbadbfe7855ec26c3f2d7e008"}, - {file = "regex-2022.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:320c2f4106962ecea0f33d8d31b985d3c185757c49c1fb735501515f963715ed"}, - {file = "regex-2022.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fbd3fe37353c62fd0eb19fb76f78aa693716262bcd5f9c14bb9e5aca4b3f0dc4"}, - {file = "regex-2022.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17e51ad1e6131c496b58d317bc9abec71f44eb1957d32629d06013a21bc99cac"}, - {file = "regex-2022.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:72bc3a5effa5974be6d965ed8301ac1e869bc18425c8a8fac179fbe7876e3aee"}, - {file = "regex-2022.3.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e5602a9b5074dcacc113bba4d2f011d2748f50e3201c8139ac5b68cf2a76bd8b"}, - {file = "regex-2022.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:729aa8ca624c42f309397c5fc9e21db90bf7e2fdd872461aabdbada33de9063c"}, - {file = "regex-2022.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d6ecfd1970b3380a569d7b3ecc5dd70dba295897418ed9e31ec3c16a5ab099a5"}, - {file = "regex-2022.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:13bbf0c9453c6d16e5867bda7f6c0c7cff1decf96c5498318bb87f8136d2abd4"}, - {file = "regex-2022.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:58ba41e462653eaf68fc4a84ec4d350b26a98d030be1ab24aba1adcc78ffe447"}, - {file = "regex-2022.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c0446b2871335d5a5e9fcf1462f954586b09a845832263db95059dcd01442015"}, - {file = "regex-2022.3.2-cp39-cp39-win32.whl", hash = "sha256:20e6a27959f162f979165e496add0d7d56d7038237092d1aba20b46de79158f1"}, - {file = "regex-2022.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:9efa41d1527b366c88f265a227b20bcec65bda879962e3fc8a2aee11e81266d7"}, - {file = "regex-2022.3.2.tar.gz", hash = "sha256:79e5af1ff258bc0fe0bdd6f69bc4ae33935a898e3cbefbbccf22e88a27fa053b"}, + {file = "regex-2022.3.15-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:42eb13b93765c6698a5ab3bcd318d8c39bb42e5fa8a7fcf7d8d98923f3babdb1"}, + {file = "regex-2022.3.15-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9beb03ff6fe509d6455971c2489dceb31687b38781206bcec8e68bdfcf5f1db2"}, + {file = "regex-2022.3.15-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0a5a1fdc9f148a8827d55b05425801acebeeefc9e86065c7ac8b8cc740a91ff"}, + {file = "regex-2022.3.15-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cb374a2a4dba7c4be0b19dc7b1adc50e6c2c26c3369ac629f50f3c198f3743a4"}, + {file = "regex-2022.3.15-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c33ce0c665dd325200209340a88438ba7a470bd5f09f7424e520e1a3ff835b52"}, + {file = "regex-2022.3.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:04c09b9651fa814eeeb38e029dc1ae83149203e4eeb94e52bb868fadf64852bc"}, + {file = "regex-2022.3.15-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ab5d89cfaf71807da93c131bb7a19c3e19eaefd613d14f3bce4e97de830b15df"}, + {file = "regex-2022.3.15-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0e2630ae470d6a9f8e4967388c1eda4762706f5750ecf387785e0df63a4cc5af"}, + {file = "regex-2022.3.15-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:df037c01d68d1958dad3463e2881d3638a0d6693483f58ad41001aa53a83fcea"}, + {file = "regex-2022.3.15-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:940570c1a305bac10e8b2bc934b85a7709c649317dd16520471e85660275083a"}, + {file = "regex-2022.3.15-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7f63877c87552992894ea1444378b9c3a1d80819880ae226bb30b04789c0828c"}, + {file = "regex-2022.3.15-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:3e265b388cc80c7c9c01bb4f26c9e536c40b2c05b7231fbb347381a2e1c8bf43"}, + {file = "regex-2022.3.15-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:058054c7a54428d5c3e3739ac1e363dc9347d15e64833817797dc4f01fb94bb8"}, + {file = "regex-2022.3.15-cp310-cp310-win32.whl", hash = "sha256:76435a92e444e5b8f346aed76801db1c1e5176c4c7e17daba074fbb46cb8d783"}, + {file = "regex-2022.3.15-cp310-cp310-win_amd64.whl", hash = "sha256:174d964bc683b1e8b0970e1325f75e6242786a92a22cedb2a6ec3e4ae25358bd"}, + {file = "regex-2022.3.15-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6e1d8ed9e61f37881c8db383a124829a6e8114a69bd3377a25aecaeb9b3538f8"}, + {file = "regex-2022.3.15-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b52771f05cff7517f7067fef19ffe545b1f05959e440d42247a17cd9bddae11b"}, + {file = "regex-2022.3.15-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:673f5a393d603c34477dbad70db30025ccd23996a2d0916e942aac91cc42b31a"}, + {file = "regex-2022.3.15-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8923e1c5231549fee78ff9b2914fad25f2e3517572bb34bfaa3aea682a758683"}, + {file = "regex-2022.3.15-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:764e66a0e382829f6ad3bbce0987153080a511c19eb3d2f8ead3f766d14433ac"}, + {file = "regex-2022.3.15-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd00859291658fe1fda48a99559fb34da891c50385b0bfb35b808f98956ef1e7"}, + {file = "regex-2022.3.15-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:aa2ce79f3889720b46e0aaba338148a1069aea55fda2c29e0626b4db20d9fcb7"}, + {file = "regex-2022.3.15-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:34bb30c095342797608727baf5c8aa122406aa5edfa12107b8e08eb432d4c5d7"}, + {file = "regex-2022.3.15-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:25ecb1dffc5e409ca42f01a2b2437f93024ff1612c1e7983bad9ee191a5e8828"}, + {file = "regex-2022.3.15-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:aa5eedfc2461c16a092a2fabc5895f159915f25731740c9152a1b00f4bcf629a"}, + {file = "regex-2022.3.15-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:7d1a6e403ac8f1d91d8f51c441c3f99367488ed822bda2b40836690d5d0059f5"}, + {file = "regex-2022.3.15-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:3e4d710ff6539026e49f15a3797c6b1053573c2b65210373ef0eec24480b900b"}, + {file = "regex-2022.3.15-cp36-cp36m-win32.whl", hash = "sha256:0100f0ded953b6b17f18207907159ba9be3159649ad2d9b15535a74de70359d3"}, + {file = "regex-2022.3.15-cp36-cp36m-win_amd64.whl", hash = "sha256:f320c070dea3f20c11213e56dbbd7294c05743417cde01392148964b7bc2d31a"}, + {file = "regex-2022.3.15-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fc8c7958d14e8270171b3d72792b609c057ec0fa17d507729835b5cff6b7f69a"}, + {file = "regex-2022.3.15-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ca6dcd17f537e9f3793cdde20ac6076af51b2bd8ad5fe69fa54373b17b48d3c"}, + {file = "regex-2022.3.15-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0214ff6dff1b5a4b4740cfe6e47f2c4c92ba2938fca7abbea1359036305c132f"}, + {file = "regex-2022.3.15-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a98ae493e4e80b3ded6503ff087a8492db058e9c68de371ac3df78e88360b374"}, + {file = "regex-2022.3.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b1cc70e31aacc152a12b39245974c8fccf313187eead559ee5966d50e1b5817"}, + {file = "regex-2022.3.15-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b4829db3737480a9d5bfb1c0320c4ee13736f555f53a056aacc874f140e98f64"}, + {file = "regex-2022.3.15-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:303b15a3d32bf5fe5a73288c316bac5807587f193ceee4eb6d96ee38663789fa"}, + {file = "regex-2022.3.15-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:dc7b7c16a519d924c50876fb152af661a20749dcbf653c8759e715c1a7a95b18"}, + {file = "regex-2022.3.15-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:ce3057777a14a9a1399b81eca6a6bfc9612047811234398b84c54aeff6d536ea"}, + {file = "regex-2022.3.15-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:48081b6bff550fe10bcc20c01cf6c83dbca2ccf74eeacbfac240264775fd7ecf"}, + {file = "regex-2022.3.15-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dcbb7665a9db9f8d7642171152c45da60e16c4f706191d66a1dc47ec9f820aed"}, + {file = "regex-2022.3.15-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c155a1a80c5e7a8fa1d9bb1bf3c8a953532b53ab1196092749bafb9d3a7cbb60"}, + {file = "regex-2022.3.15-cp37-cp37m-win32.whl", hash = "sha256:04b5ee2b6d29b4a99d38a6469aa1db65bb79d283186e8460542c517da195a8f6"}, + {file = "regex-2022.3.15-cp37-cp37m-win_amd64.whl", hash = "sha256:797437e6024dc1589163675ae82f303103063a0a580c6fd8d0b9a0a6708da29e"}, + {file = "regex-2022.3.15-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8afcd1c2297bc989dceaa0379ba15a6df16da69493635e53431d2d0c30356086"}, + {file = "regex-2022.3.15-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0066a6631c92774391f2ea0f90268f0d82fffe39cb946f0f9c6b382a1c61a5e5"}, + {file = "regex-2022.3.15-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8248f19a878c72d8c0a785a2cd45d69432e443c9f10ab924c29adda77b324ae"}, + {file = "regex-2022.3.15-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8d1f3ea0d1924feb4cf6afb2699259f658a08ac6f8f3a4a806661c2dfcd66db1"}, + {file = "regex-2022.3.15-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:794a6bc66c43db8ed06698fc32aaeaac5c4812d9f825e9589e56f311da7becd9"}, + {file = "regex-2022.3.15-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d1445824944e642ffa54c4f512da17a953699c563a356d8b8cbdad26d3b7598"}, + {file = "regex-2022.3.15-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f553a1190ae6cd26e553a79f6b6cfba7b8f304da2071052fa33469da075ea625"}, + {file = "regex-2022.3.15-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:75a5e6ce18982f0713c4bac0704bf3f65eed9b277edd3fb9d2b0ff1815943327"}, + {file = "regex-2022.3.15-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f16cf7e4e1bf88fecf7f41da4061f181a6170e179d956420f84e700fb8a3fd6b"}, + {file = "regex-2022.3.15-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:dad3991f0678facca1a0831ec1ddece2eb4d1dd0f5150acb9440f73a3b863907"}, + {file = "regex-2022.3.15-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:491fc754428514750ab21c2d294486223ce7385446f2c2f5df87ddbed32979ae"}, + {file = "regex-2022.3.15-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:6504c22c173bb74075d7479852356bb7ca80e28c8e548d4d630a104f231e04fb"}, + {file = "regex-2022.3.15-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:01c913cf573d1da0b34c9001a94977273b5ee2fe4cb222a5d5b320f3a9d1a835"}, + {file = "regex-2022.3.15-cp38-cp38-win32.whl", hash = "sha256:029e9e7e0d4d7c3446aa92474cbb07dafb0b2ef1d5ca8365f059998c010600e6"}, + {file = "regex-2022.3.15-cp38-cp38-win_amd64.whl", hash = "sha256:947a8525c0a95ba8dc873191f9017d1b1e3024d4dc757f694e0af3026e34044a"}, + {file = "regex-2022.3.15-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:591d4fba554f24bfa0421ba040cd199210a24301f923ed4b628e1e15a1001ff4"}, + {file = "regex-2022.3.15-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b9809404528a999cf02a400ee5677c81959bc5cb938fdc696b62eb40214e3632"}, + {file = "regex-2022.3.15-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f08a7e4d62ea2a45557f561eea87c907222575ca2134180b6974f8ac81e24f06"}, + {file = "regex-2022.3.15-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a86cac984da35377ca9ac5e2e0589bd11b3aebb61801204bd99c41fac516f0d"}, + {file = "regex-2022.3.15-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:286908cbe86b1a0240a867aecfe26a439b16a1f585d2de133540549831f8e774"}, + {file = "regex-2022.3.15-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b7494df3fdcc95a1f76cf134d00b54962dd83189520fd35b8fcd474c0aa616d"}, + {file = "regex-2022.3.15-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b1ceede92400b3acfebc1425937454aaf2c62cd5261a3fabd560c61e74f6da3"}, + {file = "regex-2022.3.15-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0317eb6331146c524751354ebef76a7a531853d7207a4d760dfb5f553137a2a4"}, + {file = "regex-2022.3.15-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9c144405220c5ad3f5deab4c77f3e80d52e83804a6b48b6bed3d81a9a0238e4c"}, + {file = "regex-2022.3.15-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:5b2e24f3ae03af3d8e8e6d824c891fea0ca9035c5d06ac194a2700373861a15c"}, + {file = "regex-2022.3.15-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:f2c53f3af011393ab5ed9ab640fa0876757498aac188f782a0c620e33faa2a3d"}, + {file = "regex-2022.3.15-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:060f9066d2177905203516c62c8ea0066c16c7342971d54204d4e51b13dfbe2e"}, + {file = "regex-2022.3.15-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:530a3a16e57bd3ea0dff5ec2695c09632c9d6c549f5869d6cf639f5f7153fb9c"}, + {file = "regex-2022.3.15-cp39-cp39-win32.whl", hash = "sha256:78ce90c50d0ec970bd0002462430e00d1ecfd1255218d52d08b3a143fe4bde18"}, + {file = "regex-2022.3.15-cp39-cp39-win_amd64.whl", hash = "sha256:c5adc854764732dbd95a713f2e6c3e914e17f2ccdc331b9ecb777484c31f73b6"}, + {file = "regex-2022.3.15.tar.gz", hash = "sha256:0a7b75cc7bb4cc0334380053e4671c560e31272c9d2d5a6c4b8e9ae2c9bd0f82"}, ] requests = [ {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, @@ -3407,6 +3488,34 @@ send2trash = [ {file = "Send2Trash-1.8.0-py3-none-any.whl", hash = "sha256:f20eaadfdb517eaca5ce077640cb261c7d2698385a6a0f072a4a5447fd49fa08"}, {file = "Send2Trash-1.8.0.tar.gz", hash = "sha256:d2c24762fd3759860a0aff155e45871447ea58d2be6bdd39b5c8f966a0c99c2d"}, ] +shapely = [ + {file = "Shapely-1.8.1.post1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ca96a3314b7a38a3bb385531469de1fcf2b2c2979ec2aa4f37b4c70632cf1ad"}, + {file = "Shapely-1.8.1.post1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:493902923fdd135316161a4ece5294ba3ce81accaa54540d2af3b93f7231143a"}, + {file = "Shapely-1.8.1.post1-cp310-cp310-win_amd64.whl", hash = "sha256:b82fc74d5efb11a71283c4ed69b4e036997cc70db4b73c646207ddf0476ade44"}, + {file = "Shapely-1.8.1.post1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:89bc5f3abc1ccbc7682c2e1664153c4f8f125fa9c24bff4abca48685739d5636"}, + {file = "Shapely-1.8.1.post1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:44cb895b1710f7559c28d69dfa08cafe4f58cd4b7a87091a55bdf6711ad9ad66"}, + {file = "Shapely-1.8.1.post1-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:437fff3b6274be26ffa3e450de711ee01e436324b5a405952add2146227e3eb5"}, + {file = "Shapely-1.8.1.post1-cp36-cp36m-win32.whl", hash = "sha256:dc0f46212f84c57d13189fc33cf61e13eee292704d7652e931e4b51c54b0c73c"}, + {file = "Shapely-1.8.1.post1-cp36-cp36m-win_amd64.whl", hash = "sha256:9248aad099ecf228fbdd877b0c668823dd83c48798cf04d49a1be75167e3a7ce"}, + {file = "Shapely-1.8.1.post1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:bab5ff7c576588acccd665ecce2a0fe7b47d4ce0398f2d5c1e5b2e27d09398d2"}, + {file = "Shapely-1.8.1.post1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2381ce0aff67d569eb509bcc051264aa5fbdc1fdd54f4c09963d0e09f16a8f1b"}, + {file = "Shapely-1.8.1.post1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b4d35e72022b2dbf152d476b0362596011c674ff68be9fc8f2e68e71d86502ca"}, + {file = "Shapely-1.8.1.post1-cp37-cp37m-win32.whl", hash = "sha256:5a420e7112b55a1587412a5b03ebf59e302ddd354da68516d3721718f6b8a7c5"}, + {file = "Shapely-1.8.1.post1-cp37-cp37m-win_amd64.whl", hash = "sha256:c4c366e18edf91196a399f8f0f046f93516002e6d8af0b57c23e7c7d91944b16"}, + {file = "Shapely-1.8.1.post1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2020fda37c708d44a613c020cea09e81e476f96866f348afc2601e66c0e71db1"}, + {file = "Shapely-1.8.1.post1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:69d5352fb977655c85d2f40a05ae24fc5053cccee77d0a8b1f773e54804e723e"}, + {file = "Shapely-1.8.1.post1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:83f3c8191d30ae0e3dd557434c48ca591d75342d5a3f42fc5148ec42796be624"}, + {file = "Shapely-1.8.1.post1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3e792635e92c9aacd1452a589a4fa2970114b6a9b1165e09655481f6e58970f5"}, + {file = "Shapely-1.8.1.post1-cp38-cp38-win32.whl", hash = "sha256:8cf7331f61780506976fe2175e069d898e1b04ace73be21aad55c3ee92e58e3a"}, + {file = "Shapely-1.8.1.post1-cp38-cp38-win_amd64.whl", hash = "sha256:f109064bdb0753a6bac6238538cfeeb4a09739e2d556036b343b2eabeb9520b2"}, + {file = "Shapely-1.8.1.post1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:aea1e87450adffba3d04ccbaa790df719bb7aa23b05ac797ad16be236a5d0db8"}, + {file = "Shapely-1.8.1.post1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3a3602ba2e7715ddd5d4114173dec83d3181bfb2497e8589676c284aa739fd67"}, + {file = "Shapely-1.8.1.post1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:679789d774cfe09ca05118cab78c0a6a42985b3ed23bc93606272a4509b4df28"}, + {file = "Shapely-1.8.1.post1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:363df36370f28fdc7789857929f6ff27e659f64087b4c89f7a47ed43bd3bfe4d"}, + {file = "Shapely-1.8.1.post1-cp39-cp39-win32.whl", hash = "sha256:bc6063875182515d3888180cc4cbdbaa6443e4a4386c4bb25499e9875b75dcac"}, + {file = "Shapely-1.8.1.post1-cp39-cp39-win_amd64.whl", hash = "sha256:54aeb2a57978ce731fd52289d0e1deee7c232d41aed53091f38776378f644184"}, + {file = "Shapely-1.8.1.post1.tar.gz", hash = "sha256:93ff06ff05fbe2be843b93c7b1ad8292e56e665ba01b4708f75ae8a757972e9f"}, +] shellingham = [ {file = "shellingham-1.4.0-py2.py3-none-any.whl", hash = "sha256:536b67a0697f2e4af32ab176c00a50ac2899c5a05e0d8e2dadac8e58888283f9"}, {file = "shellingham-1.4.0.tar.gz", hash = "sha256:4855c2458d6904829bd34c299f11fdeed7cfefbf8a2c522e4caea6cd76b3171e"}, @@ -3479,9 +3588,13 @@ toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] +tomli = [ + {file = "tomli-1.2.3-py3-none-any.whl", hash = "sha256:e3069e4be3ead9668e21cb9b074cd948f7b3113fd9c8bba083f48247aab8b11c"}, + {file = "tomli-1.2.3.tar.gz", hash = "sha256:05b6166bff487dc068d322585c7ea4ef78deed501cc124060e0f238e89a9231f"}, +] tomlkit = [ - {file = "tomlkit-0.10.0-py3-none-any.whl", hash = "sha256:cac4aeaff42f18fef6e07831c2c2689a51df76cf2ede07a6a4fa5fcb83558870"}, - {file = "tomlkit-0.10.0.tar.gz", hash = "sha256:d99946c6aed3387c98b89d91fb9edff8f901bf9255901081266a84fb5604adcd"}, + {file = "tomlkit-0.10.1-py3-none-any.whl", hash = "sha256:3eba517439dcb2f84cf39f4f85fd2c3398309823a3c75ac3e73003638daf7915"}, + {file = "tomlkit-0.10.1.tar.gz", hash = "sha256:3c517894eadef53e9072d343d37e4427b8f0b6200a70b7c9a19b2ebd1f53b951"}, ] tornado = [ {file = "tornado-6.1-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:d371e811d6b156d82aa5f9a4e08b58debf97c302a35714f6f45e35139c332e32"}, @@ -3527,8 +3640,8 @@ tornado = [ {file = "tornado-6.1.tar.gz", hash = "sha256:33c6e81d7bd55b468d2e793517c909b139960b6c790a60b7991b9b6b76fb9791"}, ] tqdm = [ - {file = "tqdm-4.63.0-py2.py3-none-any.whl", hash = "sha256:e643e071046f17139dea55b880dc9b33822ce21613b4a4f5ea57f202833dbc29"}, - {file = "tqdm-4.63.0.tar.gz", hash = "sha256:1d9835ede8e394bb8c9dcbffbca02d717217113adc679236873eeaac5bc0b3cd"}, + {file = "tqdm-4.64.0-py2.py3-none-any.whl", hash = "sha256:74a2cdefe14d11442cedf3ba4e21a3b84ff9a2dbdc6cfae2c34addb2a14a5ea6"}, + {file = "tqdm-4.64.0.tar.gz", hash = "sha256:40be55d30e200777a307a7585aee69e4eabb46b4ec6a4b4a5f2d9f11e7d5408d"}, ] traitlets = [ {file = "traitlets-4.3.3-py2.py3-none-any.whl", hash = "sha256:70b4c6a1d9019d7b4f6846832288f86998aa3b9207c6821f3578a6a6a467fe44"}, @@ -3571,16 +3684,16 @@ typing-extensions = [ {file = "typing_extensions-4.1.1.tar.gz", hash = "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42"}, ] unidecode = [ - {file = "Unidecode-1.3.3-py3-none-any.whl", hash = "sha256:a5a8a4b6fb033724ffba8502af2e65ca5bfc3dd53762dedaafe4b0134ad42e3c"}, - {file = "Unidecode-1.3.3.tar.gz", hash = "sha256:8521f2853fd250891dc27d156a9d30e61c4e76319da963c4a1c27083a909ac30"}, + {file = "Unidecode-1.3.4-py3-none-any.whl", hash = "sha256:afa04efcdd818a93237574791be9b2817d7077c25a068b00f8cff7baa4e59257"}, + {file = "Unidecode-1.3.4.tar.gz", hash = "sha256:8e4352fb93d5a735c788110d2e7ac8e8031eb06ccbfe8d324ab71735015f9342"}, ] urllib3 = [ - {file = "urllib3-1.26.8-py2.py3-none-any.whl", hash = "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed"}, - {file = "urllib3-1.26.8.tar.gz", hash = "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"}, + {file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"}, + {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"}, ] virtualenv = [ - {file = "virtualenv-20.13.3-py2.py3-none-any.whl", hash = "sha256:dd448d1ded9f14d1a4bfa6bfc0c5b96ae3be3f2d6c6c159b23ddcfd701baa021"}, - {file = "virtualenv-20.13.3.tar.gz", hash = "sha256:e9dd1a1359d70137559034c0f5433b34caf504af2dc756367be86a5a32967134"}, + {file = "virtualenv-20.14.0-py2.py3-none-any.whl", hash = "sha256:1e8588f35e8b42c6ec6841a13c5e88239de1e6e4e4cedfd3916b306dc826ec66"}, + {file = "virtualenv-20.14.0.tar.gz", hash = "sha256:8e5b402037287126e81ccde9432b95a8be5b19d36584f64957060a3488c11ca8"}, ] wcwidth = [ {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, @@ -3595,57 +3708,70 @@ websocket-client = [ {file = "websocket_client-1.3.1-py3-none-any.whl", hash = "sha256:074e2ed575e7c822fc0940d31c3ac9bb2b1142c303eafcf3e304e6ce035522e8"}, ] wrapt = [ - {file = "wrapt-1.13.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:e05e60ff3b2b0342153be4d1b597bbcfd8330890056b9619f4ad6b8d5c96a81a"}, - {file = "wrapt-1.13.3-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:85148f4225287b6a0665eef08a178c15097366d46b210574a658c1ff5b377489"}, - {file = "wrapt-1.13.3-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:2dded5496e8f1592ec27079b28b6ad2a1ef0b9296d270f77b8e4a3a796cf6909"}, - {file = "wrapt-1.13.3-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:e94b7d9deaa4cc7bac9198a58a7240aaf87fe56c6277ee25fa5b3aa1edebd229"}, - {file = "wrapt-1.13.3-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:498e6217523111d07cd67e87a791f5e9ee769f9241fcf8a379696e25806965af"}, - {file = "wrapt-1.13.3-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:ec7e20258ecc5174029a0f391e1b948bf2906cd64c198a9b8b281b811cbc04de"}, - {file = "wrapt-1.13.3-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:87883690cae293541e08ba2da22cacaae0a092e0ed56bbba8d018cc486fbafbb"}, - {file = "wrapt-1.13.3-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:f99c0489258086308aad4ae57da9e8ecf9e1f3f30fa35d5e170b4d4896554d80"}, - {file = "wrapt-1.13.3-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:6a03d9917aee887690aa3f1747ce634e610f6db6f6b332b35c2dd89412912bca"}, - {file = "wrapt-1.13.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:936503cb0a6ed28dbfa87e8fcd0a56458822144e9d11a49ccee6d9a8adb2ac44"}, - {file = "wrapt-1.13.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f9c51d9af9abb899bd34ace878fbec8bf357b3194a10c4e8e0a25512826ef056"}, - {file = "wrapt-1.13.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:220a869982ea9023e163ba915077816ca439489de6d2c09089b219f4e11b6785"}, - {file = "wrapt-1.13.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0877fe981fd76b183711d767500e6b3111378ed2043c145e21816ee589d91096"}, - {file = "wrapt-1.13.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:43e69ffe47e3609a6aec0fe723001c60c65305784d964f5007d5b4fb1bc6bf33"}, - {file = "wrapt-1.13.3-cp310-cp310-win32.whl", hash = "sha256:78dea98c81915bbf510eb6a3c9c24915e4660302937b9ae05a0947164248020f"}, - {file = "wrapt-1.13.3-cp310-cp310-win_amd64.whl", hash = "sha256:ea3e746e29d4000cd98d572f3ee2a6050a4f784bb536f4ac1f035987fc1ed83e"}, - {file = "wrapt-1.13.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:8c73c1a2ec7c98d7eaded149f6d225a692caa1bd7b2401a14125446e9e90410d"}, - {file = "wrapt-1.13.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:086218a72ec7d986a3eddb7707c8c4526d677c7b35e355875a0fe2918b059179"}, - {file = "wrapt-1.13.3-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:e92d0d4fa68ea0c02d39f1e2f9cb5bc4b4a71e8c442207433d8db47ee79d7aa3"}, - {file = "wrapt-1.13.3-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:d4a5f6146cfa5c7ba0134249665acd322a70d1ea61732723c7d3e8cc0fa80755"}, - {file = "wrapt-1.13.3-cp35-cp35m-win32.whl", hash = "sha256:8aab36778fa9bba1a8f06a4919556f9f8c7b33102bd71b3ab307bb3fecb21851"}, - {file = "wrapt-1.13.3-cp35-cp35m-win_amd64.whl", hash = "sha256:944b180f61f5e36c0634d3202ba8509b986b5fbaf57db3e94df11abee244ba13"}, - {file = "wrapt-1.13.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:2ebdde19cd3c8cdf8df3fc165bc7827334bc4e353465048b36f7deeae8ee0918"}, - {file = "wrapt-1.13.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:610f5f83dd1e0ad40254c306f4764fcdc846641f120c3cf424ff57a19d5f7ade"}, - {file = "wrapt-1.13.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5601f44a0f38fed36cc07db004f0eedeaadbdcec90e4e90509480e7e6060a5bc"}, - {file = "wrapt-1.13.3-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:e6906d6f48437dfd80464f7d7af1740eadc572b9f7a4301e7dd3d65db285cacf"}, - {file = "wrapt-1.13.3-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:766b32c762e07e26f50d8a3468e3b4228b3736c805018e4b0ec8cc01ecd88125"}, - {file = "wrapt-1.13.3-cp36-cp36m-win32.whl", hash = "sha256:5f223101f21cfd41deec8ce3889dc59f88a59b409db028c469c9b20cfeefbe36"}, - {file = "wrapt-1.13.3-cp36-cp36m-win_amd64.whl", hash = "sha256:f122ccd12fdc69628786d0c947bdd9cb2733be8f800d88b5a37c57f1f1d73c10"}, - {file = "wrapt-1.13.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:46f7f3af321a573fc0c3586612db4decb7eb37172af1bc6173d81f5b66c2e068"}, - {file = "wrapt-1.13.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:778fd096ee96890c10ce96187c76b3e99b2da44e08c9e24d5652f356873f6709"}, - {file = "wrapt-1.13.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0cb23d36ed03bf46b894cfec777eec754146d68429c30431c99ef28482b5c1df"}, - {file = "wrapt-1.13.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:96b81ae75591a795d8c90edc0bfaab44d3d41ffc1aae4d994c5aa21d9b8e19a2"}, - {file = "wrapt-1.13.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:7dd215e4e8514004c8d810a73e342c536547038fb130205ec4bba9f5de35d45b"}, - {file = "wrapt-1.13.3-cp37-cp37m-win32.whl", hash = "sha256:47f0a183743e7f71f29e4e21574ad3fa95676136f45b91afcf83f6a050914829"}, - {file = "wrapt-1.13.3-cp37-cp37m-win_amd64.whl", hash = "sha256:fd76c47f20984b43d93de9a82011bb6e5f8325df6c9ed4d8310029a55fa361ea"}, - {file = "wrapt-1.13.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b73d4b78807bd299b38e4598b8e7bd34ed55d480160d2e7fdaabd9931afa65f9"}, - {file = "wrapt-1.13.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ec9465dd69d5657b5d2fa6133b3e1e989ae27d29471a672416fd729b429eb554"}, - {file = "wrapt-1.13.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:dd91006848eb55af2159375134d724032a2d1d13bcc6f81cd8d3ed9f2b8e846c"}, - {file = "wrapt-1.13.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ae9de71eb60940e58207f8e71fe113c639da42adb02fb2bcbcaccc1ccecd092b"}, - {file = "wrapt-1.13.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:51799ca950cfee9396a87f4a1240622ac38973b6df5ef7a41e7f0b98797099ce"}, - {file = "wrapt-1.13.3-cp38-cp38-win32.whl", hash = "sha256:4b9c458732450ec42578b5642ac53e312092acf8c0bfce140ada5ca1ac556f79"}, - {file = "wrapt-1.13.3-cp38-cp38-win_amd64.whl", hash = "sha256:7dde79d007cd6dfa65afe404766057c2409316135cb892be4b1c768e3f3a11cb"}, - {file = "wrapt-1.13.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:981da26722bebb9247a0601e2922cedf8bb7a600e89c852d063313102de6f2cb"}, - {file = "wrapt-1.13.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:705e2af1f7be4707e49ced9153f8d72131090e52be9278b5dbb1498c749a1e32"}, - {file = "wrapt-1.13.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:25b1b1d5df495d82be1c9d2fad408f7ce5ca8a38085e2da41bb63c914baadff7"}, - {file = "wrapt-1.13.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:77416e6b17926d953b5c666a3cb718d5945df63ecf922af0ee576206d7033b5e"}, - {file = "wrapt-1.13.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:865c0b50003616f05858b22174c40ffc27a38e67359fa1495605f96125f76640"}, - {file = "wrapt-1.13.3-cp39-cp39-win32.whl", hash = "sha256:0a017a667d1f7411816e4bf214646d0ad5b1da2c1ea13dec6c162736ff25a374"}, - {file = "wrapt-1.13.3-cp39-cp39-win_amd64.whl", hash = "sha256:81bd7c90d28a4b2e1df135bfbd7c23aee3050078ca6441bead44c42483f9ebfb"}, - {file = "wrapt-1.13.3.tar.gz", hash = "sha256:1fea9cd438686e6682271d36f3481a9f3636195578bab9ca3382e2f5f01fc185"}, + {file = "wrapt-1.14.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:5a9a1889cc01ed2ed5f34574c90745fab1dd06ec2eee663e8ebeefe363e8efd7"}, + {file = "wrapt-1.14.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:9a3ff5fb015f6feb78340143584d9f8a0b91b6293d6b5cf4295b3e95d179b88c"}, + {file = "wrapt-1.14.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:4b847029e2d5e11fd536c9ac3136ddc3f54bc9488a75ef7d040a3900406a91eb"}, + {file = "wrapt-1.14.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:9a5a544861b21e0e7575b6023adebe7a8c6321127bb1d238eb40d99803a0e8bd"}, + {file = "wrapt-1.14.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:88236b90dda77f0394f878324cfbae05ae6fde8a84d548cfe73a75278d760291"}, + {file = "wrapt-1.14.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f0408e2dbad9e82b4c960274214af533f856a199c9274bd4aff55d4634dedc33"}, + {file = "wrapt-1.14.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:9d8c68c4145041b4eeae96239802cfdfd9ef927754a5be3f50505f09f309d8c6"}, + {file = "wrapt-1.14.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:22626dca56fd7f55a0733e604f1027277eb0f4f3d95ff28f15d27ac25a45f71b"}, + {file = "wrapt-1.14.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:65bf3eb34721bf18b5a021a1ad7aa05947a1767d1aa272b725728014475ea7d5"}, + {file = "wrapt-1.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:09d16ae7a13cff43660155383a2372b4aa09109c7127aa3f24c3cf99b891c330"}, + {file = "wrapt-1.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:debaf04f813ada978d7d16c7dfa16f3c9c2ec9adf4656efdc4defdf841fc2f0c"}, + {file = "wrapt-1.14.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:748df39ed634851350efa87690c2237a678ed794fe9ede3f0d79f071ee042561"}, + {file = "wrapt-1.14.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1807054aa7b61ad8d8103b3b30c9764de2e9d0c0978e9d3fc337e4e74bf25faa"}, + {file = "wrapt-1.14.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:763a73ab377390e2af26042f685a26787c402390f682443727b847e9496e4a2a"}, + {file = "wrapt-1.14.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:8529b07b49b2d89d6917cfa157d3ea1dfb4d319d51e23030664a827fe5fd2131"}, + {file = "wrapt-1.14.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:68aeefac31c1f73949662ba8affaf9950b9938b712fb9d428fa2a07e40ee57f8"}, + {file = "wrapt-1.14.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59d7d92cee84a547d91267f0fea381c363121d70fe90b12cd88241bd9b0e1763"}, + {file = "wrapt-1.14.0-cp310-cp310-win32.whl", hash = "sha256:3a88254881e8a8c4784ecc9cb2249ff757fd94b911d5df9a5984961b96113fff"}, + {file = "wrapt-1.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:9a242871b3d8eecc56d350e5e03ea1854de47b17f040446da0e47dc3e0b9ad4d"}, + {file = "wrapt-1.14.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:a65bffd24409454b889af33b6c49d0d9bcd1a219b972fba975ac935f17bdf627"}, + {file = "wrapt-1.14.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9d9fcd06c952efa4b6b95f3d788a819b7f33d11bea377be6b8980c95e7d10775"}, + {file = "wrapt-1.14.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:db6a0ddc1282ceb9032e41853e659c9b638789be38e5b8ad7498caac00231c23"}, + {file = "wrapt-1.14.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:14e7e2c5f5fca67e9a6d5f753d21f138398cad2b1159913ec9e9a67745f09ba3"}, + {file = "wrapt-1.14.0-cp35-cp35m-win32.whl", hash = "sha256:6d9810d4f697d58fd66039ab959e6d37e63ab377008ef1d63904df25956c7db0"}, + {file = "wrapt-1.14.0-cp35-cp35m-win_amd64.whl", hash = "sha256:d808a5a5411982a09fef6b49aac62986274ab050e9d3e9817ad65b2791ed1425"}, + {file = "wrapt-1.14.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b77159d9862374da213f741af0c361720200ab7ad21b9f12556e0eb95912cd48"}, + {file = "wrapt-1.14.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36a76a7527df8583112b24adc01748cd51a2d14e905b337a6fefa8b96fc708fb"}, + {file = "wrapt-1.14.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a0057b5435a65b933cbf5d859cd4956624df37b8bf0917c71756e4b3d9958b9e"}, + {file = "wrapt-1.14.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a0a4ca02752ced5f37498827e49c414d694ad7cf451ee850e3ff160f2bee9d3"}, + {file = "wrapt-1.14.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:8c6be72eac3c14baa473620e04f74186c5d8f45d80f8f2b4eda6e1d18af808e8"}, + {file = "wrapt-1.14.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:21b1106bff6ece8cb203ef45b4f5778d7226c941c83aaaa1e1f0f4f32cc148cd"}, + {file = "wrapt-1.14.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:493da1f8b1bb8a623c16552fb4a1e164c0200447eb83d3f68b44315ead3f9036"}, + {file = "wrapt-1.14.0-cp36-cp36m-win32.whl", hash = "sha256:89ba3d548ee1e6291a20f3c7380c92f71e358ce8b9e48161401e087e0bc740f8"}, + {file = "wrapt-1.14.0-cp36-cp36m-win_amd64.whl", hash = "sha256:729d5e96566f44fccac6c4447ec2332636b4fe273f03da128fff8d5559782b06"}, + {file = "wrapt-1.14.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:891c353e95bb11abb548ca95c8b98050f3620a7378332eb90d6acdef35b401d4"}, + {file = "wrapt-1.14.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23f96134a3aa24cc50614920cc087e22f87439053d886e474638c68c8d15dc80"}, + {file = "wrapt-1.14.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6807bcee549a8cb2f38f73f469703a1d8d5d990815c3004f21ddb68a567385ce"}, + {file = "wrapt-1.14.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6915682f9a9bc4cf2908e83caf5895a685da1fbd20b6d485dafb8e218a338279"}, + {file = "wrapt-1.14.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f2f3bc7cd9c9fcd39143f11342eb5963317bd54ecc98e3650ca22704b69d9653"}, + {file = "wrapt-1.14.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:3a71dbd792cc7a3d772ef8cd08d3048593f13d6f40a11f3427c000cf0a5b36a0"}, + {file = "wrapt-1.14.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5a0898a640559dec00f3614ffb11d97a2666ee9a2a6bad1259c9facd01a1d4d9"}, + {file = "wrapt-1.14.0-cp37-cp37m-win32.whl", hash = "sha256:167e4793dc987f77fd476862d32fa404d42b71f6a85d3b38cbce711dba5e6b68"}, + {file = "wrapt-1.14.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d066ffc5ed0be00cd0352c95800a519cf9e4b5dd34a028d301bdc7177c72daf3"}, + {file = "wrapt-1.14.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d9bdfa74d369256e4218000a629978590fd7cb6cf6893251dad13d051090436d"}, + {file = "wrapt-1.14.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2498762814dd7dd2a1d0248eda2afbc3dd9c11537bc8200a4b21789b6df6cd38"}, + {file = "wrapt-1.14.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f24ca7953f2643d59a9c87d6e272d8adddd4a53bb62b9208f36db408d7aafc7"}, + {file = "wrapt-1.14.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b835b86bd5a1bdbe257d610eecab07bf685b1af2a7563093e0e69180c1d4af1"}, + {file = "wrapt-1.14.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b21650fa6907e523869e0396c5bd591cc326e5c1dd594dcdccac089561cacfb8"}, + {file = "wrapt-1.14.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:354d9fc6b1e44750e2a67b4b108841f5f5ea08853453ecbf44c81fdc2e0d50bd"}, + {file = "wrapt-1.14.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1f83e9c21cd5275991076b2ba1cd35418af3504667affb4745b48937e214bafe"}, + {file = "wrapt-1.14.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:61e1a064906ccba038aa3c4a5a82f6199749efbbb3cef0804ae5c37f550eded0"}, + {file = "wrapt-1.14.0-cp38-cp38-win32.whl", hash = "sha256:28c659878f684365d53cf59dc9a1929ea2eecd7ac65da762be8b1ba193f7e84f"}, + {file = "wrapt-1.14.0-cp38-cp38-win_amd64.whl", hash = "sha256:b0ed6ad6c9640671689c2dbe6244680fe8b897c08fd1fab2228429b66c518e5e"}, + {file = "wrapt-1.14.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b3f7e671fb19734c872566e57ce7fc235fa953d7c181bb4ef138e17d607dc8a1"}, + {file = "wrapt-1.14.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:87fa943e8bbe40c8c1ba4086971a6fefbf75e9991217c55ed1bcb2f1985bd3d4"}, + {file = "wrapt-1.14.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4775a574e9d84e0212f5b18886cace049a42e13e12009bb0491562a48bb2b758"}, + {file = "wrapt-1.14.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9d57677238a0c5411c76097b8b93bdebb02eb845814c90f0b01727527a179e4d"}, + {file = "wrapt-1.14.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00108411e0f34c52ce16f81f1d308a571df7784932cc7491d1e94be2ee93374b"}, + {file = "wrapt-1.14.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d332eecf307fca852d02b63f35a7872de32d5ba8b4ec32da82f45df986b39ff6"}, + {file = "wrapt-1.14.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:01f799def9b96a8ec1ef6b9c1bbaf2bbc859b87545efbecc4a78faea13d0e3a0"}, + {file = "wrapt-1.14.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47045ed35481e857918ae78b54891fac0c1d197f22c95778e66302668309336c"}, + {file = "wrapt-1.14.0-cp39-cp39-win32.whl", hash = "sha256:2eca15d6b947cfff51ed76b2d60fd172c6ecd418ddab1c5126032d27f74bc350"}, + {file = "wrapt-1.14.0-cp39-cp39-win_amd64.whl", hash = "sha256:bb36fbb48b22985d13a6b496ea5fb9bb2a076fea943831643836c9f6febbcfdc"}, + {file = "wrapt-1.14.0.tar.gz", hash = "sha256:8323a43bd9c91f62bb7d4be74cc9ff10090e7ef820e27bfe8815c57e68261311"}, ] yarl = [ {file = "yarl-1.7.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f2a8508f7350512434e41065684076f640ecce176d262a7d54f0da41d99c5a95"}, diff --git a/pyproject.toml b/pyproject.toml index c97bc4bc..e1d6a1ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ exclude = ''' [tool.poetry] name = "scale-nucleus" -version = "0.8.4" +version = "0.9.0" description = "The official Python client library for Nucleus, the Data Platform for AI" license = "MIT" authors = ["Scale AI Nucleus Team "] @@ -45,6 +45,7 @@ click = ">=7.1.2,<9.0" # NOTE: COLAB has 7.1.2 and has problems updating rich = "^10.15.2" shellingham = "^1.4.0" scikit-learn = ">=0.24.0" +Shapely = { version = ">=1.8.0", optional = true } [tool.poetry.dev-dependencies] poetry = "^1.1.5" @@ -67,6 +68,10 @@ pytest-xdist = "^2.5.0" [tool.poetry.scripts] nu = "cli.nu:nu" +[tool.poetry.extras] +shapely = ["Shapely"] + + [tool.pytest.ini_options] markers = [ "integration: marks tests as slow (deselect with '-m \"not integration\"')", diff --git a/tests/helpers.py b/tests/helpers.py index 7c1205e8..6e16d147 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -172,6 +172,7 @@ def reference_id_from_url(url): "height": 80 + i * 10, "reference_id": reference_id_from_url(TEST_IMG_URLS[i]), "annotation_id": f"[Pytest] Box Annotation Annotation Id{i}", + "metadata": {"field_1": "string", "index": i}, } for i in range(len(TEST_IMG_URLS)) ] diff --git a/tests/metrics/__init__.py b/tests/metrics/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/metrics/test_cuboid_metrics.py b/tests/metrics/test_cuboid_metrics.py new file mode 100644 index 00000000..8c85063a --- /dev/null +++ b/tests/metrics/test_cuboid_metrics.py @@ -0,0 +1,11 @@ +import pytest + +try: + import shapely +except ModuleNotFoundError: + pytest.skip( + "Skipping metrics tests (to run install with poetry install -E shapely)", + allow_module_level=True, + ) + +# TODO(gunnar): Add Cuboid tests! diff --git a/tests/metrics/test_filtering.py b/tests/metrics/test_filtering.py new file mode 100644 index 00000000..ac8c4119 --- /dev/null +++ b/tests/metrics/test_filtering.py @@ -0,0 +1,280 @@ +import json + +import pytest + +from nucleus.metrics import FieldFilter, MetadataFilter, apply_filters +from tests.metrics.helpers import ( + TEST_BOX_ANNOTATION_LIST, + TEST_BOX_PREDICTION_LIST, +) + + +@pytest.fixture( + params=[ + TEST_BOX_ANNOTATION_LIST.box_annotations, + TEST_BOX_PREDICTION_LIST.box_predictions, + ] +) +def annotations_or_predictions(request): + yield request.param + + +def test_filter_field(annotations_or_predictions): + dnf_filters = [ + FieldFilter("label", "==", annotations_or_predictions[0].label) + ] + filtered = apply_filters(annotations_or_predictions, dnf_filters) + assert filtered == [annotations_or_predictions[0]] + + +def test_filter_metadata_field(annotations_or_predictions): + dnf_filters = [ + MetadataFilter( + "index", "==", annotations_or_predictions[0].metadata["index"] + ) + ] + filtered = apply_filters(annotations_or_predictions, dnf_filters) + assert filtered == [annotations_or_predictions[0]] + + +def test_only_and(annotations_or_predictions): + and_filters = [ + MetadataFilter( + "index", "==", annotations_or_predictions[0].metadata["index"] + ) + ] + or_filters = [and_filters] + filtered_and = apply_filters(annotations_or_predictions, and_filters) + filtered_or = apply_filters(annotations_or_predictions, or_filters) + assert filtered_and == filtered_or + + +def test_json_encoded_filters(annotations_or_predictions): + filters = [ + [MetadataFilter("index", ">", 0), MetadataFilter("index", "<", 4)] + ] + expected = apply_filters(annotations_or_predictions, filters) + json_string = json.dumps(filters) + filters_from_json = json.loads(json_string) + json_filtered = apply_filters( + annotations_or_predictions, filters_from_json + ) + assert json_filtered == expected + + +def test_json_encoded_and_filters(annotations_or_predictions): + filters = [ + MetadataFilter("index", ">", 0), + MetadataFilter("index", "<", 4), + ] + expected = apply_filters(annotations_or_predictions, filters) + json_string = json.dumps(filters) + filters_from_json = json.loads(json_string) + json_filtered = apply_filters( + annotations_or_predictions, filters_from_json + ) + assert json_filtered == expected + + +def test_or_branches(annotations_or_predictions): + index_0_or_2 = [ + [MetadataFilter("index", "==", 0)], + [MetadataFilter("index", "==", 2)], + ] + filtered = apply_filters(annotations_or_predictions, index_0_or_2) + assert filtered == [ + annotations_or_predictions[0], + annotations_or_predictions[2], + ] + + +def test_only_one_or(annotations_or_predictions): + later_matches = [ + [MetadataFilter("index", "==", -1)], + [MetadataFilter("index", "==", 2)], + ] + filtered = apply_filters(annotations_or_predictions, later_matches) + assert filtered == [ + annotations_or_predictions[2], + ] + + +def test_and_branches(annotations_or_predictions): + index_0_or_2 = [ + [ + MetadataFilter("index", "==", 0), + FieldFilter("label", "==", annotations_or_predictions[0].label), + ], + [ + MetadataFilter("index", "==", 2), + FieldFilter("label", "==", annotations_or_predictions[2].label), + ], + ] + filtered = apply_filters(annotations_or_predictions, index_0_or_2) + assert filtered == [ + annotations_or_predictions[0], + annotations_or_predictions[2], + ] + + +def test_multi_or(annotations_or_predictions): + all_match = [ + [MetadataFilter("index", "==", i)] + for i in range(len(annotations_or_predictions)) + ] + filtered = apply_filters(annotations_or_predictions, all_match) + assert filtered == annotations_or_predictions + + +def test_missing_field_raises(annotations_or_predictions): + missing_field = [[FieldFilter("i_dont_exist", "==", 1)]] + with pytest.raises(AttributeError): + apply_filters(annotations_or_predictions, missing_field) + + +def test_allow_missing_field(annotations_or_predictions): + missing_field = [ + [FieldFilter("i_dont_exist", "==", 1, allow_missing=True)] + ] + filtered = apply_filters(annotations_or_predictions, missing_field) + assert filtered == [] + + +def test_missing_metadata_raises(annotations_or_predictions): + missing_field = [[MetadataFilter("i_dont_exist", "==", 1)]] + with pytest.raises(KeyError): + apply_filters(annotations_or_predictions, missing_field) + + +def test_allow_missing_metadata_field(annotations_or_predictions): + missing_field = [ + [FieldFilter("i_dont_exist", "==", 1, allow_missing=True)] + ] + filtered = apply_filters(annotations_or_predictions, missing_field) + assert filtered == [] + + +def test_gt_metadata(annotations_or_predictions): + valid_gt = [MetadataFilter("index", ">", 0)] + filtered = apply_filters(annotations_or_predictions, valid_gt) + assert filtered == annotations_or_predictions[1:] + + +def test_gt_field(annotations_or_predictions): + valid_gt = [FieldFilter("x", ">", annotations_or_predictions[0].x)] + filtered = apply_filters(annotations_or_predictions, valid_gt) + assert filtered == annotations_or_predictions[1:] + + +def test_gte_metadata(annotations_or_predictions): + valid_gt = [MetadataFilter("index", ">=", 1)] + filtered = apply_filters(annotations_or_predictions, valid_gt) + assert filtered == annotations_or_predictions[1:] + + +def test_gte_field(annotations_or_predictions): + valid_gt = [FieldFilter("x", ">=", annotations_or_predictions[1].x)] + filtered = apply_filters(annotations_or_predictions, valid_gt) + assert filtered == annotations_or_predictions[1:] + + +def test_lt_metadata(annotations_or_predictions): + valid_gt = [MetadataFilter("index", "<", 1)] + filtered = apply_filters(annotations_or_predictions, valid_gt) + assert filtered == annotations_or_predictions[:1] + + +def test_lt_field(annotations_or_predictions): + valid_gt = [FieldFilter("x", "<", annotations_or_predictions[1].x)] + filtered = apply_filters(annotations_or_predictions, valid_gt) + assert filtered == annotations_or_predictions[:1] + + +def test_lte_metadata(annotations_or_predictions): + valid_gt = [MetadataFilter("index", "<=", 1)] + filtered = apply_filters(annotations_or_predictions, valid_gt) + assert filtered == annotations_or_predictions[:2] + + +def test_lte_field(annotations_or_predictions): + valid_gt = [FieldFilter("x", "<=", annotations_or_predictions[1].x)] + filtered = apply_filters(annotations_or_predictions, valid_gt) + assert filtered == annotations_or_predictions[:2] + + +def test_eqeq_metadata(annotations_or_predictions): + valid_gt = [MetadataFilter("index", "==", 0)] + filtered = apply_filters(annotations_or_predictions, valid_gt) + assert filtered == [annotations_or_predictions[0]] + + +def test_eqeq_field(annotations_or_predictions): + valid_gt = [FieldFilter("x", "==", annotations_or_predictions[0].x)] + filtered = apply_filters(annotations_or_predictions, valid_gt) + assert filtered == [annotations_or_predictions[0]] + + +def test_eq_metadata(annotations_or_predictions): + valid_gt = [MetadataFilter("index", "=", 0)] + filtered = apply_filters(annotations_or_predictions, valid_gt) + assert filtered == [annotations_or_predictions[0]] + + +def test_eq_field(annotations_or_predictions): + valid_gt = [FieldFilter("x", "=", annotations_or_predictions[0].x)] + filtered = apply_filters(annotations_or_predictions, valid_gt) + assert filtered == [annotations_or_predictions[0]] + + +def test_neq_metadata(annotations_or_predictions): + valid_gt = [MetadataFilter("index", "!=", 0)] + filtered = apply_filters(annotations_or_predictions, valid_gt) + assert filtered == annotations_or_predictions[1:] + + +def test_neq_field(annotations_or_predictions): + valid_gt = [FieldFilter("x", "!=", annotations_or_predictions[0].x)] + filtered = apply_filters(annotations_or_predictions, valid_gt) + assert filtered == annotations_or_predictions[1:] + + +def test_in_metadata(annotations_or_predictions): + valid_gt = [MetadataFilter("index", "in", [0, 2])] + filtered = apply_filters(annotations_or_predictions, valid_gt) + assert filtered == [ + annotations_or_predictions[0], + annotations_or_predictions[2], + ] + + +def test_in_field(annotations_or_predictions): + valid_gt = [ + FieldFilter( + "x", + "in", + [annotations_or_predictions[0].x, annotations_or_predictions[2].x], + ) + ] + filtered = apply_filters(annotations_or_predictions, valid_gt) + assert filtered == [ + annotations_or_predictions[0], + annotations_or_predictions[2], + ] + + +def test_not_in_metadata(annotations_or_predictions): + valid_gt = [MetadataFilter("index", "not in", [0, 1])] + filtered = apply_filters(annotations_or_predictions, valid_gt) + assert filtered == annotations_or_predictions[2:] + + +def test_not_in_field(annotations_or_predictions): + valid_gt = [ + FieldFilter( + "x", + "not in", + [annotations_or_predictions[0].x, annotations_or_predictions[1].x], + ) + ] + filtered = apply_filters(annotations_or_predictions, valid_gt) + assert filtered == annotations_or_predictions[2:]