fixing ignore order for deltas and tuple

Author: seperman

deepdiff/delta.py

@@ -1,4 +1,5 @@
 import logging
+from collections import defaultdict
 from collections.abc import Mapping
 from copy import deepcopy
 from decimal import Decimal
@@ -18,9 +19,10 @@
 
 
 VERIFICATION_MSG = 'Expected the previous value for {} to be {} but it is {}. Due to {}'
-ELEM_NOT_FOUND_TO_ADD_MSG = 'Key or index of {} is not found for {} for insertion operation.'
+ELEM_NOT_FOUND_TO_ADD_MSG = 'Key or index of {} is not found for {} for setting operation.'
 TYPE_CHANGE_FAIL_MSG = 'Unable to do the type change for {} from to type {} due to {}'
 VERIFY_SYMMETRY_MSG = 'that the original objects that the delta is made from must be different than what the delta is applied to.'
+FAIL_TO_REMOVE_ITEM_IGNORE_ORDER_MSG = 'Failed to remove index[{}] on {}. It was expected to be {} but got {}'
 
 
 class _NotFound:
@@ -134,6 +136,7 @@ def __add__(self, other):
         # all the other iterables to match the reverse of order of operations in DeepDiff
         self._do_iterable_item_removed()
         self._do_iterable_item_added()
+        self._do_ignore_order()
         self._do_dictionary_item_added()
         self._do_dictionary_item_removed()
         self._do_attribute_added()
@@ -169,6 +172,8 @@ def _get_elem_and_compare_to_old_value(self, obj, path_for_err_reporting, expect
         except (KeyError, IndexError, AttributeError) as e:
             current_old_value = not_found
             if self.verify_symmetry:
+                if isinstance(path_for_err_reporting, (list, tuple)):
+                    path_for_err_reporting = '.'.join([i[0] for i in path_for_err_reporting])
                 self._raise_or_log(VERIFICATION_MSG.format(
                     path_for_err_reporting,
                     expected_old_value, current_old_value, e))
@@ -357,14 +362,14 @@ def _do_attribute_removed(self):
     def _do_set_item_added(self):
         items = self.diff.get('set_item_added')
         if items:
-            self._do_set_item(items, func='union')
+            self._do_set_or_frozenset_item(items, func='union')
 
     def _do_set_item_removed(self):
         items = self.diff.get('set_item_removed')
         if items:
-            self._do_set_item(items, func='difference')
+            self._do_set_or_frozenset_item(items, func='difference')
 
-    def _do_set_item(self, items, func):
+    def _do_set_or_frozenset_item(self, items, func):
         for path, value in items.items():
             elements = _path_to_elements(path)
             parent = _get_nested_obj(obj=self, elements=elements[:-1])
@@ -374,6 +379,81 @@ def _do_set_item(self, items, func):
             new_value = getattr(obj, func)(value)
             self._simple_set_elem_value(parent, path_for_err_reporting=path, elem=elem, value=new_value, action=action)
 
+    def _do_ignore_order_get_old(self, obj, remove_indexes_per_path, fixed_indexes_values, path_for_err_reporting):
+        """
+        A generator that gets the old values in an iterable when the order was supposed to be ignored.
+        """
+        old_obj_index = -1
+        max_len = len(obj) - 1
+        while old_obj_index < max_len:
+            old_obj_index += 1
+            current_old_obj = obj[old_obj_index]
+            if current_old_obj in fixed_indexes_values:
+                continue
+            if old_obj_index in remove_indexes_per_path:
+                expected_obj_to_delete = remove_indexes_per_path.pop(old_obj_index)
+                if current_old_obj == expected_obj_to_delete:
+                    continue
+                else:
+                    self._raise_or_log(FAIL_TO_REMOVE_ITEM_IGNORE_ORDER_MSG.format(old_obj_index, path_for_err_reporting, expected_obj_to_delete, current_old_obj))
+            yield current_old_obj
+
+    def _do_ignore_order(self):
+        """
+
+            't1': [5, 1, 1, 1, 6],
+            't2': [7, 1, 1, 1, 8],
+
+            'ignore_order_fixed_indexes': {
+                'root': {
+                    0: 7,
+                    4: 8
+                }
+            },
+            'ignore_order_remove_indexes': {
+                'root': {
+                    4: 6,
+                    0: 5
+                }
+            }
+
+        """
+        fixed_indexes = self.diff.get('ignore_order_fixed_indexes', {})
+        remove_indexes = self.diff.get('ignore_order_remove_indexes', {})
+
+        paths = set(fixed_indexes.keys()) | set(remove_indexes.keys())
+        for path in paths:
+            # In the case of ignore_order reports, we are pointing to the container object.
+            # Thus we add a [0] to the elements so we can get the required objects and discard what we don't need.
+            _, parent, parent_to_obj_elem, parent_to_obj_action, obj, _, _ = self._get_elements_and_details("{}[0]".format(path))
+            fixed_indexes_per_path = fixed_indexes.get(path)
+            remove_indexes_per_path = remove_indexes.get(path)
+            fixed_indexes_values = set(fixed_indexes_per_path.values())  # TODO: this needs to be changed to use deephash
+
+            new_obj = []
+            there_are_old_items = bool(obj)
+            old_item_gen = self._do_ignore_order_get_old(
+                obj, remove_indexes_per_path, fixed_indexes_values, path_for_err_reporting=path)
+            while there_are_old_items or fixed_indexes_per_path:
+                new_obj_index = len(new_obj)
+                if new_obj_index in fixed_indexes_per_path:
+                    new_item = fixed_indexes_per_path.pop(new_obj_index)
+                    new_obj.append(new_item)
+                else:
+                    try:
+                        new_item = next(old_item_gen)
+                    except StopIteration:
+                        there_are_old_items = False
+                    else:
+                        new_obj.append(new_item)
+
+            if isinstance(obj, tuple):
+                new_obj = tuple(new_obj)
+            # Making sure that the object is re-instated inside the parent especially if it was immutable
+            # and we had to turn it into a mutable one. In such cases the object has a new id.
+            self._simple_set_elem_value(obj=parent, path_for_err_reporting=path, elem=parent_to_obj_elem,
+                                        value=new_obj, action=parent_to_obj_action)
+
 
 if __name__ == "__main__":  # pragma: no cover
     import doctest

deepdiff/diff.py

@@ -347,6 +347,11 @@ def __iterables_subscriptable(t1, t2):
 
     def __diff_iterable(self, level, parents_ids=frozenset({})):
         """Difference of iterables"""
+
+        if self.ignore_order:
+            self.__diff_iterable_with_deephash(level)
+            return
+
         # We're handling both subscriptable and non-subscriptable iterables. Which one is it?
         subscriptable = self.__iterables_subscriptable(level.t1, level.t2)
         if subscriptable:
@@ -666,10 +671,7 @@ def __diff(self, level, parents_ids=frozenset({})):
             self.__diff_numpy(level, parents_ids)
 
         elif isinstance(level.t1, Iterable):
-            if self.ignore_order:
-                self.__diff_iterable_with_deephash(level)
-            else:
-                self.__diff_iterable(level, parents_ids)
+            self.__diff_iterable(level, parents_ids)
 
         else:
             self.__diff_obj(level, parents_ids)
@@ -754,25 +756,19 @@ def to_delta_dict(self, directed=True):
 
         t1 + delta == t2
 
-
-        **Example**
-
-        Directed Delta
-            >>> class A:
-            ...     pass
-            ...
-            >>> class B:
         """
-        result = DeltaResult(tree_results=self.tree, verbose_level=2)
+        result = DeltaResult(tree_results=self.tree, ignore_order=self.ignore_order)
         result.cleanup()  # clean up text-style result dictionary
+        if self.ignore_order:
+            if not self.report_repetition:
+                raise ValueError('report_repetition must be set to True when ignore_order is True to create the delta object.')
         if directed:
             for report_key, report_value in result.items():
                 if isinstance(report_value, Mapping):
                     for path, value in report_value.items():
                         if isinstance(value, Mapping) and 'old_value' in value:
                             del value['old_value']
-        if self.ignore_order:
-            result['ignore_order'] = True
+
         return result
 
 

deepdiff/diff_doc.rst

@@ -23,7 +23,6 @@ ignore_order : Boolean, default=False
 report_repetition : Boolean, default=False
     reports repetitions when set True
     ONLY when ignore_order is set True too. This works for iterables.
-    This feature currently is experimental and is not production ready.
 
 significant_digits : int >= 0, default=None
     By default the significant_digits compares only that many digits AFTER the decimal point. However you can set override that by setting the number_format_notation="e" which will make it mean the digits in scientific notation.

deepdiff/model.py

@@ -199,28 +199,69 @@ class DeltaResult(TextResult):
 
     ADD_QUOTES_TO_STRINGS = False
 
-    def __init__(self, tree_results=None, verbose_level=1):
-        self.verbose_level = verbose_level
+    def __init__(self, tree_results=None, ignore_order=None):
+        self.ignore_order = ignore_order
 
         self.update({
             "type_changes": {},
             "dictionary_item_added": {},
-            "dictionary_item_removed": self.__set_or_dict(),
+            "dictionary_item_removed": {},
             "values_changed": {},
             "iterable_item_added": {},
-            "iterable_item_removed": self.__set_or_dict(),
+            "iterable_item_removed": {},
             "attribute_added": {},
-            "attribute_removed": self.__set_or_dict(),
+            "attribute_removed": {},
             "set_item_removed": {},
             "set_item_added": {},
-            "repetition_change": {}
+            "ignore_order_fixed_indexes": {},
+            "ignore_order_remove_indexes": {},
         })
 
         if tree_results:
             self._from_tree_results(tree_results)
 
-    def __set_or_dict(self):
-        return {} if self.verbose_level >= 2 else PrettyOrderedSet()
+    def _from_tree_results(self, tree):
+        """
+        Populate this object by parsing an existing reference-style result dictionary.
+        :param tree: A TreeResult
+        :return:
+        """
+        self._from_tree_type_changes(tree)
+        self._from_tree_default(tree, 'dictionary_item_added')
+        self._from_tree_default(tree, 'dictionary_item_removed')
+        self._from_tree_value_changed(tree)
+        if self.ignore_order:
+            self._from_tree_iterable_item_added(
+                tree, 'iterable_item_added', delta_report_key='ignore_order_fixed_indexes')
+            self._from_tree_iterable_item_added(
+                tree, 'iterable_item_removed', delta_report_key='ignore_order_remove_indexes')
+        else:
+            self._from_tree_default(tree, 'iterable_item_added')
+            self._from_tree_default(tree, 'iterable_item_removed')
+        self._from_tree_default(tree, 'attribute_added')
+        self._from_tree_default(tree, 'attribute_removed')
+        self._from_tree_set_item_removed(tree)
+        self._from_tree_set_item_added(tree)
+        self._from_tree_repetition_change(tree)
+
+    def _from_tree_iterable_item_added(self, tree, report_type, delta_report_key):
+        if report_type in tree:
+            for change in tree[report_type]:  # report each change
+                # determine change direction (added or removed)
+                # Report t2 (the new one) whenever possible.
+                # In cases where t2 doesn't exist (i.e. stuff removed), report t1.
+                if change.t2 is not notpresent:
+                    item = change.t2
+                else:
+                    item = change.t1
+
+                # do the reporting
+                path, param, _ = change.path(force=FORCE_DEFAULT, get_parent_too=True)
+                try:
+                    ignore_order_fixed_indexes = self[delta_report_key][path]
+                except KeyError:
+                    ignore_order_fixed_indexes = self[delta_report_key][path] = {}
+                ignore_order_fixed_indexes[param] = item
 
     def _from_tree_type_changes(self, tree):
         if 'type_changes' in tree:
@@ -247,7 +288,7 @@ def _from_tree_type_changes(self, tree):
                 })
                 self['type_changes'][change.path(
                     force=FORCE_DEFAULT)] = remap_dict
-                if self.verbose_level and include_values:
+                if include_values:
                     remap_dict.update(old_value=change.t1, new_value=change.t2)
 
     def _from_tree_value_changed(self, tree):
@@ -259,8 +300,26 @@ def _from_tree_value_changed(self, tree):
                 if 'diff' in change.additional:
                     the_changed.update({'diff': change.additional['diff']})
 
-    def _from_tree_unprocessed(self, tree):
-        pass
+    def _from_tree_repetition_change(self, tree):
+        if 'repetition_change' in tree:
+            for change in tree['repetition_change']:
+                path, _, _ = change.path(get_parent_too=True)
+                repetition = RemapDict(change.additional['repetition'])
+                value = change.t1
+                try:
+                    ignore_order_fixed_indexes = self['ignore_order_fixed_indexes'][path]
+                except KeyError:
+                    ignore_order_fixed_indexes = self['ignore_order_fixed_indexes'][path] = {}
+                for index in repetition['new_indexes']:
+                    ignore_order_fixed_indexes[index] = value
+                # self['repetition_change'][path][]
+                # old_indexes = set(repetition['old_indexes'])
+                # new_indexes = set(repetition['new_indexes'])
+                # value['old_indexes'] = old_indexes - new_indexes
+                # value['new_indexes'] = new_indexes - old_indexes
+                # self['repetition_change'][path] = RemapDict(change.additional[
+                #     'repetition'])
+                # self['repetition_change'][path]['value'] = change.t1
 
 
 class DiffLevel:
@@ -474,7 +533,11 @@ def all_down(self):
             level = level.down
         return level
 
-    def path(self, root="root", force=None):
+    @staticmethod
+    def _format_result(root, result):
+        return None if result is None else "{}{}".format(root, result)
+
+    def path(self, root="root", force=None, get_parent_too=False):
         """
         A python syntax string describing how to descend to this level, assuming the top level object is called root.
         Returns None if the path is not representable as a string.
@@ -496,11 +559,16 @@ def path(self, root="root", force=None):
                         This will pretend all iterables are subscriptable, for example.
         """
         # TODO: We could optimize this by building on top of self.up's path if it is cached there
-        if force in self._path:
-            result = self._path[force]
-            return None if result is None else "{}{}".format(root, result)
+        cache_key = "{}{}".format(force, get_parent_too)
+        if cache_key in self._path:
+            result = self._path[cache_key]
+            if get_parent_too:
+                # parent, param, result
+                return (self._format_result(root, result[0]), result[1], self._format_result(root, result[2]))
+            else:
+                return self._format_result(root, result)
 
-        result = ""
+        result = parent = param = ""
         level = self.all_up  # start at the root
 
         # traverse all levels of this relationship
@@ -515,6 +583,8 @@ def path(self, root="root", force=None):
             # Build path for this level
             item = next_rel.get_param_repr(force)
             if item:
+                parent = result
+                param = next_rel.param
                 result += item
             else:
                 # it seems this path is not representable as a string
@@ -525,7 +595,10 @@ def path(self, root="root", force=None):
             level = level.down
 
         self._path[force] = result
-        result = None if result is None else "{}{}".format(root, result)
+        result = self._format_result(root, result)
+        if get_parent_too:
+            parent = self._format_result(root, parent)
+            return (parent, param, result)
         return result
 
     def create_deeper(self,

deepdiff/path.py

@@ -40,7 +40,7 @@ def _path_to_elements(path, first_element=DEFAULT_FIRST_ELEMENT):
         >>> _path_to_elements(path, first_element=None)
         [(4.3, 'GET'), ('b', 'GETATTR'), ('a3', 'GET')]
     """
-    if isinstance(path, tuple):
+    if isinstance(path, (tuple, list)):
         return path
     elements = []
     if first_element: