ENH: add reduction option to hash_pandas_objects

xref pandas-dev#14729

Author: jreback

pandas/tools/hashing.py

@@ -12,7 +12,19 @@
 _default_hash_key = '0123456789123456'
 
 
-def hash_pandas_object(obj, index=True, encoding='utf8', hash_key=None):
+def reducer(arr):
+    """
+    return a scalar via our reduction function
+
+    an empty array returns np.nan
+    """
+    if len(arr):
+        return np.bitwise_xor.reduce(arr)
+    return np.nan
+
+
+def hash_pandas_object(obj, index=True, encoding='utf8', hash_key=None,
+                       reduce=False):
     """
     Return a data hash of the Index/Series/DataFrame
 
@@ -25,6 +37,8 @@ def hash_pandas_object(obj, index=True, encoding='utf8', hash_key=None):
     encoding : string, default 'utf8'
         encoding for data & key when strings
     hash_key : string key to encode, default to _default_hash_key
+    reduce : boolean, default False
+        produce a single hash result
 
     Returns
     -------
@@ -65,10 +79,14 @@ def adder(h, hashed_to_add):
         h = Series(h, index=obj.index, dtype='uint64')
     else:
         raise TypeError("Unexpected type for hashing %s" % type(obj))
+
+    if reduce:
+        h = reducer(h.values)
+
     return h
 
 
-def hash_array(vals, encoding='utf8', hash_key=None):
+def hash_array(vals, encoding='utf8', hash_key=None, reduce=False):
     """
     Given a 1d array, return an array of deterministic integers.
 
@@ -80,6 +98,8 @@ def hash_array(vals, encoding='utf8', hash_key=None):
     encoding : string, default 'utf8'
         encoding for data & key when strings
     hash_key : string key to encode, default to _default_hash_key
+    reduce : boolean, default False
+        produce a single hash result
 
     Returns
     -------
@@ -134,4 +154,8 @@ def hash_array(vals, encoding='utf8', hash_key=None):
     vals ^= vals >> 27
     vals *= np.uint64(0x94d049bb133111eb)
     vals ^= vals >> 31
+
+    if reduce:
+        vals = reducer(vals)
+
     return vals

pandas/tools/tests/test_hashing.py

@@ -3,6 +3,7 @@
 
 from pandas import DataFrame, Series, Index
 from pandas.tools.hashing import hash_array, hash_pandas_object
+from pandas.types.common import is_scalar
 import pandas.util.testing as tm
 
 
@@ -31,6 +32,20 @@ def test_consistency(self):
                           index=['foo', 'bar', 'baz'])
         tm.assert_series_equal(result, expected)
 
+    def test_reduce_consistency(self):
+        c = Series(pd.Categorical([1, 2], ordered=True))
+
+        result = hash_pandas_object(c, reduce=True)
+        self.assertEqual(result, 6505546918052763540)
+
+        c = Series(pd.Categorical([2, 1], ordered=True))
+        result = hash_pandas_object(c, reduce=True)
+        self.assertEqual(result, 6135121269729399882)
+
+        c = Series(pd.Categorical([1., 2.], ordered=True))
+        result = hash_pandas_object(c, reduce=True)
+        self.assertEqual(result, 6505546918052763540)
+
     def test_hash_array(self):
         for name, s in self.df.iteritems():
             a = s.values
@@ -46,6 +61,11 @@ def check_equal(self, obj, **kwargs):
         b = hash_pandas_object(obj, **kwargs)
         tm.assert_series_equal(a, b)
 
+        # test reducing
+        kwargs['reduce'] = True
+        result = hash_pandas_object(obj, **kwargs)
+        self.assertTrue(is_scalar(result))
+
     def check_not_equal_with_index(self, obj):
 
         # check that we are not hashing the same if