From ef777e7da4b6d5a2bfa5b4ee5237d5da299b03f6 Mon Sep 17 00:00:00 2001 From: Alex Watt Date: Sat, 30 Mar 2019 00:08:26 -0400 Subject: [PATCH 1/5] BUG: Fix #16363 - Prevent visit_BinOp from accessing value on UnaryOp --- pandas/core/computation/expr.py | 6 ++++-- pandas/tests/computation/test_eval.py | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pandas/core/computation/expr.py b/pandas/core/computation/expr.py index 16795ea8c58e9..0488d27d89be4 100644 --- a/pandas/core/computation/expr.py +++ b/pandas/core/computation/expr.py @@ -418,11 +418,13 @@ def _maybe_transform_eq_ne(self, node, left=None, right=None): def _maybe_downcast_constants(self, left, right): f32 = np.dtype(np.float32) - if left.is_scalar and not right.is_scalar and right.return_type == f32: + if (left.is_scalar and hasattr(left, 'value') and + not right.is_scalar and right.return_type == f32): # right is a float32 array, left is a scalar name = self.env.add_tmp(np.float32(left.value)) left = self.term_type(name, self.env) - if right.is_scalar and not left.is_scalar and left.return_type == f32: + if (right.is_scalar and hasattr(right, 'value') and + not left.is_scalar and left.return_type == f32): # left is a float32 array, right is a scalar name = self.env.add_tmp(np.float32(right.value)) right = self.term_type(name, self.env) diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index f2d5069ccb5d0..f67f2727b7aee 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -608,6 +608,12 @@ def test_unary_in_array(self): -False, False, ~False, +False, -37, 37, ~37, +37], dtype=np.object_)) + def test_float_comparison_bin_op(self): + # GH 16363 + df = pd.DataFrame({'x': np.array([0], dtype=np.float32)}) + res = df.eval('x < -0.1') + assert np.array_equal(res, np.array([False])), res + def test_disallow_scalar_bool_ops(self): exprs = '1 or 2', '1 and 2' exprs += 'a and b', 'a or b' From 5242befdd153fff32fe5e2e9ba30785062f8cd9e Mon Sep 17 00:00:00 2001 From: Alex Watt Date: Sat, 30 Mar 2019 08:19:37 -0400 Subject: [PATCH 2/5] Fix test and add note to whatsnew --- doc/source/whatsnew/v0.25.0.rst | 1 + pandas/tests/computation/test_eval.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index 49b2349851479..73e7b3256d14a 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -228,6 +228,7 @@ Performance Improvements Bug Fixes ~~~~~~~~~ +- Bug in :meth:`~pandas.eval` when comparing floats with scalar operators that don't have a 'value' attribute (e.g., unary operators) (:issue:`25928`) Categorical ^^^^^^^^^^^ diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index f67f2727b7aee..f46f16002cc17 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -612,7 +612,7 @@ def test_float_comparison_bin_op(self): # GH 16363 df = pd.DataFrame({'x': np.array([0], dtype=np.float32)}) res = df.eval('x < -0.1') - assert np.array_equal(res, np.array([False])), res + assert np.array_equal(res, np.array([False])) def test_disallow_scalar_bool_ops(self): exprs = '1 or 2', '1 and 2' From f0e0b3d34314c890cb646a69c5790eedd6cd7765 Mon Sep 17 00:00:00 2001 From: Alex Watt Date: Sat, 30 Mar 2019 09:06:02 -0400 Subject: [PATCH 3/5] Attempt to fix test style --- pandas/tests/computation/test_eval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index f46f16002cc17..2aa49ad2edf62 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -612,7 +612,7 @@ def test_float_comparison_bin_op(self): # GH 16363 df = pd.DataFrame({'x': np.array([0], dtype=np.float32)}) res = df.eval('x < -0.1') - assert np.array_equal(res, np.array([False])) + assert res.values == np.array([False]) def test_disallow_scalar_bool_ops(self): exprs = '1 or 2', '1 and 2' From 5d6ea064c5bfd414709fd143c313d7703f77c74f Mon Sep 17 00:00:00 2001 From: Alex Watt Date: Sat, 30 Mar 2019 18:38:18 -0400 Subject: [PATCH 4/5] Update whatsnew and add another test case --- doc/source/whatsnew/v0.25.0.rst | 4 ++-- pandas/tests/computation/test_eval.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index 73e7b3256d14a..3ec6426cdb877 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -228,7 +228,7 @@ Performance Improvements Bug Fixes ~~~~~~~~~ -- Bug in :meth:`~pandas.eval` when comparing floats with scalar operators that don't have a 'value' attribute (e.g., unary operators) (:issue:`25928`) + Categorical ^^^^^^^^^^^ @@ -269,7 +269,7 @@ Numeric - Bug in error messages in :meth:`DataFrame.corr` and :meth:`Series.corr`. Added the possibility of using a callable. (:issue:`25729`) - Bug in :meth:`Series.divmod` and :meth:`Series.rdivmod` which would raise an (incorrect) ``ValueError`` rather than return a pair of :class:`Series` objects as result (:issue:`25557`) - Raises a helpful exception when a non-numeric index is sent to :meth:`interpolate` with methods which require numeric index. (:issue:`21662`) -- +- Bug in :meth:`~pandas.eval` when comparing floats with scalar operators, for example: ``x < -0.1`` (:issue:`25928`) - diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 2aa49ad2edf62..8ea8f66a5903c 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -614,6 +614,9 @@ def test_float_comparison_bin_op(self): res = df.eval('x < -0.1') assert res.values == np.array([False]) + res = df.eval('-5 > x') + assert res.values == np.array([False]) + def test_disallow_scalar_bool_ops(self): exprs = '1 or 2', '1 and 2' exprs += 'a and b', 'a or b' From 1b88337fb79c980a6a45828e4d44b932d9ec18cb Mon Sep 17 00:00:00 2001 From: Alex Watt Date: Sat, 30 Mar 2019 18:49:34 -0400 Subject: [PATCH 5/5] Use parametrize to test both float32 and float64 --- pandas/tests/computation/test_eval.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 8ea8f66a5903c..3908a7f1f99aa 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -608,9 +608,10 @@ def test_unary_in_array(self): -False, False, ~False, +False, -37, 37, ~37, +37], dtype=np.object_)) - def test_float_comparison_bin_op(self): + @pytest.mark.parametrize('dtype', [np.float32, np.float64]) + def test_float_comparison_bin_op(self, dtype): # GH 16363 - df = pd.DataFrame({'x': np.array([0], dtype=np.float32)}) + df = pd.DataFrame({'x': np.array([0], dtype=dtype)}) res = df.eval('x < -0.1') assert res.values == np.array([False])