DEPR: allow options for using bottleneck/numexpr (pandas-dev#16157)

* DEPR: allow options for using bottleneck/numexpr

deprecate pd.computation.expressions.set_use_numexpr()

* DEPR: pandas.types.concat.union_categoricals in favor of pandas.api.type.union_categoricals

closes pandas-dev#16140

Author: jreback

doc/source/basics.rst

@@ -93,7 +93,7 @@ Accelerated operations
 ----------------------
 
 pandas has support for accelerating certain types of binary numerical and boolean operations using
-the ``numexpr`` library (starting in 0.11.0) and the ``bottleneck`` libraries.
+the ``numexpr`` library and the ``bottleneck`` libraries.
 
 These libraries are especially useful when dealing with large data sets, and provide large
 speedups. ``numexpr`` uses smart chunking, caching, and multiple cores. ``bottleneck`` is
@@ -114,6 +114,15 @@ Here is a sample (using 100 column x 100,000 row ``DataFrames``):
 You are highly encouraged to install both libraries. See the section
 :ref:`Recommended Dependencies <install.recommended_dependencies>` for more installation info.
 
+These are both enabled to be used by default, you can control this by setting the options:
+
+.. versionadded:: 0.20.0
+
+.. code-block:: python
+
+   pd.set_option('compute.use_bottleneck', False)
+   pd.set_option('compute.use_numexpr', False)
+
 .. _basics.binop:
 
 Flexible binary operations

doc/source/options.rst

@@ -425,6 +425,10 @@ mode.use_inf_as_null                False        True means treat None, NaN, -IN
                                                  INF as null (old way), False means
                                                  None and NaN are null, but INF, -INF
                                                  are not null (new way).
+compute.use_bottleneck              True         Use the bottleneck library to accelerate
+                                                 computation if it is installed
+compute.use_numexpr                 True         Use the numexpr library to accelerate
+                                                 computation if it is installed
 =================================== ============ ==================================
 
 
@@ -538,4 +542,4 @@ Only ``'display.max_rows'`` are serialized and published.
 .. ipython:: python
     :suppress:
 
-    pd.reset_option('display.html.table_schema')
+    pd.reset_option('display.html.table_schema')

doc/source/whatsnew/v0.20.0.txt

@@ -521,6 +521,7 @@ Other Enhancements
 - The ``display.show_dimensions`` option can now also be used to specify
   whether the length of a ``Series`` should be shown in its repr (:issue:`7117`).
 - ``parallel_coordinates()`` has gained a ``sort_labels`` keyword arg that sorts class labels and the colours assigned to them (:issue:`15908`)
+- Options added to allow one to turn on/off using ``bottleneck`` and ``numexpr``, see :ref:`here <basics.accelerate>` (:issue:`16157`)
 
 
 .. _ISO 8601 duration: https://en.wikipedia.org/wiki/ISO_8601#Durations
@@ -1217,7 +1218,7 @@ If indicated, a deprecation warning will be issued if you reference theses modul
 
     "pandas.lib", "pandas._libs.lib", "X"
     "pandas.tslib", "pandas._libs.tslib", "X"
-    "pandas.computation", "pandas.core.computation", ""
+    "pandas.computation", "pandas.core.computation", "X"
     "pandas.msgpack", "pandas.io.msgpack", ""
     "pandas.index", "pandas._libs.index", ""
     "pandas.algos", "pandas._libs.algos", ""

pandas/computation/__init__.py

@@ -0,0 +1,11 @@
+import warnings
+
+
+def set_use_numexpr(v=True):
+    warnings.warn("pandas.computation.expressions.set_use_numexpr is "
+                  "deprecated and will be removed in a future version.\n"
+                  "you can toggle usage of numexpr via "
+                  "pandas.get_option('compute.use_numexpr')",
+                  FutureWarning, stacklevel=2)
+    from pandas import set_option
+    set_option('compute.use_numexpr', v)

pandas/computation/expressions.py

@@ -10,6 +10,7 @@
 import numpy as np
 from pandas.core.common import _values_from_object
 from pandas.core.computation import _NUMEXPR_INSTALLED
+from pandas.core.config import get_option
 
 if _NUMEXPR_INSTALLED:
     import numexpr as ne
@@ -156,7 +157,7 @@ def _where_numexpr(cond, a, b, raise_on_error=False):
 
 
 # turn myself on
-set_use_numexpr(True)
+set_use_numexpr(get_option('compute.use_numexpr'))
 
 
 def _has_bool_dtype(x):

pandas/core/computation/expressions.py

@@ -15,8 +15,41 @@
 from pandas.core.config import (is_int, is_bool, is_text, is_instance_factory,
                                 is_one_of_factory, get_default_val,
                                 is_callable)
-from pandas.io.formats.format import detect_console_encoding
+from pandas.io.formats.console import detect_console_encoding
 
+# compute
+
+use_bottleneck_doc = """
+: bool
+    Use the bottleneck library to accelerate if it is installed,
+    the default is True
+    Valid values: False,True
+"""
+
+
+def use_bottleneck_cb(key):
+    from pandas.core import nanops
+    nanops.set_use_bottleneck(cf.get_option(key))
+
+
+use_numexpr_doc = """
+: bool
+    Use the numexpr library to accelerate computation if it is installed,
+    the default is True
+    Valid values: False,True
+"""
+
+
+def use_numexpr_cb(key):
+    from pandas.core.computation import expressions
+    expressions.set_use_numexpr(cf.get_option(key))
+
+
+with cf.config_prefix('compute'):
+    cf.register_option('use_bottleneck', True, use_bottleneck_doc,
+                       validator=is_bool, cb=use_bottleneck_cb)
+    cf.register_option('use_numexpr', True, use_numexpr_doc,
+                       validator=is_bool, cb=use_numexpr_cb)
 #
 # options from the "display" namespace
 

pandas/core/config_init.py

@@ -91,6 +91,7 @@
 import pandas.core.nanops as nanops
 import pandas.core.ops as ops
 import pandas.io.formats.format as fmt
+import pandas.io.formats.console as console
 from pandas.io.formats.printing import pprint_thing
 import pandas.plotting._core as gfx
 
@@ -513,7 +514,7 @@ def _repr_fits_horizontal_(self, ignore_width=False):
         GH3541, GH3573
         """
 
-        width, height = fmt.get_console_size()
+        width, height = console.get_console_size()
         max_columns = get_option("display.max_columns")
         nb_columns = len(self.columns)
 
@@ -577,7 +578,7 @@ def __unicode__(self):
         max_cols = get_option("display.max_columns")
         show_dimensions = get_option("display.show_dimensions")
         if get_option("display.expand_frame_repr"):
-            width, _ = fmt.get_console_size()
+            width, _ = console.get_console_size()
         else:
             width = None
         self.to_string(buf=buf, max_rows=max_rows, max_cols=max_cols,

pandas/core/frame.py

@@ -837,7 +837,8 @@ def _format_data(self):
         """
         Return the formatted data as a unicode string
         """
-        from pandas.io.formats.format import get_console_size, _get_adjustment
+        from pandas.io.formats.console import get_console_size
+        from pandas.io.formats.format import _get_adjustment
         display_width, _ = get_console_size()
         if display_width is None:
             display_width = get_option('display.width') or 80

pandas/core/indexes/base.py

@@ -1,14 +1,8 @@
 import itertools
 import functools
-import numpy as np
 import operator
 
-try:
-    import bottleneck as bn
-    _USE_BOTTLENECK = True
-except ImportError:  # pragma: no cover
-    _USE_BOTTLENECK = False
-
+import numpy as np
 from pandas import compat
 from pandas._libs import tslib, algos, lib
 from pandas.core.dtypes.common import (
@@ -23,9 +17,27 @@
     is_int_or_datetime_dtype, is_any_int_dtype)
 from pandas.core.dtypes.cast import _int64_max, maybe_upcast_putmask
 from pandas.core.dtypes.missing import isnull, notnull
-
+from pandas.core.config import get_option
 from pandas.core.common import _values_from_object
 
+try:
+    import bottleneck as bn
+    _BOTTLENECK_INSTALLED = True
+except ImportError:  # pragma: no cover
+    _BOTTLENECK_INSTALLED = False
+
+_USE_BOTTLENECK = False
+
+
+def set_use_bottleneck(v=True):
+    # set/unset to use bottleneck
+    global _USE_BOTTLENECK
+    if _BOTTLENECK_INSTALLED:
+        _USE_BOTTLENECK = v
+
+
+set_use_bottleneck(get_option('compute.use_bottleneck'))
+
 
 class disallow(object):