POC improved column-wise ops

Author: jorisvandenbossche

pandas/core/frame.py

@@ -1911,8 +1911,10 @@ def from_items(cls, items, columns=None, orient="columns"):
             raise ValueError("'orient' must be either 'columns' or 'index'")
 
     @classmethod
-    def _from_arrays(cls, arrays, columns, index, dtype=None):
-        mgr = arrays_to_mgr(arrays, columns, index, columns, dtype=dtype)
+    def _from_arrays(cls, arrays, columns, index, dtype=None, fastpath=False):
+        mgr = arrays_to_mgr(
+            arrays, columns, index, columns, dtype=dtype, fastpath=fastpath
+        )
         return cls(mgr)
 
     def to_sparse(self, fill_value=None, kind="block"):
@@ -5294,6 +5296,13 @@ def reorder_levels(self, order, axis=0):
     # ----------------------------------------------------------------------
     # Arithmetic / combination related
 
+    def _get_arrays(self):
+        res = []
+        for i in range(len(self.columns)):
+            values = self._data.iget_values(i)
+            res.append(values)
+        return res
+
     def _combine_frame(self, other, func, fill_value=None, level=None):
         this, other = self.align(other, join="outer", level=level, copy=False)
         new_index, new_columns = this.index, this.columns

pandas/core/internals/blocks.py

@@ -3043,7 +3043,9 @@ def get_block_type(values, dtype=None):
     dtype = dtype or values.dtype
     vtype = dtype.type
 
-    if is_sparse(dtype):
+    if dtype == np.bool_:
+        cls = BoolBlock
+    elif is_sparse(dtype):
         # Need this first(ish) so that Sparse[datetime] is sparse
         cls = ExtensionBlock
     elif is_categorical(values):

pandas/core/internals/construction.py

@@ -55,7 +55,7 @@
 # BlockManager Interface
 
 
-def arrays_to_mgr(arrays, arr_names, index, columns, dtype=None):
+def arrays_to_mgr(arrays, arr_names, index, columns, dtype=None, fastpath=False):
     """
     Segregate Series based on type and coerce into matrices.
 
@@ -68,7 +68,8 @@ def arrays_to_mgr(arrays, arr_names, index, columns, dtype=None):
         index = ensure_index(index)
 
     # don't force copy because getting jammed in an ndarray anyway
-    arrays = _homogenize(arrays, index, dtype)
+    if not fastpath:
+        arrays = _homogenize(arrays, index, dtype)
 
     # from BlockManager perspective
     axes = [ensure_index(columns), index]

pandas/core/internals/managers.py

@@ -987,6 +987,11 @@ def iget(self, i):
             self.axes[1],
         )
 
+    def iget_values(self, i):
+        block = self.blocks[self._blknos[i]]
+        values = block.iget(self._blklocs[i])
+        return values
+
     def delete(self, item):
         """
         Delete selected item (items if non-unique) in-place.

pandas/core/ops/__init__.py

@@ -485,7 +485,9 @@ def dispatch_to_series(left, right, func, str_rep=None, axis=None):
     if lib.is_scalar(right) or np.ndim(right) == 0:
 
         def column_op(a, b):
-            return {i: func(a.iloc[:, i], b) for i in range(len(a.columns))}
+            # return {i: func(a.iloc[:, i], b) for i in range(len(a.columns))}
+            res = [func(col, b) for col in a._get_arrays()]
+            return left._from_arrays(res, left.columns, left.index, fastpath=True)
 
     elif isinstance(right, ABCDataFrame):
         assert right._indexed_same(left)