From 01bcedde71e15333c91aebec5b36473aa2f2bd77 Mon Sep 17 00:00:00 2001 From: Brock Date: Sat, 5 Sep 2020 15:54:54 -0700 Subject: [PATCH] REF: use BlockManager.apply in csv code --- pandas/core/internals/blocks.py | 16 +++++++++------- pandas/io/formats/csvs.py | 25 +++++++++++-------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 3bcd4debbf41a..3866d6d62ed9d 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -588,7 +588,7 @@ def astype(self, dtype, copy: bool = False, errors: str = "raise"): # use native type formatting for datetime/tz/timedelta if self.is_datelike: - values = self.to_native_types() + values = self.to_native_types().values # astype formatting else: @@ -679,7 +679,7 @@ def to_native_types(self, na_rep="nan", quoting=None, **kwargs): values = np.array(values, dtype="object") values[mask] = na_rep - return values + return self.make_block(values) # block actions # def copy(self, deep: bool = True): @@ -1769,7 +1769,7 @@ def to_native_types(self, na_rep="nan", quoting=None, **kwargs): # TODO(EA2D): reshape not needed with 2D EAs # we are expected to return a 2-d ndarray - return values.reshape(1, len(values)) + return self.make_block(values) def take_nd( self, indexer, axis: int = 0, new_mgr_locs=None, fill_value=lib.no_default @@ -2016,7 +2016,7 @@ def to_native_types( values = np.array(values, dtype="object") values[mask] = na_rep - return values + return self.make_block(values) from pandas.io.formats.format import FloatArrayFormatter @@ -2028,7 +2028,8 @@ def to_native_types( quoting=quoting, fixed_width=False, ) - return formatter.get_result_as_array() + res = formatter.get_result_as_array() + return self.make_block(res) class ComplexBlock(FloatOrComplexBlock): @@ -2187,7 +2188,7 @@ def to_native_types(self, na_rep="NaT", date_format=None, **kwargs): result = dta._format_native_types( na_rep=na_rep, date_format=date_format, **kwargs ) - return np.atleast_2d(result) + return self.make_block(result) def set(self, locs, values): """ @@ -2403,7 +2404,8 @@ def fillna(self, value, **kwargs): def to_native_types(self, na_rep="NaT", **kwargs): """ convert to our native types format """ tda = self.array_values() - return tda._format_native_types(na_rep, **kwargs) + res = tda._format_native_types(na_rep, **kwargs) + return self.make_block(res) class BoolBlock(NumericBlock): diff --git a/pandas/io/formats/csvs.py b/pandas/io/formats/csvs.py index 270caec022fef..ae0f692445279 100644 --- a/pandas/io/formats/csvs.py +++ b/pandas/io/formats/csvs.py @@ -326,21 +326,18 @@ def _save_chunk(self, start_i: int, end_i: int) -> None: slicer = slice(start_i, end_i) df = self.obj.iloc[slicer] - blocks = df._mgr.blocks - - for i in range(len(blocks)): - b = blocks[i] - d = b.to_native_types( - na_rep=self.na_rep, - float_format=self.float_format, - decimal=self.decimal, - date_format=self.date_format, - quoting=self.quoting, - ) + mgr = df._mgr - for col_loc, col in zip(b.mgr_locs, d): - # self.data is a preallocated list - self.data[col_loc] = col + res = mgr.apply( + "to_native_types", + na_rep=self.na_rep, + float_format=self.float_format, + decimal=self.decimal, + date_format=self.date_format, + quoting=self.quoting, + ) + for i in range(len(res.items)): + self.data[i] = res.iget_values(i) ix = data_index.to_native_types( slicer=slicer,