Skip to content

Commit a62d90f

Browse files
Revert "Revert "REF: move shift logic from BlockManager to DataFrame (#40536)" (#42317)"
This reverts commit 027d374.
1 parent 027d374 commit a62d90f

File tree

3 files changed

+35
-44
lines changed

3 files changed

+35
-44
lines changed

pandas/core/frame.py

Lines changed: 34 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5279,28 +5279,45 @@ def shift(
52795279
axis = self._get_axis_number(axis)
52805280

52815281
ncols = len(self.columns)
5282-
if axis == 1 and periods != 0 and fill_value is lib.no_default and ncols > 0:
5283-
# We will infer fill_value to match the closest column
52845282

5285-
# Use a column that we know is valid for our column's dtype GH#38434
5286-
label = self.columns[0]
5283+
if (
5284+
axis == 1
5285+
and periods != 0
5286+
and ncols > 0
5287+
and (fill_value is lib.no_default or len(self._mgr.arrays) > 1)
5288+
):
5289+
# Exclude single-array-with-fill_value case so we issue a FutureWarning
5290+
# if an integer is passed with datetimelike dtype GH#31971
5291+
from pandas import concat
52875292

5293+
# tail: the data that is still in our shifted DataFrame
52885294
if periods > 0:
5289-
result = self.iloc[:, :-periods]
5290-
for col in range(min(ncols, abs(periods))):
5291-
# TODO(EA2D): doing this in a loop unnecessary with 2D EAs
5292-
# Define filler inside loop so we get a copy
5293-
filler = self.iloc[:, 0].shift(len(self))
5294-
result.insert(0, label, filler, allow_duplicates=True)
5295+
tail = self.iloc[:, :-periods]
52955296
else:
5296-
result = self.iloc[:, -periods:]
5297-
for col in range(min(ncols, abs(periods))):
5298-
# Define filler inside loop so we get a copy
5299-
filler = self.iloc[:, -1].shift(len(self))
5300-
result.insert(
5301-
len(result.columns), label, filler, allow_duplicates=True
5302-
)
5297+
tail = self.iloc[:, -periods:]
5298+
# pin a simple Index to avoid costly casting
5299+
tail.columns = range(len(tail.columns))
5300+
5301+
if fill_value is not lib.no_default:
5302+
# GH#35488
5303+
# TODO(EA2D): with 2D EAs we could construct other directly
5304+
ser = Series(fill_value, index=self.index)
5305+
else:
5306+
# We infer fill_value to match the closest column
5307+
if periods > 0:
5308+
ser = self.iloc[:, 0].shift(len(self))
5309+
else:
5310+
ser = self.iloc[:, -1].shift(len(self))
5311+
5312+
width = min(abs(periods), ncols)
5313+
other = concat([ser] * width, axis=1)
5314+
5315+
if periods > 0:
5316+
result = concat([other, tail], axis=1)
5317+
else:
5318+
result = concat([tail, other], axis=1)
53035319

5320+
result = cast(DataFrame, result)
53045321
result.columns = self.columns.copy()
53055322
return result
53065323

pandas/core/internals/managers.py

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -381,25 +381,6 @@ def shift(self: T, periods: int, axis: int, fill_value) -> T:
381381
if fill_value is lib.no_default:
382382
fill_value = None
383383

384-
if axis == 0 and self.ndim == 2 and self.nblocks > 1:
385-
# GH#35488 we need to watch out for multi-block cases
386-
# We only get here with fill_value not-lib.no_default
387-
ncols = self.shape[0]
388-
if periods > 0:
389-
indexer = [-1] * periods + list(range(ncols - periods))
390-
else:
391-
nper = abs(periods)
392-
indexer = list(range(nper, ncols)) + [-1] * nper
393-
result = self.reindex_indexer(
394-
self.items,
395-
indexer,
396-
axis=0,
397-
fill_value=fill_value,
398-
allow_dups=True,
399-
consolidate=False,
400-
)
401-
return result
402-
403384
return self.apply("shift", periods=periods, axis=axis, fill_value=fill_value)
404385

405386
def fillna(self: T, value, limit, inplace: bool, downcast) -> T:

pandas/tests/apply/test_frame_transform.py

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,15 +39,8 @@ def test_transform_ufunc(axis, float_frame, frame_or_series):
3939

4040

4141
@pytest.mark.parametrize("op", frame_transform_kernels)
42-
def test_transform_groupby_kernel(axis, float_frame, op, using_array_manager, request):
42+
def test_transform_groupby_kernel(axis, float_frame, op, request):
4343
# GH 35964
44-
if using_array_manager and op == "pct_change" and axis in (1, "columns"):
45-
# TODO(ArrayManager) shift with axis=1
46-
request.node.add_marker(
47-
pytest.mark.xfail(
48-
reason="shift axis=1 not yet implemented for ArrayManager"
49-
)
50-
)
5144

5245
args = [0.0] if op == "fillna" else []
5346
if axis == 0 or axis == "index":

0 commit comments

Comments
 (0)