Skip to content

BUG: negative timedeltas not printing correctly #2955

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 1, 2013
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions RELEASE.rst
Original file line number Diff line number Diff line change
Expand Up @@ -115,10 +115,12 @@ pandas 0.11.0
- Series ops with a Timestamp on the rhs was throwing an exception (GH2898_)
added tests for Series ops with datetimes,timedeltas,Timestamps, and datelike
Series on both lhs and rhs
- Series will now set its dtype automatically to ``timedelta64[ns]``
if all passed objects are timedelta objects
- Fixed subtle timedelta64 inference issue on py3
- Fixed some formatting issues on timedelta when negative
- Support null checking on timedelta64, representing (and formatting) with NaT
- Support setitem with np.nan value, converts to NaT
- Support min/max ops in a Dataframe (abs not working, nor do we error on non-supported ops)
- Support idxmin/idxmax in a Series (but with no NaT)

.. _GH622: https://github.com/pydata/pandas/issues/622
.. _GH797: https://github.com/pydata/pandas/issues/797
Expand Down
16 changes: 16 additions & 0 deletions doc/source/timeseries.rst
Original file line number Diff line number Diff line change
Expand Up @@ -961,3 +961,19 @@ Operands can also appear in a reversed order (a singluar object operated with a
s.max() - s
datetime(2011,1,1,3,5) - s
timedelta(minutes=5) + s

Some timedelta numeric like operations are supported.

.. ipython:: python

s = Series(date_range('2012-1-1', periods=3, freq='D'))
df = DataFrame(dict(A = s - Timestamp('20120101')-timedelta(minutes=5,seconds=5),
B = s - Series(date_range('2012-1-2', periods=3, freq='D'))))
df

# timedelta arithmetic
td - timedelta(minutes=5,seconds=5,microseconds=5)

# min/max operations
df.min()
df.min(axis=1)
7 changes: 7 additions & 0 deletions pandas/core/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -904,6 +904,13 @@ def _possibly_convert_platform(values):

def _possibly_cast_to_timedelta(value):
""" try to cast to timedelta64 w/o coercion """

# deal with numpy not being able to handle certain timedelta operations
if isinstance(value,np.ndarray) and value.dtype.kind == 'm':
if value.dtype != 'timedelta64[ns]':
value = value.astype('timedelta64[ns]')
return value

new_value = tslib.array_to_timedelta64(value.astype(object), coerce=False)
if new_value.dtype == 'i8':
value = np.array(new_value,dtype='timedelta64[ns]')
Expand Down
61 changes: 39 additions & 22 deletions pandas/core/nanops.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ def f(values, axis=None, skipna=True, **kwds):

def _bn_ok_dtype(dt):
# Bottleneck chokes on datetime64
return dt != np.object_ and not issubclass(dt.type, np.datetime64)
return dt != np.object_ and not issubclass(dt.type, (np.datetime64,np.timedelta64))


def _has_infs(result):
Expand All @@ -69,6 +69,34 @@ def _has_infs(result):
else:
return np.isinf(result) or np.isneginf(result)

def _isfinite(values):
if issubclass(values.dtype.type, np.timedelta64):
return isnull(values)
return -np.isfinite(values)

def _na_ok_dtype(dtype):
return not issubclass(dtype.type, (np.integer, np.datetime64, np.timedelta64))

def _view_if_needed(values):
if issubclass(values.dtype.type, (np.datetime64,np.timedelta64)):
return values.view(np.int64)
return values

def _wrap_results(result,dtype):
""" wrap our results if needed """

if issubclass(dtype.type, np.datetime64):
if not isinstance(result, np.ndarray):
result = lib.Timestamp(result)
else:
result = result.view(dtype)
elif issubclass(dtype.type, np.timedelta64):
if not isinstance(result, np.ndarray):
pass
else:
result = result.view(dtype)

return result

def nanany(values, axis=None, skipna=True):
mask = isnull(values)
Expand Down Expand Up @@ -162,13 +190,11 @@ def _nanmin(values, axis=None, skipna=True):

dtype = values.dtype

if skipna and not issubclass(dtype.type,
(np.integer, np.datetime64)):
if skipna and _na_ok_dtype(dtype):
values = values.copy()
np.putmask(values, mask, np.inf)

if issubclass(dtype.type, np.datetime64):
values = values.view(np.int64)
values = _view_if_needed(values)

# numpy 1.6.1 workaround in Python 3.x
if (values.dtype == np.object_
Expand All @@ -187,12 +213,7 @@ def _nanmin(values, axis=None, skipna=True):
else:
result = values.min(axis)

if issubclass(dtype.type, np.datetime64):
if not isinstance(result, np.ndarray):
result = lib.Timestamp(result)
else:
result = result.view(dtype)

result = _wrap_results(result,dtype)
return _maybe_null_out(result, axis, mask)


Expand All @@ -201,12 +222,11 @@ def _nanmax(values, axis=None, skipna=True):

dtype = values.dtype

if skipna and not issubclass(dtype.type, (np.integer, np.datetime64)):
if skipna and _na_ok_dtype(dtype):
values = values.copy()
np.putmask(values, mask, -np.inf)

if issubclass(dtype.type, np.datetime64):
values = values.view(np.int64)
values = _view_if_needed(values)

# numpy 1.6.1 workaround in Python 3.x
if (values.dtype == np.object_
Expand All @@ -226,20 +246,16 @@ def _nanmax(values, axis=None, skipna=True):
else:
result = values.max(axis)

if issubclass(dtype.type, np.datetime64):
if not isinstance(result, np.ndarray):
result = lib.Timestamp(result)
else:
result = result.view(dtype)

result = _wrap_results(result,dtype)
return _maybe_null_out(result, axis, mask)


def nanargmax(values, axis=None, skipna=True):
"""
Returns -1 in the NA case
"""
mask = -np.isfinite(values)
mask = _isfinite(values)
values = _view_if_needed(values)
if not issubclass(values.dtype.type, np.integer):
values = values.copy()
np.putmask(values, mask, -np.inf)
Expand All @@ -252,7 +268,8 @@ def nanargmin(values, axis=None, skipna=True):
"""
Returns -1 in the NA case
"""
mask = -np.isfinite(values)
mask = _isfinite(values)
values = _view_if_needed(values)
if not issubclass(values.dtype.type, np.integer):
values = values.copy()
np.putmask(values, mask, np.inf)
Expand Down
37 changes: 21 additions & 16 deletions pandas/core/series.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,10 +81,10 @@ def wrapper(self, other):

lvalues, rvalues = self, other

is_timedelta = com.is_timedelta64_dtype(self)
is_datetime = com.is_datetime64_dtype(self)
is_timedelta_lhs = com.is_timedelta64_dtype(self)
is_datetime_lhs = com.is_datetime64_dtype(self)

if is_datetime or is_timedelta:
if is_datetime_lhs or is_timedelta_lhs:

# convert the argument to an ndarray
def convert_to_array(values):
Expand All @@ -96,26 +96,27 @@ def convert_to_array(values):
pass
else:
values = tslib.array_to_datetime(values)
elif inferred_type in set(['timedelta','timedelta64']):
# need to convert timedelta to ns here
# safest to convert it to an object arrany to process
if isinstance(values, pa.Array) and com.is_timedelta64_dtype(values):
pass
else:
values = com._possibly_cast_to_timedelta(values)
else:
values = pa.array(values)
return values

# swap the valuesor com.is_timedelta64_dtype(self):
if is_timedelta:
lvalues, rvalues = rvalues, lvalues
lvalues = convert_to_array(lvalues)
is_timedelta = False

# convert lhs and rhs
lvalues = convert_to_array(lvalues)
rvalues = convert_to_array(rvalues)

# rhs is either a timedelta or a series/ndarray
if lib.is_timedelta_or_timedelta64_array(rvalues):
is_timedelta_rhs = com.is_timedelta64_dtype(rvalues)
is_datetime_rhs = com.is_datetime64_dtype(rvalues)

# need to convert timedelta to ns here
# safest to convert it to an object arrany to process
rvalues = tslib.array_to_timedelta64(rvalues.astype(object))
dtype = 'M8[ns]'
elif com.is_datetime64_dtype(rvalues):
# 2 datetimes or 2 timedeltas
if (is_timedelta_lhs and is_timedelta_rhs) or (is_datetime_lhs and is_datetime_rhs):

dtype = 'timedelta64[ns]'

# we may have to convert to object unfortunately here
Expand All @@ -126,6 +127,10 @@ def wrap_results(x):
np.putmask(x,mask,tslib.iNaT)
return x

# datetime and timedelta
elif (is_timedelta_lhs and is_datetime_rhs) or (is_timedelta_rhs and is_datetime_lhs):
dtype = 'M8[ns]'

else:
raise ValueError('cannot operate on a series with out a rhs '
'of a series/ndarray of type datetime64[ns] '
Expand Down
1 change: 1 addition & 0 deletions pandas/src/inference.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ try:
_TYPE_MAP[np.complex256] = 'complex'
_TYPE_MAP[np.float16] = 'floating'
_TYPE_MAP[np.datetime64] = 'datetime64'
_TYPE_MAP[np.timedelta64] = 'timedelta64'
except AttributeError:
pass

Expand Down
45 changes: 39 additions & 6 deletions pandas/tests/test_format.py
Original file line number Diff line number Diff line change
Expand Up @@ -1209,24 +1209,57 @@ def test_float_trim_zeros(self):
def test_timedelta64(self):

from pandas import date_range
from datetime import datetime
from datetime import datetime, timedelta

Series(np.array([1100, 20], dtype='timedelta64[s]')).to_string()
# check this works

s = Series(date_range('2012-1-1', periods=3, freq='D'))

# GH2146

# adding NaTs
s = Series(date_range('2012-1-1', periods=3, freq='D'))
y = s-s.shift(1)
result = y.to_string()
self.assertTrue('1 days, 00:00:00' in result)
self.assertTrue('NaT' in result)

# with frac seconds
s = Series(date_range('2012-1-1', periods=3, freq='D'))
y = s-datetime(2012,1,1,microsecond=150)
o = Series([datetime(2012,1,1,microsecond=150)]*3)
y = s-o
result = y.to_string()
self.assertTrue('-00:00:00.000150' in result)

# rounding?
o = Series([datetime(2012,1,1,1)]*3)
y = s-o
result = y.to_string()
self.assertTrue('-01:00:00' in result)
self.assertTrue('1 days, 23:00:00' in result)

o = Series([datetime(2012,1,1,1,1)]*3)
y = s-o
result = y.to_string()
self.assertTrue('-01:01:00' in result)
self.assertTrue('1 days, 22:59:00' in result)

o = Series([datetime(2012,1,1,1,1,microsecond=150)]*3)
y = s-o
result = y.to_string()
self.assertTrue('-01:01:00.000150' in result)
self.assertTrue('1 days, 22:58:59.999850' in result)

# neg time
td = timedelta(minutes=5,seconds=3)
s2 = Series(date_range('2012-1-1', periods=3, freq='D')) + td
y = s - s2
result = y.to_string()
self.assertTrue('-00:05:03' in result)

td = timedelta(microseconds=550)
s2 = Series(date_range('2012-1-1', periods=3, freq='D')) + td
y = s - td
result = y.to_string()
self.assertTrue('00:00:00.000150' in result)
self.assertTrue('2012-01-01 23:59:59.999450' in result)

def test_mixed_datetime64(self):
df = DataFrame({'A': [1, 2],
Expand Down
47 changes: 47 additions & 0 deletions pandas/tests/test_frame.py
Original file line number Diff line number Diff line change
Expand Up @@ -2884,6 +2884,53 @@ def test_timedeltas(self):
expected.sort()
assert_series_equal(result, expected)

def test_operators_timedelta64(self):

from pandas import date_range
from datetime import datetime, timedelta
df = DataFrame(dict(A = date_range('2012-1-1', periods=3, freq='D'),
B = date_range('2012-1-2', periods=3, freq='D'),
C = Timestamp('20120101')-timedelta(minutes=5,seconds=5)))

diffs = DataFrame(dict(A = df['A']-df['C'],
B = df['A']-df['B']))


# min
result = diffs.min()
self.assert_(result[0] == diffs.ix[0,'A'])
self.assert_(result[1] == diffs.ix[0,'B'])

result = diffs.min(axis=1)
self.assert_((result == diffs.ix[0,'B']).all() == True)

# max
result = diffs.max()
self.assert_(result[0] == diffs.ix[2,'A'])
self.assert_(result[1] == diffs.ix[2,'B'])

result = diffs.max(axis=1)
self.assert_((result == diffs['A']).all() == True)

# abs ###### THIS IS BROKEN NOW ###### (results are dtype=timedelta64[us]
result = np.abs(df['A']-df['B'])
result = diffs.abs()
expected = DataFrame(dict(A = df['A']-df['C'],
B = df['B']-df['A']))
#assert_frame_equal(result,expected)

# mixed frame
mixed = diffs.copy()
mixed['C'] = 'foo'
mixed['D'] = 1
mixed['E'] = 1.

# this is ok
result = mixed.min()

# this is not
result = mixed.min(axis=1)

def test_new_empty_index(self):
df1 = DataFrame(randn(0, 3))
df2 = DataFrame(randn(0, 3))
Expand Down
Loading