diff --git a/test/test_conventions.py b/test/test_conventions.py index 9a62d2e3302..56413c86ba1 100644 --- a/test/test_conventions.py +++ b/test/test_conventions.py @@ -86,7 +86,9 @@ def test_cf_datetime(self): for calendar in ['standard', 'gregorian', 'proleptic_gregorian']: expected = nc4.num2date(num_dates, units, calendar) print(num_dates, units, calendar) - actual = conventions.decode_cf_datetime(num_dates, units, calendar) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'Unable to decode time axis') + actual = conventions.decode_cf_datetime(num_dates, units, calendar) if (isinstance(actual, np.ndarray) and np.issubdtype(actual.dtype, np.datetime64)): self.assertEqual(actual.dtype, np.dtype('M8[ns]')) @@ -111,8 +113,6 @@ def test_cf_datetime(self): @requires_netCDF4 def test_decoded_cf_datetime_array(self): - import netCDF4 as nc4 - actual = conventions.DecodedCFDatetimeArray( [0, 1, 2], 'days since 1900-01-01', 'standard') expected = pd.date_range('1900-01-01', periods=3).values @@ -125,13 +125,103 @@ def test_decoded_cf_datetime_array(self): self.assertEqual(actual.dtype, np.dtype('datetime64[ns]')) self.assertArrayEqual(actual, expected) - num_dates = [722000, 720000.5] - units = 'days since 0001-01-01 0:0:0' + @requires_netCDF4 + def test_decode_non_standard_calendar(self): + import netCDF4 as nc4 + + for calendar in ['noleap', '365_day', '360_day', 'julian', 'all_leap', + '366_day']: + units = 'days since 0001-01-01' + times = pd.date_range('2001-04-01-00', end='2001-04-30-23', + freq='H') + noleap_time = nc4.date2num(times.to_pydatetime(), units, + calendar=calendar) + expected = times.values + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'Unable to decode time axis') + actual = conventions.decode_cf_datetime(noleap_time, units, + calendar=calendar) + self.assertEqual(actual.dtype, np.dtype('M8[ns]')) + self.assertArrayEqual(actual, expected) + + @requires_netCDF4 + def test_decode_non_standard_calendar_single_element(self): + units = 'days since 0001-01-01' + for calendar in ['noleap', '365_day', '360_day', 'julian', 'all_leap', + '366_day']: + for num_time in [735368, [735368], [[735368]]]: + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'Unable to decode time axis') + actual = conventions.decode_cf_datetime(num_time, units, + calendar=calendar) + self.assertEqual(actual.dtype, np.dtype('M8[ns]')) + + @requires_netCDF4 + def test_decode_non_standard_calendar_single_element_fallback(self): + import netCDF4 as nc4 + + units = 'days since 0001-01-01' + dt = nc4.netcdftime.datetime(2001, 2, 29) + for calendar in ['360_day', 'all_leap', '366_day']: + num_time = nc4.date2num(dt, units, calendar) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + actual = conventions.decode_cf_datetime(num_time, units, + calendar=calendar) + self.assertEqual(len(w), 1) + self.assertIn('Unable to decode time axis', + str(w[0].message)) + expected = np.asarray(nc4.num2date(num_time, units, calendar)) + print(num_time, calendar, actual, expected) + self.assertEqual(actual.dtype, np.dtype('O')) + self.assertEqual(expected, actual) + + @requires_netCDF4 + def test_decode_non_standard_calendar_multidim_time(self): + import netCDF4 as nc4 + calendar = 'noleap' - actual = conventions.DecodedCFDatetimeArray(num_dates, units, calendar) - expected = nc4.num2date(num_dates, units, calendar) - self.assertEqual(actual.dtype, np.dtype('O')) - self.assertArrayEqual(actual, expected) + units = 'days since 0001-01-01' + times1 = pd.date_range('2001-04-01', end='2001-04-05', freq='D') + times2 = pd.date_range('2001-05-01', end='2001-05-05', freq='D') + noleap_time1 = nc4.date2num(times1.to_pydatetime(), units, + calendar=calendar) + noleap_time2 = nc4.date2num(times2.to_pydatetime(), units, + calendar=calendar) + mdim_time = np.empty((len(noleap_time1), 2), ) + mdim_time[:, 0] = noleap_time1 + mdim_time[:, 1] = noleap_time2 + + expected1 = times1.values + expected2 = times2.values + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'Unable to decode time axis') + actual = conventions.decode_cf_datetime(mdim_time, units, + calendar=calendar) + self.assertEqual(actual.dtype, np.dtype('M8[ns]')) + self.assertArrayEqual(actual[:, 0], expected1) + self.assertArrayEqual(actual[:, 1], expected2) + + @requires_netCDF4 + def test_decode_non_standard_calendar_fallback(self): + import netCDF4 as nc4 + for year in [2010, 2011, 2012, 2013, 2014]: # insure leap year doesn't matter + for calendar in ['360_day', '366_day', 'all_leap']: + calendar = '360_day' + units = 'days since {0}-01-01'.format(year) + num_times = np.arange(100) + expected = nc4.num2date(num_times, units, calendar) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + actual = conventions.decode_cf_datetime(num_times, units, + calendar=calendar) + self.assertEqual(len(w), 1) + self.assertIn('Unable to decode time axis', + str(w[0].message)) + + self.assertEqual(actual.dtype, np.dtype('O')) + self.assertArrayEqual(actual, expected) @requires_netCDF4 def test_cf_datetime_nan(self): diff --git a/xray/conventions.py b/xray/conventions.py index 913d3317f3e..7bb0b53af86 100644 --- a/xray/conventions.py +++ b/xray/conventions.py @@ -1,5 +1,6 @@ import numpy as np import pandas as pd +import warnings from collections import defaultdict, OrderedDict from datetime import datetime @@ -88,7 +89,25 @@ def nan_safe_num2date(num): if ((calendar not in _STANDARD_CALENDARS or min_date.year < 1678 or max_date.year >= 2262) and min_date is not pd.NaT): + dates = nc4.num2date(num_dates, units, calendar) + + if min_date.year >= 1678 and max_date.year < 2262: + try: + dates = nctime_to_nptime(dates) + except ValueError as e: + warnings.warn('Unable to decode time axis into full ' + 'numpy.datetime64 objects, continuing using ' + 'dummy netCDF4.datetime objects instead, reason:' + '{0}'.format(e), RuntimeWarning, stacklevel=2) + dates = np.asarray(dates) + else: + warnings.warn('Unable to decode time axis into full ' + 'numpy.datetime64 objects, continuing using dummy ' + 'netCDF4.datetime objects instead, reason: dates out' + ' of range', RuntimeWarning, stacklevel=2) + dates = np.asarray(dates) + else: # we can safely use np.datetime64 with nanosecond precision (pandas # likes ns precision so it can directly make DatetimeIndex objects) @@ -122,6 +141,7 @@ def nan_safe_num2date(num): + np.datetime64(min_date)) # restore original shape and ensure dates are given in ns dates = dates.reshape(num_dates.shape).astype('M8[ns]') + return dates @@ -144,6 +164,16 @@ def guess_time_units(dates): return '%s since %s' % (time_unit, dates[0]) +def nctime_to_nptime(times): + """Given an array of netCDF4.datetime objects, return an array of + numpy.datetime64 objects of the same size""" + times = np.asarray(times) + new = np.empty(times.shape, dtype='M8[ns]') + for i, t in np.ndenumerate(times): + new[i] = np.datetime64(datetime(*t.timetuple()[:6])) + return new + + def encode_cf_datetime(dates, units=None, calendar=None): """Given an array of datetime objects, returns the tuple `(num, units, calendar)` suitable for a CF complient time variable. @@ -246,13 +276,7 @@ def __init__(self, array, units, calendar=None): @property def dtype(self): - if self.calendar is None or self.calendar in _STANDARD_CALENDARS: - # TODO: return the proper dtype (object) for a standard calendar - # that can't be expressed in ns precision. Perhaps we could guess - # this from the units? - return np.dtype('datetime64[ns]') - else: - return np.dtype('O') + return np.dtype('datetime64[ns]') def __getitem__(self, key): return decode_cf_datetime(self.array, units=self.units,