Merge branch 'master' of github.com:pydata/xarray into feature/interpolate

Author: Joe Hamman

.travis.yml

@@ -89,6 +89,7 @@ install:
   - python xarray/util/print_versions.py
 
 script:
+  - python -OO -c "import xarray"
   - py.test xarray --cov=xarray --cov-config ci/.coveragerc --cov-report term-missing --verbose $EXTRA_FLAGS
   - git diff upstream/master **/*py | flake8 --diff --exit-zero || true
 

doc/api.rst

@@ -490,6 +490,7 @@ Exceptions
    :toctree: generated/
 
    MergeError
+   SerializationWarning
 
 Advanced API
 ============

doc/whats-new.rst

@@ -43,12 +43,29 @@ Bug fixes
   ``NumpyIndexingAdapter``. (:issue:`1694`)
   By `Keisuke Fujii <https://github.com/fujiisoup>`_
 
+- Fix importing xarray when running Python with ``-OO`` (:issue:`1706`).
+  By `Stephan Hoyer <https://github.com/shoyer>`_.
+
+- Saving a netCDF file with a coordinates with a spaces in its names now raises
+  an appropriate warning (:issue:`1689`).
+  By `Stephan Hoyer <https://github.com/shoyer>`_.
+
 - Fix two bugs that were preventing dask arrays from being specified as
   coordinates in the DataArray constructor (:issue:`1684`).
   By `Joe Hamman <https://github.com/jhamman>`_
 
 - Fixed ``apply_ufunc`` with ``dask='parallelized'`` for scalar arguments
   (:issue:`1697`).
+- Fix "Chunksize cannot exceed dimension size" error when writing netCDF4 files
+  loaded from disk (:issue:`1225`).
+  By `Stephan Hoyer <https://github.com/shoyer>`_.
+
+- Validate the shape of coordinates with names matching dimensions in the
+  DataArray constructor (:issue:`1709`).
+  By `Stephan Hoyer <https://github.com/shoyer>`_.
+
+- Raise ``NotImplementedError`` when attempting to save a MultiIndex to a
+  netCDF file (:issue:`1547`).
   By `Stephan Hoyer <https://github.com/shoyer>`_.
 
 Testing

xarray/__init__.py

@@ -19,7 +19,7 @@
                            save_mfdataset)
 from .backends.rasterio_ import open_rasterio
 
-from .conventions import decode_cf
+from .conventions import decode_cf, SerializationWarning
 
 try:
     from .version import version as __version__

xarray/backends/netCDF4_.py

@@ -147,7 +147,11 @@ def _force_native_endianness(var):
 
 
 def _extract_nc4_variable_encoding(variable, raise_on_invalid=False,
-                                   lsd_okay=True, backend='netCDF4'):
+                                   lsd_okay=True, backend='netCDF4',
+                                   unlimited_dims=None):
+    if unlimited_dims is None:
+        unlimited_dims = ()
+
     encoding = variable.encoding.copy()
 
     safe_to_drop = set(['source', 'original_shape'])
@@ -156,10 +160,17 @@ def _extract_nc4_variable_encoding(variable, raise_on_invalid=False,
     if lsd_okay:
         valid_encodings.add('least_significant_digit')
 
-    if (encoding.get('chunksizes') is not None and
-            (encoding.get('original_shape', variable.shape) !=
-                variable.shape) and not raise_on_invalid):
-        del encoding['chunksizes']
+    if not raise_on_invalid and encoding.get('chunksizes') is not None:
+        # It's possible to get encoded chunksizes larger than a dimension size
+        # if the original file had an unlimited dimension. This is problematic
+        # if the new file no longer has an unlimited dimension.
+        chunksizes = encoding['chunksizes']
+        chunks_too_big = any(
+            c > d and dim not in unlimited_dims
+            for c, d, dim in zip(chunksizes, variable.shape, variable.dims))
+        changed_shape = encoding.get('original_shape') != variable.shape
+        if chunks_too_big or changed_shape:
+            del encoding['chunksizes']
 
     for k in safe_to_drop:
         if k in encoding:
@@ -346,7 +357,8 @@ def prepare_variable(self, name, variable, check_encoding=False,
                 'NC_CHAR type.' % name)
 
         encoding = _extract_nc4_variable_encoding(
-            variable, raise_on_invalid=check_encoding)
+            variable, raise_on_invalid=check_encoding,
+            unlimited_dims=unlimited_dims)
         nc4_var = self.ds.createVariable(
             varname=name,
             datatype=datatype,

xarray/conventions.py

@@ -17,7 +17,7 @@
 
 from .core import duck_array_ops, indexing, ops, utils
 from .core.formatting import format_timestamp, first_n_items, last_item
-from .core.variable import as_variable, Variable
+from .core.variable import as_variable, IndexVariable, Variable
 from .core.pycompat import iteritems, OrderedDict, PY3, basestring
 
 
@@ -32,6 +32,10 @@
                       'D': 1e9 * 60 * 60 * 24}
 
 
+class SerializationWarning(RuntimeWarning):
+    """Warnings about encoding/decoding issues in serialization."""
+
+
 def mask_and_scale(array, fill_value=None, scale_factor=None, add_offset=None,
                    dtype=float):
     """Scale and mask array values according to CF conventions for packed and
@@ -113,15 +117,15 @@ def _decode_datetime_with_netcdf4(num_dates, units, calendar):
         warnings.warn('Unable to decode time axis into full '
                       'numpy.datetime64 objects, continuing using dummy '
                       'netCDF4.datetime objects instead, reason: dates out'
-                      ' of range', RuntimeWarning, stacklevel=3)
+                      ' of range', SerializationWarning, stacklevel=3)
     else:
         try:
             dates = nctime_to_nptime(dates)
         except ValueError as e:
             warnings.warn('Unable to decode time axis into full '
                           'numpy.datetime64 objects, continuing using '
                           'dummy netCDF4.datetime objects instead, reason:'
-                          '{0}'.format(e), RuntimeWarning, stacklevel=3)
+                          '{0}'.format(e), SerializationWarning, stacklevel=3)
     return dates
 
 
@@ -773,7 +777,7 @@ def maybe_encode_nonstring_dtype(var, name=None):
                     warnings.warn('saving variable %s with floating '
                                   'point data as an integer dtype without '
                                   'any _FillValue to use for NaNs' % name,
-                                  RuntimeWarning, stacklevel=3)
+                                  SerializationWarning, stacklevel=3)
                 data = duck_array_ops.around(data)[...]
                 if encoding.get('_Unsigned', False):
                     signed_dtype = np.dtype('i%s' % dtype.itemsize)
@@ -828,6 +832,15 @@ def _infer_dtype(array, name=None):
 def ensure_dtype_not_object(var, name=None):
     # TODO: move this from conventions to backends? (it's not CF related)
     if var.dtype.kind == 'O':
+        if (isinstance(var, IndexVariable) and
+                isinstance(var.to_index(), pd.MultiIndex)):
+            raise NotImplementedError(
+                'variable {!r} is a MultiIndex, which cannot yet be '
+                'serialized to netCDF files '
+                '(https://github.com/pydata/xarray/issues/1077). Use '
+                'reset_index() to convert MultiIndex levels into coordinate '
+                'variables instead.'.format(name))
+
         dims, data, attrs, encoding = _var_as_tuple(var)
         missing = pd.isnull(data)
         if missing.any():
@@ -951,7 +964,7 @@ def decode_cf_variable(name, var, concat_characters=True, mask_and_scale=True,
         else:
             warnings.warn("variable %r has _Unsigned attribute but is not "
                           "of integer type. Ignoring attribute." % name,
-                          RuntimeWarning, stacklevel=3)
+                          SerializationWarning, stacklevel=3)
 
     if mask_and_scale:
         if 'missing_value' in attributes:
@@ -975,7 +988,7 @@ def decode_cf_variable(name, var, concat_characters=True, mask_and_scale=True,
             warnings.warn("variable {!r} has multiple fill values {}, "
                           "decoding all values to NaN."
                           .format(name, fill_value),
-                          RuntimeWarning, stacklevel=3)
+                          SerializationWarning, stacklevel=3)
 
         scale_factor = pop_to(attributes, encoding, 'scale_factor')
         add_offset = pop_to(attributes, encoding, 'add_offset')
@@ -1185,6 +1198,16 @@ def cf_decoder(variables, attributes,
 def _encode_coordinates(variables, attributes, non_dim_coord_names):
     # calculate global and variable specific coordinates
     non_dim_coord_names = set(non_dim_coord_names)
+
+    for name in list(non_dim_coord_names):
+        if isinstance(name, basestring) and ' ' in name:
+            warnings.warn(
+                'coordinate {!r} has a space in its name, which means it '
+                'cannot be marked as a coordinate on disk and will be '
+                'saved as a data variable instead'.format(name),
+                SerializationWarning, stacklevel=6)
+            non_dim_coord_names.discard(name)
+
     global_coordinates = non_dim_coord_names.copy()
     variable_coordinates = defaultdict(set)
     for coord_name in non_dim_coord_names:

xarray/core/dataarray.py

@@ -35,7 +35,7 @@ def _infer_coords_and_dims(shape, coords, dims):
     """All the logic for creating a new DataArray"""
 
     if (coords is not None and not utils.is_dict_like(coords) and
-        len(coords) != len(shape)):
+            len(coords) != len(shape)):
         raise ValueError('coords is not dict-like, but it has %s items, '
                          'which does not match the %s dimensions of the '
                          'data' % (len(coords), len(shape)))
@@ -50,8 +50,8 @@ def _infer_coords_and_dims(shape, coords, dims):
             if utils.is_dict_like(coords):
                 # deprecated in GH993, removed in GH1539
                 raise ValueError('inferring DataArray dimensions from '
-                                 'dictionary like ``coords`` has been '
-                                 'deprecated. Use an explicit list of '
+                                 'dictionary like ``coords`` is no longer '
+                                 'supported. Use an explicit list of '
                                  '``dims`` instead.')
             for n, (dim, coord) in enumerate(zip(dims, coords)):
                 coord = as_variable(coord,
@@ -87,6 +87,12 @@ def _infer_coords_and_dims(shape, coords, dims):
                                  'length %s on the data but length %s on '
                                  'coordinate %r' % (d, sizes[d], s, k))
 
+        if k in sizes and v.shape != (sizes[k],):
+            raise ValueError('coordinate %r is a DataArray dimension, but '
+                             'it has shape %r rather than expected shape %r '
+                             'matching the dimension size'
+                             % (k, v.shape, (sizes[k],)))
+
     assert_unique_multiindex_level_names(new_coords)
 
     return new_coords, dims

xarray/core/variable.py

@@ -96,7 +96,7 @@ def as_variable(obj, name=None):
                             '{}'.format(obj))
     elif utils.is_scalar(obj):
         obj = Variable([], obj)
-    elif (isinstance(obj, (pd.Index, IndexVariable)) and obj.name is not None):
+    elif isinstance(obj, (pd.Index, IndexVariable)) and obj.name is not None:
         obj = Variable(obj.name, obj)
     elif name is not None:
         data = as_compatible_data(obj)

xarray/plot/plot.py

@@ -402,7 +402,7 @@ def _plot2d(plotfunc):
     """
 
     # Build on the original docstring
-    plotfunc.__doc__ = '\n'.join((plotfunc.__doc__, commondoc))
+    plotfunc.__doc__ = '%s\n%s' % (plotfunc.__doc__, commondoc)
 
     @functools.wraps(plotfunc)
     def newplotfunc(darray, x=None, y=None, figsize=None, size=None,

xarray/tests/test_backends.py

@@ -376,9 +376,17 @@ def test_roundtrip_coordinates(self):
         with self.roundtrip(original) as actual:
             self.assertDatasetIdentical(original, actual)
 
-        expected = original.drop('foo')
-        with self.roundtrip(expected) as actual:
-            self.assertDatasetIdentical(expected, actual)
+    def test_roundtrip_global_coordinates(self):
+        original = Dataset({'x': [2, 3], 'y': ('a', [42]), 'z': ('x', [4, 5])})
+        with self.roundtrip(original) as actual:
+            self.assertDatasetIdentical(original, actual)
+
+    def test_roundtrip_coordinates_with_space(self):
+        original = Dataset(coords={'x': 0, 'y z': 1})
+        expected = Dataset({'y z': 1}, {'x': 0})
+        with pytest.warns(xr.SerializationWarning):
+            with self.roundtrip(original) as actual:
+                self.assertDatasetIdentical(expected, actual)
 
     def test_roundtrip_boolean_dtype(self):
         original = create_boolean_data()
@@ -711,6 +719,13 @@ def test_append_overwrite_values(self):
     def test_vectorized_indexing(self):
         self._test_vectorized_indexing(vindex_support=False)
 
+    def test_multiindex_not_implemented(self):
+        ds = (Dataset(coords={'y': ('x', [1, 2]), 'z': ('x', ['a', 'b'])})
+              .set_index(x=['y', 'z']))
+        with raises_regex(NotImplementedError, 'MultiIndex'):
+            with self.roundtrip(ds):
+                pass
+
 
 _counter = itertools.count()
 
@@ -909,6 +924,21 @@ def test_compression_encoding(self):
         with self.roundtrip(expected) as actual:
             self.assertDatasetEqual(expected, actual)
 
+    def test_encoding_chunksizes_unlimited(self):
+        # regression test for GH1225
+        ds = Dataset({'x': [1, 2, 3], 'y': ('x', [2, 3, 4])})
+        ds.variables['x'].encoding = {
+            'zlib': False,
+            'shuffle': False,
+            'complevel': 0,
+            'fletcher32': False,
+            'contiguous': False,
+            'chunksizes': (2 ** 20,),
+            'original_shape': (3,),
+        }
+        with self.roundtrip(ds) as actual:
+            self.assertDatasetEqual(ds, actual)
+
     def test_mask_and_scale(self):
         with create_tmp_file() as tmp_file:
             with nc4.Dataset(tmp_file, mode='w') as nc:
@@ -1230,6 +1260,7 @@ def test_encoding_unlimited_dims(self):
                             save_kwargs=dict(unlimited_dims=['y'])) as actual:
             self.assertEqual(actual.encoding['unlimited_dims'], set('y'))
             self.assertDatasetEqual(ds, actual)
+
         ds.encoding = {'unlimited_dims': ['y']}
         with self.roundtrip(ds) as actual:
             self.assertEqual(actual.encoding['unlimited_dims'], set('y'))
@@ -1506,8 +1537,11 @@ def roundtrip(self, data, save_kwargs={}, open_kwargs={},
                   allow_cleanup_failure=False):
         yield data.chunk()
 
+    # Override methods in DatasetIOTestCases - not applicable to dask
     def test_roundtrip_string_encoded_characters(self):
-        # Override method in DatasetIOTestCases - not applicable to dask
+        pass
+
+    def test_roundtrip_coordinates_with_space(self):
         pass
 
     def test_roundtrip_datetime_data(self):