Merge branch 'master' into 2d-coord

* master: (21 commits)
  xarray.backends refactor (pydata#2261)
  Fix indexing error for data loaded with open_rasterio (pydata#2456)
  Properly support user-provided norm. (pydata#2443)
  pep8speaks (pydata#2462)
  isort (pydata#2469)
  tests shoudn't need to pass for a PR (pydata#2471)
  Replace the last of unittest with pytest (pydata#2467)
  Add python_requires to setup.py (pydata#2465)
  Update whats-new.rst (pydata#2466)
  Clean up _parse_array_of_cftime_strings (pydata#2464)
  plot.contour: Don't make cmap if colors is a single color. (pydata#2453)
  np.AxisError was added in numpy 1.13 (pydata#2455)
  Add CFTimeIndex.shift (pydata#2431)
  Fix FutureWarning in CFTimeIndex.date_type (pydata#2448)
  fix:2445 (pydata#2446)
  Enable use of cftime.datetime coordinates with differentiate and interp (pydata#2434)
  restore ddof support in std (pydata#2447)
  Future warning for default reduction dimension of groupby (pydata#2366)
  Remove incorrect statement about "drop" in the text docs (pydata#2439)
  Use profile mechanism, not no-op mutation (pydata#2442)
  ...

Author: dcherian

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,4 +1,3 @@
  - [ ] Closes #xxxx (remove if there is no corresponding issue, which should only be the case for minor changes)
  - [ ] Tests added (for all bug fixes or enhancements)
- - [ ] Tests passed (for all non-documentation changes)
  - [ ] Fully documented, including `whats-new.rst` for all changes and `api.rst` for new API (remove if this change should not be visible to users, e.g., if it is an internal clean-up, or if this is part of a larger project that will be documented later)

.pep8speaks.yml

@@ -0,0 +1,11 @@
+# File : .pep8speaks.yml
+
+scanner:
+    diff_only: True  # If True, errors caused by only the patch are shown
+
+pycodestyle:
+    max-line-length: 79
+    ignore:  # Errors and warnings to ignore
+        - E402,  # module level import not at top of file
+        - E731,  # do not assign a lambda expression, use a def
+        - W503   # line break before binary operator

.stickler.yml

@@ -1,5 +1,5 @@
 # Based on http://conda.pydata.org/docs/travis.html
-language: python
+language: minimal
 sudo: false # use container based build
 notifications:
   email: false
@@ -10,74 +10,48 @@ branches:
 matrix:
   fast_finish: true
   include:
-  - python: 2.7
-    env: CONDA_ENV=py27-min
-  - python: 2.7
-    env: CONDA_ENV=py27-cdat+iris+pynio
-  - python: 3.5
-    env: CONDA_ENV=py35
-  - python: 3.6
-    env: CONDA_ENV=py36
-  - python: 3.6  # TODO: change this to 3.7 once https://github.com/travis-ci/travis-ci/issues/9815 is fixed
-    env: CONDA_ENV=py37
-  - python: 3.6
-    env:
+  - env: CONDA_ENV=py27-min
+  - env: CONDA_ENV=py27-cdat+iris+pynio
+  - env: CONDA_ENV=py35
+  - env: CONDA_ENV=py36
+  - env: CONDA_ENV=py37
+  - env:
     - CONDA_ENV=py36
     - EXTRA_FLAGS="--run-flaky --run-network-tests"
-  - python: 3.6
-    env: CONDA_ENV=py36-netcdf4-dev
+  - env: CONDA_ENV=py36-netcdf4-dev
     addons:
       apt_packages:
         - libhdf5-serial-dev
         - netcdf-bin
         - libnetcdf-dev
-  - python: 3.6
-    env: CONDA_ENV=py36-dask-dev
-  - python: 3.6
-    env: CONDA_ENV=py36-pandas-dev
-  - python: 3.6
-    env: CONDA_ENV=py36-bottleneck-dev
-  - python: 3.6
-    env: CONDA_ENV=py36-condaforge-rc
-  - python: 3.6
-    env: CONDA_ENV=py36-pynio-dev
-  - python: 3.6
-    env: CONDA_ENV=py36-rasterio-0.36
-  - python: 3.6
-    env: CONDA_ENV=py36-zarr-dev
-  - python: 3.5
-    env: CONDA_ENV=docs
-  - python: 3.6
-    env: CONDA_ENV=py36-hypothesis
+  - env: CONDA_ENV=py36-dask-dev
+  - env: CONDA_ENV=py36-pandas-dev
+  - env: CONDA_ENV=py36-bottleneck-dev
+  - env: CONDA_ENV=py36-condaforge-rc
+  - env: CONDA_ENV=py36-pynio-dev
+  - env: CONDA_ENV=py36-rasterio-0.36
+  - env: CONDA_ENV=py36-zarr-dev
+  - env: CONDA_ENV=docs
+  - env: CONDA_ENV=py36-hypothesis
+
   allow_failures:
-  - python: 3.6
-    env:
+  - env:
     - CONDA_ENV=py36
     - EXTRA_FLAGS="--run-flaky --run-network-tests"
-  - python: 3.6
-    env: CONDA_ENV=py36-netcdf4-dev
+  - env: CONDA_ENV=py36-netcdf4-dev
     addons:
       apt_packages:
         - libhdf5-serial-dev
         - netcdf-bin
         - libnetcdf-dev
-  - python: 3.6
-    env: CONDA_ENV=py36-pandas-dev
-  - python: 3.6
-    env: CONDA_ENV=py36-bottleneck-dev
-  - python: 3.6
-    env: CONDA_ENV=py36-condaforge-rc
-  - python: 3.6
-    env: CONDA_ENV=py36-pynio-dev
-  - python: 3.6
-    env: CONDA_ENV=py36-zarr-dev
+  - env: CONDA_ENV=py36-pandas-dev
+  - env: CONDA_ENV=py36-bottleneck-dev
+  - env: CONDA_ENV=py36-condaforge-rc
+  - env: CONDA_ENV=py36-pynio-dev
+  - env: CONDA_ENV=py36-zarr-dev
 
 before_install:
-  - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
-      wget http://repo.continuum.io/miniconda/Miniconda-3.16.0-Linux-x86_64.sh -O miniconda.sh;
-    else
-      wget http://repo.continuum.io/miniconda/Miniconda3-3.16.0-Linux-x86_64.sh -O miniconda.sh;
-    fi
+  - wget http://repo.continuum.io/miniconda/Miniconda3-3.16.0-Linux-x86_64.sh -O miniconda.sh;
   - bash miniconda.sh -b -p $HOME/miniconda
   - export PATH="$HOME/miniconda/bin:$PATH"
   - hash -r
@@ -97,6 +71,8 @@ install:
   - python xarray/util/print_versions.py
 
 script:
+  - which python
+  - python --version
   - python -OO -c "import xarray"
   - if [[ "$CONDA_ENV" == "docs" ]]; then
       conda install -c conda-forge sphinx sphinx_rtd_theme sphinx-gallery numpydoc;

.travis.yml

@@ -64,6 +64,7 @@
         "scipy": [""],
         "bottleneck": ["", null],
         "dask": [""],
+        "distributed": [""],
     },
 
 

asv_bench/asv.conf.json

@@ -1,11 +1,13 @@
 from __future__ import absolute_import, division, print_function
 
+import os
+
 import numpy as np
 import pandas as pd
 
 import xarray as xr
 
-from . import randn, randint, requires_dask
+from . import randint, randn, requires_dask
 
 try:
     import dask
@@ -14,6 +16,9 @@
     pass
 
 
+os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE'
+
+
 class IOSingleNetCDF(object):
     """
     A few examples that benchmark reading/writing a single netCDF file with
@@ -405,3 +410,39 @@ def time_open_dataset_scipy_with_time_chunks(self):
         with dask.set_options(get=dask.multiprocessing.get):
             xr.open_mfdataset(self.filenames_list, engine='scipy',
                               chunks=self.time_chunks)
+
+
+def create_delayed_write():
+    import dask.array as da
+    vals = da.random.random(300, chunks=(1,))
+    ds = xr.Dataset({'vals': (['a'], vals)})
+    return ds.to_netcdf('file.nc', engine='netcdf4', compute=False)
+
+
+class IOWriteNetCDFDask(object):
+    timeout = 60
+    repeat = 1
+    number = 5
+
+    def setup(self):
+        requires_dask()
+        self.write = create_delayed_write()
+
+    def time_write(self):
+        self.write.compute()
+
+
+class IOWriteNetCDFDaskDistributed(object):
+    def setup(self):
+        try:
+            import distributed
+        except ImportError:
+            raise NotImplementedError
+        self.client = distributed.Client()
+        self.write = create_delayed_write()
+
+    def cleanup(self):
+        self.client.shutdown()
+
+    def time_write(self):
+        self.write.compute()

asv_bench/benchmarks/dataset_io.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import, division, print_function
 
 import numpy as np
+
 import xarray as xr
 
 from . import requires_dask

asv_bench/benchmarks/unstacking.py

@@ -151,3 +151,5 @@
    plot.FacetGrid.set_titles
    plot.FacetGrid.set_ticks
    plot.FacetGrid.map
+
+   CFTimeIndex.shift

doc/api-hidden.rst

@@ -624,3 +624,6 @@ arguments for the ``from_store`` and ``dump_to_store`` Dataset methods:
    backends.H5NetCDFStore
    backends.PydapDataStore
    backends.ScipyDataStore
+   backends.FileManager
+   backends.CachingFileManager
+   backends.DummyFileManager

doc/api.rst

@@ -408,13 +408,6 @@ operations keep around coordinates:
     list(ds[['x']])
     list(ds.drop('temperature'))
 
-If a dimension name is given as an argument to ``drop``, it also drops all
-variables that use that dimension:
-
-.. ipython:: python
-
-    list(ds.drop('time'))
-
 As an alternate to dictionary-like modifications, you can use
 :py:meth:`~xarray.Dataset.assign` and :py:meth:`~xarray.Dataset.assign_coords`.
 These methods return a new dataset with additional (or replaced) or values:

doc/data-structures.rst

@@ -63,6 +63,9 @@ by specifing the time periods required.
 
     da_dt64.interp(time=pd.date_range('1/1/2000', '1/3/2000', periods=3))
 
+Interpolation of data indexed by a :py:class:`~xarray.CFTimeIndex` is also
+allowed.  See :ref:`CFTimeIndex` for examples.
+    
 .. note::
 
   Currently, our interpolation only works for regular grids.

doc/interpolation.rst

@@ -70,9 +70,9 @@ You can manual decode arrays in this form by passing a dataset to
 One unfortunate limitation of using ``datetime64[ns]`` is that it limits the
 native representation of dates to those that fall between the years 1678 and
 2262. When a netCDF file contains dates outside of these bounds, dates will be
-returned as arrays of ``cftime.datetime`` objects and a ``CFTimeIndex``
-can be used for indexing.  The ``CFTimeIndex`` enables only a subset of
-the indexing functionality of a ``pandas.DatetimeIndex`` and is only enabled
+returned as arrays of :py:class:`cftime.datetime` objects and a :py:class:`~xarray.CFTimeIndex`
+can be used for indexing.  The :py:class:`~xarray.CFTimeIndex` enables only a subset of
+the indexing functionality of a :py:class:`pandas.DatetimeIndex` and is only enabled
 when using the standalone version of ``cftime`` (not the version packaged with
 earlier versions ``netCDF4``).  See :ref:`CFTimeIndex` for more information.
 
@@ -219,20 +219,20 @@ Non-standard calendars and dates outside the Timestamp-valid range
 ------------------------------------------------------------------
 
 Through the standalone ``cftime`` library and a custom subclass of
-``pandas.Index``, xarray supports a subset of the indexing functionality enabled
-through the standard ``pandas.DatetimeIndex`` for dates from non-standard
-calendars or dates using a standard calendar, but outside the
-`Timestamp-valid range`_ (approximately between years 1678 and 2262).  This
-behavior has not yet been turned on by default; to take advantage of this
-functionality, you must have the ``enable_cftimeindex`` option set to
+:py:class:`pandas.Index`, xarray supports a subset of the indexing
+functionality enabled through the standard :py:class:`pandas.DatetimeIndex` for
+dates from non-standard calendars or dates using a standard calendar, but
+outside the `Timestamp-valid range`_ (approximately between years 1678 and
+2262).  This behavior has not yet been turned on by default; to take advantage
+of this functionality, you must have the ``enable_cftimeindex`` option set to
 ``True`` within your context (see :py:func:`~xarray.set_options` for more
 information).  It is expected that this will become the default behavior in
 xarray version 0.11.
 
 For instance, you can create a DataArray indexed by a time
 coordinate with a no-leap calendar within a context manager setting the
 ``enable_cftimeindex`` option, and the time index will be cast to a
-``CFTimeIndex``:
+:py:class:`~xarray.CFTimeIndex`:
 
 .. ipython:: python
 
@@ -247,28 +247,28 @@ coordinate with a no-leap calendar within a context manager setting the
                          
 .. note::
 
-   With the ``enable_cftimeindex`` option activated, a ``CFTimeIndex``
+   With the ``enable_cftimeindex`` option activated, a :py:class:`~xarray.CFTimeIndex`
    will be used for time indexing if any of the following are true:
 
    - The dates are from a non-standard calendar
    - Any dates are outside the Timestamp-valid range
 
-   Otherwise a ``pandas.DatetimeIndex`` will be used.  In addition, if any
+   Otherwise a :py:class:`pandas.DatetimeIndex` will be used.  In addition, if any
    variable (not just an index variable) is encoded using a non-standard
-   calendar, its times will be decoded into ``cftime.datetime`` objects,
+   calendar, its times will be decoded into :py:class:`cftime.datetime` objects,
    regardless of whether or not they can be represented using
    ``np.datetime64[ns]`` objects.
 
-xarray also includes a :py:func:`cftime_range` function, which enables creating a
-``CFTimeIndex`` with regularly-spaced dates.  For instance, we can create the
-same dates and DataArray we created above using:
+xarray also includes a :py:func:`~xarray.cftime_range` function, which enables
+creating a :py:class:`~xarray.CFTimeIndex` with regularly-spaced dates.  For instance, we can
+create the same dates and DataArray we created above using:
 
 .. ipython:: python
 
    dates = xr.cftime_range(start='0001', periods=24, freq='MS', calendar='noleap')
    da = xr.DataArray(np.arange(24), coords=[dates], dims=['time'], name='foo')
    
-For data indexed by a ``CFTimeIndex`` xarray currently supports:
+For data indexed by a :py:class:`~xarray.CFTimeIndex` xarray currently supports:
 
 - `Partial datetime string indexing`_ using strictly `ISO 8601-format`_ partial
   datetime strings:
@@ -294,7 +294,25 @@ For data indexed by a ``CFTimeIndex`` xarray currently supports:
 .. ipython:: python
 
    da.groupby('time.month').sum()
-   
+
+- Interpolation using :py:class:`cftime.datetime` objects:
+
+.. ipython:: python
+
+   da.interp(time=[DatetimeNoLeap(1, 1, 15), DatetimeNoLeap(1, 2, 15)])
+
+- Interpolation using datetime strings:
+
+.. ipython:: python
+
+   da.interp(time=['0001-01-15', '0001-02-15'])
+
+- Differentiation:
+
+.. ipython:: python
+
+   da.differentiate('time')
+
 - And serialization:
 
 .. ipython:: python
@@ -305,7 +323,7 @@ For data indexed by a ``CFTimeIndex`` xarray currently supports:
 .. note::
 
    Currently resampling along the time dimension for data indexed by a
-   ``CFTimeIndex`` is not supported.
+   :py:class:`~xarray.CFTimeIndex` is not supported.
 
 .. _Timestamp-valid range: https://pandas.pydata.org/pandas-docs/stable/timeseries.html#timestamp-limitations
 .. _ISO 8601-format: https://en.wikipedia.org/wiki/ISO_8601