Merge branch 'main' into single-matplotlib-import

Author: Illviljan

.github/workflows/ci-additional.yaml

@@ -102,7 +102,7 @@ jobs:
              $PYTEST_EXTRA_FLAGS
 
       - name: Upload code coverage to Codecov
-        uses: codecov/codecov-action@v2.0.3
+        uses: codecov/codecov-action@v2.1.0
         with:
           file: ./coverage.xml
           flags: unittests,${{ matrix.env }}

.github/workflows/ci.yaml

@@ -100,7 +100,7 @@ jobs:
           path: pytest.xml
 
       - name: Upload code coverage to Codecov
-        uses: codecov/codecov-action@v2.0.3
+        uses: codecov/codecov-action@v2.1.0
         with:
           file: ./coverage.xml
           flags: unittests

.pre-commit-config.yaml

@@ -13,7 +13,7 @@ repos:
       - id: isort
   # https://github.com/python/black#version-control-integration
   - repo: https://github.com/psf/black
-    rev: 21.8b0
+    rev: 21.9b0
     hooks:
       - id: black
   - repo: https://github.com/keewis/blackdoc
@@ -30,20 +30,21 @@ repos:
   #     - id: velin
   #       args: ["--write", "--compact"]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.910
+    rev: v0.910-1
     hooks:
       - id: mypy
-        # Copied from setup.cfg
-        exclude: "properties|asv_bench"
+        # `properies` & `asv_bench` are copied from setup.cfg.
+        # `_typed_ops.py` is added since otherwise mypy will complain (but notably only in pre-commit)
+        exclude: "properties|asv_bench|_typed_ops.py"
         additional_dependencies: [
             # Type stubs
             types-python-dateutil,
             types-pkg_resources,
             types-PyYAML,
             types-pytz,
+            typing-extensions==3.10.0.0,
             # Dependencies that are typed
             numpy,
-            typing-extensions==3.10.0.0,
           ]
   # run this occasionally, ref discussion https://github.com/pydata/xarray/pull/3194
   # - repo: https://github.com/asottile/pyupgrade

asv_bench/benchmarks/groupby.py

@@ -0,0 +1,39 @@
+import numpy as np
+
+import xarray as xr
+
+from . import parameterized, requires_dask
+
+
+class GroupBy:
+    def setup(self, *args, **kwargs):
+        self.ds = xr.Dataset(
+            {
+                "a": xr.DataArray(np.r_[np.arange(500.0), np.arange(500.0)]),
+                "b": xr.DataArray(np.arange(1000.0)),
+            }
+        )
+
+    @parameterized(["method"], [("sum", "mean")])
+    def time_agg(self, method):
+        return getattr(self.ds.groupby("a"), method)()
+
+
+class GroupByDask(GroupBy):
+    def setup(self, *args, **kwargs):
+        requires_dask()
+        super().setup(**kwargs)
+        self.ds = self.ds.chunk({"dim_0": 50})
+
+
+class GroupByDataFrame(GroupBy):
+    def setup(self, *args, **kwargs):
+        super().setup(**kwargs)
+        self.ds = self.ds.to_dataframe()
+
+
+class GroupByDaskDataFrame(GroupBy):
+    def setup(self, *args, **kwargs):
+        requires_dask()
+        super().setup(**kwargs)
+        self.ds = self.ds.chunk({"dim_0": 50}).to_dataframe()

ci/requirements/doc.yml

@@ -23,6 +23,7 @@ dependencies:
   - pooch
   - pip
   - pydata-sphinx-theme>=0.4.3
+  - pyproj
   - rasterio>=1.1
   - seaborn
   - setuptools

doc/conf.py

@@ -260,8 +260,8 @@
 ogp_image = "https://xarray.pydata.org/en/stable/_static/dataset-diagram-logo.png"
 ogp_custom_meta_tags = [
     '<meta name="twitter:card" content="summary_large_image" />',
-    '<meta property="twitter:site" content="@xarray_dev />',
-    '<meta name="image" property="og:image" content="https://xarray.pydata.org/en/stable/_static/dataset-diagram-logo.png">',
+    '<meta property="twitter:site" content="@xarray_dev" />',
+    '<meta name="image" property="og:image" content="https://xarray.pydata.org/en/stable/_static/dataset-diagram-logo.png" />',
 ]
 
 # Redirects for pages that were moved to new locations

doc/ecosystem.rst

@@ -50,6 +50,7 @@ Geosciences
 Machine Learning
 ~~~~~~~~~~~~~~~~
 - `ArviZ <https://arviz-devs.github.io/arviz/>`_: Exploratory analysis of Bayesian models, built on top of xarray.
+- `Darts <https://github.com/unit8co/darts/>`_: User-friendly modern machine learning for time series in Python.
 - `Elm <https://ensemble-learning-models.readthedocs.io>`_: Parallel machine learning on xarray data structures
 - `sklearn-xarray (1) <https://phausamann.github.io/sklearn-xarray>`_: Combines scikit-learn and xarray (1).
 - `sklearn-xarray (2) <https://sklearn-xarray.readthedocs.io/en/latest/>`_: Combines scikit-learn and xarray (2).

doc/examples/visualization_gallery.ipynb

@@ -191,7 +191,10 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "jp-MarkdownHeadingCollapsed": true,
+    "tags": []
+   },
    "source": [
     "## `imshow()` and rasterio map projections\n",
     "\n",
@@ -213,7 +216,7 @@
     "\n",
     "# The data is in UTM projection. We have to set it manually until\n",
     "# https://github.com/SciTools/cartopy/issues/813 is implemented\n",
-    "crs = ccrs.UTM('18N')\n",
+    "crs = ccrs.UTM('18')\n",
     "\n",
     "# Plot on a map\n",
     "ax = plt.subplot(projection=crs)\n",
@@ -242,20 +245,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from rasterio.warp import transform\n",
+    "from pyproj import Transformer\n",
     "import numpy as np\n",
     "\n",
     "da = xr.tutorial.open_rasterio(\"RGB.byte\")\n",
     "\n",
-    "# Compute the lon/lat coordinates with rasterio.warp.transform\n",
-    "ny, nx = len(da['y']), len(da['x'])\n",
     "x, y = np.meshgrid(da['x'], da['y'])\n",
-    "\n",
-    "# Rasterio works with 1D arrays\n",
-    "lon, lat = transform(da.crs, {'init': 'EPSG:4326'},\n",
-    "                     x.flatten(), y.flatten())\n",
-    "lon = np.asarray(lon).reshape((ny, nx))\n",
-    "lat = np.asarray(lat).reshape((ny, nx))\n",
+    "transformer = Transformer.from_crs(da.crs, \"EPSG:4326\", always_xy=True)\n",
+    "lon, lat = transformer.transform(x, y)\n",
     "da.coords['lon'] = (('y', 'x'), lon)\n",
     "da.coords['lat'] = (('y', 'x'), lat)\n",
     "\n",
@@ -265,14 +262,14 @@
     "# Plot on a map\n",
     "ax = plt.subplot(projection=ccrs.PlateCarree())\n",
     "greyscale.plot(ax=ax, x='lon', y='lat', transform=ccrs.PlateCarree(),\n",
-    "               cmap='Greys_r', add_colorbar=False)\n",
+    "               cmap='Greys_r', shading=\"auto\",add_colorbar=False)\n",
     "ax.coastlines('10m', color='r')"
    ]
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -286,7 +283,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.3"
+   "version": "3.9.7"
   }
  },
  "nbformat": 4,

doc/gallery/plot_rasterio.py

@@ -18,22 +18,17 @@
 import cartopy.crs as ccrs
 import matplotlib.pyplot as plt
 import numpy as np
-from rasterio.warp import transform
+from pyproj import Transformer
 
 import xarray as xr
 
 # Read the data
 url = "https://github.com/mapbox/rasterio/raw/master/tests/data/RGB.byte.tif"
 da = xr.open_rasterio(url)
 
-# Compute the lon/lat coordinates with rasterio.warp.transform
-ny, nx = len(da["y"]), len(da["x"])
-x, y = np.meshgrid(da["x"], da["y"])
-
-# Rasterio works with 1D arrays
-lon, lat = transform(da.crs, {"init": "EPSG:4326"}, x.flatten(), y.flatten())
-lon = np.asarray(lon).reshape((ny, nx))
-lat = np.asarray(lat).reshape((ny, nx))
+# Compute the lon/lat coordinates with pyproj
+transformer = Transformer.from_crs(da.crs, "EPSG:4326", always_xy=True)
+lon, lat = transformer.transform(*np.meshgrid(da["x"], da["y"]))
 da.coords["lon"] = (("y", "x"), lon)
 da.coords["lat"] = (("y", "x"), lat)
 

doc/howdoi.rst

@@ -43,6 +43,10 @@ How do I ...
      - :py:attr:`DataArray.data`
    * - convert to and extract the underlying NumPy array
      - :py:attr:`DataArray.values`
+   * - convert to a pandas DataFrame
+     - :py:attr:`Dataset.to_dataframe`
+   * - sort values
+     - :py:attr:`Dataset.sortby`
    * - find out if my xarray object is wrapping a Dask Array
      - :py:func:`dask.is_dask_collection`
    * - know how much memory my object requires

doc/team.rst

@@ -82,6 +82,10 @@ The current core developers team comprises:
    ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
    :link-badge:`https://github.com/andersy005,"Anderson Banihirwe",cls=btn badge-light`
 
+   ---
+   .. image:: https://avatars.githubusercontent.com/u/14371165?v=4
+   ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+   :link-badge:`https://github.com/Illviljan,"Jimmy Westling",cls=btn badge-light`
 
 
 The full list of contributors is on our `GitHub Contributors Page <https://github.com/pydata/xarray/graphs/contributors>`__.

doc/user-guide/dask.rst

@@ -13,7 +13,7 @@ dependency in a future version of xarray.
 
 For a full example of how to use xarray's Dask integration, read the
 `blog post introducing xarray and Dask`_. More up-to-date examples
-may be found at the `Pangeo project's use-cases <http://pangeo.io/use_cases/index.html>`_
+may be found at the `Pangeo project's gallery <http://gallery.pangeo.io/>`_
 and at the `Dask examples website <https://examples.dask.org/xarray.html>`_.
 
 .. _blog post introducing xarray and Dask: http://stephanhoyer.com/2015/06/11/xray-dask-out-of-core-labeled-arrays/

doc/user-guide/io.rst

@@ -742,6 +742,12 @@ GeoTIFFs and other gridded raster datasets can be opened using `rasterio`_, if
 rasterio is installed. Here is an example of how to use
 :py:func:`open_rasterio` to read one of rasterio's `test files`_:
 
+.. deprecated:: 0.19.1
+
+        Deprecated in favor of rioxarray.
+        For information about transitioning, see:
+        https://corteva.github.io/rioxarray/stable/getting_started/getting_started.html
+
 .. ipython::
     :verbatim:
 
@@ -769,12 +775,6 @@ coordinates defined in the file's projection provided by the ``crs`` attribute.
 See :ref:`/examples/visualization_gallery.ipynb#Parsing-rasterio-geocoordinates`
 for an example of how to convert these to longitudes and latitudes.
 
-.. warning::
-
-    This feature has been added in xarray v0.9.6 and should still be
-    considered experimental. Please report any bugs you may find
-    on xarray's github repository.
-
 
 Additionally, you can use `rioxarray`_ for reading in GeoTiff, netCDF or other
 GDAL readable raster data using `rasterio`_ as well as for exporting to a geoTIFF.

doc/user-guide/reshaping.rst

@@ -275,7 +275,10 @@ One may sort a DataArray/Dataset via :py:meth:`~xarray.DataArray.sortby` and
 .. ipython:: python
 
     ds = xr.Dataset(
-        {"A": (("x", "y"), [[1, 2], [3, 4]]), "B": (("x", "y"), [[5, 6], [7, 8]])},
+        {
+            "A": (("x", "y"), [[1, 2], [3, 4]]),
+            "B": (("x", "y"), [[5, 6], [7, 8]]),
+        },
         coords={"x": ["b", "a"], "y": [1, 0]},
     )
     dax = xr.DataArray([100, 99], [("x", [0, 1])])

doc/whats-new.rst

@@ -19,21 +19,23 @@ What's New
 
 v0.19.1 (unreleased)
 ---------------------
+.. TODO(by keewis): update deprecations if we decide to skip 0.19.1
 
 New Features
 ~~~~~~~~~~~~
 - Added a :py:func:`get_options` method to xarray's root namespace (:issue:`5698`, :pull:`5716`)
   By `Pushkar Kopparla <https://github.com/pkopparla>`_.
 - Xarray now does a better job rendering variable names that are long LaTeX sequences when plotting (:issue:`5681`, :pull:`5682`).
   By `Tomas Chor <https://github.com/tomchor>`_.
-- Add a option to disable the use of ``bottleneck`` (:pull:`5560`)
+- Add an option to disable the use of ``bottleneck`` (:pull:`5560`)
   By `Justus Magin <https://github.com/keewis>`_.
 - Added ``**kwargs`` argument to :py:meth:`open_rasterio` to access overviews (:issue:`3269`).
   By `Pushkar Kopparla <https://github.com/pkopparla>`_.
 - Added ``storage_options`` argument to :py:meth:`to_zarr` (:issue:`5601`).
   By `Ray Bell <https://github.com/raybellwaves>`_, `Zachary Blackwood <https://github.com/blackary>`_ and
   `Nathan Lis <https://github.com/wxman22>`_.
-
+- Histogram plots are set with a title displaying the scalar coords if any, similarly to the other plots (:issue:`5791`, :pull:`5792`).
+  By `Maxime Liquet <https://github.com/maximlt>`_.
 
 Breaking changes
 ~~~~~~~~~~~~~~~~
@@ -55,14 +57,35 @@ Breaking changes
 Deprecations
 ~~~~~~~~~~~~
 
+- Deprecate :py:func:`open_rasterio` (:issue:`4697`, :pull:`5808`).
+  By `Alan Snow <https://github.com/snowman2>`_.
+- Set the default argument for `roll_coords` to `False` for :py:meth:`DataArray.roll`
+  and :py:meth:`Dataset.roll`. (:pull:`5653`)
+  By `Tom Nicholas <https://github.com/TomNicholas>`_.
+- :py:meth:`xarray.open_mfdataset` will now error instead of warn when a value for ``concat_dim`` is
+  passed alongside ``combine='by_coords'``.
+  By `Tom Nicholas <https://github.com/TomNicholas>`_.
 
 Bug fixes
 ~~~~~~~~~
 
+- Fix ZeroDivisionError from saving dask array with empty dimension (:issue: `5741`).
+  By `Joseph K Aicher <https://github.com/jaicher>`_.
+- Fixed performance bug where ``cftime`` import attempted within various core operations if ``cftime`` not
+  installed (:pull:`5640`).
+  By `Luke Sewell <https://github.com/lusewell>`_
+- When a custom engine was used in :py:func:`~xarray.open_dataset` the engine
+  wasn't initialized properly, causing missing argument errors or inconsistent
+  method signatures. (:pull:`5684`)
+  By `Jimmy Westling <https://github.com/illviljan>`_.
+- Numbers are properly formatted in a plot's title (:issue:`5788`, :pull:`5789`).
+  By `Maxime Liquet <https://github.com/maximlt>`_.
 
 Documentation
 ~~~~~~~~~~~~~
 
+- Users are instructed to try ``use_cftime=True`` if a ``TypeError`` occurs when combining datasets and one of the types involved is a subclass of ``cftime.datetime`` (:pull:`5776`).
+  By `Zeb Nicholls <https://github.com/znicholls>`_.
 
 Internal Changes
 ~~~~~~~~~~~~~~~~
@@ -80,6 +103,10 @@ Internal Changes
   By `Jimmy Westling <https://github.com/illviljan>`_.
 - Use isort's `float_to_top` config. (:pull:`5695`).
   By `Maximilian Roos <https://github.com/max-sixty>`_.
+- Refactor `xarray.core.duck_array_ops` to no longer special-case dispatching to
+  dask versions of functions when acting on dask arrays, instead relying numpy
+  and dask's adherence to NEP-18 to dispatch automatically. (:pull:`5571`)
+  By `Tom Nicholas <https://github.com/TomNicholas>`_.
 
 .. _whats-new.0.19.0: