Merge remote-tracking branch 'upstream/main' into cmake-build

Author: WillAyd

.github/workflows/code-checks.yml

@@ -157,3 +157,32 @@ jobs:
 
       - name: Build image
         run: docker build --pull --no-cache --tag pandas-dev-env .
+
+  requirements-dev-text-installable:
+    name: Test install requirements-dev.txt
+    runs-on: ubuntu-latest
+
+    concurrency:
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-requirements-dev-text-installable
+      cancel-in-progress: true
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python
+        id: setup_python
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.8'
+          cache: 'pip'
+          cache-dependency-path: 'requirements-dev.txt'
+
+      - name: Install requirements-dev.txt
+        run: pip install -r requirements-dev.txt
+
+      - name: Check Pip Cache Hit
+        run: echo ${{ steps.setup_python.outputs.cache-hit }}

.github/workflows/comment_bot.yml

@@ -15,7 +15,6 @@ on:
 env:
   PANDAS_CI: 1
   PYTEST_TARGET: pandas
-  PYTEST_WORKERS: auto
   PATTERN: "not slow and not db and not network and not single_cpu"
 
 
@@ -36,6 +35,9 @@ jobs:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
       group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.os }}
       cancel-in-progress: true
+    env:
+      # GH 47443: PYTEST_WORKERS > 1 crashes Windows builds with memory related errors
+      PYTEST_WORKERS: ${{ matrix.os == 'macos-latest' && 'auto' || '1' }}
 
     steps:
     - name: Checkout

.github/workflows/macos-windows.yml

@@ -1,4 +1,4 @@
-name: Posix
+name: Ubuntu
 
 on:
   push:
@@ -145,7 +145,7 @@ jobs:
 
     - name: Extra installs
       # xsel for clipboard tests
-      run: sudo apt-get update && sudo apt-get install -y libc6-dev-i386 xsel ${{ env.EXTRA_APT }}
+      run: sudo apt-get update && sudo apt-get install -y xsel ${{ env.EXTRA_APT }}
 
     - name: Set up Conda
       uses: ./.github/actions/setup-conda

.github/workflows/posix.yml renamed to .github/workflows/ubuntu.yml

@@ -31,8 +31,7 @@ dependencies:
   - jinja2
   - lxml
   - matplotlib
-  # TODO: uncomment after numba supports py310
-  #- numba
+  - numba
   - numexpr
   - openpyxl
   - odfpy

ci/deps/actions-310.yaml

@@ -447,7 +447,6 @@
         "py": ("https://pylib.readthedocs.io/en/latest/", None),
         "python": ("https://docs.python.org/3/", None),
         "scipy": ("https://docs.scipy.org/doc/scipy/", None),
-        "statsmodels": ("https://www.statsmodels.org/devel/", None),
         "pyarrow": ("https://arrow.apache.org/docs/", None),
     }
 

doc/source/conf.py

@@ -326,13 +326,7 @@ Autofixing formatting errors
 ----------------------------
 
 We use several styling checks (e.g. ``black``, ``flake8``, ``isort``) which are run after
-you make a pull request. If there is a scenario where any of these checks fail then you
-can comment::
-
-    @github-actions pre-commit
-
-on that pull request. This will trigger a workflow which will autofix formatting
-errors.
+you make a pull request.
 
 To automatically fix formatting errors on each commit you make, you can
 set up pre-commit yourself. First, create a Python :ref:`environment

doc/source/development/contributing.rst

@@ -18,6 +18,7 @@ Fixed regressions
 - Fixed regression in :meth:`DataFrame.to_csv` raising error when :class:`DataFrame` contains extension dtype categorical column (:issue:`46297`, :issue:`46812`)
 - Fixed regression in representation of ``dtypes`` attribute of :class:`MultiIndex` (:issue:`46900`)
 - Fixed regression when setting values with :meth:`DataFrame.loc` updating :class:`RangeIndex` when index was set as new column and column was updated afterwards (:issue:`47128`)
+- Fixed regression in :meth:`DataFrame.fillna` and :meth:`DataFrame.update` creating a copy when updating inplace (:issue:`47188`)
 - Fixed regression in :meth:`DataFrame.nsmallest` led to wrong results when ``np.nan`` in the sorting column (:issue:`46589`)
 - Fixed regression in :func:`read_fwf` raising ``ValueError`` when ``widths`` was specified with ``usecols`` (:issue:`46580`)
 - Fixed regression in :func:`concat` not sorting columns for mixed column names (:issue:`47127`)

doc/source/whatsnew/v1.4.3.rst

@@ -810,6 +810,7 @@ Indexing
 ^^^^^^^^
 - Bug in :meth:`loc.__getitem__` with a list of keys causing an internal inconsistency that could lead to a disconnect between ``frame.at[x, y]`` vs ``frame[y].loc[x]`` (:issue:`22372`)
 - Bug in :meth:`DataFrame.iloc` where indexing a single row on a :class:`DataFrame` with a single ExtensionDtype column gave a copy instead of a view on the underlying data (:issue:`45241`)
+- Bug in :meth:`DataFrame.__getitem__` returning copy when :class:`DataFrame` has duplicated columns even if a unique column is selected (:issue:`45316`, :issue:`41062`)
 - Bug in :meth:`Series.align` does not create :class:`MultiIndex` with union of levels when both MultiIndexes intersections are identical (:issue:`45224`)
 - Bug in setting a NA value (``None`` or ``np.nan``) into a :class:`Series` with int-based :class:`IntervalDtype` incorrectly casting to object dtype instead of a float-based :class:`IntervalDtype` (:issue:`45568`)
 - Bug in indexing setting values into an ``ExtensionDtype`` column with ``df.iloc[:, i] = values`` with ``values`` having the same dtype as ``df.iloc[:, i]`` incorrectly inserting a new array instead of setting in-place (:issue:`33457`)
@@ -861,8 +862,10 @@ I/O
 - Bug in :func:`read_csv` not recognizing line break for ``on_bad_lines="warn"`` for ``engine="c"`` (:issue:`41710`)
 - Bug in :meth:`DataFrame.to_csv` not respecting ``float_format`` for ``Float64`` dtype (:issue:`45991`)
 - Bug in :func:`read_csv` not respecting a specified converter to index columns in all cases (:issue:`40589`)
+- Bug in :func:`read_csv` interpreting second row as :class:`Index` names even when ``index_col=False`` (:issue:`46569`)
 - Bug in :func:`read_parquet` when ``engine="pyarrow"`` which caused partial write to disk when column of unsupported datatype was passed (:issue:`44914`)
 - Bug in :func:`DataFrame.to_excel` and :class:`ExcelWriter` would raise when writing an empty DataFrame to a ``.ods`` file (:issue:`45793`)
+- Bug in :func:`read_excel` raising uncontrolled ``IndexError`` when ``header`` references non-existing rows (:issue:`43143`)
 - Bug in :func:`read_html` where elements surrounding ``<br>`` were joined without a space between them (:issue:`29528`)
 - Bug in :func:`read_csv` when data is longer than header leading to issues with callables in ``usecols`` expecting strings (:issue:`46997`)
 - Bug in Parquet roundtrip for Interval dtype with ``datetime64[ns]`` subtype (:issue:`45881`)

doc/source/whatsnew/v1.5.0.rst

@@ -1,22 +1,86 @@
+# Local development dependencies including docs building, website upload, ASV benchmark
 name: pandas-dev
 channels:
   - conda-forge
 dependencies:
-  # required
-  - numpy>=1.19.5
   - python=3.8
-  - python-dateutil>=2.8.1
+
+  # test dependencies
+  - cython=0.29.30
+  - pytest>=6.0
+  - pytest-cov
+  - pytest-xdist>=1.31
+  - psutil
+  - pytest-asyncio>=0.17
+  - boto3
+
+  # required dependencies
+  - python-dateutil
+  - numpy
   - pytz
 
+  # optional dependencies
+  - beautifulsoup4
+  - blosc
+  - brotlipy
+  - bottleneck
+  - fastparquet
+  - fsspec
+  - html5lib
+  - hypothesis
+  - gcsfs
+  - jinja2
+  - lxml
+  - matplotlib
+  - numba>=0.53.1
+  - numexpr>=2.8.0  # pin for "Run checks on imported code" job
+  - openpyxl
+  - odfpy
+  - pandas-gbq
+  - psycopg2
+  - pyarrow
+  - pymysql
+  - pyreadstat
+  - pytables
+  - python-snappy
+  - pyxlsb
+  - s3fs
+  - scipy
+  - sqlalchemy
+  - tabulate
+  - xarray
+  - xlrd
+  - xlsxwriter
+  - xlwt
+  - zstandard
+
+  # downstream packages
+  - aiobotocore<2.0.0  # GH#44311 pinned to fix docbuild
+  - botocore
+  - cftime
+  - dask
+  - ipython
+  - geopandas-base
+  - seaborn
+  - scikit-learn
+  - statsmodels
+  - coverage
+  - pandas-datareader
+  - pyyaml
+  - py
+  - pytorch
+
+  # local testing dependencies
+  - moto
+  - flask
+
   # benchmarks
   - asv
 
-  # building
   # The compiler packages are meta-packages and install the correct compiler (activation) packages on the respective platforms.
   - cmake>=3.18.0
   - c-compiler
   - cxx-compiler
-  - cython>=0.29.30
 
   # code checks
   - black=22.3.0
@@ -33,10 +97,11 @@ dependencies:
   # documentation
   - gitpython  # obtain contributors from git for whatsnew
   - gitdb
+  - natsort  # DataFrame.sort_values doctest
   - numpydoc
   - pandas-dev-flaker=0.5.0
   - pydata-sphinx-theme=0.8.0
-  - pytest-cython
+  - pytest-cython  # doctest
   - sphinx
   - sphinx-panels
   - types-python-dateutil
@@ -48,77 +113,14 @@ dependencies:
   - nbconvert>=6.4.5
   - nbsphinx
   - pandoc
-
-  # Dask and its dependencies (that dont install with dask)
-  - dask-core
-  - toolz>=0.7.3
-  - partd>=0.3.10
-  - cloudpickle>=0.2.1
-
-  # web (jinja2 is also needed, but it's also an optional pandas dependency)
-  - markdown
-  - feedparser
-  - pyyaml
-  - requests
-
-  # testing
-  - boto3
-  - botocore>=1.11
-  - hypothesis>=5.5.3
-  - moto  # mock S3
-  - flask
-  - pytest>=6.0
-  - pytest-cov
-  - pytest-xdist>=1.31
-  - pytest-asyncio>=0.17
-  - pytest-instafail
-
-  # downstream tests
-  - seaborn
-  - statsmodels
-
-  # unused (required indirectly may be?)
   - ipywidgets
   - nbformat
   - notebook>=6.0.3
-
-  # optional
-  - blosc
-  - bottleneck>=1.3.1
   - ipykernel
-  - ipython>=7.11.1
-  - jinja2  # pandas.Styler
-  - matplotlib>=3.3.2  # pandas.plotting, Series.plot, DataFrame.plot
-  - numexpr>=2.7.1
-  - scipy>=1.4.1
-  - numba>=0.50.1
-
-  # optional for io
-  # ---------------
-  # pd.read_html
-  - beautifulsoup4>=4.8.2
-  - html5lib
-  - lxml
-
-  # pd.read_excel, DataFrame.to_excel, pd.ExcelWriter, pd.ExcelFile
-  - openpyxl
-  - xlrd
-  - xlsxwriter
-  - xlwt
-  - odfpy
-
-  - fastparquet>=0.4.0  # pandas.read_parquet, DataFrame.to_parquet
-  - pyarrow>2.0.1  # pandas.read_parquet, DataFrame.to_parquet, pandas.read_feather, DataFrame.to_feather
-  - python-snappy  # required by pyarrow
 
-  - pytables>=3.6.1  # pandas.read_hdf, DataFrame.to_hdf
-  - s3fs>=0.4.0  # file IO when using 's3://...' path
-  - aiobotocore<2.0.0  # GH#44311 pinned to fix docbuild
-  - fsspec>=0.7.4  # for generic remote file operations
-  - gcsfs>=0.6.0  # file IO when using 'gcs://...' path
-  - sqlalchemy  # pandas.read_sql, DataFrame.to_sql
-  - xarray  # DataFrame.to_xarray
-  - cftime  # Needed for downstream xarray.CFTimeIndex test
-  - pyreadstat  # pandas.read_spss
-  - tabulate>=0.8.3  # DataFrame.to_markdown
-  - natsort  # DataFrame.sort_values
+  # web
+  - jinja2  # in optional dependencies, but documented here as needed
+  - markdown
+  - feedparser
+  - pyyaml
+  - requests