Merge branch 'main' into tguard

Author: topper-123

.github/workflows/deprecation-tracking-bot.yml

@@ -0,0 +1,31 @@
+name: Deprecations Bot
+
+on:
+  pull_request:
+    branches:
+      - main
+    types:
+      [closed]
+
+
+permissions:
+  contents: read
+
+jobs:
+  deprecation_update:
+    permissions:
+      issues: write
+    if: >-
+      contains(github.event.pull_request.labels.*.name, 'Deprecate') && github.event.pull_request.merged == true
+    runs-on: ubuntu-22.04
+    env:
+      DEPRECATION_TRACKER_ISSUE: 50578
+    steps:
+    - name: Checkout
+      run: |
+        echo "Adding deprecation PR number to deprecation tracking issue"
+        export PR=${{ github.event.pull_request.number }}
+        BODY=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/${{ github.repository }}/issues/${DEPRECATION_TRACKER_ISSUE} |
+          python3 -c "import sys, json, os; x = {'body': json.load(sys.stdin)['body']}; pr = os.environ['PR']; x['body'] += f'\n- [ ] #{pr}'; print(json.dumps(x))")
+        echo ${BODY}
+        curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -X PATCH -d "${BODY}" https://api.github.com/repos/${{ github.repository }}/issues/${DEPRECATION_TRACKER_ISSUE}

.github/workflows/docbuild-and-upload.yml

@@ -89,3 +89,10 @@ jobs:
         name: website
         path: web/build
         retention-days: 14
+
+    - name: Trigger web/doc preview
+      run: curl -X POST https://pandas.pydata.org/preview/submit/$RUN_ID/$PR_ID/
+      env:
+        RUN_ID: ${{ github.run_id }}
+        PR_ID: ${{ github.event.pull_request.number }}
+      if: github.event_name == 'pull_request'

.github/workflows/preview-docs.yml

@@ -0,0 +1,22 @@
+name: Preview docs
+on:
+  issue_comment:
+    types: created
+
+permissions:
+  contents: read
+
+jobs:
+  preview_docs:
+    permissions:
+      issues: write
+      pull-requests: write
+    runs-on: ubuntu-22.04
+    steps:
+    - if: github.event.comment.body == '/preview'
+      run: |
+        if curl --output /dev/null --silent --head --fail "https://pandas.pydata.org/preview/${{ github.event.issue.number }}/"; then
+          curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"body": "Website preview of this PR available at: https://pandas.pydata.org/preview/${{ github.event.issue.number }}/"}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments
+        else
+          curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"body": "No preview found for PR #${{ github.event.issue.number }}. Did the docs build complete?"}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments
+        fi

.github/workflows/ubuntu.yml

@@ -15,9 +15,6 @@ on:
       - "doc/**"
       - "web/**"
 
-env:
-  PANDAS_CI: 1
-
 permissions:
   contents: read
 
@@ -33,6 +30,7 @@ jobs:
         env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml, actions-311.yaml]
         pattern: ["not single_cpu", "single_cpu"]
         pyarrow_version: ["8", "9", "10"]
+        pandas_ci: [1]
         include:
           - name: "Downstream Compat"
             env_file: actions-38-downstream_compat.yaml
@@ -77,6 +75,9 @@ jobs:
             env_file: actions-310-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
             test_args: "-W error::DeprecationWarning -W error::FutureWarning"
+            # TODO(cython3): Re-enable once next-beta(after beta 1) comes out
+            # There are some warnings failing the build with -werror
+            pandas_ci: 0
         exclude:
           - env_file: actions-38.yaml
             pyarrow_version: "8"
@@ -100,6 +101,7 @@ jobs:
       LC_ALL: ${{ matrix.lc_all || '' }}
       PANDAS_DATA_MANAGER: ${{ matrix.pandas_data_manager || 'block' }}
       PANDAS_COPY_ON_WRITE: ${{ matrix.pandas_copy_on_write || '0' }}
+      PANDAS_CI: ${{ matrix.pandas_ci }}
       TEST_ARGS: ${{ matrix.test_args || '' }}
       PYTEST_WORKERS: ${{ contains(matrix.pattern, 'not single_cpu') && 'auto' || '1' }}
       PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}
@@ -140,7 +142,7 @@ jobs:
           - 5432:5432
 
       moto:
-        image: motoserver/moto
+        image: motoserver/moto:4.1.4
         env:
           AWS_ACCESS_KEY_ID: foobar_key
           AWS_SECRET_ACCESS_KEY: foobar_secret

.pep8speaks.yml

@@ -28,7 +28,7 @@ repos:
         types_or: [python, pyi]
         additional_dependencies: [black==23.1.0]
 -   repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.244
+    rev: v0.0.253
     hooks:
     -   id: ruff
 -   repo: https://github.com/jendrikseipp/vulture
@@ -42,10 +42,10 @@ repos:
     rev: v2.2.2
     hooks:
     -   id: codespell
-        types_or: [python, rst, markdown]
+        types_or: [python, rst, markdown, cython, c]
         additional_dependencies: [tomli]
 -   repo: https://github.com/MarcoGorelli/cython-lint
-    rev: v0.12.4
+    rev: v0.12.5
     hooks:
     -   id: cython-lint
     -   id: double-quote-cython-strings
@@ -78,12 +78,12 @@ repos:
             '--filter=-readability/casting,-runtime/int,-build/include_subdir,-readability/fn_size'
         ]
 -   repo: https://github.com/pycqa/pylint
-    rev: v2.16.1
+    rev: v2.16.2
     hooks:
     -   id: pylint
         stages: [manual]
 -   repo: https://github.com/pycqa/pylint
-    rev: v2.16.1
+    rev: v2.16.2
     hooks:
     -   id: pylint
         alias: redefined-outer-name
@@ -160,14 +160,6 @@ repos:
         types: [pyi]
         args: [scripts/run_stubtest.py]
         stages: [manual]
-    -   id: flake8-rst
-        name: flake8-rst
-        description: Run flake8 on code snippets in docstrings or RST files
-        language: python
-        entry: flake8-rst
-        types: [rst]
-        args: [--filename=*.rst]
-        additional_dependencies: [flake8-rst==0.7.0, flake8==3.7.9]
     -   id: inconsistent-namespace-usage
         name: 'Check for inconsistent use of pandas namespace'
         entry: python scripts/check_for_inconsistent_pandas_namespace.py
@@ -342,11 +334,6 @@ repos:
             (?x)
             ^(asv_bench|pandas/tests|doc)/
             |scripts/validate_min_versions_in_sync\.py$
-    -   id: unwanted-patterns-strings-to-concatenate
-        name: Check for use of not concatenated strings
-        language: python
-        entry: python scripts/validate_unwanted_patterns.py --validation-type="strings_to_concatenate"
-        types_or: [python, cython]
     -   id: unwanted-patterns-strings-with-misplaced-whitespace
         name: Check for strings with misplaced spaces
         language: python
@@ -441,3 +428,10 @@ repos:
         types: [python]
         files: ^pandas/tests
         language: python
+    -   id: sort-whatsnew-items
+        name: sort whatsnew entries by issue number
+        entry: python -m scripts.sort_whatsnew_note
+        types: [rst]
+        language: python
+        files: ^doc/source/whatsnew/v
+        exclude: ^doc/source/whatsnew/v(0|1|2\.0\.0)

.pre-commit-config.yaml

@@ -42,7 +42,7 @@
     // followed by the pip installed packages).
     "matrix": {
         "numpy": [],
-        "Cython": ["0.29.32"],
+        "Cython": ["0.29.33"],
         "matplotlib": [],
         "sqlalchemy": [],
         "scipy": [],

asv_bench/asv.conf.json

@@ -770,6 +770,24 @@ def time_memory_usage_object_dtype(self):
         self.df2.memory_usage(deep=True)
 
 
+class Round:
+    def setup(self):
+        self.df = DataFrame(np.random.randn(10000, 10))
+        self.df_t = self.df.transpose(copy=True)
+
+    def time_round(self):
+        self.df.round()
+
+    def time_round_transposed(self):
+        self.df_t.round()
+
+    def peakmem_round(self):
+        self.df.round()
+
+    def peakmem_round_transposed(self):
+        self.df_t.round()
+
+
 class Where:
     params = (
         [True, False],

asv_bench/benchmarks/frame_methods.py

@@ -318,7 +318,7 @@ def setup(self, sep, decimal, float_precision):
             "".join([random.choice(string.digits) for _ in range(28)])
             for _ in range(15)
         ]
-        rows = sep.join([f"0{decimal}" + "{}"] * 3) + "\n"
+        rows = sep.join([f"0{decimal}{{}}"] * 3) + "\n"
         data = rows * 5
         data = data.format(*floats) * 200  # 1000 x 3 strings csv
         self.StringIO_input = StringIO(data)

asv_bench/benchmarks/io/csv.py

@@ -386,4 +386,33 @@ def time_to_numpy_copy(self):
         self.ser.to_numpy(copy=True)
 
 
+class Replace:
+    param_names = ["num_to_replace"]
+    params = [100, 1000]
+
+    def setup(self, num_to_replace):
+        N = 1_000_000
+        self.arr = np.random.randn(N)
+        self.arr1 = self.arr.copy()
+        np.random.shuffle(self.arr1)
+        self.ser = Series(self.arr)
+
+        self.to_replace_list = np.random.choice(self.arr, num_to_replace)
+        self.values_list = np.random.choice(self.arr1, num_to_replace)
+
+        self.replace_dict = dict(zip(self.to_replace_list, self.values_list))
+
+    def time_replace_dict(self, num_to_replace):
+        self.ser.replace(self.replace_dict)
+
+    def peakmem_replace_dict(self, num_to_replace):
+        self.ser.replace(self.replace_dict)
+
+    def time_replace_list(self, num_to_replace):
+        self.ser.replace(self.to_replace_list, self.values_list)
+
+    def peakmem_replace_list(self, num_to_replace):
+        self.ser.replace(self.to_replace_list, self.values_list)
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/series_methods.py

@@ -79,8 +79,8 @@ fi
 ### DOCSTRINGS ###
 if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
-    MSG='Validate docstrings (EX04, GL01, GL02, GL03, GL04, GL05, GL06, GL07, GL09, GL10, PR03, PR04, PR05, PR06, PR08, PR09, PR10, RT01, RT02, RT04, RT05, SA02, SA03, SA04, SS01, SS02, SS03, SS04, SS05, SS06)' ; echo $MSG
-    $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX04,GL01,GL02,GL03,GL04,GL05,GL06,GL07,GL09,GL10,PR03,PR04,PR05,PR06,PR08,PR09,PR10,RT01,RT02,RT04,RT05,SA02,SA03,SA04,SS01,SS02,SS03,SS04,SS05,SS06
+    MSG='Validate docstrings (EX02, EX04, GL01, GL02, GL03, GL04, GL05, GL06, GL07, GL09, GL10, PR03, PR04, PR05, PR06, PR08, PR09, PR10, RT01, RT02, RT04, RT05, SA02, SA03, SA04, SS01, SS02, SS03, SS04, SS05, SS06)' ; echo $MSG
+    $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX02,EX04,GL01,GL02,GL03,GL04,GL05,GL06,GL07,GL09,GL10,PR03,PR04,PR05,PR06,PR08,PR09,PR10,RT01,RT02,RT04,RT05,SA02,SA03,SA04,SS01,SS02,SS03,SS04,SS05,SS06
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Partially validate docstrings (EX01)' ;  echo $MSG
@@ -93,8 +93,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas.Series.item \
         pandas.Series.pipe \
         pandas.Series.mode \
-        pandas.Series.sem \
-        pandas.Series.skew \
         pandas.Series.is_unique \
         pandas.Series.is_monotonic_increasing \
         pandas.Series.is_monotonic_decreasing \
@@ -512,6 +510,7 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas.api.extensions.ExtensionArray._from_factorized \
         pandas.api.extensions.ExtensionArray._from_sequence \
         pandas.api.extensions.ExtensionArray._from_sequence_of_strings \
+        pandas.api.extensions.ExtensionArray._hash_pandas_object \
         pandas.api.extensions.ExtensionArray._reduce \
         pandas.api.extensions.ExtensionArray._values_for_argsort \
         pandas.api.extensions.ExtensionArray._values_for_factorize \
@@ -541,8 +540,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas.DataFrame.keys \
         pandas.DataFrame.iterrows \
         pandas.DataFrame.pipe \
-        pandas.DataFrame.sem \
-        pandas.DataFrame.skew \
         pandas.DataFrame.backfill \
         pandas.DataFrame.pad \
         pandas.DataFrame.swapaxes \
@@ -557,21 +554,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas.DataFrame.__dataframe__
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
-    MSG='Partially validate docstrings (EX02)' ;  echo $MSG
-    $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX02 --ignore_functions \
-        pandas.DataFrame.plot.line \
-        pandas.Series.plot.line \
-        pandas.api.types.is_datetime64_any_dtype \
-        pandas.api.types.is_datetime64_ns_dtype \
-        pandas.api.types.is_datetime64tz_dtype \
-        pandas.plotting.andrews_curves \
-        pandas.plotting.autocorrelation_plot \
-        pandas.plotting.lag_plot \
-        pandas.plotting.parallel_coordinates \
-        pandas.plotting.radviz \
-        pandas.tseries.frequencies.to_offset
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
 fi
 
 ### DOCUMENTATION NOTEBOOKS ###

ci/code_checks.sh

@@ -6,7 +6,7 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.32
+  - cython>=0.29.33
 
   # test dependencies
   - pytest>=7.0.0

ci/deps/actions-310.yaml

@@ -6,7 +6,7 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.32
+  - cython>=0.29.33
 
   # test dependencies
   - pytest>=7.0.0

ci/deps/actions-311.yaml

@@ -7,7 +7,7 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.32
+  - cython>=0.29.33
 
   # test dependencies
   - pytest>=7.0.0

ci/deps/actions-38-downstream_compat.yaml

@@ -8,7 +8,7 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.32
+  - cython>=0.29.33
 
   # test dependencies
   - pytest>=7.0.0

ci/deps/actions-38-minimum_versions.yaml

@@ -6,7 +6,7 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.32
+  - cython>=0.29.33
 
   # test dependencies
   - pytest>=7.0.0

ci/deps/actions-38.yaml

@@ -6,7 +6,7 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.32
+  - cython>=0.29.33
 
   # test dependencies
   - pytest>=7.0.0

ci/deps/actions-39.yaml

@@ -9,7 +9,7 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.32
+  - cython>=0.29.33
 
   # test dependencies
   - pytest>=7.0.0

ci/deps/actions-pypy-38.yaml

@@ -6,7 +6,7 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.32
+  - cython>=0.29.33
 
   # test dependencies
   - pytest>=7.0.0