Merge branch 'main' into raise-on-parse-int-overflow

Author: SandroCasagrande

.github/workflows/scorecards.yml

@@ -29,7 +29,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/[email protected].3
+        uses: ossf/[email protected].6
         with:
           results_file: results.sarif
           results_format: sarif

.github/workflows/wheels.yml

@@ -52,7 +52,7 @@ jobs:
         - [windows-2019, win_amd64]
         - [windows-2019, win32]
         # TODO: support PyPy?
-        python: [["cp38", "3.8"], ["cp39", "3.9"], ["cp310", "3.10"],  ["cp311", "3.11-dev"]]# "pp38", "pp39"]
+        python: [["cp38", "3.8"], ["cp39", "3.9"], ["cp310", "3.10"],  ["cp311", "3.11"]]# "pp38", "pp39"]
     env:
       IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
       IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
@@ -73,7 +73,7 @@ jobs:
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
 
       # Used to test the built wheels
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python[1] }}
 

.pre-commit-config.yaml

@@ -102,7 +102,7 @@ repos:
         types: [python]
         stages: [manual]
         additional_dependencies: &pyright_dependencies
-        - [email protected].264
+        - [email protected].276
     -   id: pyright_reportGeneralTypeIssues
         # note: assumes python env is setup and activated
         name: pyright reportGeneralTypeIssues

asv_bench/asv.conf.json

@@ -54,7 +54,6 @@
         "openpyxl": [],
         "xlsxwriter": [],
         "xlrd": [],
-        "xlwt": [],
         "odfpy": [],
         "jinja2": [],
     },

asv_bench/benchmarks/array.py

@@ -44,6 +44,24 @@ def time_from_integer_array(self):
         pd.array(self.values_integer, dtype="Int64")
 
 
+class StringArray:
+    def setup(self):
+        N = 100_000
+        values = tm.rands_array(3, N)
+        self.values_obj = np.array(values, dtype="object")
+        self.values_str = np.array(values, dtype="U")
+        self.values_list = values.tolist()
+
+    def time_from_np_object_array(self):
+        pd.array(self.values_obj, dtype="string")
+
+    def time_from_np_str_array(self):
+        pd.array(self.values_str, dtype="string")
+
+    def time_from_list(self):
+        pd.array(self.values_list, dtype="string")
+
+
 class ArrowStringArray:
 
     params = [False, True]

asv_bench/benchmarks/groupby.py

@@ -14,6 +14,7 @@
     Timestamp,
     date_range,
     period_range,
+    to_timedelta,
 )
 
 from .pandas_vb_common import tm
@@ -35,7 +36,6 @@
         "pct_change",
         "min",
         "var",
-        "mad",
         "describe",
         "std",
         "quantile",
@@ -52,7 +52,6 @@
         "cummax",
         "pct_change",
         "var",
-        "mad",
         "describe",
         "std",
     },
@@ -311,7 +310,7 @@ def time_different_python_functions_multicol(self, df):
         df.groupby(["key1", "key2"]).agg([sum, min, max])
 
     def time_different_python_functions_singlecol(self, df):
-        df.groupby("key1").agg([sum, min, max])
+        df.groupby("key1")[["value1", "value2", "value3"]].agg([sum, min, max])
 
 
 class GroupStrings:
@@ -437,7 +436,6 @@ class GroupByMethods:
             "first",
             "head",
             "last",
-            "mad",
             "max",
             "min",
             "median",
@@ -483,7 +481,7 @@ def setup(self, dtype, method, application, ncols):
 
         if method == "describe":
             ngroups = 20
-        elif method in ["mad", "skew"]:
+        elif method == "skew":
             ngroups = 100
         else:
             ngroups = 1000
@@ -685,7 +683,7 @@ class String:
     def setup(self, dtype, method):
         cols = list("abcdefghjkl")
         self.df = DataFrame(
-            np.random.randint(0, 100, size=(1_000_000, len(cols))),
+            np.random.randint(0, 100, size=(10_000, len(cols))),
             columns=cols,
             dtype=dtype,
         )
@@ -990,4 +988,31 @@ def time_sample_weights(self):
         self.df.groupby(self.groups).sample(n=1, weights=self.weights)
 
 
+class Resample:
+    # GH 28635
+    def setup(self):
+        num_timedeltas = 20_000
+        num_groups = 3
+
+        index = MultiIndex.from_product(
+            [
+                np.arange(num_groups),
+                to_timedelta(np.arange(num_timedeltas), unit="s"),
+            ],
+            names=["groups", "timedeltas"],
+        )
+        data = np.random.randint(0, 1000, size=(len(index)))
+
+        self.df = DataFrame(data, index=index).reset_index("timedeltas")
+        self.df_multiindex = DataFrame(data, index=index)
+
+    def time_resample(self):
+        self.df.groupby(level="groups").resample("10s", on="timedeltas").mean()
+
+    def time_resample_multiindex(self):
+        self.df_multiindex.groupby(level="groups").resample(
+            "10s", level="timedeltas"
+        ).mean()
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/io/excel.py

@@ -33,7 +33,7 @@ def _generate_dataframe():
 
 class WriteExcel:
 
-    params = ["openpyxl", "xlsxwriter", "xlwt"]
+    params = ["openpyxl", "xlsxwriter"]
     param_names = ["engine"]
 
     def setup(self, engine):
@@ -68,10 +68,9 @@ def time_write_excel_style(self, engine):
 
 class ReadExcel:
 
-    params = ["xlrd", "openpyxl", "odf"]
+    params = ["openpyxl", "odf"]
     param_names = ["engine"]
     fname_excel = "spreadsheet.xlsx"
-    fname_excel_xls = "spreadsheet.xls"
     fname_odf = "spreadsheet.ods"
 
     def _create_odf(self):
@@ -92,13 +91,10 @@ def setup_cache(self):
         self.df = _generate_dataframe()
 
         self.df.to_excel(self.fname_excel, sheet_name="Sheet1")
-        self.df.to_excel(self.fname_excel_xls, sheet_name="Sheet1")
         self._create_odf()
 
     def time_read_excel(self, engine):
-        if engine == "xlrd":
-            fname = self.fname_excel_xls
-        elif engine == "odf":
+        if engine == "odf":
             fname = self.fname_odf
         else:
             fname = self.fname_excel
@@ -107,9 +103,7 @@ def time_read_excel(self, engine):
 
 class ReadExcelNRows(ReadExcel):
     def time_read_excel(self, engine):
-        if engine == "xlrd":
-            fname = self.fname_excel_xls
-        elif engine == "odf":
+        if engine == "odf":
             fname = self.fname_odf
         else:
             fname = self.fname_excel

asv_bench/benchmarks/io/sql.py

@@ -38,7 +38,7 @@ def setup(self, connection):
             },
             index=tm.makeStringIndex(N),
         )
-        self.df.loc[1000:3000, "float_with_nan"] = np.nan
+        self.df.iloc[1000:3000, 1] = np.nan
         self.df["date"] = self.df["datetime"].dt.date
         self.df["time"] = self.df["datetime"].dt.time
         self.df["datetime_string"] = self.df["datetime"].astype(str)
@@ -88,7 +88,7 @@ def setup(self, connection, dtype):
             },
             index=tm.makeStringIndex(N),
         )
-        self.df.loc[1000:3000, "float_with_nan"] = np.nan
+        self.df.iloc[1000:3000, 1] = np.nan
         self.df["date"] = self.df["datetime"].dt.date
         self.df["time"] = self.df["datetime"].dt.time
         self.df["datetime_string"] = self.df["datetime"].astype(str)
@@ -117,7 +117,7 @@ def setup(self):
             },
             index=tm.makeStringIndex(N),
         )
-        self.df.loc[1000:3000, "float_with_nan"] = np.nan
+        self.df.iloc[1000:3000, 1] = np.nan
         self.df["date"] = self.df["datetime"].dt.date
         self.df["time"] = self.df["datetime"].dt.time
         self.df["datetime_string"] = self.df["datetime"].astype(str)
@@ -164,7 +164,7 @@ def setup(self, dtype):
             },
             index=tm.makeStringIndex(N),
         )
-        self.df.loc[1000:3000, "float_with_nan"] = np.nan
+        self.df.iloc[1000:3000, 1] = np.nan
         self.df["date"] = self.df["datetime"].dt.date
         self.df["time"] = self.df["datetime"].dt.time
         self.df["datetime_string"] = self.df["datetime"].astype(str)

asv_bench/benchmarks/io/stata.py

@@ -38,13 +38,13 @@ def setup(self, convert_dates):
         )
         self.df["float32_"] = np.array(np.random.randn(N), dtype=np.float32)
         self.convert_dates = {"index": convert_dates}
-        self.df.to_stata(self.fname, self.convert_dates)
+        self.df.to_stata(self.fname, convert_dates=self.convert_dates)
 
     def time_read_stata(self, convert_dates):
         read_stata(self.fname)
 
     def time_write_stata(self, convert_dates):
-        self.df.to_stata(self.fname, self.convert_dates)
+        self.df.to_stata(self.fname, convert_dates=self.convert_dates)
 
 
 class StataMissing(Stata):
@@ -54,7 +54,7 @@ def setup(self, convert_dates):
             missing_data = np.random.randn(self.N)
             missing_data[missing_data < 0] = np.nan
             self.df[f"missing_{i}"] = missing_data
-        self.df.to_stata(self.fname, self.convert_dates)
+        self.df.to_stata(self.fname, convert_dates=self.convert_dates)
 
 
 from ..pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/io/style.py

@@ -83,11 +83,11 @@ def _style_format(self):
     def _style_apply_format_hide(self):
         self.st = self.df.style.applymap(lambda v: "color: red;")
         self.st.format("{:.3f}")
-        self.st.hide_index(self.st.index[1:])
-        self.st.hide_columns(self.st.columns[1:])
+        self.st.hide(self.st.index[1:], axis=0)
+        self.st.hide(self.st.columns[1:], axis=1)
 
     def _style_tooltips(self):
         ttips = DataFrame("abc", index=self.df.index[::2], columns=self.df.columns[::2])
         self.st = self.df.style.set_tooltips(ttips)
-        self.st.hide_index(self.st.index[12:])
-        self.st.hide_columns(self.st.columns[12:])
+        self.st.hide(self.st.index[12:], axis=0)
+        self.st.hide(self.st.columns[12:], axis=1)