Merge remote-tracking branch 'upstream/main' into pr/nikitaved/qssummer/format_iso

Author: MarcoGorelli

asv_bench/benchmarks/io/sql.py

@@ -38,7 +38,7 @@ def setup(self, connection):
             },
             index=tm.makeStringIndex(N),
         )
-        self.df.loc[1000:3000, "float_with_nan"] = np.nan
+        self.df.iloc[1000:3000, 1] = np.nan
         self.df["date"] = self.df["datetime"].dt.date
         self.df["time"] = self.df["datetime"].dt.time
         self.df["datetime_string"] = self.df["datetime"].astype(str)
@@ -88,7 +88,7 @@ def setup(self, connection, dtype):
             },
             index=tm.makeStringIndex(N),
         )
-        self.df.loc[1000:3000, "float_with_nan"] = np.nan
+        self.df.iloc[1000:3000, 1] = np.nan
         self.df["date"] = self.df["datetime"].dt.date
         self.df["time"] = self.df["datetime"].dt.time
         self.df["datetime_string"] = self.df["datetime"].astype(str)
@@ -117,7 +117,7 @@ def setup(self):
             },
             index=tm.makeStringIndex(N),
         )
-        self.df.loc[1000:3000, "float_with_nan"] = np.nan
+        self.df.iloc[1000:3000, 1] = np.nan
         self.df["date"] = self.df["datetime"].dt.date
         self.df["time"] = self.df["datetime"].dt.time
         self.df["datetime_string"] = self.df["datetime"].astype(str)
@@ -164,7 +164,7 @@ def setup(self, dtype):
             },
             index=tm.makeStringIndex(N),
         )
-        self.df.loc[1000:3000, "float_with_nan"] = np.nan
+        self.df.iloc[1000:3000, 1] = np.nan
         self.df["date"] = self.df["datetime"].dt.date
         self.df["time"] = self.df["datetime"].dt.time
         self.df["datetime_string"] = self.df["datetime"].astype(str)

asv_bench/benchmarks/reshape.py

@@ -36,7 +36,7 @@ def setup(self):
         self.df = DataFrame(data)
 
     def time_reshape_pivot_time_series(self):
-        self.df.pivot("date", "variable", "value")
+        self.df.pivot(index="date", columns="variable", values="value")
 
 
 class SimpleReshape:

doc/scripts/eval_performance.py

@@ -0,0 +1,108 @@
+from timeit import repeat as timeit
+
+import numpy as np
+import seaborn as sns
+
+from pandas import DataFrame
+
+setup_common = """from pandas import DataFrame
+from numpy.random import randn
+df = DataFrame(randn(%d, 3), columns=list('abc'))
+%s"""
+
+setup_with = "s = 'a + b * (c ** 2 + b ** 2 - a) / (a * c) ** 3'"
+
+
+def bench_with(n, times=10, repeat=3, engine="numexpr"):
+    return (
+        np.array(
+            timeit(
+                "df.eval(s, engine=%r)" % engine,
+                setup=setup_common % (n, setup_with),
+                repeat=repeat,
+                number=times,
+            )
+        )
+        / times
+    )
+
+
+setup_subset = "s = 'a <= b <= c ** 2 + b ** 2 - a and b > c'"
+
+
+def bench_subset(n, times=20, repeat=3, engine="numexpr"):
+    return (
+        np.array(
+            timeit(
+                "df.query(s, engine=%r)" % engine,
+                setup=setup_common % (n, setup_subset),
+                repeat=repeat,
+                number=times,
+            )
+        )
+        / times
+    )
+
+
+def bench(mn=3, mx=7, num=100, engines=("python", "numexpr"), verbose=False):
+    r = np.logspace(mn, mx, num=num).round().astype(int)
+
+    ev = DataFrame(np.empty((num, len(engines))), columns=engines)
+    qu = ev.copy(deep=True)
+
+    ev["size"] = qu["size"] = r
+
+    for engine in engines:
+        for i, n in enumerate(r):
+            if verbose & (i % 10 == 0):
+                print("engine: %r, i == %d" % (engine, i))
+            ev_times = bench_with(n, times=1, repeat=1, engine=engine)
+            ev.loc[i, engine] = np.mean(ev_times)
+            qu_times = bench_subset(n, times=1, repeat=1, engine=engine)
+            qu.loc[i, engine] = np.mean(qu_times)
+
+    return ev, qu
+
+
+def plot_perf(df, engines, title, filename=None):
+    from matplotlib.pyplot import figure
+
+    sns.set()
+    sns.set_palette("Set2")
+
+    fig = figure(figsize=(4, 3), dpi=120)
+    ax = fig.add_subplot(111)
+
+    for engine in engines:
+        ax.loglog(df["size"], df[engine], label=engine, lw=2)
+
+    ax.set_xlabel("Number of Rows")
+    ax.set_ylabel("Time (s)")
+    ax.set_title(title)
+    ax.legend(loc="best")
+    ax.tick_params(top=False, right=False)
+
+    fig.tight_layout()
+
+    if filename is not None:
+        fig.savefig(filename)
+
+
+if __name__ == "__main__":
+    import os
+
+    pandas_dir = os.path.dirname(
+        os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
+    )
+    static_path = os.path.join(pandas_dir, "doc", "source", "_static")
+
+    join = lambda p: os.path.join(static_path, p)
+
+    fn = join("eval-query-perf-data.h5")
+
+    engines = "python", "numexpr"
+
+    ev, qu = bench(verbose=True)  # only this one
+
+    plot_perf(ev, engines, "DataFrame.eval()", filename=join("eval-perf.png"))
+    plot_perf(qu, engines, "DataFrame.query()", filename=join("query-perf.png"))

doc/source/_static/eval-perf-small.png

@@ -236,7 +236,7 @@
 if ".dev" in version:
     switcher_version = "dev"
 elif "rc" in version:
-    switcher_version = version.split("rc")[0] + " (rc)"
+    switcher_version = version.split("rc", maxsplit=1)[0] + " (rc)"
 
 html_theme_options = {
     "external_links": [],

doc/source/_static/eval-perf.png

@@ -144,7 +144,7 @@ I want to add a new column to the ``DataFrame`` containing only the month of the
 
 By using ``Timestamp`` objects for dates, a lot of time-related
 properties are provided by pandas. For example the ``month``, but also
-``year``, ``weekofyear``, ``quarter``,… All of these properties are
+``year``, ``quarter``,… All of these properties are
 accessible by the ``dt`` accessor.
 
 .. raw:: html

doc/source/_static/query-perf-small.png

@@ -343,8 +343,6 @@ Time/date components
    DatetimeIndex.timetz
    DatetimeIndex.dayofyear
    DatetimeIndex.day_of_year
-   DatetimeIndex.weekofyear
-   DatetimeIndex.week
    DatetimeIndex.dayofweek
    DatetimeIndex.day_of_week
    DatetimeIndex.weekday