Skip to content

Commit 884b458

Browse files
author
Github Actions
committed
nabenabe0928: [feat] Add an object that realizes the perf over time viz (#331)
1 parent f9659f4 commit 884b458

File tree

54 files changed

+1131
-208
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+1131
-208
lines changed

‎development/_downloads/250b72c5b7995e4f210c161ca8b36741/example_plot_over_time.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
"""
2+
==============================
3+
Plot the Performance over Time
4+
==============================
5+
6+
Auto-Pytorch uses SMAC to fit individual machine learning algorithms
7+
and then ensembles them together using `Ensemble Selection
8+
<https://www.cs.cornell.edu/~caruana/ctp/ct.papers/caruana.icml04.icdm06long.pdf>`_.
9+
10+
The following examples shows how to plot both the performance
11+
of the individual models and their respective ensemble.
12+
13+
Additionally, as we are compatible with matplotlib,
14+
you can input any args or kwargs that are compatible with ax.plot.
15+
In the case when you would like to create multipanel visualization,
16+
please input plt.Axes obtained from matplotlib.pyplot.subplots.
17+
18+
"""
19+
import warnings
20+
21+
import numpy as np
22+
import pandas as pd
23+
24+
from sklearn import model_selection
25+
26+
import matplotlib.pyplot as plt
27+
28+
from autoPyTorch.api.tabular_classification import TabularClassificationTask
29+
from autoPyTorch.utils.results_visualizer import PlotSettingParams
30+
31+
32+
warnings.simplefilter(action='ignore', category=UserWarning)
33+
warnings.simplefilter(action='ignore', category=FutureWarning)
34+
35+
36+
############################################################################
37+
# Task Definition
38+
# ===============
39+
n_samples, dim = 100, 2
40+
X = np.random.random((n_samples, dim)) * 2 - 1
41+
y = ((X ** 2).sum(axis=-1) < 2 / np.pi).astype(np.int32)
42+
print(y)
43+
44+
X, y = pd.DataFrame(X), pd.DataFrame(y)
45+
X_train, X_test, y_train, y_test = model_selection.train_test_split(X, y)
46+
47+
############################################################################
48+
# API Instantiation and Searching
49+
# ===============================
50+
api = TabularClassificationTask(seed=42)
51+
52+
api.search(X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test,
53+
optimize_metric='accuracy', total_walltime_limit=120, func_eval_time_limit_secs=10)
54+
55+
############################################################################
56+
# Create Setting Parameters Object
57+
# ================================
58+
metric_name = 'accuracy'
59+
60+
params = PlotSettingParams(
61+
xscale='log',
62+
xlabel='Runtime',
63+
ylabel='Accuracy',
64+
title='Toy Example',
65+
show=False # If you would like to show, make it True
66+
)
67+
68+
############################################################################
69+
# Plot with the Specified Setting Parameters
70+
# ==========================================
71+
_, ax = plt.subplots()
72+
73+
api.plot_perf_over_time(
74+
ax=ax, # You do not have to provide.
75+
metric_name=metric_name,
76+
plot_setting_params=params,
77+
marker='*',
78+
markersize=10
79+
)
80+
81+
# plt.show() might cause issue depending on environments
82+
plt.savefig('example_plot_over_time.png')

‎development/_downloads/8410e1dea0ba7de106ac5e2a04cee84d/example_plot_over_time.ipynb‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": null,
6+
"metadata": {
7+
"collapsed": false
8+
},
9+
"outputs": [],
10+
"source": [
11+
"%matplotlib inline"
12+
]
13+
},
14+
{
15+
"cell_type": "markdown",
16+
"metadata": {},
17+
"source": [
18+
"\n# Plot the Performance over Time\n\nAuto-Pytorch uses SMAC to fit individual machine learning algorithms\nand then ensembles them together using `Ensemble Selection\n<https://www.cs.cornell.edu/~caruana/ctp/ct.papers/caruana.icml04.icdm06long.pdf>`_.\n\nThe following examples shows how to plot both the performance\nof the individual models and their respective ensemble.\n\nAdditionally, as we are compatible with matplotlib,\nyou can input any args or kwargs that are compatible with ax.plot.\nIn the case when you would like to create multipanel visualization,\nplease input plt.Axes obtained from matplotlib.pyplot.subplots.\n"
19+
]
20+
},
21+
{
22+
"cell_type": "code",
23+
"execution_count": null,
24+
"metadata": {
25+
"collapsed": false
26+
},
27+
"outputs": [],
28+
"source": [
29+
"import warnings\n\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn import model_selection\n\nimport matplotlib.pyplot as plt\n\nfrom autoPyTorch.api.tabular_classification import TabularClassificationTask\nfrom autoPyTorch.utils.results_visualizer import PlotSettingParams\n\n\nwarnings.simplefilter(action='ignore', category=UserWarning)\nwarnings.simplefilter(action='ignore', category=FutureWarning)"
30+
]
31+
},
32+
{
33+
"cell_type": "markdown",
34+
"metadata": {},
35+
"source": [
36+
"## Task Definition\n\n"
37+
]
38+
},
39+
{
40+
"cell_type": "code",
41+
"execution_count": null,
42+
"metadata": {
43+
"collapsed": false
44+
},
45+
"outputs": [],
46+
"source": [
47+
"n_samples, dim = 100, 2\nX = np.random.random((n_samples, dim)) * 2 - 1\ny = ((X ** 2).sum(axis=-1) < 2 / np.pi).astype(np.int32)\nprint(y)\n\nX, y = pd.DataFrame(X), pd.DataFrame(y)\nX_train, X_test, y_train, y_test = model_selection.train_test_split(X, y)"
48+
]
49+
},
50+
{
51+
"cell_type": "markdown",
52+
"metadata": {},
53+
"source": [
54+
"## API Instantiation and Searching\n\n"
55+
]
56+
},
57+
{
58+
"cell_type": "code",
59+
"execution_count": null,
60+
"metadata": {
61+
"collapsed": false
62+
},
63+
"outputs": [],
64+
"source": [
65+
"api = TabularClassificationTask(seed=42)\n\napi.search(X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test,\n optimize_metric='accuracy', total_walltime_limit=120, func_eval_time_limit_secs=10)"
66+
]
67+
},
68+
{
69+
"cell_type": "markdown",
70+
"metadata": {},
71+
"source": [
72+
"## Create Setting Parameters Object\n\n"
73+
]
74+
},
75+
{
76+
"cell_type": "code",
77+
"execution_count": null,
78+
"metadata": {
79+
"collapsed": false
80+
},
81+
"outputs": [],
82+
"source": [
83+
"metric_name = 'accuracy'\n\nparams = PlotSettingParams(\n xscale='log',\n xlabel='Runtime',\n ylabel='Accuracy',\n title='Toy Example',\n show=False # If you would like to show, make it True\n)"
84+
]
85+
},
86+
{
87+
"cell_type": "markdown",
88+
"metadata": {},
89+
"source": [
90+
"## Plot with the Specified Setting Parameters\n\n"
91+
]
92+
},
93+
{
94+
"cell_type": "code",
95+
"execution_count": null,
96+
"metadata": {
97+
"collapsed": false
98+
},
99+
"outputs": [],
100+
"source": [
101+
"_, ax = plt.subplots()\n\napi.plot_perf_over_time(\n ax=ax, # You do not have to provide.\n metric_name=metric_name,\n plot_setting_params=params,\n marker='*',\n markersize=10\n)\n\n# plt.show() might cause issue depending on environments\nplt.savefig('example_plot_over_time.png')"
102+
]
103+
}
104+
],
105+
"metadata": {
106+
"kernelspec": {
107+
"display_name": "Python 3",
108+
"language": "python",
109+
"name": "python3"
110+
},
111+
"language_info": {
112+
"codemirror_mode": {
113+
"name": "ipython",
114+
"version": 3
115+
},
116+
"file_extension": ".py",
117+
"mimetype": "text/x-python",
118+
"name": "python",
119+
"nbconvert_exporter": "python",
120+
"pygments_lexer": "ipython3",
121+
"version": "3.8.12"
122+
}
123+
},
124+
"nbformat": 4,
125+
"nbformat_minor": 0
126+
}

‎development/_downloads/bc82bea3a5dd7bdba60b65220891d9e5/examples_python.zip‎

2.7 KB
Binary file not shown.

‎development/_downloads/fb625db3c50d423b1b7881136ffdeec8/examples_jupyter.zip‎

4.36 KB
Binary file not shown.

‎development/_images/sphx_glr_example_plot_over_time_001.png‎

54.1 KB
Loading

‎development/_images/sphx_glr_example_plot_over_time_thumb.png‎

52.7 KB
Loading

‎development/_images/sphx_glr_example_visualization_001.png‎

-32 Bytes
Loading

‎development/_images/sphx_glr_example_visualization_002.png‎

456 Bytes
Loading

‎development/_images/sphx_glr_example_visualization_thumb.png‎

-958 Bytes
Loading

‎development/_modules/autoPyTorch/api/tabular_classification.html‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -486,7 +486,7 @@ <h1>Source code for autoPyTorch.api.tabular_classification</h1><div class="highl
486486
</p>
487487
<p>
488488
&copy; Copyright 2014-2021, Machine Learning Professorship Freiburg.<br/>
489-
Created using <a href="http://sphinx-doc.org/">Sphinx</a> 4.3.0.<br/>
489+
Created using <a href="http://sphinx-doc.org/">Sphinx</a> 4.3.1.<br/>
490490
</p>
491491
</div>
492492
</footer>

0 commit comments

Comments
 (0)