nabenabe0928: [feat] Support statistics print by adding results manager object (#334)

Author: Github Actions

development/_downloads/307f532dbef0476f85afc6b64b65f087/example_resampling_strategy.py

@@ -65,13 +65,15 @@
 ############################################################################
 # Print the final ensemble performance
 # ====================================
-print(api.run_history, api.trajectory)
 y_pred = api.predict(X_test)
 score = api.score(y_pred, y_test)
 print(score)
 # Print the final ensemble built by AutoPyTorch
 print(api.show_models())
 
+# Print statistics from search
+print(api.sprint_statistics())
+
 ############################################################################
 
 ############################################################################
@@ -98,13 +100,15 @@
 ############################################################################
 # Print the final ensemble performance
 # ====================================
-print(api.run_history, api.trajectory)
 y_pred = api.predict(X_test)
 score = api.score(y_pred, y_test)
 print(score)
 # Print the final ensemble built by AutoPyTorch
 print(api.show_models())
 
+# Print statistics from search
+print(api.sprint_statistics())
+
 ############################################################################
 
 ############################################################################
@@ -134,9 +138,11 @@
 ############################################################################
 # Print the final ensemble performance
 # ====================================
-print(api.run_history, api.trajectory)
 y_pred = api.predict(X_test)
 score = api.score(y_pred, y_test)
 print(score)
 # Print the final ensemble built by AutoPyTorch
 print(api.show_models())
+
+# Print statistics from search
+print(api.sprint_statistics())

development/_downloads/38ebc52de63d1626596d1647c695c721/example_tabular_regression.ipynb

@@ -98,7 +98,7 @@
       },
       "outputs": [],
       "source": [
-        "print(api.run_history, api.trajectory)\ny_pred = api.predict(X_test)\n\n# Rescale the Neural Network predictions into the original target range\nscore = api.score(y_pred, y_test)\n\nprint(score)\n# Print the final ensemble built by AutoPyTorch\nprint(api.show_models())"
+        "y_pred = api.predict(X_test)\n\n# Rescale the Neural Network predictions into the original target range\nscore = api.score(y_pred, y_test)\n\nprint(score)\n# Print the final ensemble built by AutoPyTorch\nprint(api.show_models())\n\n# Print statistics from search\nprint(api.sprint_statistics())"
       ]
     }
   ],

development/_downloads/3b0b756ccfcac69e6a1673e56f2f543f/example_visualization.ipynb

@@ -98,7 +98,7 @@
       },
       "outputs": [],
       "source": [
-        "# We will plot the search incumbent through time.\n\n# Collect the performance of individual machine learning algorithms\n# found by SMAC\nindividual_performances = []\nfor run_key, run_value in estimator.run_history.data.items():\n    if run_value.status != StatusType.SUCCESS:\n        # Ignore crashed runs\n        continue\n    individual_performances.append({\n        'Timestamp': pd.Timestamp(\n            time.strftime(\n                '%Y-%m-%d %H:%M:%S',\n                time.localtime(run_value.endtime)\n            )\n        ),\n        'single_best_optimization_accuracy': accuracy._optimum - run_value.cost,\n        'single_best_test_accuracy': np.nan if run_value.additional_info is None else\n        accuracy._optimum - run_value.additional_info['test_loss'],\n    })\nindividual_performance_frame = pd.DataFrame(individual_performances)\n\n# Collect the performance of the ensemble through time\n# This ensemble is built from the machine learning algorithms\n# found by SMAC\nensemble_performance_frame = pd.DataFrame(estimator.ensemble_performance_history)\n\n# As we are tracking the incumbent, we are interested in the cummax() performance\nensemble_performance_frame['ensemble_optimization_accuracy'] = ensemble_performance_frame[\n    'train_accuracy'\n].cummax()\nensemble_performance_frame['ensemble_test_accuracy'] = ensemble_performance_frame[\n    'test_accuracy'\n].cummax()\nensemble_performance_frame.drop(columns=['test_accuracy', 'train_accuracy'], inplace=True)\nindividual_performance_frame['single_best_optimization_accuracy'] = individual_performance_frame[\n    'single_best_optimization_accuracy'\n].cummax()\nindividual_performance_frame['single_best_test_accuracy'] = individual_performance_frame[\n    'single_best_test_accuracy'\n].cummax()\n\npd.merge(\n    ensemble_performance_frame,\n    individual_performance_frame,\n    on=\"Timestamp\", how='outer'\n).sort_values('Timestamp').fillna(method='ffill').plot(\n    x='Timestamp',\n    kind='line',\n    legend=True,\n    title='Auto-PyTorch accuracy over time',\n    grid=True,\n)\nplt.show()\n\n# We then can understand the importance of each input feature using\n# a permutation importance analysis. This is done as a proof of concept, to\n# showcase that we can leverage of scikit-learn API.\nresult = permutation_importance(estimator, X_train, y_train, n_repeats=5,\n                                scoring='accuracy',\n                                random_state=seed)\nsorted_idx = result.importances_mean.argsort()\n\nfig, ax = plt.subplots()\nax.boxplot(result.importances[sorted_idx].T,\n           vert=False, labels=X_test.columns[sorted_idx])\nax.set_title(\"Permutation Importances (Train set)\")\nfig.tight_layout()\nplt.show()"
+        "# We will plot the search incumbent through time.\n\n# Collect the performance of individual machine learning algorithms\n# found by SMAC\nindividual_performances = []\nfor run_key, run_value in estimator.run_history.data.items():\n    if run_value.status != StatusType.SUCCESS:\n        # Ignore crashed runs\n        continue\n    individual_performances.append({\n        'Timestamp': pd.Timestamp(\n            time.strftime(\n                '%Y-%m-%d %H:%M:%S',\n                time.localtime(run_value.endtime)\n            )\n        ),\n        'single_best_optimization_accuracy': accuracy._optimum - run_value.cost,\n        'single_best_test_accuracy': np.nan if run_value.additional_info is None else\n        accuracy._optimum - run_value.additional_info['test_loss']['accuracy'],\n    })\nindividual_performance_frame = pd.DataFrame(individual_performances)\n\n# Collect the performance of the ensemble through time\n# This ensemble is built from the machine learning algorithms\n# found by SMAC\nensemble_performance_frame = pd.DataFrame(estimator.ensemble_performance_history)\n\n# As we are tracking the incumbent, we are interested in the cummax() performance\nensemble_performance_frame['ensemble_optimization_accuracy'] = ensemble_performance_frame[\n    'train_accuracy'\n].cummax()\nensemble_performance_frame['ensemble_test_accuracy'] = ensemble_performance_frame[\n    'test_accuracy'\n].cummax()\nensemble_performance_frame.drop(columns=['test_accuracy', 'train_accuracy'], inplace=True)\nindividual_performance_frame['single_best_optimization_accuracy'] = individual_performance_frame[\n    'single_best_optimization_accuracy'\n].cummax()\nindividual_performance_frame['single_best_test_accuracy'] = individual_performance_frame[\n    'single_best_test_accuracy'\n].cummax()\n\npd.merge(\n    ensemble_performance_frame,\n    individual_performance_frame,\n    on=\"Timestamp\", how='outer'\n).sort_values('Timestamp').fillna(method='ffill').plot(\n    x='Timestamp',\n    kind='line',\n    legend=True,\n    title='Auto-PyTorch accuracy over time',\n    grid=True,\n)\nplt.show()\n\n# We then can understand the importance of each input feature using\n# a permutation importance analysis. This is done as a proof of concept, to\n# showcase that we can leverage of scikit-learn API.\nresult = permutation_importance(estimator, X_train, y_train, n_repeats=5,\n                                scoring='accuracy',\n                                random_state=seed)\nsorted_idx = result.importances_mean.argsort()\n\nfig, ax = plt.subplots()\nax.boxplot(result.importances[sorted_idx].T,\n           vert=False, labels=X_test.columns[sorted_idx])\nax.set_title(\"Permutation Importances (Train set)\")\nfig.tight_layout()\nplt.show()"
       ]
     }
   ],

development/_downloads/3f9c66ebcc4532fdade3cdaa4d769bde/example_custom_configuration_space.ipynb

@@ -26,7 +26,7 @@
       },
       "outputs": [],
       "source": [
-        "import os\nimport tempfile as tmp\nimport warnings\n\nos.environ['JOBLIB_TEMP_FOLDER'] = tmp.gettempdir()\nos.environ['OMP_NUM_THREADS'] = '1'\nos.environ['OPENBLAS_NUM_THREADS'] = '1'\nos.environ['MKL_NUM_THREADS'] = '1'\n\nwarnings.simplefilter(action='ignore', category=UserWarning)\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nimport sklearn.datasets\nimport sklearn.model_selection\n\nfrom autoPyTorch.api.tabular_classification import TabularClassificationTask\nfrom autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates\n\n\ndef get_search_space_updates():\n    \"\"\"\n    Search space updates to the task can be added using HyperparameterSearchSpaceUpdates\n    Returns:\n        HyperparameterSearchSpaceUpdates\n    \"\"\"\n    updates = HyperparameterSearchSpaceUpdates()\n    updates.append(node_name=\"data_loader\",\n                   hyperparameter=\"batch_size\",\n                   value_range=[16, 512],\n                   default_value=32)\n    updates.append(node_name=\"lr_scheduler\",\n                   hyperparameter=\"CosineAnnealingLR:T_max\",\n                   value_range=[50, 60],\n                   default_value=55)\n    updates.append(node_name='network_backbone',\n                   hyperparameter='ResNetBackbone:dropout',\n                   value_range=[0, 0.5],\n                   default_value=0.2)\n    return updates\n\n\nif __name__ == '__main__':\n\n    ############################################################################\n    # Data Loading\n    # ============\n    X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True)\n    X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(\n        X,\n        y,\n        random_state=1,\n    )\n\n    ############################################################################\n    # Build and fit a classifier with include components\n    # ==================================================\n    api = TabularClassificationTask(\n        search_space_updates=get_search_space_updates(),\n        include_components={'network_backbone': ['MLPBackbone', 'ResNetBackbone'],\n                            'encoder': ['OneHotEncoder']}\n    )\n\n    ############################################################################\n    # Search for an ensemble of machine learning algorithms\n    # =====================================================\n    api.search(\n        X_train=X_train.copy(),\n        y_train=y_train.copy(),\n        X_test=X_test.copy(),\n        y_test=y_test.copy(),\n        optimize_metric='accuracy',\n        total_walltime_limit=150,\n        func_eval_time_limit_secs=30\n    )\n\n    ############################################################################\n    # Print the final ensemble performance\n    # ====================================\n    print(api.run_history, api.trajectory)\n    y_pred = api.predict(X_test)\n    score = api.score(y_pred, y_test)\n    print(score)\n    print(api.show_models())\n\n    ############################################################################\n    # Build and fit a classifier with exclude components\n    # ==================================================\n    api = TabularClassificationTask(\n        search_space_updates=get_search_space_updates(),\n        exclude_components={'network_backbone': ['MLPBackbone'],\n                            'encoder': ['OneHotEncoder']}\n    )\n\n    ############################################################################\n    # Search for an ensemble of machine learning algorithms\n    # =====================================================\n    api.search(\n        X_train=X_train,\n        y_train=y_train,\n        X_test=X_test.copy(),\n        y_test=y_test.copy(),\n        optimize_metric='accuracy',\n        total_walltime_limit=150,\n        func_eval_time_limit_secs=30\n    )\n\n    ############################################################################\n    # Print the final ensemble performance\n    # ====================================\n    print(api.run_history, api.trajectory)\n    y_pred = api.predict(X_test)\n    score = api.score(y_pred, y_test)\n    print(score)\n    print(api.show_models())"
+        "import os\nimport tempfile as tmp\nimport warnings\n\nos.environ['JOBLIB_TEMP_FOLDER'] = tmp.gettempdir()\nos.environ['OMP_NUM_THREADS'] = '1'\nos.environ['OPENBLAS_NUM_THREADS'] = '1'\nos.environ['MKL_NUM_THREADS'] = '1'\n\nwarnings.simplefilter(action='ignore', category=UserWarning)\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nimport sklearn.datasets\nimport sklearn.model_selection\n\nfrom autoPyTorch.api.tabular_classification import TabularClassificationTask\nfrom autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates\n\n\ndef get_search_space_updates():\n    \"\"\"\n    Search space updates to the task can be added using HyperparameterSearchSpaceUpdates\n    Returns:\n        HyperparameterSearchSpaceUpdates\n    \"\"\"\n    updates = HyperparameterSearchSpaceUpdates()\n    updates.append(node_name=\"data_loader\",\n                   hyperparameter=\"batch_size\",\n                   value_range=[16, 512],\n                   default_value=32)\n    updates.append(node_name=\"lr_scheduler\",\n                   hyperparameter=\"CosineAnnealingLR:T_max\",\n                   value_range=[50, 60],\n                   default_value=55)\n    updates.append(node_name='network_backbone',\n                   hyperparameter='ResNetBackbone:dropout',\n                   value_range=[0, 0.5],\n                   default_value=0.2)\n    return updates\n\n\nif __name__ == '__main__':\n\n    ############################################################################\n    # Data Loading\n    # ============\n    X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True)\n    X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(\n        X,\n        y,\n        random_state=1,\n    )\n\n    ############################################################################\n    # Build and fit a classifier with include components\n    # ==================================================\n    api = TabularClassificationTask(\n        search_space_updates=get_search_space_updates(),\n        include_components={'network_backbone': ['MLPBackbone', 'ResNetBackbone'],\n                            'encoder': ['OneHotEncoder']}\n    )\n\n    ############################################################################\n    # Search for an ensemble of machine learning algorithms\n    # =====================================================\n    api.search(\n        X_train=X_train.copy(),\n        y_train=y_train.copy(),\n        X_test=X_test.copy(),\n        y_test=y_test.copy(),\n        optimize_metric='accuracy',\n        total_walltime_limit=150,\n        func_eval_time_limit_secs=30\n    )\n\n    ############################################################################\n    # Print the final ensemble performance\n    # ====================================\n    y_pred = api.predict(X_test)\n    score = api.score(y_pred, y_test)\n    print(score)\n    print(api.show_models())\n\n    # Print statistics from search\n    print(api.sprint_statistics())\n\n    ############################################################################\n    # Build and fit a classifier with exclude components\n    # ==================================================\n    api = TabularClassificationTask(\n        search_space_updates=get_search_space_updates(),\n        exclude_components={'network_backbone': ['MLPBackbone'],\n                            'encoder': ['OneHotEncoder']}\n    )\n\n    ############################################################################\n    # Search for an ensemble of machine learning algorithms\n    # =====================================================\n    api.search(\n        X_train=X_train,\n        y_train=y_train,\n        X_test=X_test.copy(),\n        y_test=y_test.copy(),\n        optimize_metric='accuracy',\n        total_walltime_limit=150,\n        func_eval_time_limit_secs=30\n    )\n\n    ############################################################################\n    # Print the final ensemble performance\n    # ====================================\n    y_pred = api.predict(X_test)\n    score = api.score(y_pred, y_test)\n    print(score)\n    print(api.show_models())\n\n    # Print statistics from search\n    print(api.sprint_statistics())"
       ]
     }
   ],

development/_downloads/4cbefcc88d68bf84110d315dc5fdb8e1/example_resampling_strategy.ipynb

@@ -98,7 +98,7 @@
       },
       "outputs": [],
       "source": [
-        "print(api.run_history, api.trajectory)\ny_pred = api.predict(X_test)\nscore = api.score(y_pred, y_test)\nprint(score)\n# Print the final ensemble built by AutoPyTorch\nprint(api.show_models())"
+        "y_pred = api.predict(X_test)\nscore = api.score(y_pred, y_test)\nprint(score)\n# Print the final ensemble built by AutoPyTorch\nprint(api.show_models())\n\n# Print statistics from search\nprint(api.sprint_statistics())"
       ]
     },
     {
@@ -152,7 +152,7 @@
       },
       "outputs": [],
       "source": [
-        "print(api.run_history, api.trajectory)\ny_pred = api.predict(X_test)\nscore = api.score(y_pred, y_test)\nprint(score)\n# Print the final ensemble built by AutoPyTorch\nprint(api.show_models())"
+        "y_pred = api.predict(X_test)\nscore = api.score(y_pred, y_test)\nprint(score)\n# Print the final ensemble built by AutoPyTorch\nprint(api.show_models())\n\n# Print statistics from search\nprint(api.sprint_statistics())"
       ]
     },
     {
@@ -206,7 +206,7 @@
       },
       "outputs": [],
       "source": [
-        "print(api.run_history, api.trajectory)\ny_pred = api.predict(X_test)\nscore = api.score(y_pred, y_test)\nprint(score)\n# Print the final ensemble built by AutoPyTorch\nprint(api.show_models())"
+        "y_pred = api.predict(X_test)\nscore = api.score(y_pred, y_test)\nprint(score)\n# Print the final ensemble built by AutoPyTorch\nprint(api.show_models())\n\n# Print statistics from search\nprint(api.sprint_statistics())"
       ]
     }
   ],

development/_downloads/5517f58caf37183d75ea0dc7cedf8b58/example_custom_configuration_space.py

@@ -94,12 +94,14 @@ def get_search_space_updates():
     ############################################################################
     # Print the final ensemble performance
     # ====================================
-    print(api.run_history, api.trajectory)
     y_pred = api.predict(X_test)
     score = api.score(y_pred, y_test)
     print(score)
     print(api.show_models())
 
+    # Print statistics from search
+    print(api.sprint_statistics())
+
     ############################################################################
     # Build and fit a classifier with exclude components
     # ==================================================
@@ -125,8 +127,10 @@ def get_search_space_updates():
     ############################################################################
     # Print the final ensemble performance
     # ====================================
-    print(api.run_history, api.trajectory)
     y_pred = api.predict(X_test)
     score = api.score(y_pred, y_test)
     print(score)
     print(api.show_models())
+
+    # Print statistics from search
+    print(api.sprint_statistics())

development/_downloads/6e1b08ae2c5784892d9641cc2992248d/example_run_with_portfolio.py

@@ -63,9 +63,11 @@
     ############################################################################
     # Print the final ensemble performance
     # ====================================
-    print(api.run_history, api.trajectory)
     y_pred = api.predict(X_test)
     score = api.score(y_pred, y_test)
     print(score)
     # Print the final ensemble built by AutoPyTorch
     print(api.show_models())
+
+    # Print statistics from search
+    print(api.sprint_statistics())