Address comments from shuhei

Author: ravinkohli

autoPyTorch/evaluation/abstract_evaluator.py

@@ -16,11 +16,6 @@
 
 from smac.tae import StatusType
 
-import autoPyTorch.pipeline.image_classification
-import autoPyTorch.pipeline.tabular_classification
-import autoPyTorch.pipeline.tabular_regression
-import autoPyTorch.pipeline.traditional_tabular_classification
-import autoPyTorch.pipeline.traditional_tabular_regression
 from autoPyTorch.automl_common.common.utils.backend import Backend
 from autoPyTorch.constants import (
     CLASSIFICATION_TASKS,
@@ -42,6 +37,11 @@
     calculate_loss,
     get_metrics,
 )
+from autoPyTorch.pipeline.image_classification import ImageClassificationPipeline
+from autoPyTorch.pipeline.tabular_classification import TabularClassificationPipeline
+from autoPyTorch.pipeline.tabular_regression import TabularRegressionPipeline
+from autoPyTorch.pipeline.traditional_tabular_classification import TraditionalTabularClassificationPipeline
+from autoPyTorch.pipeline.traditional_tabular_regression import TraditionalTabularRegressionPipeline
 from autoPyTorch.utils.common import subsampler
 from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates
 from autoPyTorch.utils.logging_ import PicklableClientLogger, get_named_client_logger
@@ -65,7 +65,7 @@ class MyTraditionalTabularClassificationPipeline(BaseEstimator):
     Attributes:
         dataset_properties (Dict[str, Any]):
             A dictionary containing dataset specific information
-        random_state (Optional[Union[int, np.random.RandomState]]):
+        random_state (Optional[np.random.RandomState]):
             Object that contains a seed and allows for reproducible results
         init_params  (Optional[Dict]):
             An optional dictionary that is passed to the pipeline's steps. It complies
@@ -74,15 +74,14 @@ class MyTraditionalTabularClassificationPipeline(BaseEstimator):
 
     def __init__(self, config: str,
                  dataset_properties: Dict[str, Any],
-                 random_state: Optional[Union[int, np.random.RandomState]] = None,
+                 random_state: Optional[np.random.RandomState] = None,
                  init_params: Optional[Dict] = None):
         self.config = config
         self.dataset_properties = dataset_properties
         self.random_state = random_state
         self.init_params = init_params
-        self.pipeline = autoPyTorch.pipeline.traditional_tabular_classification.\
-            TraditionalTabularClassificationPipeline(dataset_properties=dataset_properties,
-                                                     random_state=self.random_state)
+        self.pipeline = TraditionalTabularClassificationPipeline(dataset_properties=dataset_properties,
+                                                                 random_state=self.random_state)
         configuration_space = self.pipeline.get_hyperparameter_search_space()
         default_configuration = configuration_space.get_default_configuration().get_dictionary()
         default_configuration['model_trainer:tabular_traditional_model:traditional_learner'] = config
@@ -120,8 +119,7 @@ def get_pipeline_representation(self) -> Dict[str, str]:
 
     @staticmethod
     def get_default_pipeline_options() -> Dict[str, Any]:
-        return autoPyTorch.pipeline.traditional_tabular_classification. \
-            TraditionalTabularClassificationPipeline.get_default_pipeline_options()
+        return TraditionalTabularClassificationPipeline.get_default_pipeline_options()
 
 
 class MyTraditionalTabularRegressionPipeline(BaseEstimator):
@@ -136,23 +134,22 @@ class MyTraditionalTabularRegressionPipeline(BaseEstimator):
     Attributes:
         dataset_properties (Dict[str, Any]):
             A dictionary containing dataset specific information
-        random_state (Optional[Union[int, np.random.RandomState]]):
+        random_state (Optional[np.random.RandomState]):
             Object that contains a seed and allows for reproducible results
         init_params  (Optional[Dict]):
             An optional dictionary that is passed to the pipeline's steps. It complies
             a similar function as the kwargs
     """
     def __init__(self, config: str,
                  dataset_properties: Dict[str, Any],
-                 random_state: Optional[Union[int, np.random.RandomState]] = None,
+                 random_state: Optional[np.random.RandomState] = None,
                  init_params: Optional[Dict] = None):
         self.config = config
         self.dataset_properties = dataset_properties
         self.random_state = random_state
         self.init_params = init_params
-        self.pipeline = autoPyTorch.pipeline.traditional_tabular_regression.\
-            TraditionalTabularRegressionPipeline(dataset_properties=dataset_properties,
-                                                 random_state=self.random_state)
+        self.pipeline = TraditionalTabularRegressionPipeline(dataset_properties=dataset_properties,
+                                                             random_state=self.random_state)
         configuration_space = self.pipeline.get_hyperparameter_search_space()
         default_configuration = configuration_space.get_default_configuration().get_dictionary()
         default_configuration['model_trainer:tabular_traditional_model:traditional_learner'] = config
@@ -185,8 +182,7 @@ def get_pipeline_representation(self) -> Dict[str, str]:
 
     @staticmethod
     def get_default_pipeline_options() -> Dict[str, Any]:
-        return autoPyTorch.pipeline.traditional_tabular_regression. \
-            TraditionalTabularRegressionPipeline.get_default_pipeline_options()
+        return TraditionalTabularRegressionPipeline.get_default_pipeline_options()
 
 
 class DummyClassificationPipeline(DummyClassifier):
@@ -460,7 +456,7 @@ def __init__(self, backend: Backend,
             elif isinstance(self.configuration, str):
                 self.pipeline_class = MyTraditionalTabularRegressionPipeline
             elif isinstance(self.configuration, Configuration):
-                self.pipeline_class = autoPyTorch.pipeline.tabular_regression.TabularRegressionPipeline
+                self.pipeline_class = TabularRegressionPipeline
             else:
                 raise ValueError('task {} not available'.format(self.task_type))
             self.predict_function = self._predict_regression
@@ -474,9 +470,9 @@ def __init__(self, backend: Backend,
                     raise ValueError("Only tabular tasks are currently supported with traditional methods")
             elif isinstance(self.configuration, Configuration):
                 if self.task_type in TABULAR_TASKS:
-                    self.pipeline_class = autoPyTorch.pipeline.tabular_classification.TabularClassificationPipeline
+                    self.pipeline_class = TabularClassificationPipeline
                 elif self.task_type in IMAGE_TASKS:
-                    self.pipeline_class = autoPyTorch.pipeline.image_classification.ImageClassificationPipeline
+                    self.pipeline_class = ImageClassificationPipeline
                 else:
                     raise ValueError('task {} not available'.format(self.task_type))
             self.predict_function = self._predict_proba

autoPyTorch/pipeline/components/setup/traditional_ml/traditional_learner/learners.py

@@ -20,7 +20,7 @@
 from autoPyTorch.pipeline.components.setup.traditional_ml.traditional_learner.base_traditional_learner import \
     BaseTraditionalLearner
 from autoPyTorch.pipeline.components.setup.traditional_ml.traditional_learner.utils import (
-    AUTOPYTORCH_TO_CATBOOST_METRICS
+    AutoPyTorchToCatboostMetrics
 )
 
 
@@ -112,11 +112,11 @@ def _prepare_model(self,
                        y_train: np.ndarray
                        ) -> None:
         if not self.is_classification:
-            self.config['eval_metric'] = AUTOPYTORCH_TO_CATBOOST_METRICS.get(self.metric.name, 'R2')
+            self.config['eval_metric'] = AutoPyTorchToCatboostMetrics[self.metric.name].value
             # CatBoost Cannot handle a random state object, just the seed
             self.model = CatBoostRegressor(**self.config, random_state=self.random_state.get_state()[1][0])
         else:
-            self.config['eval_metric'] = AUTOPYTORCH_TO_CATBOOST_METRICS.get(self.metric.name, 'Accuracy')
+            self.config['eval_metric'] = AutoPyTorchToCatboostMetrics[self.metric.name].value
             # CatBoost Cannot handle a random state object, just the seed
             self.model = CatBoostClassifier(**self.config, random_state=self.random_state.get_state()[1][0])
 

autoPyTorch/pipeline/components/setup/traditional_ml/traditional_learner/utils.py

@@ -1,13 +1,15 @@
-AUTOPYTORCH_TO_CATBOOST_METRICS = {
-    "mean_absolute_error": "MAE",
-    "root_mean_squared_error": "RMSE",
-    "mean_squared_log_error": "MSLE",
-    "r2": "R2",
-    "accuracy": "Accuracy",
-    "balanced_accuracy": "BalancedAccuracy",
-    "f1": "F1",
-    "roc_auc": "AUC",
-    "precision": "Precision",
-    "recall": "Recall",
-    "log_loss": "Logloss"
-}
+from enum import Enum
+
+
+class AutoPyTorchToCatboostMetrics(Enum):
+    mean_absolute_error = "MAE"
+    root_mean_squared_error = "RMSE"
+    mean_squared_log_error = "MSLE"
+    r2 = "R2"
+    accuracy = "Accuracy"
+    balanced_accuracy = "BalancedAccuracy"
+    f1 = "F1"
+    roc_auc = "AUC"
+    precision = "Precision"
+    recall = "Recall"
+    log_loss = "Logloss"

autoPyTorch/pipeline/components/training/trainer/__init__.py

@@ -250,7 +250,7 @@ def _fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> 'TrainerChoic
         # Support additional user metrics
         additional_metrics = X['additional_metrics'] if 'additional_metrics' in X else None
         if 'optimize_metric' in X:
-            additional_metrics = additional_metrics.append(X['optimize_metric']) if additional_metrics is not None\
+            additional_metrics = additional_metrics.append(X['optimize_metric']) if additional_metrics is not None \
                 else [X['optimize_metric']]
         additional_losses = X['additional_losses'] if 'additional_losses' in X else None
         self.choice.prepare(

autoPyTorch/pipeline/traditional_tabular_regression.py

@@ -9,7 +9,7 @@
 
 from autoPyTorch.pipeline.base_pipeline import BasePipeline
 from autoPyTorch.pipeline.components.base_choice import autoPyTorchChoice
-from autoPyTorch.pipeline.components.setup.traditional_ml.base_model_choice import ModelChoice
+from autoPyTorch.pipeline.components.setup.traditional_ml import ModelChoice
 
 
 class TraditionalTabularRegressionPipeline(RegressorMixin, BasePipeline):

test/test_api/test_api.py

@@ -32,8 +32,8 @@
     HoldoutValTypes,
 )
 from autoPyTorch.datasets.tabular_dataset import TabularDataset
-from autoPyTorch.pipeline.components.setup.traditional_ml.traditional_learner import _traditional_learners
 from autoPyTorch.optimizer.smbo import AutoMLSMBO
+from autoPyTorch.pipeline.components.setup.traditional_ml.traditional_learner import _traditional_learners
 from autoPyTorch.pipeline.components.training.metrics.metrics import accuracy
 
 

test/test_pipeline/components/setup/test_setup_traditional_models.py

@@ -7,7 +7,7 @@
 
 import pytest
 
-from autoPyTorch.pipeline.components.setup.traditional_ml.base_model_choice import ModelChoice
+from autoPyTorch.pipeline.components.setup.traditional_ml import ModelChoice
 from autoPyTorch.pipeline.components.setup.traditional_ml.tabular_traditional_model import TabularTraditionalModel
 
 
@@ -83,52 +83,51 @@ def test_get_set_config_space(self, dataset_properties):
                                                     "regression_categorical_only",
                                                     "regression_numerical_and_categorical"
                                                     ], indirect=True)
-class TestTraditionalModels:
-    def test_model_fit_predict_score(self, traditional_learner, fit_dictionary_tabular):
-
-        if len(fit_dictionary_tabular['dataset_properties']['numerical_columns']) == 0 and traditional_learner == 'knn':
-            pytest.skip("knn can not work with categorical only data")
-
-        model = TabularTraditionalModel(traditional_learner=traditional_learner)
-
-        blockPrint()
-        model.fit(X=fit_dictionary_tabular)
-        enablePrint()
-
-        assert isinstance(model.fit_output, dict)
-        assert 'val_preds' in model.fit_output.keys()
-        assert isinstance(model.fit_output['val_preds'], list)
-        assert len(model.fit_output['val_preds']) == len(fit_dictionary_tabular['val_indices'])
-        if model.model.is_classification:
-            assert len(model.fit_output['val_preds'][0]) == len(np.unique(fit_dictionary_tabular['y_train']))
-        assert len(np.argwhere(0 > np.array(model.fit_output['val_preds']).all() > 1)) == 0
-        assert 'labels' in model.fit_output.keys()
-        assert len(model.fit_output['labels']) == len(fit_dictionary_tabular['val_indices'])
-        assert 'train_score' in model.fit_output.keys()
-        assert isinstance(model.fit_output['train_score'], float)
-        assert 'val_score' in model.fit_output.keys()
-        assert isinstance(model.fit_output['val_score'], float)
-
-        # Test if traditional model can predict on val set
-        if model.model.is_classification:
-            y_pred = model.predict_proba(fit_dictionary_tabular['X_train'][fit_dictionary_tabular['val_indices']])
-        else:
-            y_pred = model.predict(fit_dictionary_tabular['X_train'][fit_dictionary_tabular['val_indices']])
-
-        assert np.allclose(y_pred.squeeze(), model.fit_output['val_preds'], atol=1e-04)
-        assert y_pred.shape[0] == len(fit_dictionary_tabular['val_indices'])
-        # Test if classifier can score and
-        # the result is same as in results
-        score = model.score(fit_dictionary_tabular['X_train'][fit_dictionary_tabular['val_indices']],
-                            fit_dictionary_tabular['y_train'][fit_dictionary_tabular['val_indices']])
-        assert np.allclose(score, model.fit_output['val_score'], atol=1e-6)
-
-        if sys.version_info >= (3, 7):
-            dump_file = os.path.join(fit_dictionary_tabular['backend'].temporary_directory, 'dump.pkl')
-
-            with open(dump_file, 'wb') as f:
-                pickle.dump(model, f)
-
-            with open(dump_file, 'rb') as f:
-                restored_estimator = pickle.load(f)
-            restored_estimator.predict(fit_dictionary_tabular['X_train'])
+def test_model_fit_predict_score(traditional_learner, fit_dictionary_tabular):
+
+    if len(fit_dictionary_tabular['dataset_properties']['numerical_columns']) == 0 and traditional_learner == 'knn':
+        pytest.skip("knn can not work with categorical only data")
+
+    model = TabularTraditionalModel(traditional_learner=traditional_learner)
+
+    blockPrint()
+    model.fit(X=fit_dictionary_tabular)
+    enablePrint()
+
+    assert isinstance(model.fit_output, dict)
+    assert 'val_preds' in model.fit_output.keys()
+    assert isinstance(model.fit_output['val_preds'], list)
+    assert len(model.fit_output['val_preds']) == len(fit_dictionary_tabular['val_indices'])
+    if model.model.is_classification:
+        assert len(model.fit_output['val_preds'][0]) == len(np.unique(fit_dictionary_tabular['y_train']))
+    assert len(np.argwhere(0 > np.array(model.fit_output['val_preds']).all() > 1)) == 0
+    assert 'labels' in model.fit_output.keys()
+    assert len(model.fit_output['labels']) == len(fit_dictionary_tabular['val_indices'])
+    assert 'train_score' in model.fit_output.keys()
+    assert isinstance(model.fit_output['train_score'], float)
+    assert 'val_score' in model.fit_output.keys()
+    assert isinstance(model.fit_output['val_score'], float)
+
+    # Test if traditional model can predict on val set
+    if model.model.is_classification:
+        y_pred = model.predict_proba(fit_dictionary_tabular['X_train'][fit_dictionary_tabular['val_indices']])
+    else:
+        y_pred = model.predict(fit_dictionary_tabular['X_train'][fit_dictionary_tabular['val_indices']])
+
+    assert np.allclose(y_pred.squeeze(), model.fit_output['val_preds'], atol=1e-04)
+    assert y_pred.shape[0] == len(fit_dictionary_tabular['val_indices'])
+    # Test if classifier can score and
+    # the result is same as in results
+    score = model.score(fit_dictionary_tabular['X_train'][fit_dictionary_tabular['val_indices']],
+                        fit_dictionary_tabular['y_train'][fit_dictionary_tabular['val_indices']])
+    assert np.allclose(score, model.fit_output['val_score'], atol=1e-6)
+
+    if sys.version_info >= (3, 7):
+        dump_file = os.path.join(fit_dictionary_tabular['backend'].temporary_directory, 'dump.pkl')
+
+        with open(dump_file, 'wb') as f:
+            pickle.dump(model, f)
+
+        with open(dump_file, 'rb') as f:
+            restored_estimator = pickle.load(f)
+        restored_estimator.predict(fit_dictionary_tabular['X_train'])