Ravin Kohli: [ADD] Test evaluator (#368)

Author: Github Actions

development/_downloads/bc82bea3a5dd7bdba60b65220891d9e5/examples_python.zip

@@ -126,8 +126,8 @@ <h1>Source code for autoPyTorch.api.tabular_classification</h1><div class="highl
 <span class="kn">from</span> <span class="nn">autoPyTorch.data.tabular_validator</span> <span class="kn">import</span> <span class="n">TabularInputValidator</span>
 <span class="kn">from</span> <span class="nn">autoPyTorch.datasets.base_dataset</span> <span class="kn">import</span> <span class="n">BaseDatasetPropertiesType</span>
 <span class="kn">from</span> <span class="nn">autoPyTorch.datasets.resampling_strategy</span> <span class="kn">import</span> <span class="p">(</span>
-    <span class="n">CrossValTypes</span><span class="p">,</span>
     <span class="n">HoldoutValTypes</span><span class="p">,</span>
+    <span class="n">ResamplingStrategies</span><span class="p">,</span>
 <span class="p">)</span>
 <span class="kn">from</span> <span class="nn">autoPyTorch.datasets.tabular_dataset</span> <span class="kn">import</span> <span class="n">TabularDataset</span>
 <span class="kn">from</span> <span class="nn">autoPyTorch.evaluation.utils</span> <span class="kn">import</span> <span class="n">DisableFileOutputParameters</span>
@@ -177,8 +177,15 @@ <h1>Source code for autoPyTorch.api.tabular_classification</h1><div class="highl
 <span class="sd">            name and Value is an Iterable of the names of the components</span>
 <span class="sd">            to exclude. All except these components will be present in</span>
 <span class="sd">            the search space.</span>
+<span class="sd">        resampling_strategy resampling_strategy (RESAMPLING_STRATEGIES),</span>
+<span class="sd">                (default=HoldoutValTypes.holdout_validation):</span>
+<span class="sd">                strategy to split the training data.</span>
+<span class="sd">        resampling_strategy_args (Optional[Dict[str, Any]]): arguments</span>
+<span class="sd">            required for the chosen resampling strategy. If None, uses</span>
+<span class="sd">            the default values provided in DEFAULT_RESAMPLING_PARAMETERS</span>
+<span class="sd">            in ```datasets/resampling_strategy.py```.</span>
 <span class="sd">        search_space_updates (Optional[HyperparameterSearchSpaceUpdates]):</span>
-<span class="sd">            search space updates that can be used to modify the search</span>
+<span class="sd">            Search space updates that can be used to modify the search</span>
 <span class="sd">            space of particular components or choice modules of the pipeline</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
@@ -196,7 +203,7 @@ <h1>Source code for autoPyTorch.api.tabular_classification</h1><div class="highl
         <span class="n">delete_output_folder_after_terminate</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
         <span class="n">include_components</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
         <span class="n">exclude_components</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-        <span class="n">resampling_strategy</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">CrossValTypes</span><span class="p">,</span> <span class="n">HoldoutValTypes</span><span class="p">]</span> <span class="o">=</span> <span class="n">HoldoutValTypes</span><span class="o">.</span><span class="n">holdout_validation</span><span class="p">,</span>
+        <span class="n">resampling_strategy</span><span class="p">:</span> <span class="n">ResamplingStrategies</span> <span class="o">=</span> <span class="n">HoldoutValTypes</span><span class="o">.</span><span class="n">holdout_validation</span><span class="p">,</span>
         <span class="n">resampling_strategy_args</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
         <span class="n">backend</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Backend</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
         <span class="n">search_space_updates</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">HyperparameterSearchSpaceUpdates</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
@@ -266,7 +273,7 @@ <h1>Source code for autoPyTorch.api.tabular_classification</h1><div class="highl
         <span class="n">y_train</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">,</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">],</span>
         <span class="n">X_test</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">,</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
         <span class="n">y_test</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">,</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-        <span class="n">resampling_strategy</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">CrossValTypes</span><span class="p">,</span> <span class="n">HoldoutValTypes</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">resampling_strategy</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">ResamplingStrategies</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
         <span class="n">resampling_strategy_args</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
         <span class="n">dataset_name</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">TabularDataset</span><span class="p">,</span> <span class="n">TabularInputValidator</span><span class="p">]:</span>
@@ -283,7 +290,7 @@ <h1>Source code for autoPyTorch.api.tabular_classification</h1><div class="highl
 <span class="sd">                Testing feature set</span>
 <span class="sd">            y_test (Optional[Union[List, pd.DataFrame, np.ndarray]]):</span>
 <span class="sd">                Testing target set</span>
-<span class="sd">            resampling_strategy (Optional[Union[CrossValTypes, HoldoutValTypes]]):</span>
+<span class="sd">            resampling_strategy (Optional[RESAMPLING_STRATEGIES]):</span>
 <span class="sd">                Strategy to split the training data. if None, uses</span>
 <span class="sd">                HoldoutValTypes.holdout_validation.</span>
 <span class="sd">            resampling_strategy_args (Optional[Dict[str, Any]]):</span>

development/_downloads/fb625db3c50d423b1b7881136ffdeec8/examples_jupyter.zip

@@ -126,8 +126,8 @@ <h1>Source code for autoPyTorch.api.tabular_regression</h1><div class="highlight
 <span class="kn">from</span> <span class="nn">autoPyTorch.data.tabular_validator</span> <span class="kn">import</span> <span class="n">TabularInputValidator</span>
 <span class="kn">from</span> <span class="nn">autoPyTorch.datasets.base_dataset</span> <span class="kn">import</span> <span class="n">BaseDatasetPropertiesType</span>
 <span class="kn">from</span> <span class="nn">autoPyTorch.datasets.resampling_strategy</span> <span class="kn">import</span> <span class="p">(</span>
-    <span class="n">CrossValTypes</span><span class="p">,</span>
     <span class="n">HoldoutValTypes</span><span class="p">,</span>
+    <span class="n">ResamplingStrategies</span><span class="p">,</span>
 <span class="p">)</span>
 <span class="kn">from</span> <span class="nn">autoPyTorch.datasets.tabular_dataset</span> <span class="kn">import</span> <span class="n">TabularDataset</span>
 <span class="kn">from</span> <span class="nn">autoPyTorch.evaluation.utils</span> <span class="kn">import</span> <span class="n">DisableFileOutputParameters</span>
@@ -177,8 +177,15 @@ <h1>Source code for autoPyTorch.api.tabular_regression</h1><div class="highlight
 <span class="sd">            name and Value is an Iterable of the names of the components</span>
 <span class="sd">            to exclude. All except these components will be present in</span>
 <span class="sd">            the search space.</span>
+<span class="sd">        resampling_strategy resampling_strategy (RESAMPLING_STRATEGIES),</span>
+<span class="sd">                (default=HoldoutValTypes.holdout_validation):</span>
+<span class="sd">                strategy to split the training data.</span>
+<span class="sd">        resampling_strategy_args (Optional[Dict[str, Any]]): arguments</span>
+<span class="sd">            required for the chosen resampling strategy. If None, uses</span>
+<span class="sd">            the default values provided in DEFAULT_RESAMPLING_PARAMETERS</span>
+<span class="sd">            in ```datasets/resampling_strategy.py```.</span>
 <span class="sd">        search_space_updates (Optional[HyperparameterSearchSpaceUpdates]):</span>
-<span class="sd">            search space updates that can be used to modify the search</span>
+<span class="sd">            Search space updates that can be used to modify the search</span>
 <span class="sd">            space of particular components or choice modules of the pipeline</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
@@ -197,7 +204,7 @@ <h1>Source code for autoPyTorch.api.tabular_regression</h1><div class="highlight
         <span class="n">delete_output_folder_after_terminate</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
         <span class="n">include_components</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
         <span class="n">exclude_components</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-        <span class="n">resampling_strategy</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">CrossValTypes</span><span class="p">,</span> <span class="n">HoldoutValTypes</span><span class="p">]</span> <span class="o">=</span> <span class="n">HoldoutValTypes</span><span class="o">.</span><span class="n">holdout_validation</span><span class="p">,</span>
+        <span class="n">resampling_strategy</span><span class="p">:</span> <span class="n">ResamplingStrategies</span> <span class="o">=</span> <span class="n">HoldoutValTypes</span><span class="o">.</span><span class="n">holdout_validation</span><span class="p">,</span>
         <span class="n">resampling_strategy_args</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
         <span class="n">backend</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Backend</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
         <span class="n">search_space_updates</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">HyperparameterSearchSpaceUpdates</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
@@ -267,7 +274,7 @@ <h1>Source code for autoPyTorch.api.tabular_regression</h1><div class="highlight
         <span class="n">y_train</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">,</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">],</span>
         <span class="n">X_test</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">,</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
         <span class="n">y_test</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">,</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-        <span class="n">resampling_strategy</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">CrossValTypes</span><span class="p">,</span> <span class="n">HoldoutValTypes</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">resampling_strategy</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">ResamplingStrategies</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
         <span class="n">resampling_strategy_args</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
         <span class="n">dataset_name</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">TabularDataset</span><span class="p">,</span> <span class="n">TabularInputValidator</span><span class="p">]:</span>
@@ -284,7 +291,7 @@ <h1>Source code for autoPyTorch.api.tabular_regression</h1><div class="highlight
 <span class="sd">                Testing feature set</span>
 <span class="sd">            y_test (Optional[Union[List, pd.DataFrame, np.ndarray]]):</span>
 <span class="sd">                Testing target set</span>
-<span class="sd">            resampling_strategy (Optional[Union[CrossValTypes, HoldoutValTypes]]):</span>
+<span class="sd">            resampling_strategy (Optional[RESAMPLING_STRATEGIES]):</span>
 <span class="sd">                Strategy to split the training data. if None, uses</span>
 <span class="sd">                HoldoutValTypes.holdout_validation.</span>
 <span class="sd">            resampling_strategy_args (Optional[Dict[str, Any]]):</span>

development/_images/sphx_glr_example_plot_over_time_001.png

@@ -85,20 +85,13 @@ Image Classification
     Pipeline Random Config:
      ________________________________________ 
     Configuration:
-      image_augmenter:GaussianBlur:sigma_min, Value: 0.7252880928545244
-      image_augmenter:GaussianBlur:sigma_offset, Value: 1.521238468454762
-      image_augmenter:GaussianBlur:use_augmenter, Value: True
-      image_augmenter:GaussianNoise:sigma_offset, Value: 1.3019933264526289
+      image_augmenter:GaussianBlur:use_augmenter, Value: False
+      image_augmenter:GaussianNoise:sigma_offset, Value: 1.1139913028820034
       image_augmenter:GaussianNoise:use_augmenter, Value: True
-      image_augmenter:RandomAffine:rotate, Value: 273
-      image_augmenter:RandomAffine:scale_offset, Value: 0.32933509809921574
-      image_augmenter:RandomAffine:shear, Value: 18
-      image_augmenter:RandomAffine:translate_percent_offset, Value: 0.12689372744350622
-      image_augmenter:RandomAffine:use_augmenter, Value: True
-      image_augmenter:RandomCutout:p, Value: 0.2567527903290805
-      image_augmenter:RandomCutout:use_augmenter, Value: True
+      image_augmenter:RandomAffine:use_augmenter, Value: False
+      image_augmenter:RandomCutout:use_augmenter, Value: False
       image_augmenter:Resize:use_augmenter, Value: False
-      image_augmenter:ZeroPadAndCrop:percent, Value: 0.38501712446074526
+      image_augmenter:ZeroPadAndCrop:percent, Value: 0.261983365182016
       normalizer:__choice__, Value: 'NoNormalizer'
 
     Fitting the pipeline...
@@ -178,7 +171,7 @@ Image Classification
 
 .. rst-class:: sphx-glr-timing
 
-   **Total running time of the script:** ( 0 minutes  8.384 seconds)
+   **Total running time of the script:** ( 0 minutes  5.513 seconds)
 
 
 .. _sphx_glr_download_examples_20_basics_example_image_classification.py: