feat(cli): support --with in ado create (#262)

* feat(cli): add --with to ado create

* feat(cli): return identifier of the resource created in ado create commands

* fix(cli): check create space options correctly

* feat(cli): support --with samplestore in ado create space

* feat(cli): support --with space and --with ac in ado create operation

* chore: OperationFunctionConf was renamed to OperatorFunctionConf

* feat(cli): allow overriding values and performing other manipulations in ado create space

* test: add new tests for ado create space --with

* refactor(examples): use default sample store

* fix(cli): de-indent code

* test(cli): add test for ado create operation --with space

* test: do not use active ado context

* refactor(test): move test

* refactor(cli): reorder create parameters

* docs(website): update cli documentation

* chore(examples): remove sample store identifier

Co-authored-by: Michael Johnston <[email protected]>
Signed-off-by: Alessandro Pomponio <[email protected]>

* fix(cli): override resources when using with

Before it would just append to spaces and actuator configuration identifiers, causing issues

* docs(cli): update help string for ado create --with

Co-Authored-By: Michael Johnston <[email protected]>

* docs(cli): update cli help strings

* docs(cli): add missing space in --use-default-sample-store

Co-authored-by: Michael Johnston <[email protected]>
Signed-off-by: Alessandro Pomponio <[email protected]>

---------

Signed-off-by: Alessandro Pomponio <[email protected]>
Co-authored-by: Michael Johnston <[email protected]>

Author: AlessandroPomponio

examples/optimization_test_functions/space.yaml

@@ -1,7 +1,6 @@
 # Copyright (c) IBM Corporation
 # SPDX-License-Identifier: MIT
 
-sampleStoreIdentifier: dfe035
 entitySpace:
 - identifier: x2
   propertyDomain:

orchestrator/cli/commands/create.py

@@ -1,6 +1,6 @@
 # Copyright (c) IBM Corporation
 # SPDX-License-Identifier: MIT
-
+import os
 import pathlib
 from typing import Annotated
 
@@ -14,7 +14,10 @@
 )
 from orchestrator.cli.models.choice import HiddenPluralChoice, HiddenShorthandChoice
 from orchestrator.cli.models.parameters import AdoCreateCommandParameters
-from orchestrator.cli.models.types import AdoCreateSupportedResourceTypes
+from orchestrator.cli.models.types import (
+    AdoCreateSupportedResourceTypes,
+    AdoCreateWithResourceSupportedResourceTypes,
+)
 from orchestrator.cli.resources.actuator_configuration.create import (
     create_actuator_configuration,
 )
@@ -24,10 +27,14 @@
 from orchestrator.cli.resources.sample_store.create import create_sample_store
 from orchestrator.cli.utils.input.parsers import (
     parse_key_value_pairs,
+    resource_shorthands_to_full_names,
 )
 from orchestrator.cli.utils.output.prints import (
     ERROR,
+    HINT,
+    bold,
     console_print,
+    magenta,
 )
 from orchestrator.core import CoreResourceKinds
 from orchestrator.metastore.base import (
@@ -107,20 +114,9 @@ def create_resource(
         typer.Option(
             "--new-sample-store",
             help="Request and use a new, empty sample store. Available only for space and sample store. "
-            "Ignored if --set or --use-latest are used.",
+            "Ignored if --with or --use-latest are used.",
         ),
     ] = False,
-    use_latest: Annotated[
-        list[CoreResourceKinds] | None,
-        typer.Option(
-            show_default=False,
-            click_type=HiddenShorthandChoice(CoreResourceKinds),
-            help="""
-            Reuse the latest identifier of a resource kind. Can be used multiple times.
-
-            Only supported for spaces and operations. Ignored if --set is used.""",
-        ),
-    ] = None,
     set_values: Annotated[
         list[str] | None,
         typer.Option(
@@ -136,13 +132,34 @@ def create_resource(
             """,
         ),
     ] = None,
+    with_resources: Annotated[
+        list[str] | None,
+        typer.Option(
+            "--with",
+            help="""Specify additional resources to use or create via key=value pairs,
+            where the key is a resource type and the value is a resource ID or a resource configuration file.
+
+            Supports creating spaces with sample stores and operations with spaces and actuator configurations.""",
+        ),
+    ] = None,
+    use_latest: Annotated[
+        list[CoreResourceKinds] | None,
+        typer.Option(
+            show_default=False,
+            click_type=HiddenShorthandChoice(CoreResourceKinds),
+            help="""
+            Reuse the latest identifier of a resource kind. Can be used multiple times.
+
+            Only supported for spaces and operations. Ignored if --with is used.""",
+        ),
+    ] = None,
     use_default_sample_store: Annotated[
         bool,
         typer.Option(
             "--use-default-sample-store",
             rich_help_panel=CREATE_SPACE_PANEL_NAME,
             help="Request and use the default sample store. Available only for spaces. "
-            "Ignored if --set, --use-latest, or --new-sample-store are used."
+            "Ignored if --with, --use-latest, or --new-sample-store are used. "
             "Alias for --set sampleStoreIdentifier=default.",
         ),
     ] = False,
@@ -212,6 +229,7 @@ def create_resource(
 
     ado_configuration: AdoConfiguration = ctx.obj
     override_values = parse_key_value_pairs(set_values)
+    with_resources = parse_with_resource_options(with_resources)
 
     parameters = AdoCreateCommandParameters(
         ado_configuration=ado_configuration,
@@ -222,6 +240,7 @@ def create_resource(
         resource_type=resource_type,
         use_default_sample_store=use_default_sample_store,
         use_latest=use_latest,
+        with_resources=with_resources,
     )
 
     method_mapping = {
@@ -255,3 +274,63 @@ def register_create_command(app: typer.Typer):
         options_metavar="[-f | --file <configuration>] [--set <path=document> ...] "
         "[--new-sample-store] [--dry-run]",
     )(create_resource)
+
+
+def parse_with_resource_options(
+    user_provided_options: list[str] | None,
+) -> dict[CoreResourceKinds, pathlib.Path | str]:
+
+    parsed_options = {}
+    if not user_provided_options:
+        return parsed_options
+
+    # To simplify checking whether the resource type is supported
+    supported_resource_types = {
+        v.value for v in AdoCreateWithResourceSupportedResourceTypes
+    }
+
+    for with_option in parse_key_value_pairs(user_provided_options):
+        for resource_type, value in with_option.items():
+
+            resource_type = resource_shorthands_to_full_names(resource_type)
+            if resource_type not in supported_resource_types:
+                console_print(
+                    f"{ERROR}{bold(resource_type)} is not a supported resource type "
+                    f"for the {magenta('--with')} option.\n"
+                    f"{HINT}The only supported resource types are "
+                    f"[b cyan]{', '.join(supported_resource_types)}[/b cyan] and their shorthands.",
+                    stderr=True,
+                )
+                raise typer.Exit(1)
+
+            # Now we know the resource_type can be parsed as a CoreResourceKinds
+            resource_type = CoreResourceKinds(resource_type)
+
+            # Since we are dealing with a dictionary, we can only support one --with option per resource type
+            if resource_type in parsed_options:
+                console_print(
+                    f"{ERROR}The {resource_type.value} resource type was specified more than once.",
+                    stderr=True,
+                )
+                raise typer.Exit(1)
+
+            # We allow users to provide either paths to a resource configuration file or a resource identifier.
+            # In this function we only check whether it's one or the other (with validation on the path). Both
+            # schema validation for configuration files and resource retrieval for ids will be performed in the
+            # resource-specific create functions.
+            #
+            # If a dot or a path separator character is present in the value, we assume it must be a path and
+            # fail if it doesn't exist, or it's not a file.
+            value_must_be_path = "." in value or os.sep in value
+            value_as_path = pathlib.Path(value)
+            if value_as_path.exists() and value_as_path.is_file():
+                parsed_options[resource_type] = value_as_path
+            elif value_must_be_path:
+                console_print(
+                    f"{ERROR}{value} does not exist or is not a file.", stderr=True
+                )
+                raise typer.Exit(1)
+            else:
+                parsed_options[resource_type] = value
+
+    return parsed_options

orchestrator/cli/models/parameters.py

@@ -54,6 +54,7 @@ class AdoCreateCommandParameters(pydantic.BaseModel):
     resource_type: AdoCreateSupportedResourceTypes
     use_default_sample_store: bool
     use_latest: list[CoreResourceKinds] | None
+    with_resources: dict[CoreResourceKinds, pathlib.Path | str] | None
 
 
 class AdoDeleteCommandParameters(pydantic.BaseModel):

orchestrator/cli/models/types.py

@@ -53,6 +53,12 @@ class AdoCreateSupportedResourceTypes(Enum):
     SAMPLE_STORE = _SAMPLE_STORE_SINGULAR
 
 
+class AdoCreateWithResourceSupportedResourceTypes(Enum):
+    ACTUATOR_CONFIGURATION = _ACTUATOR_CONFIGURATION_SINGULAR
+    DISCOVERY_SPACE = _DISCOVERY_SPACE_SINGULAR
+    SAMPLE_STORE = _SAMPLE_STORE_SINGULAR
+
+
 #################### ado delete ####################
 class AdoDeleteSupportedResourceTypes(Enum):
     ACTUATOR_CONFIGURATION = _ACTUATOR_CONFIGURATION_SINGULAR

orchestrator/cli/resources/actuator_configuration/create.py

@@ -44,7 +44,7 @@ def create_actuator_configuration(parameters: AdoCreateCommandParameters):
 
     if parameters.dry_run:
         console_print(ADO_CREATE_DRY_RUN_CONFIG_VALID, stderr=True)
-        return
+        return None
 
     resource_to_be_created = ActuatorConfigurationResource(
         config=actuatorconfig_configuration
@@ -65,3 +65,5 @@ def create_actuator_configuration(parameters: AdoCreateCommandParameters):
         f"{magenta(resource_to_be_created.identifier)}",
         stderr=True,
     )
+
+    return resource_to_be_created.identifier

orchestrator/cli/resources/context/create.py

@@ -40,7 +40,7 @@ def create_context(parameters: AdoCreateCommandParameters):
 
     if parameters.dry_run:
         console_print(ADO_CREATE_DRY_RUN_CONFIG_VALID, stderr=True)
-        return
+        return None
 
     destination = parameters.ado_configuration.project_context_path_for_context(
         context_configuration.project
@@ -80,3 +80,5 @@ def create_context(parameters: AdoCreateCommandParameters):
             f"\t{cyan('ado context ' + context_configuration.project)}",
             stderr=True,
         )
+
+    return context_configuration.project

orchestrator/cli/resources/discovery_space/create.py

@@ -7,6 +7,7 @@
 from rich.status import Status
 
 from orchestrator.cli.models.parameters import AdoCreateCommandParameters
+from orchestrator.cli.resources.sample_store.create import create_sample_store
 from orchestrator.cli.utils.generic.wrappers import get_sql_store
 from orchestrator.cli.utils.output.prints import (
     ADO_CREATE_DRY_RUN_CONFIG_VALID,
@@ -30,14 +31,21 @@
 
 def create_discovery_space(parameters: AdoCreateCommandParameters):
 
-    if (
-        parameters.new_sample_store
-        and parameters.use_latest
-        and CoreResourceKinds.SAMPLESTORE in parameters.use_latest
-    ):
+    # Fail early if there is an invalid combination of parameters
+    mutually_exclusive_options = [
+        parameters.new_sample_store,
+        parameters.use_latest is not None
+        and len(parameters.use_latest) >= 0
+        and CoreResourceKinds.SAMPLESTORE in parameters.use_latest,
+        parameters.with_resources is not None
+        and CoreResourceKinds.SAMPLESTORE in parameters.with_resources,
+    ]
+
+    if sum(mutually_exclusive_options) >= 2:
         console_print(
-            f"{ERROR}You can only set one of --new-sample-store "
-            f"and --use-latest {CoreResourceKinds.SAMPLESTORE.value}",
+            f"{ERROR}You can only set one of --new-sample-store, "
+            f"--use-latest {CoreResourceKinds.SAMPLESTORE.value}, "
+            f"--with {CoreResourceKinds.SAMPLESTORE.value}=path/id",
             stderr=True,
         )
         raise typer.Exit(1)
@@ -57,6 +65,35 @@ def create_discovery_space(parameters: AdoCreateCommandParameters):
         space_configuration = override_values_in_pydantic_model(
             model=space_configuration, override_values=parameters.override_values
         )
+
+    if (
+        parameters.with_resources
+        and CoreResourceKinds.SAMPLESTORE in parameters.with_resources
+    ):
+
+        if isinstance(parameters.with_resources[CoreResourceKinds.SAMPLESTORE], str):
+            space_configuration.sampleStoreIdentifier = parameters.with_resources[
+                CoreResourceKinds.SAMPLESTORE
+            ]
+        else:
+            from orchestrator.cli.models.types import AdoCreateSupportedResourceTypes
+
+            space_configuration.sampleStoreIdentifier = create_sample_store(
+                AdoCreateCommandParameters(
+                    ado_configuration=parameters.ado_configuration,
+                    dry_run=False,
+                    new_sample_store=False,
+                    override_values=[],
+                    resource_configuration_file=parameters.with_resources[
+                        CoreResourceKinds.SAMPLESTORE
+                    ],
+                    use_default_sample_store=False,
+                    use_latest=[],
+                    with_resources={},
+                    resource_type=AdoCreateSupportedResourceTypes.SAMPLE_STORE,
+                )
+            )
+
     elif (
         parameters.use_latest and CoreResourceKinds.SAMPLESTORE in parameters.use_latest
     ):
@@ -82,6 +119,7 @@ def create_discovery_space(parameters: AdoCreateCommandParameters):
             stderr=True,
         )
         space_configuration.sampleStoreIdentifier = latest_recorded_sample_store
+
     elif parameters.new_sample_store:
 
         # Replay experiments cannot use --new-sample-store
@@ -139,12 +177,13 @@ def create_discovery_space(parameters: AdoCreateCommandParameters):
         parameters.ado_configuration.latest_resource_ids[
             CoreResourceKinds.SAMPLESTORE
         ] = sample_store_resource.identifier
+
     elif parameters.use_default_sample_store:
         space_configuration.sampleStoreIdentifier = "default"
 
     if parameters.dry_run:
         console_print(ADO_CREATE_DRY_RUN_CONFIG_VALID, stderr=True)
-        return
+        return None
 
     if space_configuration.sampleStoreIdentifier == "default":
         info_message = (
@@ -212,3 +251,5 @@ def create_discovery_space(parameters: AdoCreateCommandParameters):
     console_print(
         f"{SUCCESS}Created space with identifier: {magenta(space.uri)}", stderr=True
     )
+
+    return space.uri