Add the cluster stack name to the metadata table for quick identification (#6385)

Also add the cw log group name as the stack name for now.  Adds a dummy cluster test for quick iteration as well.

Author: dreambeyondorange

tests/integration-tests/conftest.py

@@ -392,16 +392,19 @@ def clusters_factory(request, region):
 
     def _cluster_factory(cluster_config, upper_case_cluster_name=False, custom_cli_credentials=None, **kwargs):
         cluster_config = _write_config_to_outdir(request, cluster_config, "clusters_configs")
+        cluster_name = (
+            request.config.getoption("cluster")
+            if request.config.getoption("cluster")
+            else "integ-tests-{0}{1}{2}".format(
+                random_alphanumeric().upper() if upper_case_cluster_name else random_alphanumeric(),
+                "-" if request.config.getoption("stackname_suffix") else "",
+                request.config.getoption("stackname_suffix"),
+            )
+        )
+        request.node.user_properties.append(("cluster_stack_name", f"{cluster_name}"))
+        request.node.user_properties.append(("cw_log_group_name", f"{cluster_name}"))
         cluster = Cluster(
-            name=(
-                request.config.getoption("cluster")
-                if request.config.getoption("cluster")
-                else "integ-tests-{0}{1}{2}".format(
-                    random_alphanumeric().upper() if upper_case_cluster_name else random_alphanumeric(),
-                    "-" if request.config.getoption("stackname_suffix") else "",
-                    request.config.getoption("stackname_suffix"),
-                )
-            ),
+            name=cluster_name,
             config_file=cluster_config,
             ssh_key=request.config.getoption("key_path"),
             region=region,

tests/integration-tests/conftest_utils.py

@@ -192,15 +192,15 @@ def publish_test_metadata(item: pytest.Item, rep: pytest.TestReport):
             cli_commit=item.config.getoption("--pcluster-git-ref"),
             cookbook_commit=item.config.getoption("--cookbook-git-ref"),
             node_commit=item.config.getoption("--node-git-ref"),
+            cluster_stack_name="none",
+            cw_log_group_name="none",
             setup_metadata=PhaseMetadata(
                 rep.when,
                 status=rep.outcome,
                 start_time=get_user_prop(item, f"start_time_{rep.when}"),
                 end_time=get_user_prop(item, f"end_time_{rep.when}"),
             ),
         )
-        # Create the metadata table on the fly if it doesn't exist
-        metadata_table_mgr.create_metadata_table()
     if rep.when == "call":
         # Update the call test data
         test_metadata = jsonpickle.decode(get_user_prop(item, "metadata"))
@@ -210,6 +210,8 @@ def publish_test_metadata(item: pytest.Item, rep: pytest.TestReport):
             start_time=get_user_prop(item, f"start_time_{rep.when}"),
             end_time=get_user_prop(item, f"end_time_{rep.when}"),
         )
+        test_metadata.cluster_stack_name = get_user_prop(item, "cluster_stack_name")
+        test_metadata.cw_log_group_name = get_user_prop(item, "cw_log_group_name")
     if rep.when == "teardown":
         # Update the teardown test data
         test_metadata = jsonpickle.decode(get_user_prop(item, "metadata"))

tests/integration-tests/framework/metadata_table_manager.py

@@ -41,7 +41,7 @@ class TestMetadata:
     cli_commit: str = ""
     cookbook_commit: str = ""
     node_commit: str = ""
-    cfn_stack_name: str = ""
+    cluster_stack_name: str = ""
     cw_log_group_name: str = ""
     global_build_number: int = 0
 
@@ -118,7 +118,7 @@ def publish_metadata(self, metadata: List[TestMetadata]):
                         "cli_commit": {"S": datum.cli_commit if datum.cli_commit else "None"},
                         "cookbook_commit": {"S": datum.cookbook_commit if datum.cookbook_commit else "None"},
                         "node_commit": {"S": datum.node_commit if datum.node_commit else "None"},
-                        "cfn_stack_name": {"S": datum.cfn_stack_name if datum.cfn_stack_name else "None"},
+                        "cluster_stack_name": {"S": datum.cluster_stack_name if datum.cluster_stack_name else "None"},
                         "cw_log_group_name": {"S": datum.cw_log_group_name if datum.cw_log_group_name else "None"},
                         "global_build_number": {
                             "N": str(datum.global_build_number) if datum.global_build_number else "0"
@@ -166,7 +166,7 @@ def get_metadata(self, ids: List[str]) -> List[TestMetadata]:
                         cli_commit=response["Item"]["cli_commit"]["S"],
                         cookbook_commit=response["Item"]["cookbook_commit"]["S"],
                         node_commit=response["Item"]["node_commit"]["S"],
-                        cfn_stack_name=response["Item"]["cfn_stack_name"]["S"],
+                        cluster_stack_name=response["Item"]["cluster_stack_name"]["S"],
                         cw_log_group_name=response["Item"]["cw_log_group_name"]["S"],
                         global_build_number=int(response["Item"]["global_build_number"]["N"]),
                     )

tests/integration-tests/requirements.txt

@@ -4,12 +4,14 @@ assertpy
 boto3
 cfn_flip
 click
+decorator
 #https://github.com/fabric/fabric/issues/2204
 fabric==2.6.0
 filelock
 jinja2
 jsonpickle
 junitparser
+lexicon
 matplotlib
 pexpect
 psutil

tests/integration-tests/tests/dummy/test_dummy.py

@@ -26,3 +26,19 @@ def test_dummy(
 ):
     """Do nothing"""
     logging.info("Do nothing but test hooks")
+
+
+@pytest.mark.usefixtures("os", "scheduler", "instance")
+def test_dummy_cluster(
+    region,
+    scheduler,
+    pcluster_config_reader,
+    vpc_stack,
+    s3_bucket_factory,
+    test_datadir,
+    clusters_factory,
+):
+    """Do nothing"""
+    logging.info("Do nothing but launch a cluster")
+    cluster_config = pcluster_config_reader()
+    clusters_factory(cluster_config)

tests/integration-tests/tests/dummy/test_dummy/test_dummy_cluster/pcluster.config.yaml

@@ -0,0 +1,25 @@
+Image:
+  Os: {{ os }}
+HeadNode:
+  InstanceType: {{ instance }}
+  Networking:
+    SubnetId: {{ public_subnet_id }}
+  Ssh:
+    KeyName: {{ key_name }}
+  Imds:
+    Secured: {{ imds_secured }}
+Scheduling:
+  Scheduler: {{ scheduler }}
+  {{ scheduler_prefix }}Queues:
+    - Name: ondemand1
+      Networking:
+        SubnetIds:
+          - {{ private_subnet_id }}
+      ComputeResources:
+        - Name: compute-resource-11
+          Instances:
+            - InstanceType: c5.large
+        - Name: compute-resource-12
+          Instances:
+            - InstanceType: {{ instance }}
+          MinCount: 0