Automatically approves change requests decreasing quotas for Openshift

ci/run_functional_tests_openshift.sh

@@ -17,6 +17,8 @@ fi
 export DJANGO_SETTINGS_MODULE="local_settings"
 export FUNCTIONAL_TESTS="True"
 export OS_AUTH_URL="https://onboarding-onboarding.cluster.local"
+export OS_API_URL="https://onboarding-onboarding.cluster.local:6443"
+
 
 coverage run --source="." -m django test coldfront_plugin_cloud.tests.functional.openshift
 coverage report

src/coldfront_plugin_cloud/attributes.py

@@ -19,6 +19,7 @@ class CloudAllocationAttribute:
 
 
 RESOURCE_AUTH_URL = 'Identity Endpoint URL'
+RESOURCE_API_URL = 'OpenShift API Endpoint URL'
 RESOURCE_IDENTITY_NAME = 'OpenShift Identity Provider Name'
 RESOURCE_ROLE = 'Role for User in Project'
 
@@ -33,6 +34,7 @@ class CloudAllocationAttribute:
 
 RESOURCE_ATTRIBUTES = [
     CloudResourceAttribute(name=RESOURCE_AUTH_URL),
+    CloudResourceAttribute(name=RESOURCE_API_URL),
     CloudResourceAttribute(name=RESOURCE_IDENTITY_NAME),
     CloudResourceAttribute(name=RESOURCE_FEDERATION_PROTOCOL),
     CloudResourceAttribute(name=RESOURCE_IDP),

src/coldfront_plugin_cloud/management/commands/add_openshift_resource.py

@@ -17,6 +17,10 @@ def add_arguments(self, parser):
                             help='Name of OpenShift resource')
         parser.add_argument('--auth-url', type=str, required=True,
                             help='URL of the openshift-acct-mgt endpoint')
+        parser.add_argument('--api-url', type=str, required=True,
+                            help='API URL of the openshift cluster')
+        parser.add_argument('--idp', type=str, required=True,
+                            help='Name of Openshift identity provider')
         parser.add_argument('--role', type=str, default='edit',
                             help='Role for user when added to project (default: edit)')
 
@@ -37,6 +41,18 @@ def handle(self, *args, **options):
             resource=openshift,
             value=options['auth_url']
         )
+        ResourceAttribute.objects.get_or_create(
+            resource_attribute_type=ResourceAttributeType.objects.get(
+                name=attributes.RESOURCE_API_URL),
+            resource=openshift,
+            value=options['api_url']
+        )
+        ResourceAttribute.objects.get_or_create(
+            resource_attribute_type=ResourceAttributeType.objects.get(
+                name=attributes.RESOURCE_IDENTITY_NAME),
+            resource=openshift,
+            value=options['idp']
+        )
         ResourceAttribute.objects.get_or_create(
             resource_attribute_type=ResourceAttributeType.objects.get(
                 name=attributes.RESOURCE_ROLE),

src/coldfront_plugin_cloud/management/commands/validate_allocations.py

@@ -197,50 +197,7 @@ def handle(self, *args, **options):
                     expected_value = allocation.get_attribute(attr.name)
                     current_value = quota.get(key, None)
 
-                    PATTERN = r"([0-9]+)(m|Ki|Mi|Gi|Ti|Pi|Ei|K|M|G|T|P|E)?"
-
-                    suffix = {
-                        "Ki": 2**10,
-                        "Mi": 2**20,
-                        "Gi": 2**30,
-                        "Ti": 2**40,
-                        "Pi": 2**50,
-                        "Ei": 2**60,
-                        "m": 10**-3,
-                        "K": 10**3,
-                        "M": 10**6,
-                        "G": 10**9,
-                        "T": 10**12,
-                        "P": 10**15,
-                        "E": 10**18,
-                    }
-
-                    if current_value and current_value != "0":
-                        result = re.search(PATTERN, current_value)
-
-                        if result is None:
-                            raise CommandError(
-                                f"Unable to parse current_value = '{current_value}' for {attr.name}"
-                            )
-
-                        value = int(result.groups()[0])
-                        unit = result.groups()[1]
-
-                        # Convert to number i.e. without any unit suffix
-
-                        if unit is not None:
-                            current_value = value * suffix[unit]
-                        else:
-                            current_value = value
-
-                        # Convert some attributes to units that coldfront uses
-
-                        if "RAM" in attr.name:
-                            current_value = round(current_value / suffix["Mi"])
-                        elif "Storage" in attr.name:
-                            current_value = round(current_value / suffix["Gi"])
-                    elif current_value and current_value == "0":
-                        current_value = 0
+                    current_value = utils.parse_openshift_quota_value(attr.name, current_value)
 
                     if expected_value is None and current_value is not None:
                         msg = (

src/coldfront_plugin_cloud/openshift.py

@@ -36,12 +36,12 @@ def clean_openshift_metadata(obj):
     return obj
 
 QUOTA_KEY_MAPPING = {
-    attributes.QUOTA_LIMITS_CPU: lambda x: {":limits.cpu": f"{x * 1000}m"},
-    attributes.QUOTA_LIMITS_MEMORY: lambda x: {":limits.memory": f"{x}Mi"},
-    attributes.QUOTA_LIMITS_EPHEMERAL_STORAGE_GB: lambda x: {":limits.ephemeral-storage": f"{x}Gi"},
-    attributes.QUOTA_REQUESTS_STORAGE: lambda x: {":requests.storage": f"{x}Gi"},
-    attributes.QUOTA_REQUESTS_GPU: lambda x: {":requests.nvidia.com/gpu": f"{x}"},
-    attributes.QUOTA_PVC: lambda x: {":persistentvolumeclaims": f"{x}"},
+    attributes.QUOTA_LIMITS_CPU: lambda x: {"limits.cpu": f"{x * 1000}m"},
+    attributes.QUOTA_LIMITS_MEMORY: lambda x: {"limits.memory": f"{x}Mi"},
+    attributes.QUOTA_LIMITS_EPHEMERAL_STORAGE_GB: lambda x: {"limits.ephemeral-storage": f"{x}Gi"},
+    attributes.QUOTA_REQUESTS_STORAGE: lambda x: {"requests.storage": f"{x}Gi"},
+    attributes.QUOTA_REQUESTS_GPU: lambda x: {"requests.nvidia.com/gpu": f"{x}"},
+    attributes.QUOTA_PVC: lambda x: {"persistentvolumeclaims": f"{x}"},
 }
 
 
@@ -77,7 +77,7 @@ def __init__(self, resource, allocation):
     def k8_client(self):
         # Load Endpoint URL and Auth token for new k8 client
         openshift_token = os.getenv(f"OPENSHIFT_{self.safe_resource_name}_TOKEN")
-        openshift_url = self.resource.get_attribute(attributes.RESOURCE_AUTH_URL)
+        openshift_url = self.resource.get_attribute(attributes.RESOURCE_API_URL)
 
         k8_config = kubernetes.client.Configuration()
         k8_config.api_key["authorization"] = openshift_token
@@ -146,20 +146,79 @@ def create_project(self, suggested_project_name):
         project_name = project_id
         self._create_project(project_name, project_id)
         return self.Project(project_name, project_id)
+
+    def delete_moc_quotas(self, project_id):
+        """deletes all resourcequotas from an openshift project"""
+        resourcequotas = self._openshift_get_resourcequotas(project_id)
+        for resourcequota in resourcequotas:
+            self._openshift_delete_resourcequota(project_id, resourcequota["metadata"]["name"])
+
+        logger.info(f"All quotas for {project_id} successfully deleted")
 
     def set_quota(self, project_id):
-        url = f"{self.auth_url}/projects/{project_id}/quota"
-        payload = dict()
+        """Sets the quota for a project, creating a minimal resourcequota
+        object in the project namespace with no extra scopes"""
+
+        quota_spec = {}
         for key, func in QUOTA_KEY_MAPPING.items():
             if (x := self.allocation.get_attribute(key)) is not None:
-                payload.update(func(x))
-        r = self.session.put(url, data=json.dumps({'Quota': payload}))
-        self.check_response(r)
+                quota_spec.update(func(x))
+
+        quota_def = {
+            "metadata": {"name": f"{project_id}-project"},
+            "spec": {"hard": quota_spec},
+        }
+
+        self.delete_moc_quotas(project_id)
+        self._openshift_create_resourcequota(project_id, quota_def)
+
+        logger.info(f"Quota for {project_id} successfully created")
+
+    def _get_moc_quota_from_resourcequotas(self, project_id):
+        """This returns a dictionary suitable for merging in with the
+        specification from Adjutant/ColdFront"""
+        resourcequotas = self._openshift_get_resourcequotas(project_id)
+        moc_quota = {}
+        for rq in resourcequotas:
+            name, spec = rq["metadata"]["name"], rq["spec"]
+            logger.info(f"processing resourcequota: {project_id}:{name}")
+            scope_list = spec.get("scopes", [""])
+            for quota_name, quota_value in spec.get("hard", {}).items():
+                for scope_item in scope_list:
+                    moc_quota_name = f"{scope_item}:{quota_name}"
+                    moc_quota.setdefault(moc_quota_name, quota_value)
+        return moc_quota
 
     def get_quota(self, project_id):
-        url = f"{self.auth_url}/projects/{project_id}/quota"
-        r = self.session.get(url)
-        return self.check_response(r)
+        quota_from_project = self._get_moc_quota_from_resourcequotas(project_id)
+
+        quota = {}
+        for quota_name, quota_value in quota_from_project.items():
+            if quota_value:
+                quota[quota_name] = quota_value
+
+        quota_object = {
+            "Version": "0.9",
+            "Kind": "MocQuota",
+            "ProjectName": project_id,
+            "Quota": quota,
+        }
+        return quota_object
+
+    def _get_moc_quota_used_from_resourcequotas(self, project_id):
+        resourcequotas = self._openshift_get_resourcequotas(project_id)
+        moc_quota_used = {}
+        for rq in resourcequotas:
+            moc_quota_used.update(rq["status"]["used"])
+        return moc_quota_used
+
+    def get_quota_used(self, project_id):
+        resourcequotas = self._openshift_get_resourcequotas(project_id)
+        moc_quota_used = {}
+        # TODO Any concerns about this being a list? Can a project have multiple resourcequotas?
+        for rq in resourcequotas:
+            moc_quota_used.update(rq["status"]["used"])
+        return moc_quota_used
 
     def create_project_defaults(self, project_id):
         pass
@@ -181,7 +240,7 @@ def reactivate_project(self, project_id):
     def get_federated_user(self, username):
         if (
             self._openshift_user_exists(username)
-            and self._openshift_get_identity(username)
+            and self._openshift_identity_exists(username)
             and self._openshift_useridentitymapping_exists(username, username)
         ):
             return {'username': username}
@@ -264,25 +323,81 @@ def _openshift_get_identity(self, id_user):
     def _openshift_user_exists(self, user_name):
         try:
             self._openshift_get_user(user_name)
-        except kexc.NotFoundError:
-            return False
+        except kexc.NotFoundError as e:
+            # Ensures error raise because resource not found, 
+            # not because of other reasons, like incorrect url
+            e_info = json.loads(e.body)
+            if e_info.get("reason") == "NotFound":
+                return False
+            raise e
         return True
 
     def _openshift_identity_exists(self, id_user):
         try:
             self._openshift_get_identity(id_user)
-        except kexc.NotFoundError:
-            return False
+        except kexc.NotFoundError as e:
+            e_info = json.loads(e.body)
+            if e_info.get("reason") == "NotFound":
+                return False
+            raise e
         return True
 
     def _openshift_useridentitymapping_exists(self, user_name, id_user):
         try:
             user = self._openshift_get_user(user_name)
-        except kexc.NotFoundError:
-            return False
+        except kexc.NotFoundError as e:
+            e_info = json.loads(e.body)
+            if e_info.get("reason") == "NotFound":
+                return False
+            raise e
 
         return any(
             identity == self.qualified_id_user(id_user)
             for identity in user.get("identities", [])
         )
 
+    def _openshift_get_project(self, project_name):
+        api = self.get_resource_api(API_PROJECT, "Project")
+        return clean_openshift_metadata(api.get(name=project_name).to_dict())
+
+    def _openshift_get_resourcequotas(self, project_id):
+        """Returns a list of all of the resourcequota objects"""
+        # Raise a NotFound error if the project doesn't exist
+        self._openshift_get_project(project_id)
+        api = self.get_resource_api(API_CORE, "ResourceQuota")
+        res = clean_openshift_metadata(api.get(namespace=project_id).to_dict())
+
+        return res["items"]
+
+    def _wait_for_quota_to_settle(self, project_id, resource_quota):
+        """Wait for quota on resourcequotas to settle.
+
+        When creating a new resourcequota that sets a quota on resourcequota objects, we need to
+        wait for OpenShift to calculate the quota usage before we attempt to create any new
+        resourcequota objects.
+        """
+
+        if "resourcequotas" in resource_quota["spec"]["hard"]:
+            logger.info("waiting for resourcequota quota")
+
+            api = self.get_resource_api(API_CORE, "ResourceQuota")
+            while True:
+                resp = clean_openshift_metadata(
+                    api.get(
+                        namespace=project_id, name=resource_quota["metadata"]["name"]
+                    ).to_dict()
+                )
+                if "resourcequotas" in resp["status"].get("used", {}):
+                    break
+                time.sleep(0.1)
+
+    def _openshift_create_resourcequota(self, project_id, quota_def):
+        api = self.get_resource_api(API_CORE, "ResourceQuota")
+        res = api.create(namespace=project_id, body=quota_def).to_dict()
+        self._wait_for_quota_to_settle(project_id, res)
+
+    def _openshift_delete_resourcequota(self, project_id, resourcequota_name):
+        """In an openshift namespace {project_id) delete a specified resourcequota"""
+        api = self.get_resource_api(API_CORE, "ResourceQuota")
+        return api.delete(namespace=project_id, name=resourcequota_name).to_dict()
+

src/coldfront_plugin_cloud/signals.py

@@ -6,11 +6,15 @@
 from coldfront_plugin_cloud.tasks import (activate_allocation,
                                           add_user_to_allocation,
                                           disable_allocation,
-                                          remove_user_from_allocation)
+                                          remove_user_from_allocation,
+                                          get_allocation_usage,
+                                          approve_change_request)
+from coldfront_plugin_cloud import utils
 from coldfront.core.allocation.signals import (allocation_activate,
                                                allocation_activate_user,
                                                allocation_disable,
                                                allocation_remove_user,
+                                               allocation_change_created,
                                                allocation_change_approved)
 
 
@@ -52,3 +56,20 @@ def activate_allocation_user_receiver(sender, **kwargs):
 def allocation_remove_user_receiver(sender, **kwargs):
     allocation_user_pk = kwargs.get('allocation_user_pk')
     remove_user_from_allocation(allocation_user_pk)
+
+# TODO (Quan): How to/should we do the functional test for this?
+@receiver(allocation_change_created)
+def allocation_change_created_receiver(sender, **kwargs):
+    allocation_pk = kwargs.get('allocation_pk')
+    allocation_change_pk = kwargs.get('allocation_change_pk')
+
+    if not utils.check_cr_only_decreases(allocation_change_pk):
+        return
+
+    if utils.check_cr_set_to_zero(allocation_change_pk):
+        return
+
+    allocation_quota_usage = get_allocation_usage(allocation_pk)
+    if allocation_quota_usage and utils.check_usage_is_lower(allocation_change_pk, allocation_quota_usage):
+        approve_change_request(allocation_change_pk) # Updates attributes on Coldfront side
+        allocation_change_approved.send(None, allocation_pk=allocation_pk, allocation_change_pk=allocation_change_pk)