Skip to content

Refactored validate_allocation to use tasks.UNIT_QUOTA_MULTIPLIERS when determining what attributes an allocation should have #201

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
QuanMPhm merged 1 commit into from
May 7, 2025
Merged

Refactored validate_allocation to use tasks.UNIT_QUOTA_MULTIPLIERS when determining what attributes an allocation should have #201

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
212 changes: 103 additions & 109 deletions src/coldfront_plugin_cloud/management/commands/validate_allocations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,39 +103,38 @@ def handle(self, *args, **options):

obj_key = openstack.OpenStackResourceAllocator.QUOTA_KEY_MAPPING['object']['keys'][attributes.QUOTA_OBJECT_GB]

for attr in attributes.ALLOCATION_QUOTA_ATTRIBUTES:
if 'OpenStack' in attr.name:
key = allocator.QUOTA_KEY_MAPPING_ALL_KEYS.get(attr.name, None)
if not key:
# Note(knikolla): Some attributes are only maintained
# for bookkeeping purposes and do not have a
# corresponding quota set on the service.
continue

expected_value = allocation.get_attribute(attr.name)
current_value = quota.get(key, None)
if key == obj_key and expected_value <= 0:
expected_obj_value = 1
current_value = int(allocator.object(project_id).head_account().get(obj_key))
if current_value != expected_obj_value:
failed_validation = True
msg = (f'Value for quota for {attr.name} = {current_value} does not match expected'
f' value of {expected_obj_value} on allocation {allocation_str}')
logger.warning(msg)
elif expected_value is None and current_value:
msg = (f'Attribute "{attr.name}" expected on allocation {allocation_str} but not set.'
f' Current quota is {current_value}.')
if options['apply']:
utils.set_attribute_on_allocation(
allocation, attr.name, current_value
)
msg = f'{msg} Attribute set to match current quota.'
logger.warning(msg)
elif not current_value == expected_value:
for attr in tasks.get_expected_attributes(allocator):
key = allocator.QUOTA_KEY_MAPPING_ALL_KEYS.get(attr, None)
if not key:
# Note(knikolla): Some attributes are only maintained
# for bookkeeping purposes and do not have a
# corresponding quota set on the service.
continue

expected_value = allocation.get_attribute(attr)
current_value = quota.get(key, None)
if key == obj_key and expected_value <= 0:
expected_obj_value = 1
current_value = int(allocator.object(project_id).head_account().get(obj_key))
if current_value != expected_obj_value:
failed_validation = True
msg = (f'Value for quota for {attr.name} = {current_value} does not match expected'
f' value of {expected_value} on allocation {allocation_str}')
msg = (f'Value for quota for {attr} = {current_value} does not match expected'
f' value of {expected_obj_value} on allocation {allocation_str}')
logger.warning(msg)
elif expected_value is None and current_value:
msg = (f'Attribute "{attr}" expected on allocation {allocation_str} but not set.'
f' Current quota is {current_value}.')
if options['apply']:
utils.set_attribute_on_allocation(
allocation, attr, current_value
)
msg = f'{msg} Attribute set to match current quota.'
logger.warning(msg)
elif not current_value == expected_value:
failed_validation = True
msg = (f'Value for quota for {attr} = {current_value} does not match expected'
f' value of {expected_value} on allocation {allocation_str}')
logger.warning(msg)

if failed_validation and options['apply']:
try:
Expand Down Expand Up @@ -185,89 +184,84 @@ def handle(self, *args, **options):

failed_validation = Command.sync_users(project_id, allocation, allocator, options["apply"])

for attr in attributes.ALLOCATION_QUOTA_ATTRIBUTES:
if "OpenShift" in attr.name:
key_with_lambda = openshift.OpenShiftResourceAllocator.QUOTA_KEY_MAPPING.get(attr.name, None)

# Openshift and Openshift VM allocations have different quota attributes
if not key_with_lambda:
continue

# This gives me just the plain key
key = list(key_with_lambda(1).keys())[0]

expected_value = allocation.get_attribute(attr.name)
current_value = quota.get(key, None)

PATTERN = r"([0-9]+)(m|Ki|Mi|Gi|Ti|Pi|Ei|K|M|G|T|P|E)?"

suffix = {
"Ki": 2**10,
"Mi": 2**20,
"Gi": 2**30,
"Ti": 2**40,
"Pi": 2**50,
"Ei": 2**60,
"m": 10**-3,
"K": 10**3,
"M": 10**6,
"G": 10**9,
"T": 10**12,
"P": 10**15,
"E": 10**18,
}

if current_value and current_value != "0":
result = re.search(PATTERN, current_value)

if result is None:
raise CommandError(
f"Unable to parse current_value = '{current_value}' for {attr.name}"
)
for attr in tasks.get_expected_attributes(allocator):
key_with_lambda = allocator.QUOTA_KEY_MAPPING.get(attr, None)

# This gives me just the plain key
key = list(key_with_lambda(1).keys())[0]

expected_value = allocation.get_attribute(attr)
current_value = quota.get(key, None)

PATTERN = r"([0-9]+)(m|Ki|Mi|Gi|Ti|Pi|Ei|K|M|G|T|P|E)?"

suffix = {
"Ki": 2**10,
"Mi": 2**20,
"Gi": 2**30,
"Ti": 2**40,
"Pi": 2**50,
"Ei": 2**60,
"m": 10**-3,
"K": 10**3,
"M": 10**6,
"G": 10**9,
"T": 10**12,
"P": 10**15,
"E": 10**18,
}

if current_value and current_value != "0":
result = re.search(PATTERN, current_value)

if result is None:
raise CommandError(
f"Unable to parse current_value = '{current_value}' for {attr}"
)

value = int(result.groups()[0])
unit = result.groups()[1]
value = int(result.groups()[0])
unit = result.groups()[1]

# Convert to number i.e. without any unit suffix
# Convert to number i.e. without any unit suffix

if unit is not None:
current_value = value * suffix[unit]
else:
current_value = value
if unit is not None:
current_value = value * suffix[unit]
else:
current_value = value

# Convert some attributes to units that coldfront uses
# Convert some attributes to units that coldfront uses

if "RAM" in attr.name:
current_value = round(current_value / suffix["Mi"])
elif "Storage" in attr.name:
current_value = round(current_value / suffix["Gi"])
elif current_value and current_value == "0":
current_value = 0
if "RAM" in attr:
current_value = round(current_value / suffix["Mi"])
elif "Storage" in attr:
current_value = round(current_value / suffix["Gi"])
elif current_value and current_value == "0":
current_value = 0

if expected_value is None and current_value is not None:
msg = (
f'Attribute "{attr.name}" expected on allocation {allocation_str} but not set.'
f" Current quota is {current_value}."
)
if options["apply"]:
utils.set_attribute_on_allocation(
allocation, attr.name, current_value
)
msg = f"{msg} Attribute set to match current quota."
logger.warning(msg)
elif not (current_value == expected_value):
msg = (
f"Value for quota for {attr.name} = {current_value} does not match expected"
f" value of {expected_value} on allocation {allocation_str}"
if expected_value is None and current_value is not None:
msg = (
f'Attribute "{attr}" expected on allocation {allocation_str} but not set.'
f" Current quota is {current_value}."
)
if options["apply"]:
utils.set_attribute_on_allocation(
allocation, attr, current_value
)
logger.warning(msg)
msg = f"{msg} Attribute set to match current quota."
logger.warning(msg)
elif not (current_value == expected_value):
msg = (
f"Value for quota for {attr} = {current_value} does not match expected"
f" value of {expected_value} on allocation {allocation_str}"
)
logger.warning(msg)

if options["apply"]:
try:
allocator.set_quota(project_id)
logger.warning(
f"Quota for allocation {project_id} was out of date. Reapplied!"
)
except Exception as e:
logger.error(f'setting openshift quota failed: {e}')
continue
if options["apply"]:
try:
allocator.set_quota(project_id)
logger.warning(
f"Quota for allocation {project_id} was out of date. Reapplied!"
)
except Exception as e:
logger.error(f'setting openshift quota failed: {e}')
continue
11 changes: 8 additions & 3 deletions src/coldfront_plugin_cloud/tasks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,11 @@
},
}

def get_expected_attributes(allocator: base.ResourceAllocator):
"""Based on the allocator's resource type, return the expected quotas attributes the allocation should have"""
resource_name = allocator.resource_type
return list(UNIT_QUOTA_MULTIPLIERS[resource_name].keys())


def find_allocator(allocation) -> base.ResourceAllocator:
allocators = {
Expand All @@ -99,10 +104,10 @@ def set_quota_attributes():
allocation.quantity = 1

# Calculate the quota for the project, and set the attribute for each element
uqm = UNIT_QUOTA_MULTIPLIERS[allocator.resource_type]
for coldfront_attr in uqm.keys():
expected_coldfront_attrs = get_expected_attributes(allocator)
for coldfront_attr in expected_coldfront_attrs:
if not allocation.get_attribute(coldfront_attr):
value = allocation.quantity * uqm.get(coldfront_attr, 0)
value = allocation.quantity * UNIT_QUOTA_MULTIPLIERS[allocator.resource_type].get(coldfront_attr, 0)
value += STATIC_QUOTA[allocator.resource_type].get(coldfront_attr, 0)
utils.set_attribute_on_allocation(allocation,
coldfront_attr,
Expand Down