diff --git a/Makefile b/Makefile index 3d960a54b..748391255 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,17 @@ GIT_BRANCH:=$(shell git symbolic-ref --short HEAD 2>&1 | grep -v fatal) #define the GO_BUILD_ARGS if you need to pass additional arguments to the go build GO_BUILD_ARGS?= +## Location to install dependencies to +LOCALBIN ?= $(shell pwd)/bin +$(LOCALBIN): + mkdir -p $(LOCALBIN) + +## Tool Versions +CONTROLLER_TOOLS_VERSION ?= v0.9.2 + +## Tool Binaries +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen + # Reset branch name if this a Travis CI environment ifneq ($(strip $(TRAVIS_BRANCH)),) GIT_BRANCH:=${TRAVIS_BRANCH} @@ -28,7 +39,7 @@ endif .PHONY: print-global-variables -# Build the controler executable for use in docker image build +# Build the controller executable for use in docker image build mcad-controller: init generate-code ifeq ($(strip $(GO_BUILD_ARGS)),) $(info Compiling controller) @@ -60,6 +71,15 @@ verify-tag-name: print-global-variables # Check for invalid tag name t=${TAG} && [ $${#t} -le 128 ] || { echo "Target name $$t has 128 or more chars"; false; } +.PHONY: controller-gen +controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. +$(CONTROLLER_GEN): $(LOCALBIN) + test -s $(LOCALBIN)/controller-gen || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) + +.PHONY: manifests +manifests: controller-gen ## Generate CustomResourceDefinition objects. + $(CONTROLLER_GEN) crd:allowDangerousTypes=true paths="./pkg/apis/..." output:crd:artifacts:config=config/crd/bases + generate-code: pkg/apis/controller/v1beta1/zz_generated.deepcopy.go pkg/apis/controller/v1beta1/zz_generated.deepcopy.go: ${BIN_DIR}/deepcopy-gen @@ -139,18 +159,17 @@ clean: #CRD file maintenance rules DEPLOYMENT_CRD_DIR=deployment/mcad-controller/crds CRD_BASE_DIR=config/crd/bases -MCAD_CRDS= ${DEPLOYMENT_CRD_DIR}/ibm.com_quotasubtree-v1.yaml \ +MCAD_CRDS= ${DEPLOYMENT_CRD_DIR}/ibm.com_quotasubtrees.yaml \ ${DEPLOYMENT_CRD_DIR}/mcad.ibm.com_appwrappers.yaml \ ${DEPLOYMENT_CRD_DIR}/mcad.ibm.com_queuejobs.yaml \ ${DEPLOYMENT_CRD_DIR}/mcad.ibm.com_schedulingspecs.yaml update-deployment-crds: ${MCAD_CRDS} -${DEPLOYMENT_CRD_DIR}/mcad.ibm.com_schedulingspecs.yaml : ${CRD_BASE_DIR}/mcad.ibm.com_schedulingspecs.yaml +${DEPLOYMENT_CRD_DIR}/ibm.com_quotasubtrees.yaml : ${CRD_BASE_DIR}/ibm.com_quotasubtrees.yaml ${DEPLOYMENT_CRD_DIR}/mcad.ibm.com_appwrappers.yaml : ${CRD_BASE_DIR}/mcad.ibm.com_appwrappers.yaml ${DEPLOYMENT_CRD_DIR}/mcad.ibm.com_queuejobs.yaml : ${CRD_BASE_DIR}/mcad.ibm.com_queuejobs.yaml ${DEPLOYMENT_CRD_DIR}/mcad.ibm.com_schedulingspecs.yaml : ${CRD_BASE_DIR}/mcad.ibm.com_schedulingspecs.yaml - $(DEPLOYMENT_CRD_DIR)/%: ${CRD_BASE_DIR}/% - cp $< $@ \ No newline at end of file + cp $< $@ diff --git a/config/crd/bases/ibm.com_quotasubtree-v1.yaml b/config/crd/bases/ibm.com_quotasubtree-v1.yaml deleted file mode 100644 index 38bd14197..000000000 --- a/config/crd/bases/ibm.com_quotasubtree-v1.yaml +++ /dev/null @@ -1,62 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: quotasubtrees.ibm.com - finalizers: [] -spec: - group: ibm.com - scope: Namespaced - names: - kind: QuotaSubtree - singular: quotasubtree - plural: quotasubtrees - shortNames: - - qst - versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - schema: - openAPIV3Schema: - type: object - properties: - status: - type: object - x-kubernetes-preserve-unknown-fields: true - spec: - type: object - properties: - parent: - type: string - parentNamespace: - type: string - children: - type: array - items: - type: object - properties: - name: - type: string - namespace: - type: string - quotas: - type: object - properties: - disabled: - type: boolean - hardLimit: - type: boolean - requests: - type: object - properties: - cpu: - x-kubernetes-int-or-string: true - pattern: '^[0-9]*(m)?$' - memory: - x-kubernetes-int-or-string: true - pattern: '^[0-9]*(Ei|Pi|Ti|Gi|Mi|Ki|E|P|T|G|M|K)?$' - nvidia.com/gpu: - x-kubernetes-int-or-string: true - pattern: '^[0-9]?$' diff --git a/config/crd/bases/ibm.com_quotasubtrees.yaml b/config/crd/bases/ibm.com_quotasubtrees.yaml new file mode 100644 index 000000000..a410ab505 --- /dev/null +++ b/config/crd/bases/ibm.com_quotasubtrees.yaml @@ -0,0 +1,122 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: quotasubtrees.ibm.com +spec: + group: ibm.com + names: + kind: QuotaSubtree + listKind: QuotaSubtreeList + plural: quotasubtrees + singular: quotasubtree + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: QuotaSubtree is a specification for a quota subtree resource + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: QuotaSubtreeSpec is the spec for a resource plan + properties: + children: + items: + description: Child is the spec for a QuotaSubtree resource + properties: + name: + type: string + namespace: + type: string + path: + type: string + quotas: + description: Quota is the spec for a QuotaSubtree resource + properties: + disabled: + type: boolean + hardLimit: + type: boolean + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + type: array + parent: + type: string + parentNamespace: + type: string + type: object + status: + description: QuotaSubtreeStatus is the status for a QuotaSubtree resource + properties: + children: + items: + description: ResourceAllocation is the spec for the child status + properties: + allocated: + description: ResourceAllocationStatus is the spec for the child + resource usage + properties: + requests: + additionalProperties: + type: string + type: object + type: object + name: + type: string + namespace: + type: string + path: + type: string + type: object + type: array + totalAllocation: + description: ResourceAllocation is the spec for the child status + properties: + allocated: + description: ResourceAllocationStatus is the spec for the child + resource usage + properties: + requests: + additionalProperties: + type: string + type: object + type: object + name: + type: string + namespace: + type: string + path: + type: string + type: object + required: + - children + - totalAllocation + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/config/crd/bases/mcad.ibm.com_appwrappers.yaml b/config/crd/bases/mcad.ibm.com_appwrappers.yaml index 3280f8a14..cf3032a5e 100644 --- a/config/crd/bases/mcad.ibm.com_appwrappers.yaml +++ b/config/crd/bases/mcad.ibm.com_appwrappers.yaml @@ -1,8 +1,10 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.1 + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null name: appwrappers.mcad.ibm.com spec: group: mcad.ibm.com @@ -44,33 +46,33 @@ spec: properties: GenericItems: items: - description: AppWrapperResource is App Wrapper aggregation resource + description: AppWrapperGenericResource is App Wrapper aggregation + resource properties: allocated: description: The number of allocated replicas from this resource type format: int32 type: integer - apiVersion: - description: 'APIVersion defines the versioned schema of - this representation of an object. Servers should convert - recognized schemas to the latest internal value, and may - reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string completionstatus: - description: Optional field that drives completion status of appwrapper. - This field within an item of an appwrapper determines the full state of appwrapper. - The completionstatus field contains a list of conditions that make the associate item considered - completed, for instance :- completion conditions could be "Complete" or "Failed". - The associated item's level .status.conditions[].type field is monitored for any one of these conditions. Once all items with this - option is set and the conditionstatus is met the entire appwrapper state will be changed to one of the valid appwrapper completion state. Note :- this is an AND - operation for all items where this option is set. See the list of appwrapper states for a list of valid complete states. + description: 'Optional field that drives completion status + of this AppWrapper. This field within an item of an AppWrapper + determines the full state of the AppWrapper. The completionstatus + field contains a list of conditions that make the associate + item considered completed, for instance: - completion + conditions could be "Complete" or "Failed". The associated + item''s level .status.conditions[].type field is monitored + for any one of these conditions. Once all items with this + option is set and the conditionstatus is met the entire + AppWrapper state will be changed to one of the valid AppWrapper + completion state. Note: - this is an AND operation for + all items where this option is set. See the list of AppWrapper + states for a list of valid complete states.' type: string - custompodresources: description: Optional section that specifies resource requirements for non-standard k8s resources, follows same format as - that of standard k8s resources + that of standard k8s resources. items: properties: limits: @@ -107,14 +109,6 @@ spec: type: object x-kubernetes-embedded-resource: true x-kubernetes-preserve-unknown-fields: true - kind: - description: 'Kind is a string value representing the REST - resource this object represents. Servers may infer this - from the endpoint the client submits requests to. Cannot - be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object minavailable: description: The minimal available pods to run for this AppWrapper; the default value is nil @@ -135,182 +129,76 @@ spec: type: integer type: object type: array - Items: - items: - description: 'AppWrapperResource is App Wrapper aggregation - resource todo: To be depricated' - properties: - allocatedreplicas: - description: The number of allocated replicas from this - resource type - format: int32 - type: integer - apiVersion: - description: 'APIVersion defines the versioned schema of - this representation of an object. Servers should convert - recognized schemas to the latest internal value, and may - reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST - resource this object represents. Servers may infer this - from the endpoint the client submits requests to. Cannot - be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - minavailable: - description: The minimal available pods to run for this - AppWrapper; the default value is nil - format: int32 - type: integer - priority: - description: The priority of this resource - format: int32 - type: integer - priorityslope: - description: The increasing rate of priority value for this - resource - format: float - type: number - replicas: - description: Replicas is the number of desired replicas - format: int32 - type: integer - template: - description: The template for the resource; it is now a - raw text because we don't know for what resource it should - be instantiated - type: object - x-kubernetes-preserve-unknown-fields: true - type: - description: The type of the resource (is the resource a - Pod, a ReplicaSet, a ... ?) - type: string - required: - - template - type: object - type: array - apiVersion: - description: 'APIVersion defines the versioned schema of this - representation of an object. Servers should convert recognized - schemas to the latest internal value, and may reject unrecognized - values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource - this object represents. Servers may infer this from the endpoint - the client submits requests to. Cannot be updated. In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - description: ListMeta describes metadata that synthetic resources - must have, including lists and various status objects. A resource - may have only one of {ObjectMeta, ListMeta}. - properties: - continue: - description: continue may be set if the user set a limit on - the number of items returned, and indicates that the server - has more data available. The value is opaque and may be - used to issue another request to the endpoint that served - this list to retrieve the next set of available objects. - Continuing a consistent list may not be possible if the - server configuration has changed or more than a few minutes - have passed. The resourceVersion field returned when using - this continue value will be identical to the value in the - first response, unless you have received this token from - an error message. - type: string - remainingItemCount: - description: remainingItemCount is the number of subsequent - items in the list which are not included in this list response. - If the list request contained label or field selectors, - then the number of remaining items is unknown and the field - will be left unset and omitted during serialization. If - the list is complete (either because it is not chunking - or because this is the last chunk), then there are no more - remaining items and this field will be left unset and omitted - during serialization. Servers older than v1.15 do not set - this field. The intended use of the remainingItemCount is - *estimating* the size of a collection. Clients should not - rely on the remainingItemCount to be set or to be exact. - format: int64 - type: integer - resourceVersion: - description: 'String that identifies the server''s internal - version of this object that can be used by clients to determine - when objects have changed. Value must be treated as opaque - by clients and passed unmodified back to the server. Populated - by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' - type: string - selfLink: - description: "selfLink is a URL representing this object. - Populated by the system. Read-only. \n DEPRECATED Kubernetes - will stop propagating this field in 1.20 release and the - field is planned to be removed in 1.21 release." - type: string - type: object type: object schedulingSpec: - description: Field 'schedulingSpec' specifies the parameters used for scheduling generic items - wrapped inside AppWrappers. It defines the policy for requeuing jobs based on the number - of running pods. + description: SchedSpec specifies the parameters used for scheduling + generic items wrapped inside AppWrappers. It defines the policy + for requeuing jobs based on the number of running pods. properties: + dispatchDuration: + description: Wall clock duration time of appwrapper in seconds. + properties: + expected: + type: integer + limit: + type: integer + overrun: + type: boolean + type: object minAvailable: - description: Expected number of pods in running and/or completed state. Requeuing is triggered - when the number of running/completed pods is not equal to this value. When not specified, - requeuing is disabled and no check is performed. + description: Expected number of pods in running and/or completed + state. Requeuing is triggered when the number of running/completed + pods is not equal to this value. When not specified, requeuing + is disabled and no check is performed. type: integer + nodeSelector: + additionalProperties: + type: string + type: object requeuing: - description: Specification of the requeuing strategy based on waiting time. Values in this field - control how often the pod check should happen and if requeuing has reached its maximum number of - times. + description: Specification of the requeuing strategy based on + waiting time. Values in this field control how often the pod + check should happen, and if requeuing has reached its maximum + number of times. properties: + growthType: + default: exponential + description: Growth strategy to increase the waiting time + between requeuing checks. The values available are 'exponential', + 'linear', or 'none'. For example, 'exponential' growth would + double the 'timeInSeconds' value every time a requeuing + event is triggered. If the string value is misspelled or + not one of the possible options, the growth behavior is + defaulted to 'none'. + type: string initialTimeInSeconds: - description: Value to keep track of the initial wait time. Users cannot set this as it is - taken from 'timeInSeconds'. - type: integer - timeInSeconds: - description: Initial waiting time before requeuing conditions are checked. This value is - specified by the user, but it may grow as requeuing events happen. - type: integer - default: 300 - maxTimeInSeconds: - description: Maximum waiting time for requeuing checks + description: Value to keep track of the initial wait time. + Users cannot set this as it is taken from 'timeInSeconds'. type: integer + maxNumRequeuings: default: 0 - growthType: - description: Growth strategy to increase the waiting time between requeuing checks. The values - available are 'exponential', 'linear', or 'none'. For example, 'exponential' growth would - double the 'timeInSeconds' value every time a requeuing event is triggered. If the string value - is misspelled or not one of the possible options, the growth behavior is defaulted to 'none'. - type: string - default: "exponential" - numRequeuings: - description: Field to keep track of how many times a requeuing event has been triggered + description: Maximum number of requeuing events allowed. Once + this value is reached (e.g., 'numRequeuings = maxNumRequeuings', + no more requeuing checks are performed and the generic items + are stopped and removed from the cluster (AppWrapper remains + deployed). type: integer + maxTimeInSeconds: default: 0 - maxNumRequeuings: - description: Maximum number of requeuing events allowed. Once this value is reached (e.g., - 'numRequeuings = maxNumRequeuings', no more requeuing checks are performed and the generic - items are stopped and removed from the cluster (AppWrapper remains deployed). + description: Maximum waiting time for requeuing checks. type: integer + numRequeuings: default: 0 - type: object - dispatchDuration: - description: Wall clock duration time of appwrapper in seconds - properties: - expected: - format: int32 + description: Field to keep track of how many times a requeuing + event has been triggered. type: integer - limit: - format: int32 + timeInSeconds: + default: 300 + description: Initial waiting time before requeuing conditions + are checked. This value is specified by the user, but it + may grow as requeuing events happen. type: integer type: object - nodeSelector: - additionalProperties: - type: string - type: object type: object selector: description: A label selector is a label query over a set of resources. @@ -359,6 +247,7 @@ spec: are ANDed. type: object type: object + x-kubernetes-map-type: atomic service: description: AppWrapperService is App Wrapper service definition properties: @@ -695,10 +584,10 @@ spec: description: Can run? type: boolean conditions: - description: Represents the latest available observations of a appwrapper's + description: Represents the latest available observations of the AppWrapper's current condition. items: - description: DeploymentCondition describes the state of a deployment + description: AppWrapperCondition describes the state of an AppWrapper at a certain point. properties: lastTransitionMicroTime: @@ -711,7 +600,7 @@ spec: format: date-time type: string message: - description: A human readable message indicating details about + description: A human-readable message indicating details about the transition. type: string reason: @@ -721,65 +610,23 @@ spec: description: Status of the condition, one of True, False, Unknown. type: string type: - description: Type of appwrapper condition. + description: Type of AppWrapper condition. type: string required: - status - type type: object type: array - pendingpodconditions: - description: Represent conditions of pod(s) that failed scheduling. - items: - description: Describes scheduling failed condition(s) of a pod - properties: - podname: - description: Name of the pod - type: string - conditions: - description: Failed condition(s) of a pod - items: - description: DeploymentCondition describes the state of a deployment - at a certain point. - properties: - LastProbeTime: - description: Last time the condition transitioned from one status - to another. - format: date-time - type: string - LastTransitionTime: - description: The last time this condition was updated. - format: date-time - type: string - message: - description: A human readable message indicating details about - the transition. - type: string - reason: - description: The reason for the condition's last transition. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of appwrapper condition. - type: string - required: - - status - - type - type: object - type: array - type: object - type: array + controllerfirstdispatchtimestamp: + description: Microsecond level timestamp when controller first dispatches + the AppWrapper + format: date-time + type: string controllerfirsttimestamp: description: Microsecond level timestamp when controller first sees QueueJob (by Informer) format: date-time type: string - controllerfirstdispatchtimestamp: - description: Microsecond level timestamp when controller first dispatches appwrapper - format: date-time - type: string failed: description: The number of resources which reached phase Failed. format: int32 @@ -792,34 +639,77 @@ spec: description: Is Dispatched? type: boolean local: - description: Indicate if message is a duplicate (for Informer to recognize duplicate messages) + description: Indicate if message is a duplicate (for Informer to recognize + duplicate messages) type: boolean message: type: string - totalcpu: - description: The number of cpu consumed by all pods belonging to an appwrapper. - format: int32 - type: integer - totalmemory: - description: The amount of memory consumed by all pods belonging to an appwrapper. - format: int32 - type: integer - totalgpu: - description: The total number of GPUs consumed by all pods belonging to an appwrapper. - format: int32 + numberOfRequeueings: + description: Field to keep track of how many times a requeuing event + has been triggered type: integer pending: description: The number of pending pods. format: int32 type: integer + pendingpodconditions: + description: Represents the latest available observations of pods + belonging to the AppWrapper. + items: + properties: + conditions: + items: + description: PodCondition contains details for the current + condition of this pod. + properties: + lastProbeTime: + description: Last time we probed the condition. + format: date-time + type: string + lastTransitionTime: + description: Last time the condition transitioned from + one status to another. + format: date-time + type: string + message: + description: Human-readable message indicating details + about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the + condition's last transition. + type: string + status: + description: 'Status is the status of the condition. Can + be True, False, Unknown. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + type: string + type: + description: 'Type is the type of the condition. More + info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + type: string + required: + - status + - type + type: object + type: array + podname: + type: string + type: object + type: array queuejobstate: - description: State of QueueJob - Init, Queueing, HeadOfLine, Rejoining ... + description: State of QueueJob - Init, Queueing, HeadOfLine, Rejoining, + ... type: string + requeueingTimeInSeconds: + description: Field to keep track of total number of seconds spent + in requeueing + type: integer running: format: int32 type: integer sender: - description: Indicate sender of this message (extremely useful for debugging) + description: Indicate sender of this message (extremely useful for + debugging) type: string state: description: State - Pending, Running, Failed, Deleted @@ -833,16 +723,21 @@ spec: (is this different from the MinAvailable from JobStatus) format: int32 type: integer - numberOfRequeueings: - description: Field to keep track of how many times a requeuing event has been triggered + totalcpu: + description: The number of CPU consumed by all pods belonging to the + AppWrapper. format: int32 type: integer - default: 0 - requeueingTimeInSeconds: - description: Field to keep track of total number of seconds spent in requeueing + totalgpu: + description: The total number of GPUs consumed by all pods belonging + to the AppWrapper. + format: int32 + type: integer + totalmemory: + description: The amount of memory consumed by all pods belonging to + the AppWrapper. format: int32 type: integer - default: 0 type: object required: - spec @@ -851,4 +746,3 @@ spec: storage: true subresources: status: {} - diff --git a/config/crd/bases/mcad.ibm.com_queuejobs.yaml b/config/crd/bases/mcad.ibm.com_queuejobs.yaml index 6c60855de..9a7344f68 100644 --- a/config/crd/bases/mcad.ibm.com_queuejobs.yaml +++ b/config/crd/bases/mcad.ibm.com_queuejobs.yaml @@ -1,10 +1,10 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.1 + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null name: queuejobs.mcad.ibm.com spec: group: mcad.ibm.com @@ -33,70 +33,77 @@ spec: type: object spec: description: Specification of the desired behavior of a cron job, including - the minAvailable and the requeuing strategy + the minAvailable properties: schedulerName: type: string schedulingSpec: - description: Field 'schedulingSpec' specifies the parameters used for scheduling generic items - wrapped inside AppWrappers. It defines the policy for requeuing jobs based on the number - of running pods. + description: SchedSpec specifies the parameters for scheduling. properties: + dispatchDuration: + description: Wall clock duration time of appwrapper in seconds. + properties: + expected: + type: integer + limit: + type: integer + overrun: + type: boolean + type: object minAvailable: - description: Expected number of pods in running and/or completed state. Requeuing is triggered - when the number of running/completed pods is not equal to this value. When not specified, - requeuing is disabled and no check is performed. + description: Expected number of pods in running and/or completed + state. Requeuing is triggered when the number of running/completed + pods is not equal to this value. When not specified, requeuing + is disabled and no check is performed. type: integer + nodeSelector: + additionalProperties: + type: string + type: object requeuing: - description: Specification of the requeuing strategy based on waiting time. Values in this field - control how often the pod check should happen and if requeuing has reached its maximum number of - times. + description: Specification of the requeuing strategy based on + waiting time. Values in this field control how often the pod + check should happen, and if requeuing has reached its maximum + number of times. properties: + growthType: + default: exponential + description: Growth strategy to increase the waiting time + between requeuing checks. The values available are 'exponential', + 'linear', or 'none'. For example, 'exponential' growth would + double the 'timeInSeconds' value every time a requeuing + event is triggered. If the string value is misspelled or + not one of the possible options, the growth behavior is + defaulted to 'none'. + type: string initialTimeInSeconds: - description: Value to keep track of the initial wait time. Users cannot set this as it is - taken from 'timeInSeconds'. - type: integer - timeInSeconds: - description: Initial waiting time before requeuing conditions are checked. This value is - specified by the user, but it may grow as requeuing events happen. - type: integer - default: 300 - maxTimeInSeconds: - description: Maximum waiting time for requeuing checks + description: Value to keep track of the initial wait time. + Users cannot set this as it is taken from 'timeInSeconds'. type: integer + maxNumRequeuings: default: 0 - growthType: - description: Growth strategy to increase the waiting time between requeuing checks. The values - available are 'exponential', 'linear', or 'none'. For example, 'exponential' growth would - double the 'timeInSeconds' value every time a requeuing event is triggered. If the string value - is misspelled or not one of the possible options, the growth behavior is defaulted to 'none'. - type: string - default: "exponential" - numRequeuings: - description: Field to keep track of how many times a requeuing event has been triggered + description: Maximum number of requeuing events allowed. Once + this value is reached (e.g., 'numRequeuings = maxNumRequeuings', + no more requeuing checks are performed and the generic items + are stopped and removed from the cluster (AppWrapper remains + deployed). type: integer + maxTimeInSeconds: default: 0 - maxNumRequeuings: - description: Maximum number of requeuing events allowed. Once this value is reached (e.g., - 'numRequeuings = maxNumRequeuings', no more requeuing checks are performed and the generic - items are stopped and removed from the cluster (AppWrapper remains deployed). + description: Maximum waiting time for requeuing checks. type: integer + numRequeuings: default: 0 - type: object - dispatchDuration: - description: Wall clock duration time of appwrapper in seconds - properties: - expected: - format: int32 + description: Field to keep track of how many times a requeuing + event has been triggered. type: integer - limit: - format: int32 + timeInSeconds: + default: 300 + description: Initial waiting time before requeuing conditions + are checked. This value is specified by the user, but it + may grow as requeuing events happen. type: integer type: object - nodeSelector: - additionalProperties: - type: string - type: object type: object taskSpecs: description: TaskSpecs specifies the task specification of QueueJob @@ -153,6 +160,7 @@ spec: only "value". The requirements are ANDed. type: object type: object + x-kubernetes-map-type: atomic template: description: Specifies the pod that will be created for this TaskSpec when executing a QueueJob @@ -497,6 +505,7 @@ spec: are ANDed. type: object type: object + x-kubernetes-map-type: atomic namespaces: description: namespaces specifies which namespaces the labelSelector @@ -612,6 +621,7 @@ spec: ANDed. type: object type: object + x-kubernetes-map-type: atomic namespaces: description: namespaces specifies which namespaces the labelSelector applies @@ -728,6 +738,7 @@ spec: are ANDed. type: object type: object + x-kubernetes-map-type: atomic namespaces: description: namespaces specifies which namespaces the labelSelector @@ -843,6 +854,7 @@ spec: ANDed. type: object type: object + x-kubernetes-map-type: atomic namespaces: description: namespaces specifies which namespaces the labelSelector applies @@ -5110,6 +5122,7 @@ spec: "value". The requirements are ANDed. type: object type: object + x-kubernetes-map-type: atomic maxSkew: description: 'MaxSkew describes the degree to which pods may be unevenly distributed. When @@ -5145,12 +5158,12 @@ spec: spread constraint. - DoNotSchedule (default) tells the scheduler not to schedule it. - ScheduleAnyway tells the scheduler to schedule the pod in any - location, but giving higher precedence to - topologies that would help reduce the skew. - A constraint is considered "Unsatisfiable" for - an incoming pod if and only if every possible - node assigment for that pod would violate "MaxSkew" - on some topology. For example, in a 3-zone cluster, + location, but giving higher precedence to topologies + that would help reduce the skew. A constraint + is considered "Unsatisfiable" for an incoming + pod if and only if every possible node assigment + for that pod would violate "MaxSkew" on some + topology. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | If WhenUnsatisfiable @@ -5625,12 +5638,12 @@ spec: pod is removed. \n Use this if: a) the volume is only needed while the pod runs, b) features of normal volumes like restoring from snapshot - or capacity tracking are needed, c) the storage + or capacity tracking are needed, c) the storage driver is specified through a storage class, and d) the storage driver supports dynamic volume - provisioning through a PersistentVolumeClaim - (see EphemeralVolumeSource for more information - on the connection between this volume type and + provisioning through a PersistentVolumeClaim + (see EphemeralVolumeSource for more information + on the connection between this volume type and PersistentVolumeClaim). \n Use PersistentVolumeClaim or one of the vendor-specific APIs for volumes that persist for longer than the lifecycle of @@ -5825,6 +5838,7 @@ spec: are ANDed. type: object type: object + x-kubernetes-map-type: atomic storageClassName: description: 'Name of the StorageClass required by the claim. More info: diff --git a/config/crd/bases/mcad.ibm.com_schedulingspecs.yaml b/config/crd/bases/mcad.ibm.com_schedulingspecs.yaml index 3de2af124..e91a18ffd 100644 --- a/config/crd/bases/mcad.ibm.com_schedulingspecs.yaml +++ b/config/crd/bases/mcad.ibm.com_schedulingspecs.yaml @@ -1,10 +1,10 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.1 + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null name: schedulingspecs.mcad.ibm.com spec: group: mcad.ibm.com @@ -33,70 +33,71 @@ spec: type: object spec: properties: + dispatchDuration: + description: Wall clock duration time of appwrapper in seconds. + properties: + expected: + type: integer + limit: + type: integer + overrun: + type: boolean + type: object minAvailable: - description: Expected number of pods in running and/or completed state. Requeuing is triggered - when the number of running/completed pods is not equal to this value. When not specified, - requeuing is disabled and no check is performed. + description: Expected number of pods in running and/or completed state. + Requeuing is triggered when the number of running/completed pods + is not equal to this value. When not specified, requeuing is disabled + and no check is performed. type: integer + nodeSelector: + additionalProperties: + type: string + type: object requeuing: - description: Specification of the requeuing strategy based on waiting time. Values in this field - control how often the pod check should happen and if requeuing has reached its maximum number of - times. + description: Specification of the requeuing strategy based on waiting + time. Values in this field control how often the pod check should + happen, and if requeuing has reached its maximum number of times. properties: + growthType: + default: exponential + description: Growth strategy to increase the waiting time between + requeuing checks. The values available are 'exponential', 'linear', + or 'none'. For example, 'exponential' growth would double the + 'timeInSeconds' value every time a requeuing event is triggered. + If the string value is misspelled or not one of the possible + options, the growth behavior is defaulted to 'none'. + type: string initialTimeInSeconds: - description: Value to keep track of the initial wait time. Users cannot set this as it is - taken from 'timeInSeconds'. - type: integer - timeInSeconds: - description: Initial waiting time before requeuing conditions are checked. This value is - specified by the user, but it may grow as requeuing events happen. - type: integer - default: 300 - maxTimeInSeconds: - description: Maximum waiting time for requeuing checks + description: Value to keep track of the initial wait time. Users + cannot set this as it is taken from 'timeInSeconds'. type: integer + maxNumRequeuings: default: 0 - growthType: - description: Growth strategy to increase the waiting time between requeuing checks. The values - available are 'exponential', 'linear', or 'none'. For example, 'exponential' growth would - double the 'timeInSeconds' value every time a requeuing event is triggered. If the string value - is misspelled or not one of the possible options, the growth behavior is defaulted to 'none'. - type: string - default: "exponential" - numRequeuings: - description: Field to keep track of how many times a requeuing event has been triggered + description: Maximum number of requeuing events allowed. Once + this value is reached (e.g., 'numRequeuings = maxNumRequeuings', + no more requeuing checks are performed and the generic items + are stopped and removed from the cluster (AppWrapper remains + deployed). type: integer + maxTimeInSeconds: default: 0 - maxNumRequeuings: - description: Maximum number of requeuing events allowed. Once this value is reached (e.g., - 'numRequeuings = maxNumRequeuings', no more requeuing checks are performed and the generic - items are stopped and removed from the cluster (AppWrapper remains deployed). + description: Maximum waiting time for requeuing checks. type: integer + numRequeuings: default: 0 - type: object - dispatchDuration: - description: Wall clock duration time of appwrapper in seconds - properties: - expected: - format: int32 + description: Field to keep track of how many times a requeuing + event has been triggered. type: integer - limit: - format: int32 + timeInSeconds: + default: 300 + description: Initial waiting time before requeuing conditions + are checked. This value is specified by the user, but it may + grow as requeuing events happen. type: integer type: object - nodeSelector: - additionalProperties: - type: string - type: object type: object required: - metadata type: object served: true storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/deployment/mcad-controller/crds/ibm.com_quotasubtree-v1.yaml b/deployment/mcad-controller/crds/ibm.com_quotasubtree-v1.yaml deleted file mode 100644 index 38bd14197..000000000 --- a/deployment/mcad-controller/crds/ibm.com_quotasubtree-v1.yaml +++ /dev/null @@ -1,62 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: quotasubtrees.ibm.com - finalizers: [] -spec: - group: ibm.com - scope: Namespaced - names: - kind: QuotaSubtree - singular: quotasubtree - plural: quotasubtrees - shortNames: - - qst - versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - schema: - openAPIV3Schema: - type: object - properties: - status: - type: object - x-kubernetes-preserve-unknown-fields: true - spec: - type: object - properties: - parent: - type: string - parentNamespace: - type: string - children: - type: array - items: - type: object - properties: - name: - type: string - namespace: - type: string - quotas: - type: object - properties: - disabled: - type: boolean - hardLimit: - type: boolean - requests: - type: object - properties: - cpu: - x-kubernetes-int-or-string: true - pattern: '^[0-9]*(m)?$' - memory: - x-kubernetes-int-or-string: true - pattern: '^[0-9]*(Ei|Pi|Ti|Gi|Mi|Ki|E|P|T|G|M|K)?$' - nvidia.com/gpu: - x-kubernetes-int-or-string: true - pattern: '^[0-9]?$' diff --git a/deployment/mcad-controller/crds/ibm.com_quotasubtrees.yaml b/deployment/mcad-controller/crds/ibm.com_quotasubtrees.yaml new file mode 100644 index 000000000..a410ab505 --- /dev/null +++ b/deployment/mcad-controller/crds/ibm.com_quotasubtrees.yaml @@ -0,0 +1,122 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: quotasubtrees.ibm.com +spec: + group: ibm.com + names: + kind: QuotaSubtree + listKind: QuotaSubtreeList + plural: quotasubtrees + singular: quotasubtree + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: QuotaSubtree is a specification for a quota subtree resource + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: QuotaSubtreeSpec is the spec for a resource plan + properties: + children: + items: + description: Child is the spec for a QuotaSubtree resource + properties: + name: + type: string + namespace: + type: string + path: + type: string + quotas: + description: Quota is the spec for a QuotaSubtree resource + properties: + disabled: + type: boolean + hardLimit: + type: boolean + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + type: array + parent: + type: string + parentNamespace: + type: string + type: object + status: + description: QuotaSubtreeStatus is the status for a QuotaSubtree resource + properties: + children: + items: + description: ResourceAllocation is the spec for the child status + properties: + allocated: + description: ResourceAllocationStatus is the spec for the child + resource usage + properties: + requests: + additionalProperties: + type: string + type: object + type: object + name: + type: string + namespace: + type: string + path: + type: string + type: object + type: array + totalAllocation: + description: ResourceAllocation is the spec for the child status + properties: + allocated: + description: ResourceAllocationStatus is the spec for the child + resource usage + properties: + requests: + additionalProperties: + type: string + type: object + type: object + name: + type: string + namespace: + type: string + path: + type: string + type: object + required: + - children + - totalAllocation + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/deployment/mcad-controller/crds/mcad.ibm.com_appwrappers.yaml b/deployment/mcad-controller/crds/mcad.ibm.com_appwrappers.yaml index 3280f8a14..cf3032a5e 100644 --- a/deployment/mcad-controller/crds/mcad.ibm.com_appwrappers.yaml +++ b/deployment/mcad-controller/crds/mcad.ibm.com_appwrappers.yaml @@ -1,8 +1,10 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.1 + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null name: appwrappers.mcad.ibm.com spec: group: mcad.ibm.com @@ -44,33 +46,33 @@ spec: properties: GenericItems: items: - description: AppWrapperResource is App Wrapper aggregation resource + description: AppWrapperGenericResource is App Wrapper aggregation + resource properties: allocated: description: The number of allocated replicas from this resource type format: int32 type: integer - apiVersion: - description: 'APIVersion defines the versioned schema of - this representation of an object. Servers should convert - recognized schemas to the latest internal value, and may - reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string completionstatus: - description: Optional field that drives completion status of appwrapper. - This field within an item of an appwrapper determines the full state of appwrapper. - The completionstatus field contains a list of conditions that make the associate item considered - completed, for instance :- completion conditions could be "Complete" or "Failed". - The associated item's level .status.conditions[].type field is monitored for any one of these conditions. Once all items with this - option is set and the conditionstatus is met the entire appwrapper state will be changed to one of the valid appwrapper completion state. Note :- this is an AND - operation for all items where this option is set. See the list of appwrapper states for a list of valid complete states. + description: 'Optional field that drives completion status + of this AppWrapper. This field within an item of an AppWrapper + determines the full state of the AppWrapper. The completionstatus + field contains a list of conditions that make the associate + item considered completed, for instance: - completion + conditions could be "Complete" or "Failed". The associated + item''s level .status.conditions[].type field is monitored + for any one of these conditions. Once all items with this + option is set and the conditionstatus is met the entire + AppWrapper state will be changed to one of the valid AppWrapper + completion state. Note: - this is an AND operation for + all items where this option is set. See the list of AppWrapper + states for a list of valid complete states.' type: string - custompodresources: description: Optional section that specifies resource requirements for non-standard k8s resources, follows same format as - that of standard k8s resources + that of standard k8s resources. items: properties: limits: @@ -107,14 +109,6 @@ spec: type: object x-kubernetes-embedded-resource: true x-kubernetes-preserve-unknown-fields: true - kind: - description: 'Kind is a string value representing the REST - resource this object represents. Servers may infer this - from the endpoint the client submits requests to. Cannot - be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object minavailable: description: The minimal available pods to run for this AppWrapper; the default value is nil @@ -135,182 +129,76 @@ spec: type: integer type: object type: array - Items: - items: - description: 'AppWrapperResource is App Wrapper aggregation - resource todo: To be depricated' - properties: - allocatedreplicas: - description: The number of allocated replicas from this - resource type - format: int32 - type: integer - apiVersion: - description: 'APIVersion defines the versioned schema of - this representation of an object. Servers should convert - recognized schemas to the latest internal value, and may - reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST - resource this object represents. Servers may infer this - from the endpoint the client submits requests to. Cannot - be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - minavailable: - description: The minimal available pods to run for this - AppWrapper; the default value is nil - format: int32 - type: integer - priority: - description: The priority of this resource - format: int32 - type: integer - priorityslope: - description: The increasing rate of priority value for this - resource - format: float - type: number - replicas: - description: Replicas is the number of desired replicas - format: int32 - type: integer - template: - description: The template for the resource; it is now a - raw text because we don't know for what resource it should - be instantiated - type: object - x-kubernetes-preserve-unknown-fields: true - type: - description: The type of the resource (is the resource a - Pod, a ReplicaSet, a ... ?) - type: string - required: - - template - type: object - type: array - apiVersion: - description: 'APIVersion defines the versioned schema of this - representation of an object. Servers should convert recognized - schemas to the latest internal value, and may reject unrecognized - values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource - this object represents. Servers may infer this from the endpoint - the client submits requests to. Cannot be updated. In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - description: ListMeta describes metadata that synthetic resources - must have, including lists and various status objects. A resource - may have only one of {ObjectMeta, ListMeta}. - properties: - continue: - description: continue may be set if the user set a limit on - the number of items returned, and indicates that the server - has more data available. The value is opaque and may be - used to issue another request to the endpoint that served - this list to retrieve the next set of available objects. - Continuing a consistent list may not be possible if the - server configuration has changed or more than a few minutes - have passed. The resourceVersion field returned when using - this continue value will be identical to the value in the - first response, unless you have received this token from - an error message. - type: string - remainingItemCount: - description: remainingItemCount is the number of subsequent - items in the list which are not included in this list response. - If the list request contained label or field selectors, - then the number of remaining items is unknown and the field - will be left unset and omitted during serialization. If - the list is complete (either because it is not chunking - or because this is the last chunk), then there are no more - remaining items and this field will be left unset and omitted - during serialization. Servers older than v1.15 do not set - this field. The intended use of the remainingItemCount is - *estimating* the size of a collection. Clients should not - rely on the remainingItemCount to be set or to be exact. - format: int64 - type: integer - resourceVersion: - description: 'String that identifies the server''s internal - version of this object that can be used by clients to determine - when objects have changed. Value must be treated as opaque - by clients and passed unmodified back to the server. Populated - by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' - type: string - selfLink: - description: "selfLink is a URL representing this object. - Populated by the system. Read-only. \n DEPRECATED Kubernetes - will stop propagating this field in 1.20 release and the - field is planned to be removed in 1.21 release." - type: string - type: object type: object schedulingSpec: - description: Field 'schedulingSpec' specifies the parameters used for scheduling generic items - wrapped inside AppWrappers. It defines the policy for requeuing jobs based on the number - of running pods. + description: SchedSpec specifies the parameters used for scheduling + generic items wrapped inside AppWrappers. It defines the policy + for requeuing jobs based on the number of running pods. properties: + dispatchDuration: + description: Wall clock duration time of appwrapper in seconds. + properties: + expected: + type: integer + limit: + type: integer + overrun: + type: boolean + type: object minAvailable: - description: Expected number of pods in running and/or completed state. Requeuing is triggered - when the number of running/completed pods is not equal to this value. When not specified, - requeuing is disabled and no check is performed. + description: Expected number of pods in running and/or completed + state. Requeuing is triggered when the number of running/completed + pods is not equal to this value. When not specified, requeuing + is disabled and no check is performed. type: integer + nodeSelector: + additionalProperties: + type: string + type: object requeuing: - description: Specification of the requeuing strategy based on waiting time. Values in this field - control how often the pod check should happen and if requeuing has reached its maximum number of - times. + description: Specification of the requeuing strategy based on + waiting time. Values in this field control how often the pod + check should happen, and if requeuing has reached its maximum + number of times. properties: + growthType: + default: exponential + description: Growth strategy to increase the waiting time + between requeuing checks. The values available are 'exponential', + 'linear', or 'none'. For example, 'exponential' growth would + double the 'timeInSeconds' value every time a requeuing + event is triggered. If the string value is misspelled or + not one of the possible options, the growth behavior is + defaulted to 'none'. + type: string initialTimeInSeconds: - description: Value to keep track of the initial wait time. Users cannot set this as it is - taken from 'timeInSeconds'. - type: integer - timeInSeconds: - description: Initial waiting time before requeuing conditions are checked. This value is - specified by the user, but it may grow as requeuing events happen. - type: integer - default: 300 - maxTimeInSeconds: - description: Maximum waiting time for requeuing checks + description: Value to keep track of the initial wait time. + Users cannot set this as it is taken from 'timeInSeconds'. type: integer + maxNumRequeuings: default: 0 - growthType: - description: Growth strategy to increase the waiting time between requeuing checks. The values - available are 'exponential', 'linear', or 'none'. For example, 'exponential' growth would - double the 'timeInSeconds' value every time a requeuing event is triggered. If the string value - is misspelled or not one of the possible options, the growth behavior is defaulted to 'none'. - type: string - default: "exponential" - numRequeuings: - description: Field to keep track of how many times a requeuing event has been triggered + description: Maximum number of requeuing events allowed. Once + this value is reached (e.g., 'numRequeuings = maxNumRequeuings', + no more requeuing checks are performed and the generic items + are stopped and removed from the cluster (AppWrapper remains + deployed). type: integer + maxTimeInSeconds: default: 0 - maxNumRequeuings: - description: Maximum number of requeuing events allowed. Once this value is reached (e.g., - 'numRequeuings = maxNumRequeuings', no more requeuing checks are performed and the generic - items are stopped and removed from the cluster (AppWrapper remains deployed). + description: Maximum waiting time for requeuing checks. type: integer + numRequeuings: default: 0 - type: object - dispatchDuration: - description: Wall clock duration time of appwrapper in seconds - properties: - expected: - format: int32 + description: Field to keep track of how many times a requeuing + event has been triggered. type: integer - limit: - format: int32 + timeInSeconds: + default: 300 + description: Initial waiting time before requeuing conditions + are checked. This value is specified by the user, but it + may grow as requeuing events happen. type: integer type: object - nodeSelector: - additionalProperties: - type: string - type: object type: object selector: description: A label selector is a label query over a set of resources. @@ -359,6 +247,7 @@ spec: are ANDed. type: object type: object + x-kubernetes-map-type: atomic service: description: AppWrapperService is App Wrapper service definition properties: @@ -695,10 +584,10 @@ spec: description: Can run? type: boolean conditions: - description: Represents the latest available observations of a appwrapper's + description: Represents the latest available observations of the AppWrapper's current condition. items: - description: DeploymentCondition describes the state of a deployment + description: AppWrapperCondition describes the state of an AppWrapper at a certain point. properties: lastTransitionMicroTime: @@ -711,7 +600,7 @@ spec: format: date-time type: string message: - description: A human readable message indicating details about + description: A human-readable message indicating details about the transition. type: string reason: @@ -721,65 +610,23 @@ spec: description: Status of the condition, one of True, False, Unknown. type: string type: - description: Type of appwrapper condition. + description: Type of AppWrapper condition. type: string required: - status - type type: object type: array - pendingpodconditions: - description: Represent conditions of pod(s) that failed scheduling. - items: - description: Describes scheduling failed condition(s) of a pod - properties: - podname: - description: Name of the pod - type: string - conditions: - description: Failed condition(s) of a pod - items: - description: DeploymentCondition describes the state of a deployment - at a certain point. - properties: - LastProbeTime: - description: Last time the condition transitioned from one status - to another. - format: date-time - type: string - LastTransitionTime: - description: The last time this condition was updated. - format: date-time - type: string - message: - description: A human readable message indicating details about - the transition. - type: string - reason: - description: The reason for the condition's last transition. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of appwrapper condition. - type: string - required: - - status - - type - type: object - type: array - type: object - type: array + controllerfirstdispatchtimestamp: + description: Microsecond level timestamp when controller first dispatches + the AppWrapper + format: date-time + type: string controllerfirsttimestamp: description: Microsecond level timestamp when controller first sees QueueJob (by Informer) format: date-time type: string - controllerfirstdispatchtimestamp: - description: Microsecond level timestamp when controller first dispatches appwrapper - format: date-time - type: string failed: description: The number of resources which reached phase Failed. format: int32 @@ -792,34 +639,77 @@ spec: description: Is Dispatched? type: boolean local: - description: Indicate if message is a duplicate (for Informer to recognize duplicate messages) + description: Indicate if message is a duplicate (for Informer to recognize + duplicate messages) type: boolean message: type: string - totalcpu: - description: The number of cpu consumed by all pods belonging to an appwrapper. - format: int32 - type: integer - totalmemory: - description: The amount of memory consumed by all pods belonging to an appwrapper. - format: int32 - type: integer - totalgpu: - description: The total number of GPUs consumed by all pods belonging to an appwrapper. - format: int32 + numberOfRequeueings: + description: Field to keep track of how many times a requeuing event + has been triggered type: integer pending: description: The number of pending pods. format: int32 type: integer + pendingpodconditions: + description: Represents the latest available observations of pods + belonging to the AppWrapper. + items: + properties: + conditions: + items: + description: PodCondition contains details for the current + condition of this pod. + properties: + lastProbeTime: + description: Last time we probed the condition. + format: date-time + type: string + lastTransitionTime: + description: Last time the condition transitioned from + one status to another. + format: date-time + type: string + message: + description: Human-readable message indicating details + about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the + condition's last transition. + type: string + status: + description: 'Status is the status of the condition. Can + be True, False, Unknown. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + type: string + type: + description: 'Type is the type of the condition. More + info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + type: string + required: + - status + - type + type: object + type: array + podname: + type: string + type: object + type: array queuejobstate: - description: State of QueueJob - Init, Queueing, HeadOfLine, Rejoining ... + description: State of QueueJob - Init, Queueing, HeadOfLine, Rejoining, + ... type: string + requeueingTimeInSeconds: + description: Field to keep track of total number of seconds spent + in requeueing + type: integer running: format: int32 type: integer sender: - description: Indicate sender of this message (extremely useful for debugging) + description: Indicate sender of this message (extremely useful for + debugging) type: string state: description: State - Pending, Running, Failed, Deleted @@ -833,16 +723,21 @@ spec: (is this different from the MinAvailable from JobStatus) format: int32 type: integer - numberOfRequeueings: - description: Field to keep track of how many times a requeuing event has been triggered + totalcpu: + description: The number of CPU consumed by all pods belonging to the + AppWrapper. format: int32 type: integer - default: 0 - requeueingTimeInSeconds: - description: Field to keep track of total number of seconds spent in requeueing + totalgpu: + description: The total number of GPUs consumed by all pods belonging + to the AppWrapper. + format: int32 + type: integer + totalmemory: + description: The amount of memory consumed by all pods belonging to + the AppWrapper. format: int32 type: integer - default: 0 type: object required: - spec @@ -851,4 +746,3 @@ spec: storage: true subresources: status: {} - diff --git a/deployment/mcad-controller/crds/mcad.ibm.com_queuejobs.yaml b/deployment/mcad-controller/crds/mcad.ibm.com_queuejobs.yaml index 6c60855de..9a7344f68 100644 --- a/deployment/mcad-controller/crds/mcad.ibm.com_queuejobs.yaml +++ b/deployment/mcad-controller/crds/mcad.ibm.com_queuejobs.yaml @@ -1,10 +1,10 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.1 + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null name: queuejobs.mcad.ibm.com spec: group: mcad.ibm.com @@ -33,70 +33,77 @@ spec: type: object spec: description: Specification of the desired behavior of a cron job, including - the minAvailable and the requeuing strategy + the minAvailable properties: schedulerName: type: string schedulingSpec: - description: Field 'schedulingSpec' specifies the parameters used for scheduling generic items - wrapped inside AppWrappers. It defines the policy for requeuing jobs based on the number - of running pods. + description: SchedSpec specifies the parameters for scheduling. properties: + dispatchDuration: + description: Wall clock duration time of appwrapper in seconds. + properties: + expected: + type: integer + limit: + type: integer + overrun: + type: boolean + type: object minAvailable: - description: Expected number of pods in running and/or completed state. Requeuing is triggered - when the number of running/completed pods is not equal to this value. When not specified, - requeuing is disabled and no check is performed. + description: Expected number of pods in running and/or completed + state. Requeuing is triggered when the number of running/completed + pods is not equal to this value. When not specified, requeuing + is disabled and no check is performed. type: integer + nodeSelector: + additionalProperties: + type: string + type: object requeuing: - description: Specification of the requeuing strategy based on waiting time. Values in this field - control how often the pod check should happen and if requeuing has reached its maximum number of - times. + description: Specification of the requeuing strategy based on + waiting time. Values in this field control how often the pod + check should happen, and if requeuing has reached its maximum + number of times. properties: + growthType: + default: exponential + description: Growth strategy to increase the waiting time + between requeuing checks. The values available are 'exponential', + 'linear', or 'none'. For example, 'exponential' growth would + double the 'timeInSeconds' value every time a requeuing + event is triggered. If the string value is misspelled or + not one of the possible options, the growth behavior is + defaulted to 'none'. + type: string initialTimeInSeconds: - description: Value to keep track of the initial wait time. Users cannot set this as it is - taken from 'timeInSeconds'. - type: integer - timeInSeconds: - description: Initial waiting time before requeuing conditions are checked. This value is - specified by the user, but it may grow as requeuing events happen. - type: integer - default: 300 - maxTimeInSeconds: - description: Maximum waiting time for requeuing checks + description: Value to keep track of the initial wait time. + Users cannot set this as it is taken from 'timeInSeconds'. type: integer + maxNumRequeuings: default: 0 - growthType: - description: Growth strategy to increase the waiting time between requeuing checks. The values - available are 'exponential', 'linear', or 'none'. For example, 'exponential' growth would - double the 'timeInSeconds' value every time a requeuing event is triggered. If the string value - is misspelled or not one of the possible options, the growth behavior is defaulted to 'none'. - type: string - default: "exponential" - numRequeuings: - description: Field to keep track of how many times a requeuing event has been triggered + description: Maximum number of requeuing events allowed. Once + this value is reached (e.g., 'numRequeuings = maxNumRequeuings', + no more requeuing checks are performed and the generic items + are stopped and removed from the cluster (AppWrapper remains + deployed). type: integer + maxTimeInSeconds: default: 0 - maxNumRequeuings: - description: Maximum number of requeuing events allowed. Once this value is reached (e.g., - 'numRequeuings = maxNumRequeuings', no more requeuing checks are performed and the generic - items are stopped and removed from the cluster (AppWrapper remains deployed). + description: Maximum waiting time for requeuing checks. type: integer + numRequeuings: default: 0 - type: object - dispatchDuration: - description: Wall clock duration time of appwrapper in seconds - properties: - expected: - format: int32 + description: Field to keep track of how many times a requeuing + event has been triggered. type: integer - limit: - format: int32 + timeInSeconds: + default: 300 + description: Initial waiting time before requeuing conditions + are checked. This value is specified by the user, but it + may grow as requeuing events happen. type: integer type: object - nodeSelector: - additionalProperties: - type: string - type: object type: object taskSpecs: description: TaskSpecs specifies the task specification of QueueJob @@ -153,6 +160,7 @@ spec: only "value". The requirements are ANDed. type: object type: object + x-kubernetes-map-type: atomic template: description: Specifies the pod that will be created for this TaskSpec when executing a QueueJob @@ -497,6 +505,7 @@ spec: are ANDed. type: object type: object + x-kubernetes-map-type: atomic namespaces: description: namespaces specifies which namespaces the labelSelector @@ -612,6 +621,7 @@ spec: ANDed. type: object type: object + x-kubernetes-map-type: atomic namespaces: description: namespaces specifies which namespaces the labelSelector applies @@ -728,6 +738,7 @@ spec: are ANDed. type: object type: object + x-kubernetes-map-type: atomic namespaces: description: namespaces specifies which namespaces the labelSelector @@ -843,6 +854,7 @@ spec: ANDed. type: object type: object + x-kubernetes-map-type: atomic namespaces: description: namespaces specifies which namespaces the labelSelector applies @@ -5110,6 +5122,7 @@ spec: "value". The requirements are ANDed. type: object type: object + x-kubernetes-map-type: atomic maxSkew: description: 'MaxSkew describes the degree to which pods may be unevenly distributed. When @@ -5145,12 +5158,12 @@ spec: spread constraint. - DoNotSchedule (default) tells the scheduler not to schedule it. - ScheduleAnyway tells the scheduler to schedule the pod in any - location, but giving higher precedence to - topologies that would help reduce the skew. - A constraint is considered "Unsatisfiable" for - an incoming pod if and only if every possible - node assigment for that pod would violate "MaxSkew" - on some topology. For example, in a 3-zone cluster, + location, but giving higher precedence to topologies + that would help reduce the skew. A constraint + is considered "Unsatisfiable" for an incoming + pod if and only if every possible node assigment + for that pod would violate "MaxSkew" on some + topology. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | If WhenUnsatisfiable @@ -5625,12 +5638,12 @@ spec: pod is removed. \n Use this if: a) the volume is only needed while the pod runs, b) features of normal volumes like restoring from snapshot - or capacity tracking are needed, c) the storage + or capacity tracking are needed, c) the storage driver is specified through a storage class, and d) the storage driver supports dynamic volume - provisioning through a PersistentVolumeClaim - (see EphemeralVolumeSource for more information - on the connection between this volume type and + provisioning through a PersistentVolumeClaim + (see EphemeralVolumeSource for more information + on the connection between this volume type and PersistentVolumeClaim). \n Use PersistentVolumeClaim or one of the vendor-specific APIs for volumes that persist for longer than the lifecycle of @@ -5825,6 +5838,7 @@ spec: are ANDed. type: object type: object + x-kubernetes-map-type: atomic storageClassName: description: 'Name of the StorageClass required by the claim. More info: diff --git a/deployment/mcad-controller/crds/mcad.ibm.com_schedulingspecs.yaml b/deployment/mcad-controller/crds/mcad.ibm.com_schedulingspecs.yaml index 3de2af124..e91a18ffd 100644 --- a/deployment/mcad-controller/crds/mcad.ibm.com_schedulingspecs.yaml +++ b/deployment/mcad-controller/crds/mcad.ibm.com_schedulingspecs.yaml @@ -1,10 +1,10 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.1 + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null name: schedulingspecs.mcad.ibm.com spec: group: mcad.ibm.com @@ -33,70 +33,71 @@ spec: type: object spec: properties: + dispatchDuration: + description: Wall clock duration time of appwrapper in seconds. + properties: + expected: + type: integer + limit: + type: integer + overrun: + type: boolean + type: object minAvailable: - description: Expected number of pods in running and/or completed state. Requeuing is triggered - when the number of running/completed pods is not equal to this value. When not specified, - requeuing is disabled and no check is performed. + description: Expected number of pods in running and/or completed state. + Requeuing is triggered when the number of running/completed pods + is not equal to this value. When not specified, requeuing is disabled + and no check is performed. type: integer + nodeSelector: + additionalProperties: + type: string + type: object requeuing: - description: Specification of the requeuing strategy based on waiting time. Values in this field - control how often the pod check should happen and if requeuing has reached its maximum number of - times. + description: Specification of the requeuing strategy based on waiting + time. Values in this field control how often the pod check should + happen, and if requeuing has reached its maximum number of times. properties: + growthType: + default: exponential + description: Growth strategy to increase the waiting time between + requeuing checks. The values available are 'exponential', 'linear', + or 'none'. For example, 'exponential' growth would double the + 'timeInSeconds' value every time a requeuing event is triggered. + If the string value is misspelled or not one of the possible + options, the growth behavior is defaulted to 'none'. + type: string initialTimeInSeconds: - description: Value to keep track of the initial wait time. Users cannot set this as it is - taken from 'timeInSeconds'. - type: integer - timeInSeconds: - description: Initial waiting time before requeuing conditions are checked. This value is - specified by the user, but it may grow as requeuing events happen. - type: integer - default: 300 - maxTimeInSeconds: - description: Maximum waiting time for requeuing checks + description: Value to keep track of the initial wait time. Users + cannot set this as it is taken from 'timeInSeconds'. type: integer + maxNumRequeuings: default: 0 - growthType: - description: Growth strategy to increase the waiting time between requeuing checks. The values - available are 'exponential', 'linear', or 'none'. For example, 'exponential' growth would - double the 'timeInSeconds' value every time a requeuing event is triggered. If the string value - is misspelled or not one of the possible options, the growth behavior is defaulted to 'none'. - type: string - default: "exponential" - numRequeuings: - description: Field to keep track of how many times a requeuing event has been triggered + description: Maximum number of requeuing events allowed. Once + this value is reached (e.g., 'numRequeuings = maxNumRequeuings', + no more requeuing checks are performed and the generic items + are stopped and removed from the cluster (AppWrapper remains + deployed). type: integer + maxTimeInSeconds: default: 0 - maxNumRequeuings: - description: Maximum number of requeuing events allowed. Once this value is reached (e.g., - 'numRequeuings = maxNumRequeuings', no more requeuing checks are performed and the generic - items are stopped and removed from the cluster (AppWrapper remains deployed). + description: Maximum waiting time for requeuing checks. type: integer + numRequeuings: default: 0 - type: object - dispatchDuration: - description: Wall clock duration time of appwrapper in seconds - properties: - expected: - format: int32 + description: Field to keep track of how many times a requeuing + event has been triggered. type: integer - limit: - format: int32 + timeInSeconds: + default: 300 + description: Initial waiting time before requeuing conditions + are checked. This value is specified by the user, but it may + grow as requeuing events happen. type: integer type: object - nodeSelector: - additionalProperties: - type: string - type: object type: object required: - metadata type: object served: true storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/pkg/apis/controller/v1beta1/appwrapper.go b/pkg/apis/controller/v1beta1/appwrapper.go index 4349ff1cc..b7811e1ca 100644 --- a/pkg/apis/controller/v1beta1/appwrapper.go +++ b/pkg/apis/controller/v1beta1/appwrapper.go @@ -1,19 +1,4 @@ /* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -/* Copyright 2019, 2021 The Multi-Cluster App Dispatcher Authors. Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package v1beta1 import ( @@ -44,9 +30,9 @@ const AppWrapperAnnotationKey = "appwrapper.mcad.ibm.com/appwrapper-name" // +kubebuilder:object:root=true // +kubebuilder:subresource:status +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // Definition of AppWrapper class -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object type AppWrapper struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` @@ -55,9 +41,9 @@ type AppWrapper struct { } // +kubebuilder:object:root=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // AppWrapperList is a collection of AppWrappers. -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object type AppWrapperList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata"` @@ -80,15 +66,13 @@ type AppWrapperSpec struct { Selector *metav1.LabelSelector `json:"selector,omitempty" protobuf:"bytes,1,opt,name=selector"` - // SchedSpec specifies the parameters for scheduling. + // SchedSpec specifies the parameters used for scheduling generic items wrapped inside AppWrappers. + // It defines the policy for requeuing jobs based on the number of running pods. SchedSpec SchedulingSpecTemplate `json:"schedulingSpec,omitempty" protobuf:"bytes,2,opt,name=schedulingSpec"` } // a collection of AppWrapperResource type AppWrapperResourceList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata"` - // +optional GenericItems []AppWrapperGenericResource `json:"GenericItems"` } @@ -100,8 +84,6 @@ type AppWrapperService struct { // AppWrapperGenericResource is App Wrapper aggregation resource type AppWrapperGenericResource struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata"` // Replicas is the number of desired replicas DesiredAvailable int32 `json:"replicas,omitempty" protobuf:"bytes,2,opt,name=desiredavailable"` @@ -114,14 +96,12 @@ type AppWrapperGenericResource struct { // The priority of this resource // +optional - // +kubebuilder:validation:Type=number - // +kubebuilder:validation:Format=float - Priority float64 `json:"priority"` + Priority int32 `json:"priority"` // The increasing rate of priority value for this resource - // +optional // +kubebuilder:validation:Type=number // +kubebuilder:validation:Format=float + // +optional PrioritySlope float64 `json:"priorityslope"` // The template for the resource; it is now a raw text because we don't know for what resource @@ -131,11 +111,20 @@ type AppWrapperGenericResource struct { // +kubebuilder:validation:EmbeddedResource GenericTemplate runtime.RawExtension `json:"generictemplate"` - // Optional section that specifies resource requirements for non-standard k8s resources, follows same format as that - // of standard k8s resources + // Optional section that specifies resource requirements for non-standard k8s resources, + // follows same format as that of standard k8s resources. CustomPodResources []CustomPodResourceTemplate `json:"custompodresources,omitempty"` - // Optional field for users to determine completion status of item + // Optional field that drives completion status of this AppWrapper. + // This field within an item of an AppWrapper determines the full state of the AppWrapper. + // The completionstatus field contains a list of conditions that make the associate item considered + // completed, for instance: + // - completion conditions could be "Complete" or "Failed". + // The associated item's level .status.conditions[].type field is monitored for any one of these conditions. + // Once all items with this option is set and the conditionstatus is met the entire AppWrapper state will be changed to one of the valid AppWrapper completion state. + // Note: + // - this is an AND operation for all items where this option is set. + // See the list of AppWrapper states for a list of valid complete states. CompletionStatus string `json:"completionstatus,omitempty"` } @@ -199,7 +188,7 @@ type AppWrapperStatus struct { // Microsecond level timestamp when controller first sees QueueJob (by Informer) ControllerFirstTimestamp metav1.MicroTime `json:"controllerfirsttimestamp,omitempty"` - // Microsecond level timestamp when controller first dispatches appwrapper + // Microsecond level timestamp when controller first dispatches the AppWrapper ControllerFirstDispatchTimestamp metav1.MicroTime `json:"controllerfirstdispatchtimestamp,omitempty"` // Tell Informer to ignore this update message (do not generate a controller event) @@ -211,20 +200,22 @@ type AppWrapperStatus struct { // Indicate if message is a duplicate (for Informer to recognize duplicate messages) Local bool `json:"local,omitempty"` - // Represents the latest available observations of a appwrapper's current condition. + // Represents the latest available observations of the AppWrapper's current condition. Conditions []AppWrapperCondition `json:"conditions,omitempty"` - // Represents the latest available observations of pods under appwrapper - PendingPodConditions []PendingPodSpec `json:"pendingpodconditions"` + // Represents the latest available observations of pods belonging to the AppWrapper. + PendingPodConditions []PendingPodSpec `json:"pendingpodconditions,omitempty"` + + // Resources consumed - // Represents the number of cpu consumed by all pods belonging to an appwrapper. - TotalCPU float64 `json:"totalcpu,omitempty"` + // The number of CPU consumed by all pods belonging to the AppWrapper. + TotalCPU int32 `json:"totalcpu,omitempty"` - // Represents the amount of memory consumed by all pods belonging to an appwrapper. - TotalMemory float64 `json:"totalmemory,omitempty"` + // The amount of memory consumed by all pods belonging to the AppWrapper. + TotalMemory int32 `json:"totalmemory,omitempty"` - // Represents the total number of GPUs consumed by all pods belonging to an appwrapper. - TotalGPU int64 `json:"totalgpu,omitempty"` + // The total number of GPUs consumed by all pods belonging to the AppWrapper. + TotalGPU int32 `json:"totalgpu,omitempty"` // Field to keep track of total number of seconds spent in requeueing RequeueingTimeInSeconds int `json:"requeueingTimeInSeconds,omitempty"` @@ -262,9 +253,9 @@ const ( AppWrapperCondRunningHoldCompletion AppWrapperConditionType = "RunningHoldCompletion" ) -// DeploymentCondition describes the state of a deployment at a certain point. +// AppWrapperCondition describes the state of an AppWrapper at a certain point. type AppWrapperCondition struct { - // Type of appwrapper condition. + // Type of AppWrapper condition. Type AppWrapperConditionType `json:"type"` // Status of the condition, one of True, False, Unknown. Status v1.ConditionStatus `json:"status"` @@ -274,7 +265,7 @@ type AppWrapperCondition struct { LastTransitionMicroTime metav1.MicroTime `json:"lastTransitionMicroTime,omitempty"` // The reason for the condition's last transition. Reason string `json:"reason,omitempty"` - // A human readable message indicating details about the transition. + // A human-readable message indicating details about the transition. Message string `json:"message,omitempty"` } diff --git a/pkg/apis/controller/v1beta1/doc.go b/pkg/apis/controller/v1beta1/doc.go index 9e57c941f..e064d70e7 100644 --- a/pkg/apis/controller/v1beta1/doc.go +++ b/pkg/apis/controller/v1beta1/doc.go @@ -1,19 +1,4 @@ /* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -/* Copyright 2019, 2021 The Multi-Cluster App Dispatcher Authors. Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,5 +13,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + // +k8s:deepcopy-gen=package +// +kubebuilder:object:generate=true +// +groupName=mcad.ibm.com + package v1beta1 diff --git a/pkg/apis/controller/v1beta1/schedulingspec.go b/pkg/apis/controller/v1beta1/schedulingspec.go index 457cd1540..90f5d65fb 100644 --- a/pkg/apis/controller/v1beta1/schedulingspec.go +++ b/pkg/apis/controller/v1beta1/schedulingspec.go @@ -1,19 +1,4 @@ /* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -/* Copyright 2019, 2021 The Multi-Cluster App Dispatcher Authors. Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package v1beta1 import ( @@ -46,21 +32,46 @@ type SchedulingSpec struct { } type SchedulingSpecTemplate struct { - NodeSelector map[string]string `json:"nodeSelector,omitempty" protobuf:"bytes,1,rep,name=nodeSelector"` - MinAvailable int `json:"minAvailable,omitempty" protobuf:"bytes,2,rep,name=minAvailable"` - Requeuing RequeuingTemplate `json:"requeuing,omitempty" protobuf:"bytes,1,rep,name=requeuing"` - ClusterScheduling ClusterSchedulingSpec `json:"clusterScheduling,omitempty"` - DispatchingWindow DispatchingWindowSpec `json:"dispatchingWindow,omitempty"` - DispatchDuration DispatchDurationSpec `json:"dispatchDuration,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty" protobuf:"bytes,1,rep,name=nodeSelector"` + // Expected number of pods in running and/or completed state. + // Requeuing is triggered when the number of running/completed pods is not equal to this value. + // When not specified, requeuing is disabled and no check is performed. + MinAvailable int `json:"minAvailable,omitempty" protobuf:"bytes,2,rep,name=minAvailable"` + // Specification of the requeuing strategy based on waiting time. + // Values in this field control how often the pod check should happen, + // and if requeuing has reached its maximum number of times. + Requeuing RequeuingTemplate `json:"requeuing,omitempty" protobuf:"bytes,1,rep,name=requeuing"` + // Wall clock duration time of appwrapper in seconds. + DispatchDuration DispatchDurationSpec `json:"dispatchDuration,omitempty"` } type RequeuingTemplate struct { - InitialTimeInSeconds int `json:"initialTimeInSeconds,omitempty" protobuf:"bytes,1,rep,name=initialTimeInSeconds"` - TimeInSeconds int `json:"timeInSeconds,omitempty" protobuf:"bytes,2,rep,name=timeInSeconds"` - MaxTimeInSeconds int `json:"maxTimeInSeconds,omitempty" protobuf:"bytes,3,rep,name=maxTimeInSeconds"` - GrowthType string `json:"growthType,omitempty" protobuf:"bytes,4,rep,name=growthType"` - NumRequeuings int `json:"numRequeuings,omitempty" protobuf:"bytes,5,rep,name=numRequeuings"` - MaxNumRequeuings int `json:"maxNumRequeuings,omitempty" protobuf:"bytes,6,rep,name=maxNumRequeuings"` + // Value to keep track of the initial wait time. + // Users cannot set this as it is taken from 'timeInSeconds'. + InitialTimeInSeconds int `json:"initialTimeInSeconds,omitempty" protobuf:"bytes,1,rep,name=initialTimeInSeconds"` + // Initial waiting time before requeuing conditions are checked. This value is + // specified by the user, but it may grow as requeuing events happen. + // +kubebuilder:default=300 + TimeInSeconds int `json:"timeInSeconds,omitempty" protobuf:"bytes,2,rep,name=timeInSeconds"` + // Maximum waiting time for requeuing checks. + // +kubebuilder:default=0 + MaxTimeInSeconds int `json:"maxTimeInSeconds,omitempty" protobuf:"bytes,3,rep,name=maxTimeInSeconds"` + // Growth strategy to increase the waiting time between requeuing checks. + // The values available are 'exponential', 'linear', or 'none'. + // For example, 'exponential' growth would double the 'timeInSeconds' value + // every time a requeuing event is triggered. + // If the string value is misspelled or not one of the possible options, + // the growth behavior is defaulted to 'none'. + // +kubebuilder:default=exponential + GrowthType string `json:"growthType,omitempty" protobuf:"bytes,4,rep,name=growthType"` + // Field to keep track of how many times a requeuing event has been triggered. + // +kubebuilder:default=0 + NumRequeuings int `json:"numRequeuings,omitempty" protobuf:"bytes,5,rep,name=numRequeuings"` + // Maximum number of requeuing events allowed. Once this value is reached (e.g., + // 'numRequeuings = maxNumRequeuings', no more requeuing checks are performed and the generic + // items are stopped and removed from the cluster (AppWrapper remains deployed). + // +kubebuilder:default=0 + MaxNumRequeuings int `json:"maxNumRequeuings,omitempty" protobuf:"bytes,6,rep,name=maxNumRequeuings"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/controller/v1beta1/zz_generated.deepcopy.go b/pkg/apis/controller/v1beta1/zz_generated.deepcopy.go index 056c6e604..b06d11b4d 100644 --- a/pkg/apis/controller/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/controller/v1beta1/zz_generated.deepcopy.go @@ -91,8 +91,6 @@ func (in *AppWrapperCondition) DeepCopy() *AppWrapperCondition { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AppWrapperGenericResource) DeepCopyInto(out *AppWrapperGenericResource) { *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) if in.MinAvailable != nil { in, out := &in.MinAvailable, &out.MinAvailable *out = new(int32) @@ -155,8 +153,6 @@ func (in *AppWrapperList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AppWrapperResourceList) DeepCopyInto(out *AppWrapperResourceList) { *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) if in.GenericItems != nil { in, out := &in.GenericItems, &out.GenericItems *out = make([]AppWrapperGenericResource, len(*in)) @@ -586,8 +582,6 @@ func (in *SchedulingSpecTemplate) DeepCopyInto(out *SchedulingSpecTemplate) { } } out.Requeuing = in.Requeuing - in.ClusterScheduling.DeepCopyInto(&out.ClusterScheduling) - in.DispatchingWindow.DeepCopyInto(&out.DispatchingWindow) out.DispatchDuration = in.DispatchDuration return } diff --git a/pkg/apis/quotaplugins/quotasubtree/v1/doc.go b/pkg/apis/quotaplugins/quotasubtree/v1/doc.go index 632034959..a5f1a2300 100755 --- a/pkg/apis/quotaplugins/quotasubtree/v1/doc.go +++ b/pkg/apis/quotaplugins/quotasubtree/v1/doc.go @@ -1,2 +1,21 @@ +/* +Copyright 2023 The Multi-Cluster App Dispatcher Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + // +k8s:deepcopy-gen=package +// +kubebuilder:object:generate=true +// +groupName=ibm.com + package v1 diff --git a/pkg/controller/clusterstate/api/resource_info.go b/pkg/controller/clusterstate/api/resource_info.go index e38374ce2..a74646840 100644 --- a/pkg/controller/clusterstate/api/resource_info.go +++ b/pkg/controller/clusterstate/api/resource_info.go @@ -1,19 +1,4 @@ /* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -/* Copyright 2019, 2021 The Multi-Cluster App Dispatcher Authors. Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package api import ( @@ -116,12 +102,12 @@ func (r *Resource) Replace(rr *Resource) *Resource { return r } -//Sub subtracts two Resource objects. +// Sub subtracts two Resource objects. func (r *Resource) Sub(rr *Resource) (*Resource, error) { return r.NonNegSub(rr) } -//Sub subtracts two Resource objects and return zero for negative subtractions. +// Sub subtracts two Resource objects and return zero for negative subtractions. func (r *Resource) NonNegSub(rr *Resource) (*Resource, error) { // Check for negative calculation var isNegative bool diff --git a/pkg/controller/queuejob/queuejob_controller_ex.go b/pkg/controller/queuejob/queuejob_controller_ex.go index dbe86710d..0bc84d108 100644 --- a/pkg/controller/queuejob/queuejob_controller_ex.go +++ b/pkg/controller/queuejob/queuejob_controller_ex.go @@ -702,12 +702,11 @@ func (qjm *XController) getDispatchedAppWrappers() (map[string]*clusterstateapi. return awrRetVal, awsRetVal } -func (qjm *XController) addTotalSnapshotResourcesConsumedByAw(totalgpu int64, totalcpu float64, totalmemory float64) *clusterstateapi.Resource { - +func (qjm *XController) addTotalSnapshotResourcesConsumedByAw(totalgpu int32, totalcpu int32, totalmemory int32) *clusterstateapi.Resource { totalResource := clusterstateapi.EmptyResource() - totalResource.GPU = totalgpu - totalResource.MilliCPU = totalcpu - totalResource.Memory = totalmemory + totalResource.GPU = int64(totalgpu) + totalResource.MilliCPU = float64(totalcpu) + totalResource.Memory = float64(totalmemory) return totalResource diff --git a/pkg/controller/queuejobresources/pod/pod.go b/pkg/controller/queuejobresources/pod/pod.go index 62abd4783..da56aac19 100644 --- a/pkg/controller/queuejobresources/pod/pod.go +++ b/pkg/controller/queuejobresources/pod/pod.go @@ -214,9 +214,9 @@ func (qjrPod *QueueJobResPod) UpdateQueueJobStatus(queuejob *arbv1.AppWrapper) e queuejob.Status.Succeeded = succeeded queuejob.Status.Failed = failed // Total resources by all running pods - queuejob.Status.TotalGPU = totalResourcesConsumedForPodPhases.GPU - queuejob.Status.TotalCPU = totalResourcesConsumedForPodPhases.MilliCPU - queuejob.Status.TotalMemory = totalResourcesConsumedForPodPhases.Memory + queuejob.Status.TotalGPU = int32(totalResourcesConsumedForPodPhases.GPU) + queuejob.Status.TotalCPU = int32(totalResourcesConsumedForPodPhases.MilliCPU) + queuejob.Status.TotalMemory = int32(totalResourcesConsumedForPodPhases.Memory) queuejob.Status.PendingPodConditions = nil for podName, cond := range podsConditionMap { diff --git a/test/e2e-kuttl-deployment-01/steps/03-install.yaml b/test/e2e-kuttl-deployment-01/steps/03-install.yaml index 9827b963a..63ef37e2f 100644 --- a/test/e2e-kuttl-deployment-01/steps/03-install.yaml +++ b/test/e2e-kuttl-deployment-01/steps/03-install.yaml @@ -11,7 +11,6 @@ spec: growthType: "none" maxNumRequeuings: 3 resources: - Items: [] GenericItems: - replicas: 1 completionstatus: Complete diff --git a/test/e2e-kuttl-deployment-01/steps/06-install.yaml b/test/e2e-kuttl-deployment-01/steps/06-install.yaml index 318a504f5..2f9268645 100644 --- a/test/e2e-kuttl-deployment-01/steps/06-install.yaml +++ b/test/e2e-kuttl-deployment-01/steps/06-install.yaml @@ -11,7 +11,6 @@ spec: growthType: "none" maxNumRequeuings: 1 resources: - Items: [] GenericItems: - replicas: 1 completionstatus: Complete diff --git a/test/e2e-kuttl/quota-forest/04-install.yaml b/test/e2e-kuttl/quota-forest/04-install.yaml index 8d27e067d..882e7c4db 100644 --- a/test/e2e-kuttl/quota-forest/04-install.yaml +++ b/test/e2e-kuttl/quota-forest/04-install.yaml @@ -10,11 +10,8 @@ spec: service: spec: {} resources: - metadata: {} - Items: [] GenericItems: - - metadata: {} - replicas: 1 + - replicas: 1 generictemplate: apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 kind: StatefulSet diff --git a/test/e2e-kuttl/quota-forest/05-install.yaml b/test/e2e-kuttl/quota-forest/05-install.yaml index 48443bf1a..6cc9cbba9 100644 --- a/test/e2e-kuttl/quota-forest/05-install.yaml +++ b/test/e2e-kuttl/quota-forest/05-install.yaml @@ -10,10 +10,8 @@ spec: service: spec: {} resources: - metadata: {} GenericItems: - - metadata: {} - replicas: 1 + - replicas: 1 generictemplate: apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 kind: StatefulSet diff --git a/test/e2e-kuttl/quota-forest/06-install.yaml b/test/e2e-kuttl/quota-forest/06-install.yaml index 692dcbabb..c42f6bf9c 100644 --- a/test/e2e-kuttl/quota-forest/06-install.yaml +++ b/test/e2e-kuttl/quota-forest/06-install.yaml @@ -11,10 +11,8 @@ spec: spec: {} priority: 1000 resources: - metadata: {} GenericItems: - - metadata: {} - replicas: 1 + - replicas: 1 generictemplate: apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 kind: StatefulSet diff --git a/test/e2e-kuttl/quota-forest/07-install.yaml b/test/e2e-kuttl/quota-forest/07-install.yaml index d0ce84e6f..702177491 100644 --- a/test/e2e-kuttl/quota-forest/07-install.yaml +++ b/test/e2e-kuttl/quota-forest/07-install.yaml @@ -11,10 +11,8 @@ spec: spec: {} priority: 1000 resources: - metadata: {} GenericItems: - - metadata: {} - replicas: 1 + - replicas: 1 generictemplate: apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 kind: StatefulSet diff --git a/test/e2e-kuttl/quota-forest/08-install.yaml b/test/e2e-kuttl/quota-forest/08-install.yaml index 252ce7cdf..6eb041280 100644 --- a/test/e2e-kuttl/quota-forest/08-install.yaml +++ b/test/e2e-kuttl/quota-forest/08-install.yaml @@ -11,10 +11,8 @@ spec: spec: {} priority: 1000 resources: - metadata: {} GenericItems: - - metadata: {} - replicas: 1 + - replicas: 1 generictemplate: apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 kind: StatefulSet diff --git a/test/e2e/util.go b/test/e2e/util.go index 62ace9a18..feac21a08 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -250,10 +250,6 @@ func createGenericAWTimeoutWithStatus(context *context, name string) *arbv1.AppW AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "aw-test-jobtimeout-with-comp-1-job"), - Namespace: "test", - }, DesiredAvailable: 1, GenericTemplate: runtime.RawExtension{ Raw: rb, @@ -645,10 +641,6 @@ func createJobAWWithInitContainer(context *context, name string, requeuingTimeIn AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: context.namespace, - }, DesiredAvailable: 1, GenericTemplate: runtime.RawExtension{ Raw: rb, @@ -721,10 +713,6 @@ func createDeploymentAW(context *context, name string) *arbv1.AppWrapper { AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "item1"), - Namespace: context.namespace, - }, DesiredAvailable: 1, GenericTemplate: runtime.RawExtension{ Raw: rb, @@ -801,10 +789,6 @@ func createDeploymentAWwith550CPU(context *context, name string) *arbv1.AppWrapp AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "item1"), - Namespace: context.namespace, - }, DesiredAvailable: 1, GenericTemplate: runtime.RawExtension{ Raw: rb, @@ -881,10 +865,6 @@ func createDeploymentAWwith350CPU(context *context, name string) *arbv1.AppWrapp AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "item1"), - Namespace: context.namespace, - }, DesiredAvailable: 1, GenericTemplate: runtime.RawExtension{ Raw: rb, @@ -961,10 +941,6 @@ func createDeploymentAWwith426CPU(context *context, name string) *arbv1.AppWrapp AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "item1"), - Namespace: context.namespace, - }, DesiredAvailable: 1, GenericTemplate: runtime.RawExtension{ Raw: rb, @@ -1041,10 +1017,6 @@ func createDeploymentAWwith425CPU(context *context, name string) *arbv1.AppWrapp AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "item1"), - Namespace: context.namespace, - }, DesiredAvailable: 1, GenericTemplate: runtime.RawExtension{ Raw: rb, @@ -1116,10 +1088,6 @@ func createGenericDeploymentAW(context *context, name string) *arbv1.AppWrapper AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "aw-generic-deployment-3-item1"), - Namespace: context.namespace, - }, DesiredAvailable: 1, GenericTemplate: runtime.RawExtension{ Raw: rb, @@ -1199,10 +1167,6 @@ func createGenericJobAWWithStatus(context *context, name string) *arbv1.AppWrapp AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-comp-1"), - Namespace: "test", - }, DesiredAvailable: 1, GenericTemplate: runtime.RawExtension{ Raw: rb, @@ -1326,10 +1290,6 @@ func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1. AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-comp-ms-21-1"), - Namespace: "test", - }, DesiredAvailable: 1, GenericTemplate: runtime.RawExtension{ Raw: rb, @@ -1337,10 +1297,6 @@ func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1. CompletionStatus: "Complete", }, { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-comp-ms-21-2"), - Namespace: "test", - }, DesiredAvailable: 1, GenericTemplate: runtime.RawExtension{ Raw: rb2, @@ -1386,10 +1342,6 @@ func createAWGenericItemWithoutStatus(context *context, name string) *arbv1.AppW AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-comp-44"), - Namespace: "test", - }, DesiredAvailable: 1, GenericTemplate: runtime.RawExtension{ Raw: rb, @@ -1469,10 +1421,6 @@ func createGenericJobAWWithScheduleSpec(context *context, name string) *arbv1.Ap AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-scheduling-spec"), - Namespace: "test", - }, GenericTemplate: runtime.RawExtension{ Raw: rb, }, @@ -1552,10 +1500,6 @@ func createGenericJobAWtWithLargeCompute(context *context, name string) *arbv1.A AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-large-comp-1"), - Namespace: "test", - }, DesiredAvailable: 1, GenericTemplate: runtime.RawExtension{ Raw: rb, @@ -1627,10 +1571,6 @@ func createGenericServiceAWWithNoStatus(context *context, name string) *arbv1.Ap AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-comp-1"), - Namespace: "test", - }, DesiredAvailable: 1, GenericTemplate: runtime.RawExtension{ Raw: rb, @@ -1757,10 +1697,6 @@ func createGenericDeploymentAWWithMultipleItems(context *context, name string) * AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "deployment-1"), - Namespace: "test", - }, DesiredAvailable: 1, GenericTemplate: runtime.RawExtension{ Raw: rb, @@ -1768,10 +1704,6 @@ func createGenericDeploymentAWWithMultipleItems(context *context, name string) * CompletionStatus: "Progressing", }, { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "deployment-2"), - Namespace: "test", - }, DesiredAvailable: 1, GenericTemplate: runtime.RawExtension{ Raw: rb1, @@ -1850,10 +1782,6 @@ func createGenericDeploymentWithCPUAW(context *context, name string, cpuDemand s AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "item1"), - Namespace: context.namespace, - }, DesiredAvailable: 1, GenericTemplate: runtime.RawExtension{ Raw: rb, @@ -1936,10 +1864,6 @@ func createGenericDeploymentCustomPodResourcesWithCPUAW(context *context, name s AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "item1"), - Namespace: context.namespace, - }, CustomPodResources: []arbv1.CustomPodResourceTemplate{ { Replicas: replicas, @@ -2093,10 +2017,6 @@ func createStatefulSetAW(context *context, name string) *arbv1.AppWrapper { AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "item1"), - Namespace: context.namespace, - }, DesiredAvailable: 1, GenericTemplate: runtime.RawExtension{ Raw: rb, @@ -2169,10 +2089,6 @@ func createGenericStatefulSetAW(context *context, name string) *arbv1.AppWrapper AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "item1"), - Namespace: context.namespace, - }, DesiredAvailable: 2, GenericTemplate: runtime.RawExtension{ Raw: rb, @@ -2232,10 +2148,6 @@ func createBadPodTemplateAW(context *context, name string) *arbv1.AppWrapper { AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "item"), - Namespace: context.namespace, - }, DesiredAvailable: 2, GenericTemplate: runtime.RawExtension{ Raw: rb, @@ -2314,19 +2226,11 @@ func createPodTemplateAW(context *context, name string) *arbv1.AppWrapper { AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "item"), - Namespace: context.namespace, - }, GenericTemplate: runtime.RawExtension{ Raw: rb, }, }, { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "item1"), - Namespace: context.namespace, - }, GenericTemplate: runtime.RawExtension{ Raw: rb1, }, @@ -2390,10 +2294,6 @@ func createPodCheckFailedStatusAW(context *context, name string) *arbv1.AppWrapp AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "item"), - Namespace: context.namespace, - }, DesiredAvailable: 1, GenericTemplate: runtime.RawExtension{ Raw: rb, @@ -2466,10 +2366,6 @@ func createGenericPodAWCustomDemand(context *context, name string, cpuDemand str AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "item"), - Namespace: context.namespace, - }, GenericTemplate: runtime.RawExtension{ Raw: rb, }, @@ -2540,10 +2436,6 @@ func createGenericPodAW(context *context, name string) *arbv1.AppWrapper { AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "item"), - Namespace: context.namespace, - }, GenericTemplate: runtime.RawExtension{ Raw: rb, }, @@ -2616,10 +2508,6 @@ func createGenericPodTooBigAW(context *context, name string) *arbv1.AppWrapper { AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "item"), - Namespace: context.namespace, - }, GenericTemplate: runtime.RawExtension{ Raw: rb, }, @@ -2675,10 +2563,6 @@ func createBadGenericPodAW(context *context, name string) *arbv1.AppWrapper { AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "item"), - Namespace: context.namespace, - }, GenericTemplate: runtime.RawExtension{ Raw: rb, }, @@ -2710,10 +2594,6 @@ func createBadGenericItemAW(context *context, name string) *arbv1.AppWrapper { AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "item"), - Namespace: context.namespace, - }, // GenericTemplate: runtime.RawExtension{ // Raw: rb, // }, @@ -2775,10 +2655,6 @@ func createBadGenericPodTemplateAW(context *context, name string) (*arbv1.AppWra AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "item"), - Namespace: context.namespace, - }, DesiredAvailable: 2, GenericTemplate: runtime.RawExtension{ Raw: rb,