diff --git a/demos/argo-cd/applications/airflow-postgres.yaml b/demos/argo-cd/applications/airflow-postgres.yaml new file mode 100644 index 00000000..7052b64e --- /dev/null +++ b/demos/argo-cd/applications/airflow-postgres.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: airflow-postgres +spec: + project: airflow + destination: + server: https://kubernetes.default.svc + namespace: stackable-airflow + sources: + - repoURL: "registry-1.docker.io/bitnamicharts" + path: postgresql + # helm inspect chart oci://registry-1.docker.io/bitnamicharts/postgresql + targetRevision: 16.6.3 # 17.4.0 + chart: postgresql + helm: + # TODO this breaks naming as long as we use the airflow stack yaml which needs this svc name + releaseName: postgresql-airflow + valuesObject: + commonLabels: + stackable.tech/vendor: Stackable + auth: + database: airflow + username: airflow + existingSecret: postgresql-credentials + - repoURL: https://github.com/stackabletech/demos.git + # TODO: adapt to release-25.3 + targetRevision: "spike/argocd-demo" + path: demos/argo-cd/manifests/airflow-postgres/ + syncPolicy: + syncOptions: + - CreateNamespace=true + automated: + selfHeal: true + prune: true diff --git a/demos/argo-cd/applications/airflow.yaml b/demos/argo-cd/applications/airflow.yaml new file mode 100644 index 00000000..82c1b71c --- /dev/null +++ b/demos/argo-cd/applications/airflow.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: airflow +spec: + project: airflow + destination: + server: https://kubernetes.default.svc + namespace: stackable-airflow + source: + repoURL: https://github.com/stackabletech/demos.git + # TODO: adapt to release-25.3 + targetRevision: "spike/argocd-demo" + path: demos/argo-cd/manifests/airflow/ + syncPolicy: + syncOptions: + - CreateNamespace=true + automated: + selfHeal: true + prune: true diff --git a/demos/argo-cd/applications/minio.yaml b/demos/argo-cd/applications/minio.yaml new file mode 100644 index 00000000..42ca5407 --- /dev/null +++ b/demos/argo-cd/applications/minio.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: minio +spec: + project: minio + destination: + server: https://kubernetes.default.svc + namespace: minio + source: + repoURL: https://github.com/stackabletech/demos.git + # TODO: adapt to release-25.3 + targetRevision: "spike/argocd-demo" + path: demos/argo-cd/manifests/minio/ + syncPolicy: + syncOptions: + - CreateNamespace=true + automated: + selfHeal: true + prune: true diff --git a/demos/argo-cd/dags/date_demo.py b/demos/argo-cd/dags/date_demo.py new file mode 100644 index 00000000..704aac51 --- /dev/null +++ b/demos/argo-cd/dags/date_demo.py @@ -0,0 +1,20 @@ +"""Example DAG returning the current date""" +from datetime import datetime, timedelta + +from airflow import DAG +from airflow.operators.bash import BashOperator + +with DAG( + dag_id='date_demo', + schedule_interval='0-59 * * * *', + start_date=datetime(2021, 1, 1), + catchup=False, + dagrun_timeout=timedelta(minutes=5), + tags=['example'], + params={}, +) as dag: + + run_this = BashOperator( + task_id='run_every_minute', + bash_command='date', + ) diff --git a/demos/argo-cd/dags/pyspark_pi.py b/demos/argo-cd/dags/pyspark_pi.py new file mode 100644 index 00000000..35bd4c38 --- /dev/null +++ b/demos/argo-cd/dags/pyspark_pi.py @@ -0,0 +1,177 @@ +"""Example DAG demonstrating how to apply a Kubernetes Resource from Airflow running in-cluster""" +from datetime import datetime, timedelta +from airflow import DAG +from typing import TYPE_CHECKING, Optional, Sequence, Dict +from kubernetes import client +from airflow.exceptions import AirflowException +from airflow.sensors.base import BaseSensorOperator +from airflow.models import BaseOperator +from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook +import yaml +from airflow.utils import yaml +import os + +if TYPE_CHECKING: + from airflow.utils.context import Context + +class SparkKubernetesOperator(BaseOperator): + template_fields: Sequence[str] = ('application_file', 'namespace') + template_ext: Sequence[str] = ('.yaml', '.yml', '.json') + ui_color = '#f4a460' + + def __init__( + self, + *, + application_file: str, + namespace: Optional[str] = None, + kubernetes_conn_id: str = 'kubernetes_in_cluster', + api_group: str = 'spark.stackable.tech', + api_version: str = 'v1alpha1', + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.application_file = application_file + self.namespace = namespace + self.kubernetes_conn_id = kubernetes_conn_id + self.api_group = api_group + self.api_version = api_version + self.plural = "sparkapplications" + + def execute(self, context: 'Context'): + hook = KubernetesHook(conn_id=self.kubernetes_conn_id) + self.log.info("Creating SparkApplication...") + response = hook.create_custom_object( + group=self.api_group, + version=self.api_version, + plural=self.plural, + body=self.application_file, + namespace=self.namespace, + ) + return response + + +class SparkKubernetesSensor(BaseSensorOperator): + template_fields = ("application_name", "namespace") + # See https://github.com/stackabletech/spark-k8s-operator/pull/460/files#diff-d737837121132af6b60f50279a78464b05dcfd06c05d1d090f4198a5e962b5f6R371 + # Unknown is set immediately so it must be excluded from the failed states. + FAILURE_STATES = ("Failed") + SUCCESS_STATES = ("Succeeded") + + def __init__( + self, + *, + application_name: str, + attach_log: bool = False, + namespace: Optional[str] = None, + kubernetes_conn_id: str = 'kubernetes_in_cluster', + api_group: str = 'spark.stackable.tech', + api_version: str = 'v1alpha1', + poke_interval: float = 60, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.application_name = application_name + self.attach_log = attach_log + self.namespace = namespace + self.kubernetes_conn_id = kubernetes_conn_id + self.hook = KubernetesHook(conn_id=self.kubernetes_conn_id) + self.api_group = api_group + self.api_version = api_version + self.poke_interval = poke_interval + + def _log_driver(self, application_state: str, response: dict) -> None: + if not self.attach_log: + return + status_info = response["status"] + if "driverInfo" not in status_info: + return + driver_info = status_info["driverInfo"] + if "podName" not in driver_info: + return + driver_pod_name = driver_info["podName"] + namespace = response["metadata"]["namespace"] + log_method = self.log.error if application_state in self.FAILURE_STATES else self.log.info + try: + log = "" + for line in self.hook.get_pod_logs(driver_pod_name, namespace=namespace): + log += line.decode() + log_method(log) + except client.rest.ApiException as e: + self.log.warning( + "Could not read logs for pod %s. It may have been disposed.\n" + "Make sure timeToLiveSeconds is set on your SparkApplication spec.\n" + "underlying exception: %s", + driver_pod_name, + e, + ) + + def poke(self, context: Dict) -> bool: + self.log.info("Poking: %s", self.application_name) + response = self.hook.get_custom_object( + group=self.api_group, + version=self.api_version, + plural="sparkapplications", + name=self.application_name, + namespace=self.namespace, + ) + try: + application_state = response["status"]["phase"] + except KeyError: + self.log.debug(f"SparkApplication status could not be established: {response}") + return False + if self.attach_log and application_state in self.FAILURE_STATES + self.SUCCESS_STATES: + self._log_driver(application_state, response) + if application_state in self.FAILURE_STATES: + raise AirflowException(f"SparkApplication failed with state: {application_state}") + elif application_state in self.SUCCESS_STATES: + self.log.info("SparkApplication ended successfully") + return True + else: + self.log.info("SparkApplication is still in state: %s", application_state) + return False + +with DAG( + dag_id='sparkapp_dag', + schedule_interval=None, + start_date=datetime(2022, 1, 1), + catchup=False, + dagrun_timeout=timedelta(minutes=60), + tags=['example'], + params={"example_key": "example_value"}, +) as dag: + + def load_body_to_dict(body): + try: + body_dict = yaml.safe_load(body) + except yaml.YAMLError as e: + raise AirflowException(f"Exception when loading resource definition: {e}\n") + return body_dict + + yaml_path = os.path.join(os.environ.get('AIRFLOW__CORE__DAGS_FOLDER'), '../manifests/spark-k8s/pyspark_pi.yaml') + + with open(yaml_path, 'r') as file: + crd = file.read() + with open('/run/secrets/kubernetes.io/serviceaccount/namespace', 'r') as file: + ns = file.read() + + document=load_body_to_dict(crd) + application_name='pyspark-pi-'+datetime.utcnow().strftime('%Y%m%d%H%M%S') + document.update({'metadata': {'name': application_name, 'namespace': ns}}) + + t1 = SparkKubernetesOperator( + task_id='spark_pi_submit', + namespace=ns, + application_file=document, + do_xcom_push=True, + dag=dag, + ) + + t2 = SparkKubernetesSensor( + task_id='spark_pi_monitor', + namespace=ns, + application_name="{{ task_instance.xcom_pull(task_ids='spark_pi_submit')['metadata']['name'] }}", + poke_interval=5, + dag=dag, + ) + + t1 >> t2 diff --git a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml new file mode 100644 index 00000000..51bd342f --- /dev/null +++ b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: postgresql-credentials + namespace: stackable-airflow +spec: + encryptedData: + database: AgAlOcdpjYAaeiHbIwmGM3lb7okU4DROr+4bs7oQBPBY924XdamqlSw4Y7V8svvHRqD5Hz8SRx0SY2dkLaBQZBaXKV8qaMaVdqX0zK8zvoIbw/bQFXMTFpPq/kUD/kBdTwGJdAd/VpdY1m4Dcn247AhsQuNbqcZ8xibH2IW1e41f1/5nidHY+0a0O2qrSjAU8RxBAuBlLS6ZZiT2vmgahe6CUarJ3Dzr8InZRmtgwT7F8Zzli6cqgOPDsUg768KwDOIxpO+tnOT9b57SAbLUn5SOedFbAodWgQAg913URTiPOCrt5Loyp9B4tQXv1s74WFmgPlEkQmavkIyQg3R2Yki0QhkFysOFtp41ilRFTvgauxV3irryTv/frgA+XJDwfYexG2whgWH/VbT/HHIMoHBuRD864lpWZ/qDdLhFx7A5Eflli+YO+xuOQP4eFEsWeVYtcBNEi8Epy86LBiDCttCYWJ96XEmriqTpBebjxpaihxfCnPEbq5bkLv9RhO9vgzgYPgceLvPJ9GSpSJFv2s+ohvvJYyeAiuNLRkPR++tHFaofCVnuIfbeFgS1WVi/F4TrNsq2JlNfUnrzGTkc1CIkx/u02cE7K+Smfi/Dz30iDE7A+TLxDz7pguozMHX0eQCYDJ4ehehkArfH/XXe/ZH7Hb5MV6nMV32ZxkNi4x5iMdIsuLUdXJHWqSi0Ue9cZFBsP2rqmr5S + password: AgB/x1/DNC9uYWCTH1xu17qSr9GrnTcQ4LD5BCTu/5wUd/TK2rWM8zHce0mF97e6km+FyT+G96utiRThutNkAOpzT7xrZwkp45F0HqF0aD+NOYMzMn4duVLaK7JXLm6u7187Ci6kFMkmZ+Npb94wX+iUa+kgTQTkMP1HsdgQj4c8gj1ldpnoUDQt8/wsTF/gEukXD14E0BbEMkRTPh8PzkiirlLVWCff3YauQVRKXL2yWa79Fo47zkZ/WMJXyB8PVBQot2OIDL1ygddTBklBoP6ts2MOo3G7blmQWu5cgS/+IAw837pEWpDg5+UnUkM1+OQ+52PWLRXPbiSMFFlcbZIHwuASx7lRRm482q+kfJBb4oALd7EPHmjcBg1lV1iR74NetJ2wkMMcqnvkITxw8U8To1ZKseBpNH7mBWB9u2lvpjT5EWMOveVLCq6NwU2IcdgHhUaMByrxq5I3u1k1Y1hZvM0txQp8Mr/Zas/EEJ/w3sN+i9O/JWIonRx84Y9OJxIQEXManz7LzzzIxXikCvenm/8oQMfPHa7w5Scdb6sEQKsFXZ+aMQMcUTp7xuZbYzuX/7CmVdPHlFOhAXGAbRE2J739uWmyMVQkJ9cHSKzbCJdXDD3+oJB42Xz37tJg08xvMGBTPJjJoKnXfHIPS2yQISg1VgfyWNg/T2LZLuqt5htpUbQsWsL6K9/z7qtsama5pUiVTsmu + postgres-password: AgBpGdazaNMQ2j2jI1sK5D1PCFvhXn/kH7Xhj2P2GgHesCxml2WN8B99dtk2O81BDt7IYNqadhwdXe+pmvJU48qkbfcmgE+Kbjbbdo8bv9GxMQcYPPH0ahqIfFAxNgMBHnbQ0Zxy+a3vzNiXLeD4FAwU0Pxa7kSwED0/EiyfmZcvph2J5JTi4FwhU2ITycMwsjQnt7KVw1SGWMZkro6HoJ9wGSj7GRSeHX7HEMBCehf8TOMEo1Emy6IqYHAywdcNgfBEoLvgUUFElBxYLshCP2NTTCU2o374Xw2fiErjiR7Q075Mshpxl4bqGf2HgGT44Lsn5KDupPNnV7FPx0xTk3OmmE5Ja5gUAeLYU3bF8B7DSm61ltQnkL+FQ5sy3tjCTsH3wyYbg4zVoQlJuyRXJdG48Y0aHkQHwMpZlZEkB8IWGZZ9Kd9UKTTFnS3odca8OhdWYQ28ARz4Mj0Jio0dEO/63EyCJZiL4/Ck4rj8I49Wz832GjHVkMNFqthK6jN+HmlGcKXRKRMFLyIrvH7mUv1EuOsvh2+qtrsGGWSYwfETmb1J/iUp7X2+KhPnn6GP/poXLKeeDtiRfRSXTP70vRDllOMgSRTorRxS44FD/ac3KVA4RlpA+Ho+XU4QhYnNx8Q6haTQINY8Yypw/JRAPO4l379EFZtFdCSrKZ9XmRPsSZkZPZZJSVUe2ByGs/SScOR1xf1pwdPjWoOd + username: AgC716fxKUrMXCsRsWeiSTliHKP8NrwBr0TD7/w2F1HO93pqjSVU5/6hKOiFdNllbyPRudSHAeYWqJhQxxl5tS9kSaEp29zkxAUnmNwfLL3MKdCohxz2LZzC7saaOdjDkX2x8jcy68STPupsdg5bX40D4tO/s8qXnWWZJrW0ht/J+orgjbnfHk51JBMRBh6WD43yKLOBTHGO5U/TqRKg20kn+jDm2hcdec8axcjn2E57B3wrL8tvUcVRElO0XOcLHHBrav/BumRkwS4bMGqa+Vcarrr8U9MuQXVUzbCuwt326q/Kg862ozYY9uWAVtUy+Yfb4SAcEUC9Y+Ioo776H3YIr6q8P6RI+lFNmz4Q8AFDN9D5IRNjQ/D73llHw944dYRJl8CVjoJJYvRXv8z+4GzL7O+eF+smx4EhcY3sFxjkNCv+s+EDbUXvQR0ULqTEc5WZlBqp9TQEvzBwN3CVYO0k76FQYdjoU6xb/32gALe6zMxOLTliR9axqDvCL3OCj97/HJy8RLL2sYH82NZ1sGHqfTkTKYGk24Zfzd+a+p0PDbU14ZBuXKWBDkzNaVdpLvWhtdBczarfVp4Z2b6IutbT2C43fcmi7WYJOtV5em1sGhKOtbEy+wC+r5GKr6qRV58jm88cwtqZgm6IirDt/bzJDqhPndBzW5iQZzkko/1Tr4y/6oaTleTcmkDlSg2Cn8q/R7MNYUjT + template: + metadata: + creationTimestamp: null + name: postgresql-credentials + namespace: stackable-airflow + type: Opaque diff --git a/demos/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml b/demos/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml new file mode 100644 index 00000000..66abed2f --- /dev/null +++ b/demos/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: airflow-spark-clusterrole +rules: + - apiGroups: + - spark.stackable.tech + resources: + - sparkapplications + verbs: + - create + - get + - list + - apiGroups: + - airflow.stackable.tech + resources: + - airflowdbs + verbs: + - create + - get + - list + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - watch + - list + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - list diff --git a/demos/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml b/demos/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml new file mode 100644 index 00000000..1f9e1b5d --- /dev/null +++ b/demos/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: airflow-spark-clusterrole-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: airflow-spark-clusterrole +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:serviceaccounts diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml new file mode 100644 index 00000000..8ebbb9bf --- /dev/null +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -0,0 +1,92 @@ +--- +apiVersion: airflow.stackable.tech/v1alpha1 +kind: AirflowCluster +metadata: + name: airflow +spec: + image: + productVersion: 2.10.4 + clusterConfig: + listenerClass: external-unstable + loadExamples: false + exposeConfig: false + credentialsSecret: airflow-credentials + dagsGitSync: + - repo: https://github.com/stackabletech/demos/ + branch: spike/argocd-demo + gitFolder: "demos/argo-cd/dags" + depth: 2 + volumes: + - name: minio-tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: pod,node + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech + volumeMounts: + - name: minio-tls + mountPath: /stackable/minio-tls + webservers: + envOverrides: &envOverrides + AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" + # Via sealed secrets, just kept for reference here + # AIRFLOW_CONN_MINIO: "aws://admin:adminadmin@/?endpoint_url=https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000" + AWS_CA_BUNDLE: "/stackable/minio-tls/ca.crt" + AIRFLOW__LOGGING__REMOTE_LOGGING: "True" + AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER: s3://demo/airflow-task-logs/ + AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio + podOverrides: &podOverrides + spec: + containers: + - name: airflow + env: + - name: AIRFLOW_CONN_MINIO + valueFrom: + secretKeyRef: + name: airflow-minio-connection + key: airflow-minio-connection + config: + gracefulShutdownTimeout: 30s + resources: + cpu: + min: 400m + max: "1" + memory: + limit: 1Gi + roleGroups: + default: + replicas: 1 + kubernetesExecutors: + envOverrides: *envOverrides + podOverrides: + spec: + containers: + - name: base + env: + - name: AIRFLOW_CONN_MINIO + valueFrom: + secretKeyRef: + name: airflow-minio-connection + key: airflow-minio-connection + schedulers: + envOverrides: *envOverrides + podOverrides: *podOverrides + config: + gracefulShutdownTimeout: 30s + resources: + cpu: + min: 400m + max: "1" + memory: + limit: 1Gi + roleGroups: + default: + replicas: 1 diff --git a/demos/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml b/demos/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml new file mode 100644 index 00000000..9f763340 --- /dev/null +++ b/demos/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: airflow-credentials + namespace: stackable-airflow +spec: + encryptedData: + adminUser.email: AgBJq/dTwq0biM8b8bl2/daQSbWPRXSnOHOMnrJRjxM7A5KX68HI0Ifp4r47gSTZDQuKxNx5PaFuyU6W0tT9CvbFSPdhTXb6G5xLXITafrpd2he9LiTRzI0wzh/xa8sOChTNHK3rXBpU8n0Y7aYVDp7mibFk6CQKhMMq8s/OL5Scp2R4mbiajAtXMwQ2wt1fF3CAqR4QgxqGGr0cEugGf8zl/Z3GryUjJsqjNyhkjDxBN2E6oiP8Q1NCMV7AGGxdqHyi3PJH6hFddk07fTKu24oV1/IkdPZ7QgTXzhV2dk57KcSX8wB9gksWsCiFGlL4RnKwkmMbNzXq5m9zAk9YkN0QyMu0pRBI/NqhFTfhXIBcVsIbEa/37rGxV6oIaLqndneSG4fDTu0HP6ga98pM/nD2Nrzlj1gS+KwaBSGWq+CWs3HSah35treHxne3U2nuG98+nnIvUQUYI36b7MAP9xQ5Koykk4iU6LrVoxyck42k3VxeWNLc4W1axOBLGyjSxxUmZMWF4lmlz4Z045/SuBF8O4mmIHj/3cK5EEcAlme3lebNKht9/w3iaxDd+dG04CP8RMQGE7i3OdHpteilZMszXyjYz8m08u/wgiKG8aEkh8ad9fNeR4RnGP5CMpifrLZeCsismj4R9JN6B3lr3q2s1Ot/aedPsk4JMj0RgJAnLkSQHbYI3twJAOq4C/6HH+fSsripLIL4CjC39Hevp6Nku6ef + adminUser.firstname: AgC+TLraEdynGLvTTBkzXgBEvnK0c+97PsdcVTK3BIFD0L4APn3HM6V2WiUcyBH//YJqUiNvg5mnJZx3EA1EakPdzkXlitYNFy6XoMDTkrMZTgiD/VAiABBmPyVJesGz5xxlovqFGpbGWQbXjr+fGOINqSciQi1JMzN/jQ9EJQRnyxr7Xq3rmF4/NsWo8s+fEbQHx8SmNrS8mu9BFd+QSL5pTskZJkcaUuzduCXIheacJcRE+YwiufQVOFSo2AkgGcbwzLBRJuL6+u6MST3qr/xapsBuGSwHtEoVcI/NApTYsGZCeNN6H8e083BnRGCunfnw/R9/5Xg4uxdUEbGT16gP4H5/o8V22AmH2opSgPgxVSntAn6h5cxva2YUc5Bezn+pKFZmK0KCmC8tUSDusje3X0emMIs9+fC+ksj81UwSikC+hKbuZz+HuzS23UCy+w5R9X2SDGk802zxdLVhsXn/46dgaEqzOEa1pYJEyCRZUUv4KTdM0/XXfkid4r1uP0d6bZ7jlS9tSDQ/ajm5dAhl5W7nIYdRVAqM5KhvvY3mWpqIdscdr+bhpE46mJPstP6O4FM3/0OYPntmjxSVTFf0Uc8mHY4RRAM2sN/ViqcDGumn3Q8xlQtG4vMO2Tx+GAStzIu9AGsTFYcak3fI8kpZhs7mhYCXeigSusaIcsH5bfC7uekMrqhrKraY5hOU/xaxjydPIzN6 + adminUser.lastname: AgBzqzvmZLHUVDcG6f5cyBxUef4Ov0PNBGaqrYLKPMk4a681XpPlfJnNvMk87aV8Ts1xunoozWs4uM2Z6pQa6ZIYoR5oRkO4bLqK16frBaVypKVsKRPkol+bSyWPLGuI0izIKdAcHrtuaEffb6Sr98i6ColIqGLEYlzUalnPIuK/W7QqKXO91jHYnTv4m6LUMere1N/x1cmeZ3+W+BukWLkfiLko4wgCSauUCene9azY5ehtmlIAUo9xNC3oDA+rTGTI1SkG2Xe/ytI92sedCEMmVX7oU9PI0KGfckchc6azmOTsWN0hSnUdIHPPDn+9L0x77HwHeMfiVgO2Bai76lC5fO2b9Pw0OxID5EKY4jvIrtQ2uQANCtYupge6Z37njpzI070U/H65NDtCKAGmcuy4pFgdF60VrEPrfAN+uaVJXmJGu75QmaKqGWE0RAW9UcjiKD9YaQf7pN2tx6v2oga/OR/cDTR+LJYIcwAC2HI+v9t2cb3LqznnoixVwU83qOOqDelq/Pd1vV3Sf1R2Kj5E4UlYm/lEtSczKJyxX4Rim34kzMqZnmYP/56bs4NwXjJ8vmMHEeFkO5tL3NaVgeJ3yKvLG4yUtasXxlrSUgLdXXTu6wN7PuBUkzCN4maYHrDjJYApo1iIXhObtgRbDatosCUCJOSvE0BEEf9CKP++SkUdipWVk84QK3ryNYQOZ12Lc4rnkw== + adminUser.password: AgAXw2meNuLCnWIDeBdrUI1osShroov1kjvRJ5+mLtGHD9cOTfQlAJpGXN+QwVWXz/hsRl0zhMxI1EomJhXOOyNI67ptlaMnSV/Wtt5X92ISmHReR/1R2cXv0e/NQLbD4M3AmCpvSz0agYMBxtIFh6mVeMah2RRjVD74Xc3P3SrHjnY7CiLYe58JWVkxGAp6WO2JTpopUJI1Iagr9MHfkGzPp3ygFVxwiMjPIVCM+IeAbtHLNi2w6slrcKa3IJduLSaGOQEr/p8YcPm2Qx2TGHOpiM6Lw9api8JtN28mS5hArc7qj5kkwkfEiHW2exYMXVSdWy7glgEGFuTr7MRi0LpLqS72DT55E9uxAObnlWuiKVmk6UfExX2mMfze+Iz4sKXrqOFoLAIXx8+yjO3t5oDvJf9pxLNihl4rYyLg9v8+UkkGJLQZPOqjQoeuH9ighDkHdfkiUDdprnG9iZAQBFzKN7On+tMA09McwHkC6jV+RIv47zty1CMoEPQ2wEM+zDHU6DG2HvtHQYov1pCpXzyvMz48hcKiAwIjZQM1RZBL0tN5qd8Tp38MqtAvVt7PvllwAN2LKt9HdV5j82kyWseE6MPYkB1l8Q6ctUn5YY0kw+tWwgtmYbeRY9+kSIUphxntX+unzhCKCAY1oBOc/cdim3YQInGGe6lx3JGUQvz5LpP6+aVBPyy8HOPnUSyxdu2ZaBkG5SKhKvfH + adminUser.username: AgCVVTX2c+4moS0v6mXuNQ5v6EtxZ43hN0QYg8wgYG+GPR3qZ2Ep+FkUJq86q9dFcuX6EXXtS2h8BJr2sUrRW6QySFnhtmg7A1d4anslN3yAjLipwrreNT9cn1t4PsdIhU0q5Z2uPmMsnLF28D73AxWA59iiveIId2KSuB2lsaC8S8CeW3Df48LW3cnukHtOvc+j84YQgFdagujBtwAMCiym2pdBJG5GojTHQ/5P3AnV+iM5wzXaYIeF5UZ0wLs5WU3CqHLxsY3q7NrLKGzS0xTqqdc6H++tJQFZY5G3q4NcCx+jq9oRErE0ARz3ok2pOHPArUEbyas9+iibA9HJcCmrKguQ8QBvaIQh9L0YtHCtwvaMbR12ZmMfRtMW6pZnM/d2K648JFekuezGTOVEjLk+ZvVm45L4USZpB0MD5LXEy3m8cDdkn2wqTWvblfjgfi7opoLgpwLyf/BZuunrFWT08x9K1WmNQ7OkM4WxscLRyccFOtZQfoUtT8hW+K5/mxLjOQDpryWPCUeBDAMnBvbGG/0AABp7a+vWG9KVus2ZyfUHVFIHgJBsHXXDtONIkmRvTal5bjlRsVk4Y0O3DoJa+UjXYoIBSm1JwdUJMOnL1MoS7Ijk+iXKPMc/3fcbrnfqYxw3ymr485bL8fKh2n99MsCdR1yX/mTxw3f6bRZdN5O7IUTkkKVxx1b08FD1svSwNKHBZQ== + connections.celeryBrokerUrl: AgAD1Iu/Oggx5NbNY29aJweDDVHlnL2KnsSJwBufPSxsaQouGdIIMIsWr/y2z+qCtd+TBG7f/ZK6vvTlT8kGeq9PGguGTOs+jQObFsqOWoWMn+zbZl+H3SsXHE9ohlrq36n6b68vRrpR/zIvI/QcFg7FC8qhH7GhxzYoUfXbIVEg9ZUyg8W9GgznIM8pJD7v1blz16fXerC3D3wcY449wvQuygZucGyGNfCK9+y75/HQldYVg5lG0Gb914uT5hQ4lATlH7xnHQcxwXhT+30qphNcIJSNZW2D6S0kovtoZ8xk6P/xoO5uZNzNyFnRxIu/AhVi3T2L/dzlO9ztbxwjCu19xxDo16/vhXQ5awGcyJjHqZc8Pu4v5e7sJTElguAHooQraL/dcx9fJvVOxk+3GUMwixfzSR3R/BD2Na4l9mT3fwiW53UTYIcDm/jRwZ8EPZz6GtWVPtwoOkpe6y/OLoRtGNlJEW2fR2d4VwsrqnPV57716FuI5ETwvCSJcqaoMk1kPZBzWGjgf02owizSD7gKONBLX8G34981JsTu/vloPD9uPYQoQ9oFlKVtdmYKhL6u08ekiwLoojp51T/JB5I0RYLKSu4nR8O8SK49RH2CF/4viBs5cCezP8Cfq4qxSBHsd12PXHocUWK3v5e7xgQXi4wJzoLifNcS2FXwV+97DvHaYCM9KUogdL3K/MnPSfb6SGI5WHJkExK5Hr2WqKG+3zFIr+TfIf8V5QcrXaFEz6m0iK49x5hIvjgmnQ== + connections.celeryResultBackend: AgAEg3o3zzMI1fqVt119TiNZ2U9Uvg5ua1gSW1klHtI8P49n/ZTmYftFJcPkJfw8mjawxuSzwTvPWNirkcTyF0jpauyn2W3L7q2SwuMMmS96z4R0FBwa4a1AjP7Rovq9r4lPjtjn45IPX5VPDG2gBiscVhBFLzozFucgXVk6bLo3bu8/lXuf+5PXaTH7TtRfYuTipiY4jbvlCx5jR3omzgnLThOWaoplnfIQPQbhnLF4t9ydipLxV+Dr2ccGnT+yehHxbmROK5kRD8DgPe0oe1+qFS5zIRyG6KZBgI3+ru3wAkbREeuE/Bq9nDWmRS5JVDlOKaFqFtYU2h12LB/f+MKPhwPdvzCiWLmT+XU8WW25pjUXHDakXRm4RJI2J+24cTjue2jv9GbNF4+n+ywunUpNqEagJ3q0VFZ8o5evZWjw55qMnsE6eMpSoPygssD1sfQ5jf4W00onxjSEijR7STqi/6zBEZzKzPSij3fH5HeWmsd9MwnQ+1/J8VIighxKm2RSRn6ClRsXggMvzCCf/wg5zwYClMWage65fytKYH12lR4Dl2fNHFYexTIWvrL/36xuuKhP/Glpz6dJHKm5bHOjeNo+RMtKLTWvr3NVqMQMzK9yl0JdPJKByoqLy+xR5pKy9TacihXJXZhpEPd2IDK2/zxCI+71OIN0iNqL1KnfI5VMtFJu/jEmkgzD+Hqg4UlI3a2frgxtv5chMvIusHIxf/2fv3OoWVhfIjObD9nlnbwFQHhAfnMemeMhaxlnxQ34aI4gFZJ1m3hg + connections.secretKey: AgB9N48o16//pMoE7YPHGHQUDd6n5gQGJcsDK2kmsxM1fj4POpG8eRwxXpRlNwBcH8VhgJAWJKLWUrIy5cQ4DM77gDGcG1YOpHIXDMDzTtocLmLJmpxoFGrswv0lEB0dCZwSpOph3btBzCDuKNA/nzHD/iwoVkZiDao/CyC+wjRrudamIfC0aJJsnSo1NAzcmsP1IEaURF0LoWdAPXWCqzEHbJ6FjCdQe/5F2md30PHGt1p0x3/ro4i4gv54ZePWBAStZml/a5i219UBvGROJgBCAJYEb0wlQ5Aut+7PVzjgUfLgWpvGEl1XBXGjuR5NUOiEwcYok+QsonQcUGaNScMzzqrfWRIagAlcGnSzs4TsLUgA2opBlKClVdEqnyS1QEH062ZTrxoFJ0mGg6T1oh6rElGfzmOTMhU95nRIF3wtYh4Si20IDAbWkIJgk1j5+Vvg31KHOTQXR6WdXc1CqYuo3LVzjP4JeIvrsdyqIJC4dg2WAKDTrQ3qixDv9Yq+bT9jMHrdQPQBz1clb8g6OEqtuDeobkTJFU/vdlZRSz86/mtsujpBNHr2SPa5IwIr2ALBx61NXfalc+gp2lUDTQSZPRf/oHIyD/SbARFDCNGmf0rPO1yQ2dg85+kLLSE2+K6uPYPwqippSIyiazIzCi5FLa6B6IEqAxjdrPyBdrMu3Qgkzdhp23UaL7yRnXJkHnPtb+URhrbnqPVIDiNeF0Nt + connections.sqlalchemyDatabaseUri: AgBGW3Nva7kn/GxNKinig8hj/b0KSa6nESiU2zKPwW6ZDhwTA3Kf3jcJCfKE2ISSLY3L7qNeihVnYP3f6S00y+b7oixLxvYxUxCGaVvNON/xtK+dFYoYQfh3hXevLWYuf7Bg5et6Rr2hoDJbkq6Nix2uIEMDiRjWyll3ma3ajlVSxbt4xdboXb/OqRzlCLS433nAruu2woSvw2d4ypoDHYd0uGw/W4ushxOEIDVC61kJU3T9K7UqzUK4bTMljiYxrwMAb3cZ0dnPw2FU/fpT9SDomCWtjnktMkfbzgVWgD/oIh+2IZH0p3Y9GoIDJUhHlBFudDu/t+tQ91F2Xpy3gqmWFSHkfnCex699hx4dBqAeSlf2Frwfz0ZR0CM4aSIKlPH1eyumz8VA5z8AmK8fDcWCvwYeRdS67qPcu0SDKodAlhA0XR3CSj+357TmId8KHS+DZCisrxAuMJGMt6FbCbTRxY6lM4yJHrkP+L9Hv3474deDApZix4h64pqiH5P1hYIsiDM02EkdI5JwrRuZewcfVMDzGhve/u9nSa5dMADrDdjKZAys8hCUgi//ynfJzpOKi/WbfvNeGcYbQdvDX9dyXTbvsSYQ/+jq8DnU69RwQWF3lYIwZAWeoGAThX4jvSppuDY+FtS/1TsrymLQSpPH4lXjgSw7yomRvNx2x2n4Pa3mPpgQ2SR4qKQX4lLeE2t+R39xKQreA4FO3XItLFnZ8ddLj1mcmatGhSMmD6AWI2hmxxxJJ3Of4AnBDrCoV68fYdPk/HeY3C8JfHQnSKcG + template: + metadata: + creationTimestamp: null + name: airflow-credentials + namespace: stackable-airflow + type: Opaque diff --git a/demos/argo-cd/manifests/airflow/sealed-airflow-minio-connection.yaml b/demos/argo-cd/manifests/airflow/sealed-airflow-minio-connection.yaml new file mode 100644 index 00000000..8e9f4f8d --- /dev/null +++ b/demos/argo-cd/manifests/airflow/sealed-airflow-minio-connection.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: airflow-minio-connection + namespace: stackable-airflow +spec: + encryptedData: + airflow-minio-connection: AgAHQL1NIeZiWH1opQYqHINixw0muWaNsYKMhtgJqqdVMLeKIU/sckZ7Z6aokaEYXGVQRe+Tp+D9AaYaF5CdKeMQKQ3MmhxXYZh0EuAcNRyvz4u4K696KEIa4XpaT5k6oRbIwRUGCc4hiRVApmMOgGYl4iEwVQ/1Twd5ffEAohMsySgZQBn17zB+hKC3u2AcIrZ1btHvBrVhC4pil/kyytBwjTkMq1iAkF8lwt7gatLukD9ldLaVdYTe21R+nP/UT7DwHRnBlI1dzkfrMdplFud2XIEmSrFF7ihzc5r+Lsm2vsnjaV5OgzDjIPI3RRA7g8/06Lebpru/E9h6HB5RIu59ga72g02QIHh5HffGUCnbEsLTDDbBcuAQB2jSodotfsofVKOhZChOIJDchvohhrQ86wqlkQyjfNMq2Gh46Nc7ZjomlpL/p0afmak34GFoPYMCD7IJ2DafEsmKA8dYfjTI1kVzLhYuoeDB3f82tu8xL9cXCEUsw7Y7EosZjeITeLcV1oMm+jeHzNHGaLmCI9k1sY5bL6xjbG59sOvuEgI302tS9ua592u9uqOOKKjE4eZM7CAmEjgMmDPbeteXGnZcOqfj8FWmlp0f4gm9s+68KH5PDtkKdtBckNjDeLj3QhMC2gstXnPJR3Hnm/bFQ4i/6XpnkuOFCvQT/Baar5scw4vhJ0tN8TGeMd651YXU7N2MEB+2WtiO0StbzhcWIoiGmvN+SRLawPZ87Sqlt26/bs5+jnuEv4Fy1oXhxbZwuf/YsZjEaKFI+HbFVvwM0K2d00/z9DYAB44oY2PmrZwsk0KqP2hkuumq + template: + metadata: + creationTimestamp: null + name: airflow-minio-connection + namespace: stackable-airflow + type: Opaque diff --git a/demos/argo-cd/manifests/minio/minio.yaml b/demos/argo-cd/manifests/minio/minio.yaml new file mode 100644 index 00000000..687e37a7 --- /dev/null +++ b/demos/argo-cd/manifests/minio/minio.yaml @@ -0,0 +1,697 @@ +--- +# Source: minio/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "minio-sa" +--- +# Source: minio/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: minio + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm +data: + initialize: |- + #!/bin/sh + set -e # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # connectToMinio + # Use a check-sleep-check loop to wait for MinIO service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 + LIMIT=29 # Allow 30 attempts + set -e # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) + SECRET=$(cat /config/rootPassword) + set +e # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" + $MC_COMMAND + STATUS=$? + until [ $STATUS = 0 ]; do + ATTEMPTS=$(expr $ATTEMPTS + 1) + echo \"Failed attempts: $ATTEMPTS\" + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 + fi + sleep 2 # 1 second intervals between attempts + $MC_COMMAND + STATUS=$? + done + set -e # reset `e` as active + return 0 + } + + # checkBucketExists ($bucket) + # Check if the bucket exists, by using the exit code of `mc ls` + checkBucketExists() { + BUCKET=$1 + CMD=$(${MC} stat myminio/$BUCKET >/dev/null 2>&1) + return $? + } + + # createBucket ($bucket, $policy, $purge) + # Ensure bucket exists, purging if asked to + createBucket() { + BUCKET=$1 + POLICY=$2 + PURGE=$3 + VERSIONING=$4 + OBJECTLOCKING=$5 + + # Purge the bucket, if set & exists + # Since PURGE is user input, check explicitly for `true` + if [ $PURGE = true ]; then + if checkBucketExists $BUCKET; then + echo "Purging bucket '$BUCKET'." + set +e # don't exit if this fails + ${MC} rm -r --force myminio/$BUCKET + set -e # reset `e` as active + else + echo "Bucket '$BUCKET' does not exist, skipping purge." + fi + fi + + # Create the bucket if it does not exist and set objectlocking if enabled (NOTE: versioning will be not changed if OBJECTLOCKING is set because it enables versioning to the Buckets created) + if ! checkBucketExists $BUCKET; then + if [ ! -z $OBJECTLOCKING ]; then + if [ $OBJECTLOCKING = true ]; then + echo "Creating bucket with OBJECTLOCKING '$BUCKET'" + ${MC} mb --with-lock myminio/$BUCKET + elif [ $OBJECTLOCKING = false ]; then + echo "Creating bucket '$BUCKET'" + ${MC} mb myminio/$BUCKET + fi + elif [ -z $OBJECTLOCKING ]; then + echo "Creating bucket '$BUCKET'" + ${MC} mb myminio/$BUCKET + else + echo "Bucket '$BUCKET' already exists." + fi + fi + + # set versioning for bucket if objectlocking is disabled or not set + if [ $OBJECTLOCKING = false ]; then + if [ ! -z $VERSIONING ]; then + if [ $VERSIONING = true ]; then + echo "Enabling versioning for '$BUCKET'" + ${MC} version enable myminio/$BUCKET + elif [ $VERSIONING = false ]; then + echo "Suspending versioning for '$BUCKET'" + ${MC} version suspend myminio/$BUCKET + fi + fi + else + echo "Bucket '$BUCKET' versioning unchanged." + fi + + # At this point, the bucket should exist, skip checking for existence + # Set policy on the bucket + echo "Setting policy of bucket '$BUCKET' to '$POLICY'." + ${MC} anonymous set $POLICY myminio/$BUCKET + } + + # Try connecting to MinIO instance + scheme=https + connectToMinio $scheme + + + + # Create the buckets + createBucket demo "public" false false false + + add-user: |- + #!/bin/sh + set -e ; # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. + # Special characters for example : ',",<,>,{,} + MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_tmp" + + # connectToMinio + # Use a check-sleep-check loop to wait for MinIO service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 1 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 + } + + # checkUserExists () + # Check if the user exists, by using the exit code of `mc admin user info` + checkUserExists() { + CMD=$(${MC} admin user info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) + return $? + } + + # createUser ($policy) + createUser() { + POLICY=$1 + #check accessKey_and_secretKey_tmp file + if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then + echo "credentials file does not exist" + return 1 + fi + if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then + echo "credentials file is invalid" + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + return 1 + fi + USER=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) + # Create the user if it does not exist + if ! checkUserExists ; then + echo "Creating user '$USER'" + cat $MINIO_ACCESSKEY_SECRETKEY_TMP | ${MC} admin user add myminio + else + echo "User '$USER' already exists." + fi + #clean up credentials files. + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + + # set policy for user + if [ ! -z $POLICY -a $POLICY != " " ] ; then + echo "Adding policy '$POLICY' for '$USER'" + set +e ; # policy already attach errors out, allow it. + ${MC} admin policy attach myminio $POLICY --user=$USER + set -e + else + echo "User '$USER' has no policy attached." + fi + } + + # Try connecting to MinIO instance + scheme=https + connectToMinio $scheme + + + + # Create the users + echo console > $MINIO_ACCESSKEY_SECRETKEY_TMP + echo console123 >> $MINIO_ACCESSKEY_SECRETKEY_TMP + createUser consoleAdmin + + add-policy: |- + #!/bin/sh + set -e ; # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # connectToMinio + # Use a check-sleep-check loop to wait for MinIO service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 1 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 + } + + # checkPolicyExists ($policy) + # Check if the policy exists, by using the exit code of `mc admin policy info` + checkPolicyExists() { + POLICY=$1 + CMD=$(${MC} admin policy info myminio $POLICY > /dev/null 2>&1) + return $? + } + + # createPolicy($name, $filename) + createPolicy () { + NAME=$1 + FILENAME=$2 + + # Create the name if it does not exist + echo "Checking policy: $NAME (in /config/$FILENAME.json)" + if ! checkPolicyExists $NAME ; then + echo "Creating policy '$NAME'" + else + echo "Policy '$NAME' already exists." + fi + ${MC} admin policy create myminio $NAME /config/$FILENAME.json + + } + + # Try connecting to MinIO instance + scheme=https + connectToMinio $scheme + + add-svcacct: |- + #!/bin/sh + set -e ; # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. + # Special characters for example : ',",<,>,{,} + MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_svcacct_tmp" + + # connectToMinio + # Use a check-sleep-check loop to wait for MinIO service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 2 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 + } + + # checkSvcacctExists () + # Check if the svcacct exists, by using the exit code of `mc admin user svcacct info` + checkSvcacctExists() { + CMD=$(${MC} admin user svcacct info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) + return $? + } + + # createSvcacct ($user) + createSvcacct () { + USER=$1 + FILENAME=$2 + #check accessKey_and_secretKey_tmp file + if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then + echo "credentials file does not exist" + return 1 + fi + if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then + echo "credentials file is invalid" + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + return 1 + fi + SVCACCT=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) + # Create the svcacct if it does not exist + if ! checkSvcacctExists ; then + echo "Creating svcacct '$SVCACCT'" + # Check if policy file is define + if [ -z $FILENAME ]; then + ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) myminio $USER + else + ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --policy /config/$FILENAME.json myminio $USER + fi + else + echo "Svcacct '$SVCACCT' already exists." + fi + #clean up credentials files. + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + } + + # Try connecting to MinIO instance + scheme=https + connectToMinio $scheme + + custom-command: |- + #!/bin/sh + set -e ; # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # connectToMinio + # Use a check-sleep-check loop to wait for MinIO service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 1 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 + } + + # runCommand ($@) + # Run custom mc command + runCommand() { + ${MC} "$@" + return $? + } + + # Try connecting to MinIO instance + scheme=https + connectToMinio $scheme +--- +# Source: minio/templates/pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm +spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "10Gi" +--- +# Source: minio/templates/console-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: minio-console + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm +spec: + type: NodePort + externalTrafficPolicy: "Cluster" + ports: + - name: https + port: 9001 + protocol: TCP + targetPort: 9001 + selector: + app: minio + release: minio +--- +# Source: minio/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: minio + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm + monitoring: "true" +spec: + type: NodePort + externalTrafficPolicy: "Cluster" + ports: + - name: https + port: 9000 + protocol: TCP + targetPort: 9000 + selector: + app: minio + release: minio +--- +# Source: minio/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm + stackable.tech/vendor: Stackable +spec: + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 100% + maxUnavailable: 0 + replicas: 1 + selector: + matchLabels: + app: minio + release: minio + template: + metadata: + name: minio + labels: + app: minio + release: minio + stackable.tech/vendor: Stackable + annotations: + checksum/secrets: fa63e34a92c817c84057e2d452fa683e66462a57b0529388fb96a57e05f38e57 + checksum/config: ebea49cc4c1bfbd1b156a58bf770a776ff87fe199f642d31c2816b5515112e72 + spec: + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 1000 + runAsUser: 1000 + + serviceAccountName: minio-sa + containers: + - name: minio + image: "quay.io/minio/minio:RELEASE.2024-12-18T13-15-44Z" + imagePullPolicy: IfNotPresent + command: + - "/bin/sh" + - "-ce" + - | + # minio requires the TLS key pair to be specially named + # mkdir -p /etc/minio/certs + cp -v /etc/minio/original_certs/tls.crt /etc/minio/certs/public.crt + cp -v /etc/minio/original_certs/tls.key /etc/minio/certs/private.key + + /usr/bin/docker-entrypoint.sh minio server /export -S /etc/minio/certs/ --address :9000 --console-address :9001 + volumeMounts: + - name: minio-user + mountPath: "/tmp/credentials" + readOnly: true + - name: export + mountPath: /export + - mountPath: /etc/minio/original_certs + name: tls + - mountPath: /etc/minio/certs + name: certs + ports: + - name: https + containerPort: 9000 + - name: https-console + containerPort: 9001 + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: minio + key: rootUser + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: minio + key: rootPassword + - name: MINIO_PROMETHEUS_AUTH_TYPE + value: "public" + resources: + requests: + cpu: 1 + memory: 2Gi + securityContext: + readOnlyRootFilesystem: false + volumes: + - name: export + persistentVolumeClaim: + claimName: minio + - name: minio-user + secret: + secretName: minio + + - ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: service=minio + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1 + storageClassName: secrets.stackable.tech + name: tls + - emptyDir: + medium: Memory + sizeLimit: 5Mi + name: certs +--- +# Source: minio/templates/post-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: minio-post-job + labels: + app: minio-post-job + chart: minio-5.4.0 + release: minio + heritage: Helm + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation +spec: + template: + metadata: + labels: + app: minio-job + release: minio + stackable.tech/vendor: Stackable + spec: + restartPolicy: OnFailure + volumes: + - name: etc-path + emptyDir: {} + - name: tmp + emptyDir: {} + - name: minio-configuration + projected: + sources: + - configMap: + name: minio + - secret: + name: minio + - ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: service=minio + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1 + storageClassName: secrets.stackable.tech + name: tls + - emptyDir: + medium: Memory + sizeLimit: 5Mi + name: certs + serviceAccountName: minio-sa + containers: + - name: minio-make-bucket + image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" + imagePullPolicy: IfNotPresent + command: + - "/bin/sh" + - "-ce" + - | + # Copy the CA cert from the "tls" SecretClass + # mkdir -p /etc/minio/mc/certs/CAs + cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt + + . /config/initialize + env: + - name: MINIO_ENDPOINT + value: minio + - name: MINIO_PORT + value: "9000" + volumeMounts: + - name: etc-path + mountPath: /etc/minio/mc + - name: tmp + mountPath: /tmp + - name: minio-configuration + mountPath: /config + - name: tls + mountPath: /etc/minio/mc/original_certs + - name: certs + mountPath: /etc/minio/mc/certs/CAs + resources: + requests: + memory: 128Mi + - name: minio-make-user + image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" + imagePullPolicy: IfNotPresent + command: + - "/bin/sh" + - "-ce" + - | + # Copy the CA cert from the "tls" SecretClass + # mkdir -p /etc/minio/mc/certs/CAs + cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt + + . /config/add-user + env: + - name: MINIO_ENDPOINT + value: minio + - name: MINIO_PORT + value: "9000" + volumeMounts: + - name: etc-path + mountPath: /etc/minio/mc + - name: tmp + mountPath: /tmp + - name: minio-configuration + mountPath: /config + - name: tls + mountPath: /etc/minio/mc/original_certs + - name: certs + mountPath: /etc/minio/mc/certs/CAs + resources: + requests: + memory: 128Mi diff --git a/demos/argo-cd/manifests/minio/sealed-minio-credentials.yaml b/demos/argo-cd/manifests/minio/sealed-minio-credentials.yaml new file mode 100644 index 00000000..5a030480 --- /dev/null +++ b/demos/argo-cd/manifests/minio/sealed-minio-credentials.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: minio + namespace: minio +spec: + encryptedData: + rootPassword: AgBSBE4tAAJHV28O13Z+lg4CAzk/snvC+2WAM/ErBUeg4JiQZBIT+9o0MAIwM/IPcT7TpjrIpjx9UUiM5o8b/xjBIkxYBkpr8dNPE57IX+yVyWSxe5CPmBqazxEHU6cdlVT+s2NrydxyEsreSd6gs3+sHRd88AHQUhf3QJiD+hV2UE33jRe/so1if3OG9om5mNa/8EOqXeBnNBTCynaMDDuGyUcCUyzh5MEdaLG1YUu9flYCqlXNEsIkDR+1vgmbX1gcYdGcAIF7C71g8KMnGSt5dtAzxyuDhtDTCaq4+z3FbwEoJ4JQpmiDZzqtR5FDVUD8yS9b01Weg2cSlIEeNq83qN40wn8PkDQ6GEkl8Dou6yBpVklgnNJ+zgr+CxmFjrE6tGJnKQK1eOZmVCW43QFOOBvhyqXE0gpIlTO6rTzwfawHyWT1Wl83qCCcslb+/YylQx4tl/7fdFda5aNRN+1KQrfhkO9kc1zqeaJgO2nI31OpZOxu2U86WF9cQ/acFtOcF4/DDfYOu2dvvzyfzazfj6NIP0P2DjIpqIt7lTZ5t0SYgWB+cf5afsR8Y2TrO4K5B4Ys1Q9RTojqTuL/lyjuhNxYm2b/sIaQbBoJthntNrwaQO/KJ2nWngeRA+lFZTgbWigR6eQaLZXMIA+GioKKLiQxDiLeIvqfVMO6fwQWpeqznBruuQjVcB1aH18wLejD89ze9ztBI9JB + rootUser: AgAydeD6HzjMO94+dVqBLipjRRMLqHBDnAgqvDc+GUwT7dUuenMl2Qjx6D4G9Lk6HIuoqGpW1ZNVs65w+IEIETUGBR23I+E4drp2DFkGamXng+flkCJIvvvnmVoHYSDvO1mGEaqiF109vPOqj8vHtNHDjM89DdVWvdMkLQzM/XWK8pjIY/9g+K+a2te9yM3MzzlQBh7DPw5WdL5Nn97Ip5dUMA9D6fSLNidSqz24TLBbS9cLvRHERy9BLow7MqxyK0017iNH8TnpIzWphE+6iH9qPXGLNVlmeq8Ex4L74x3BIyas+yC2ErXPlvBNb/PG+L/788d4ZHW8JRNnQRhI2u8tvrf/Gg55/9a6hu2DQ+oqgT5jSJTPUE0meHIwAIVtLfxHozHYPD0IIdsgZgj89yfzaM3uIv02TnqAaDU113DVq5r7yZhh/THz9euw82/lPJYpuYwuSOuarddz18mOYWgQrhfBgOINxOr/OsFBl2HAfxnJyE2J4N50o4g9P5idr8+Px8P2bIKmrIbYI/YWVkHuGrslyXJFvdQvo3VJVXp7qGVJLOu3BTujxHc13CfMMDQGWXP3LvUFtDYH2fn1LHl0TlSlzyNzFdRir/b75tNKWRRuyOQ4qUJGSES1sDCmK7C8BA7d4oD0Sn0HZt7n4xp+3+b/8Rmicm/w8Uh1syTaxgW/2KQA9iXmHT8Un6gET71NF9E8mg== + template: + metadata: + creationTimestamp: null + name: minio + namespace: minio + type: Opaque diff --git a/demos/argo-cd/manifests/spark-k8s/pyspark_pi.yaml b/demos/argo-cd/manifests/spark-k8s/pyspark_pi.yaml new file mode 100644 index 00000000..559f4822 --- /dev/null +++ b/demos/argo-cd/manifests/spark-k8s/pyspark_pi.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: spark.stackable.tech/v1alpha1 +kind: SparkApplication +metadata: + name: pyspark-pi +spec: + version: "1.0" + sparkImage: + productVersion: 3.5.2 + mode: cluster + mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py + job: + config: + resources: + cpu: + min: 500m + max: 500m + memory: + limit: 512Mi + driver: + config: + resources: + cpu: + min: 1000m + max: 1200m + memory: + limit: 1024Mi + executor: + config: + resources: + cpu: + min: 500m + max: 1000m + memory: + limit: 1024Mi + replicas: 3 diff --git a/demos/argo-cd/projects/airflow.yaml b/demos/argo-cd/projects/airflow.yaml new file mode 100644 index 00000000..f518fd55 --- /dev/null +++ b/demos/argo-cd/projects/airflow.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: airflow +spec: + description: Project to manage Stackable Airflow via Git + sourceRepos: + - "*" + destinations: + - namespace: stackable-airflow + server: https://kubernetes.default.svc + clusterResourceWhitelist: + - group: "*" + kind: "*" diff --git a/demos/argo-cd/projects/minio.yaml b/demos/argo-cd/projects/minio.yaml new file mode 100644 index 00000000..38fee1df --- /dev/null +++ b/demos/argo-cd/projects/minio.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: minio +spec: + description: Project to manage Minio + sourceRepos: + - "*" + destinations: + - namespace: minio + server: https://kubernetes.default.svc + clusterResourceWhitelist: + - group: "*" + kind: "*" diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml index bc4d2591..00fc8d22 100644 --- a/demos/demos-v2.yaml +++ b/demos/demos-v2.yaml @@ -1,5 +1,33 @@ --- demos: + argocd: + description: Deploy Stackable operators and Airflow via ArgoCD and activate a simple Airflow DAG. + stackableStack: argocd + labels: + - argocd + - airflow + - job-scheduling + manifests: + ################################ + # projects + ################################ + - plainYaml: demos/argo-cd/projects/airflow.yaml + - plainYaml: demos/argo-cd/projects/minio.yaml + ################################ + # prerequisites + ################################ + - plainYaml: demos/argo-cd/applications/airflow-postgres.yaml + - plainYaml: demos/argo-cd/applications/minio.yaml + ################################ + # products + ################################ + - plainYaml: demos/argo-cd/applications/airflow.yaml + supportedNamespaces: + - argo-cd + resourceRequests: + cpu: 2401m + memory: 9010Mi + pvc: 24Gi airflow-scheduled-job: description: Activate a simple Airflow DAG to run continuously at a set interval stackableStack: airflow diff --git a/stacks/_templates/argo-cd.yaml b/stacks/_templates/argo-cd.yaml new file mode 100644 index 00000000..1c72ab9c --- /dev/null +++ b/stacks/_templates/argo-cd.yaml @@ -0,0 +1,18 @@ +--- +releaseName: argocd +name: argo-cd +repo: + name: argo-cd + url: https://argoproj.github.io/argo-helm +version: v7.8.23 +options: + configs: + secret: + argocdServerAdminPassword: "{{ argocdAdminPassword }}" + # We have to set the time otherwise error message: + # invalid session: Account password has changed since token issued + argocdServerAdminPasswordMtime: "2025-01-01T00:00:00Z" + dex: + enabled: false + notifications: + enabled: false diff --git a/stacks/argo-cd/applications/sealed-secrets.yaml b/stacks/argo-cd/applications/sealed-secrets.yaml new file mode 100644 index 00000000..450e7225 --- /dev/null +++ b/stacks/argo-cd/applications/sealed-secrets.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: sealed-secrets +spec: + project: default + sources: + - repoURL: "registry-1.docker.io/bitnamicharts" + path: sealed-secrets + targetRevision: 2.5.9 # 0.29.0 + chart: sealed-secrets + helm: + releaseName: sealed-secrets-controller + valuesObject: + secretName: sealed-secrets-key + - repoURL: https://github.com/stackabletech/demos.git + # TODO: adapt to release-25.3 + targetRevision: "spike/argocd-demo" + path: stacks/argo-cd/secrets/ + destination: + server: https://kubernetes.default.svc + namespace: kube-system + syncPolicy: + syncOptions: + - CreateNamespace=true + automated: + selfHeal: true + prune: true diff --git a/stacks/argo-cd/applicationsets/stackable-operators.yaml b/stacks/argo-cd/applicationsets/stackable-operators.yaml new file mode 100644 index 00000000..4ef1670a --- /dev/null +++ b/stacks/argo-cd/applicationsets/stackable-operators.yaml @@ -0,0 +1,61 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: stackable-operators +spec: + generators: + - list: + elements: + # Left out as currently installed via stackablectl + # operator: commons + # operator: listener + # operator: secret + - operator: airflow + - operator: druid + - operator: hbase + - operator: hdfs + - operator: hive + - operator: kafka + - operator: nifi + - operator: opa + - operator: spark-k8s + - operator: superset + - operator: trino + - operator: zookeeper + template: + metadata: + # {% raw %} + name: "{{ operator }}-operator" + # {% endraw %} + spec: + project: stackable-operators + ignoreDifferences: + # mitigating: https://github.com/stackabletech/hdfs-operator/issues/626 + - group: "apiextensions.k8s.io" + kind: "CustomResourceDefinition" + jqPathExpressions: + - .spec.names.categories | select(. == []) + - .spec.names.shortNames | select(. == []) + - .spec.versions[].additionalPrinterColumns | select(. == []) + source: + repoURL: https://repo.stackable.tech/repository/helm-stable/ + targetRevision: "25.3.0" + # {% raw %} + chart: "{{ operator }}-operator" + # {% endraw %} + helm: + # {% raw %} + releaseName: "{{ operator }}-operator" + # {% endraw %} + destination: + server: https://kubernetes.default.svc + namespace: stackable-operators + syncPolicy: + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + automated: + selfHeal: true + prune: true diff --git a/stacks/argo-cd/projects/stackable-operators.yaml b/stacks/argo-cd/projects/stackable-operators.yaml new file mode 100644 index 00000000..4acfc6cd --- /dev/null +++ b/stacks/argo-cd/projects/stackable-operators.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: stackable-operators +spec: + description: Project to manage Stackable Operators via Helm + sourceRepos: + - "*" + destinations: + - namespace: stackable-operators + server: https://kubernetes.default.svc + # required to deploy clusterwide resources like clusteroles etc. + clusterResourceWhitelist: + - group: "*" + kind: "*" diff --git a/stacks/argo-cd/secrets/sealed-secrets-key.yaml b/stacks/argo-cd/secrets/sealed-secrets-key.yaml new file mode 100644 index 00000000..7196e082 --- /dev/null +++ b/stacks/argo-cd/secrets/sealed-secrets-key.yaml @@ -0,0 +1,93 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: sealed-secrets-key +stringData: + # Generated on 2025/04/12 + # openssl req -x509 -nodes -newkey rsa:4096 -keyout tls.key -out tls.crt -subj "/CN=sealed-secrets" -days 3650 + # required for sealed secrets decryption + tls.crt: | + -----BEGIN CERTIFICATE----- + MIIFEzCCAvugAwIBAgIUFWEYhsQ0cTg6XATOSu70TgA/BGQwDQYJKoZIhvcNAQEL + BQAwGTEXMBUGA1UEAwwOc2VhbGVkLXNlY3JldHMwHhcNMjUwNDEyMDIxODEyWhcN + MzUwNDEwMDIxODEyWjAZMRcwFQYDVQQDDA5zZWFsZWQtc2VjcmV0czCCAiIwDQYJ + KoZIhvcNAQEBBQADggIPADCCAgoCggIBAMbJgXSu4ELhULYyraB4l/fnGHf8NXNM + z/PG452XrKBGdElOScxYhkentBtGUlfsYeoanOwZikKLr3ngoMzvYIuoLCqkQbA2 + u5jlP8VUtTsMc0amYPAJeEvW/KLbH/6/BFJC9Qrt/kglvVRyDWOb2VN1Dx8RbY0Q + uWqdF460OGacStyHnRgpp43a7vb4YRKxNkb7Na1s+KYUIa6hJPgIIfLGxksiQKAr + zuRKFATImFDVzXiXLHP8c7qcQa4+bg9C194CUoWt+hC2SbrGGFEJ9M6c4TzU7/5y + aE9f4TTZfadH77XpukraI26fDnLKuspK5r3VFmEWE7d8qft6J8i9Dwl9OI7DJfUx + /dHOFAb/6oiN8TmIRtR6fnreuBv5qWq3B8wnfxmVNf22C0rS8KQaj019IQdZk1Sm + hOXfTtoZ4vZk9pw+gFdkoNF1rMqzmQnoR+IcKmWRydLx0pftQ3IpGYBqncfNnaK0 + tCag+dZbmPVtL8M5ovWX1WUoAfKSczBaxMnYO3I57R5jEEOR4BfWIdPEVVMgni2u + tO560gYMHpQHGzkaH8Doa0ZyV64rctEQhCIU4bV6JzwvDnYE8O96ej3n75NIYwKF + NmSNslExd5DP2fLlIK9mnh5q8FwFqGjqefTVUJ+0pCCMRCvID8FjcqCPEF4Xvpf7 + 6LRNSF5gerelAgMBAAGjUzBRMB0GA1UdDgQWBBTuOIW0EZNXykIIvU0L8xhOfU5z + mzAfBgNVHSMEGDAWgBTuOIW0EZNXykIIvU0L8xhOfU5zmzAPBgNVHRMBAf8EBTAD + AQH/MA0GCSqGSIb3DQEBCwUAA4ICAQBQlrQ0pfAjUSu44ZWwXpkSxNERwKI1eMUO + XroKPRSAmdB/tB7JgH3wdT/Yz7pdzvt0LKu1SlcIQXRJ5VsG9k7clhJXQ1Ogse7w + g7TvQWp49uw8OlVY/mJXlia7dgKiyREGSGjpSsUC+pHcFRmZ4N2los7agY62hZS0 + 68esldNIHLPR9PfM84jGfkk1CkFefayROL7kITVemBKHBPw5Jo8JSLnahSmfiJ1e + K7SY7u/j5PY3MobnjB0MtxaDovmJjjvZ9eQZsTSRnDsHOWZEcHZ1Eoga6NEjaGG2 + kYX8RFp8TM1xhq77/+uTWxhXyekrlxmhBY/ft/C34cm4jM0qw8db8fMXsYtPo/G1 + Cq3ry8+XpqM28TtqHy8hj9nK/WQNvJ9z892FVk7nbBcFzctomdXO9vh+p9C5PW6u + dkofuMW3EC0HuinXZoUiFMwVSGF6lm7lxgkNnn9JJa7v1OCJVqQxFGoJk2gDh+Ub + zHCuy/s4nu5MsBX3xJgp2h7/DPAr1DO04tmAUZ9OwpEyeYzWogM2cIm6yeMRNl5+ + xqhWZGh7/5s5iik1g0wB5o08IwKlTMI/b1cNbJBymgjwqGlJRRAKP2IwvFGae3sX + no3+9FB1FJjEX7sKIHFqN03w1GyrPW4qqc/9is6UaLmxTlu98QeMikel+Wjpd8bP + DAIkrU/MPw== + -----END CERTIFICATE----- + tls.key: | + -----BEGIN PRIVATE KEY----- + MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDGyYF0ruBC4VC2 + Mq2geJf35xh3/DVzTM/zxuOdl6ygRnRJTknMWIZHp7QbRlJX7GHqGpzsGYpCi695 + 4KDM72CLqCwqpEGwNruY5T/FVLU7DHNGpmDwCXhL1vyi2x/+vwRSQvUK7f5IJb1U + cg1jm9lTdQ8fEW2NELlqnReOtDhmnErch50YKaeN2u72+GESsTZG+zWtbPimFCGu + oST4CCHyxsZLIkCgK87kShQEyJhQ1c14lyxz/HO6nEGuPm4PQtfeAlKFrfoQtkm6 + xhhRCfTOnOE81O/+cmhPX+E02X2nR++16bpK2iNunw5yyrrKSua91RZhFhO3fKn7 + eifIvQ8JfTiOwyX1Mf3RzhQG/+qIjfE5iEbUen563rgb+alqtwfMJ38ZlTX9tgtK + 0vCkGo9NfSEHWZNUpoTl307aGeL2ZPacPoBXZKDRdazKs5kJ6EfiHCplkcnS8dKX + 7UNyKRmAap3HzZ2itLQmoPnWW5j1bS/DOaL1l9VlKAHyknMwWsTJ2DtyOe0eYxBD + keAX1iHTxFVTIJ4trrTuetIGDB6UBxs5Gh/A6GtGcleuK3LREIQiFOG1eic8Lw52 + BPDveno95++TSGMChTZkjbJRMXeQz9ny5SCvZp4eavBcBaho6nn01VCftKQgjEQr + yA/BY3KgjxBeF76X++i0TUheYHq3pQIDAQABAoICAAi5ieTG6Ha5qxWgF6DyLLYC + TtTfUoeUZhEnQtC6ejwQ847PG0FB27zpNlK3Iqs+ElXDJBg0Y8OEHCgfgjc/Gg/v + H5HMjUnbER9GupE3j0+/k2SqWJlYdQ7xE+a1nYvMXaJwUrnPfx2yswKx5yFe0v+i + ejmCcI+PMxrOjnI35JIpBxvIQ6PD75keiZFN4/wnyn/Ri/DSwP67EKJjQLs2fNT0 + wU8aulboAqfgPMzb8rF453svjrwq0aKDCVqeM1hDS7RbCj0DSTUyvFLbWcW0HT+f + QOnZrQyQKsCxxDGSCn2NKaR56nBZbtIutCSOKNDIT6R30IhjfsCK0xluP7DuuVTn + u8jHZ/Lw3WT6YdkCC/XamQ78sz8YbsccvcIGL1IOoh41DsJ71A4De+L6awqj6d2v + n/YUx+qNrAQPC/qXU5FEb+KUOaHuCL31UjdRCjQSyFmu8OH5Z0IuYClO9c0gW3ej + QfLcMpASnQft/jbwhmK16SceEpssu7XSFbfSNDCeK+Au7jXXvl3joEr8sqfWTryR + LYRVEZH1nzpycl2Q6B7Lr8GAj3BEIpmYUiz+5Iz7n/Ocl7m0pLO0rsriL3p5nMcs + ZN3/J5vrelx+t8NJZ1idg7wUjHpRnFpvTIfdfZactZH1M22jVH2kaMsN78shIr7Z + NKCEPW3/jZPDQ1Qw58y5AoIBAQDlOsoUWwHbS2ZHCYYZ6Ef8CXpJHYpGnqfu6dEK + VOavL5HtxrrAhot/DYodATn5LdsU9iI6j+TBRgafgrcyhVSzXdpNDYCeCCroQAyl + P+p4MeC5YTKnqEOHeCEJ2hXbxL67umjykP5tAGSgNianUKO3MUm6qyJY3mpuotFj + t4zxstVxjk4xwROzSReN6VvNS8NbGZ5AkIiVaXiHk48jNOY7KwByRHh+RLk2juiK + UTLeIsVL193kF26WTFKzz+4OImk5OWmIS4bW901qsEeTMk5DEZf+YalFCs/iXjKF + BjlPkkA4U52lF8ps2RRBl97p5AuypFFFWTL6qu+5K+nkhKUZAoIBAQDeAJXjnbNQ + BZT4I1JpuUBwTQVkedDMch7aCsnxQ86I7Enu39w+5UKvgW5b4XOGGvzHpqrwNvXa + tTLjcS5o0ucL2dwAuQ2k31TxC1mHyzPsf9gFBr/qm3ej/qFM+F4qFDH5JpHRMxSR + LU4jX1Z9jk3IjDvQYgCAcQ9Q63BANoaMwIw8oitiysaM9oRe5NXGBURuegre8xGQ + mM1867z67vigG9+oWAvOiCYibq8AqS7QH4azP39H0w840SDkxPNpmyfjo/cIkKoQ + X4h8TbWHSDAWZZYzqLB9pSHfLZXZpvIaJLyqc8UnkvRbrUQuwB3nORlIMB5lc1L0 + TMnDHfb1hUxtAoIBAQC+/TiZtSna2PuifGDq/QndCNiGSsKHDHHHMbEl9o8tr2ja + wBWhi3h2p4VHnVn8LyS2Vk02leV20y15gaO5yDVGLRBR2soq4Ys+bv8aQCLPel9O + lZlq71s0Z/a/CqHjtg1kxG/kisIxd35frj6OcTW20K4phaTcfpQXEjl759bbGrGU + J4knE0OiwWeKqb//UPy3zqR4DvR96J6v91Xvc4fQDIgWrM3oBsJ6np9Jf1XulvHj + uzv5azyMpZLSlpA13p3lO9yd3q++M9ag6bb5AZULdIE7RYuPJu8GII3fH8xoa+Zu + B9z1TNkvz7582Fi2b3b/pMDCEwvSptGNavU5nPshAoIBABT5mS/YRojY/iozrCWx + CxWRMWE84S1v2tpfCpVxAqKNMK8RF44ZuyM552Nm+bNB+YAjjyMDDleLpzM2Dlx5 + ELtd6htnYJKjRzM/D0Z1PIKr0Y0oBhn0dmlYNYVAMiVnoxEXQ8+wM9FDpo5RA6r5 + NwArv7sAh2ubQwcyN3jZrfy+dIGGEPNUlU3cPMV6eXgsJzsI7N4DuTUgOsoCu+g5 + i+QUvhz5tKPe2TuHmaFGWtXvJJrq9OukVVvY5nK26QihUDOB2mVExcymOg8EiJIv + j9uN3ZMj7QjyATFE7VOTImy1691u8sG6cLv4suh33ltqWakoYrbTHpNWbfmxaxPn + dEECggEALNKGY0JSHzuwA8wL7XoiOawHJdfBx5BenpM8NpsKAFVACsVZcY4OzJHB + Dwdr4bhBcRaHrHdM1RrqIvnWnF2gaiNNifNZ9MwHberWhjqQMqdRiv7RD6wp/jPe + jeIFj/endqSWp3LhyayK8pHZDAEbx57sex0doiosyhCvqYWlcDe2XwvTxGkdpM8J + V9nqyQh7zNYvVZUby43Gl+Bdbxirolq5Jcx+Z2wxiXt2pL3JArGtnbwHFNP0oZio + 2RmFjD0ZqLQzYmWObQeYKMHcz/8kMNtNhgUax0ZXTGOGEt4bO6Iu2S/V/thVg/Wn + hnSzVDPZAGg6LcqDPW9csv6W+BCo9w== + -----END PRIVATE KEY----- diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index d013c95f..6355b5bd 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -1,5 +1,47 @@ --- stacks: + argocd: + description: Deploying Stackable Demos with ArgoCD + stackableRelease: dev + # TODO: We actually want to deploy all operators via ArgoCD, but we currently *have to* install + # operators with stackablectl. Therefore we install the internal operators via stackablectl. + # stackableOperators: [] + stackableOperators: + - listener + - commons + - secret + labels: + - argocd + - sealed secrets + - airflow + manifests: + - helmChart: stacks/_templates/argo-cd.yaml + ################################ + # projects + ################################ + - plainYaml: stacks/argo-cd/projects/stackable-operators.yaml + ################################ + # operators + ################################ + - plainYaml: stacks/argo-cd/applicationsets/stackable-operators.yaml + ################################ + # prerequisites + ################################ + - plainYaml: stacks/argo-cd/applications/sealed-secrets.yaml + supportedNamespaces: + - argo-cd + resourceRequests: + memory: 10000Mi + cpu: 8000m + pvc: 20Gi + parameters: + - name: stackableReleaseVersion + description: Stackable release to be installed via Argo + default: 25.3.0 + - name: argocdAdminPassword + description: Password of the ArgoCD admin user + # generated via: `htpasswd -nbBC 10 "" adminadmin | tr -d ':\n'` + default: $2y$10$HhJC3pGHTlk8RyBoS39N/.wC72mdWxV2X8QS1wROUwCFxl.2tGfky monitoring: description: Stack containing Prometheus and Grafana stackableRelease: dev