From b6f5ae58abe32355ab36be2cedad04b86eba220d Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 11 Apr 2025 18:40:55 +0200 Subject: [PATCH 01/37] wip --- stacks/_templates/argo-cd.yaml | 18 ++++++++ .../applications/commons-operator.yaml | 42 ++++++++++++++++++ .../argo-cd/applications/secret-operator.yaml | 43 +++++++++++++++++++ .../argo-cd/projects/stackable-operators.yaml | 19 ++++++++ stacks/stacks-v2.yaml | 30 +++++++++++++ 5 files changed, 152 insertions(+) create mode 100644 stacks/_templates/argo-cd.yaml create mode 100644 stacks/argo-cd/applications/commons-operator.yaml create mode 100644 stacks/argo-cd/applications/secret-operator.yaml create mode 100644 stacks/argo-cd/projects/stackable-operators.yaml diff --git a/stacks/_templates/argo-cd.yaml b/stacks/_templates/argo-cd.yaml new file mode 100644 index 00000000..1c72ab9c --- /dev/null +++ b/stacks/_templates/argo-cd.yaml @@ -0,0 +1,18 @@ +--- +releaseName: argocd +name: argo-cd +repo: + name: argo-cd + url: https://argoproj.github.io/argo-helm +version: v7.8.23 +options: + configs: + secret: + argocdServerAdminPassword: "{{ argocdAdminPassword }}" + # We have to set the time otherwise error message: + # invalid session: Account password has changed since token issued + argocdServerAdminPasswordMtime: "2025-01-01T00:00:00Z" + dex: + enabled: false + notifications: + enabled: false diff --git a/stacks/argo-cd/applications/commons-operator.yaml b/stacks/argo-cd/applications/commons-operator.yaml new file mode 100644 index 00000000..97fafa90 --- /dev/null +++ b/stacks/argo-cd/applications/commons-operator.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: commons-operator +spec: + # this interferes with stackablectl's templating and + # cannot really be used here properly but must be provided. + generators: + - list: + elements: + - cluster: development + template: + metadata: + name: commons-operator + spec: + project: stackable-operators + ignoreDifferences: + # mitigating: https://github.com/stackabletech/hdfs-operator/issues/626 + - group: "apiextensions.k8s.io" + kind: "CustomResourceDefinition" + jqPathExpressions: + - .spec.names.categories | select(. == []) + - .spec.names.shortNames | select(. == []) + - .spec.versions[].additionalPrinterColumns | select(. == []) + source: + repoURL: https://repo.stackable.tech/repository/helm-stable/ + targetRevision: "{{ stackableVersion }}" + chart: commons-operator + helm: + releaseName: commons-operator + destination: + server: https://kubernetes.default.svc + namespace: "{{ stackableOperatorNamespace }}" + syncPolicy: + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + automated: + selfHeal: true + prune: true diff --git a/stacks/argo-cd/applications/secret-operator.yaml b/stacks/argo-cd/applications/secret-operator.yaml new file mode 100644 index 00000000..e8e46e42 --- /dev/null +++ b/stacks/argo-cd/applications/secret-operator.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: secret-operator +spec: + generators: + - list: + elements: + - cluster: development + url: https://kubernetes.default.svc + helmChartRevision: 25.3.0 + targetRevision: HEAD + template: + metadata: + name: secret-operator + spec: + project: stackable-operators + ignoreDifferences: + # mitigating: https://github.com/stackabletech/hdfs-operator/issues/626 + - group: "apiextensions.k8s.io" + kind: "CustomResourceDefinition" + jqPathExpressions: + - .spec.names.categories | select(. == []) + - .spec.names.shortNames | select(. == []) + - .spec.versions[].additionalPrinterColumns | select(. == []) + source: + repoURL: https://repo.stackable.tech/repository/helm-stable/ + targetRevision: 25.3.0 + chart: secret-operator + helm: + releaseName: secret-operator + destination: + server: "{{url}}" + namespace: stackable-operators + syncPolicy: + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + automated: + selfHeal: true + prune: true diff --git a/stacks/argo-cd/projects/stackable-operators.yaml b/stacks/argo-cd/projects/stackable-operators.yaml new file mode 100644 index 00000000..3e3e82a0 --- /dev/null +++ b/stacks/argo-cd/projects/stackable-operators.yaml @@ -0,0 +1,19 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: stackable-operators +spec: + description: Stackable operators ArgoCD Project + sourceRepos: + - "*" + destinations: + - namespace: argo-cd + server: https://kubernetes.default.svc + - namespace: stackable-operators + server: https://kubernetes.default.svc + - namespace: stackable-products + server: https://kubernetes.default.svc + # required to deploy clusterwide resources like clusteroles etc. + clusterResourceWhitelist: + - group: "*" + kind: "*" diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index d013c95f..d48df9dc 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -1,5 +1,35 @@ --- stacks: + argocd: + description: TODO + stackableRelease: dev + # we dont not want to install operators... + stackableOperators: + - listener + labels: + - argocd + manifests: + #- helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/argocd.yaml + - helmChart: stacks/_templates/argo-cd.yaml + - plainYaml: stacks/argo-cd/projects/stackable-operators.yaml + - plainYaml: stacks/argo-cd/applications/commons-operator.yaml + supportedNamespaces: + - argo-cd + resourceRequests: + memory: 2000Mi + cpu: 2000m + pvc: 20Gi + parameters: + - name: stackableVersion + description: Stackable release to be installed via Argo + default: 25.3.0 + - name: stackableOperatorNamespace + description: Stackable namespace for the operators + default: stackable-operators + - name: argocdAdminPassword + description: Password of the ArgoCD admin user + # generated via: `htpasswd -nbBC 10 "" adminadmin | tr -d ':\n'` + default: $2y$10$HhJC3pGHTlk8RyBoS39N/.wC72mdWxV2X8QS1wROUwCFxl.2tGfky monitoring: description: Stack containing Prometheus and Grafana stackableRelease: dev From 3e983a028465c5a3bb8ec8b4194dcc4c7d0fd0ca Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 11 Apr 2025 20:02:36 +0200 Subject: [PATCH 02/37] argocd / airflow stack working --- .../applications/airflow-postgres.yaml | 31 ++++++++++++++ stacks/argo-cd/applications/airflow.yaml | 20 +++++++++ .../applicationsets/airflow-operator.yaml | 42 +++++++++++++++++++ .../commons-operator.yaml | 2 +- .../applicationsets/listener-operator.yaml | 42 +++++++++++++++++++ .../secret-operator.yaml | 11 +++-- stacks/argo-cd/projects/airflow.yaml | 14 +++++++ .../argo-cd/projects/stackable-operators.yaml | 6 +-- stacks/stacks-v2.yaml | 23 +++++++++- 9 files changed, 177 insertions(+), 14 deletions(-) create mode 100644 stacks/argo-cd/applications/airflow-postgres.yaml create mode 100644 stacks/argo-cd/applications/airflow.yaml create mode 100644 stacks/argo-cd/applicationsets/airflow-operator.yaml rename stacks/argo-cd/{applications => applicationsets}/commons-operator.yaml (95%) create mode 100644 stacks/argo-cd/applicationsets/listener-operator.yaml rename stacks/argo-cd/{applications => applicationsets}/secret-operator.yaml (79%) create mode 100644 stacks/argo-cd/projects/airflow.yaml diff --git a/stacks/argo-cd/applications/airflow-postgres.yaml b/stacks/argo-cd/applications/airflow-postgres.yaml new file mode 100644 index 00000000..d59f9961 --- /dev/null +++ b/stacks/argo-cd/applications/airflow-postgres.yaml @@ -0,0 +1,31 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: airflow-postgres +spec: + project: airflow + destination: + server: https://kubernetes.default.svc + namespace: stackable-airflow + source: + repoURL: "registry-1.docker.io/bitnamicharts" + path: postgresql + # helm inspect chart oci://registry-1.docker.io/bitnamicharts/postgresql + targetRevision: 16.6.3 # 17.4.0 + chart: postgresql + helm: + # TODO this breaks naming as long as we use the airflow stack yaml which needs this svc name + releaseName: postgresql-airflow + valuesObject: + commonLabels: + stackable.tech/vendor: Stackable + auth: + username: airflow + password: airflow + database: airflow + syncPolicy: + syncOptions: + - CreateNamespace=true + automated: + selfHeal: true + prune: true diff --git a/stacks/argo-cd/applications/airflow.yaml b/stacks/argo-cd/applications/airflow.yaml new file mode 100644 index 00000000..4789efe9 --- /dev/null +++ b/stacks/argo-cd/applications/airflow.yaml @@ -0,0 +1,20 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: airflow +spec: + project: airflow + destination: + namespace: stackable-airflow + server: https://kubernetes.default.svc + source: + repoURL: https://github.com/stackabletech/demos.git + targetRevision: "{{ demoTargetRevision }}" + # TODO: change to other directory + path: stacks/airflow/ + syncPolicy: + syncOptions: + - CreateNamespace=true + automated: + selfHeal: true + prune: true diff --git a/stacks/argo-cd/applicationsets/airflow-operator.yaml b/stacks/argo-cd/applicationsets/airflow-operator.yaml new file mode 100644 index 00000000..109ecc94 --- /dev/null +++ b/stacks/argo-cd/applicationsets/airflow-operator.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: airflow-operator +spec: + # this interferes with stackablectl's templating and + # cannot really be used here properly but must be provided. + generators: + - list: + elements: + - cluster: development + template: + metadata: + name: airflow-operator + spec: + project: stackable-operators + ignoreDifferences: + # mitigating: https://github.com/stackabletech/hdfs-operator/issues/626 + - group: "apiextensions.k8s.io" + kind: "CustomResourceDefinition" + jqPathExpressions: + - .spec.names.categories | select(. == []) + - .spec.names.shortNames | select(. == []) + - .spec.versions[].additionalPrinterColumns | select(. == []) + source: + repoURL: https://repo.stackable.tech/repository/helm-stable/ + targetRevision: "{{ stackableReleaseVersion }}" + chart: airflow-operator + helm: + releaseName: airflow-operator + destination: + server: https://kubernetes.default.svc + namespace: "{{ stackableOperatorNamespace }}" + syncPolicy: + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + automated: + selfHeal: true + prune: true diff --git a/stacks/argo-cd/applications/commons-operator.yaml b/stacks/argo-cd/applicationsets/commons-operator.yaml similarity index 95% rename from stacks/argo-cd/applications/commons-operator.yaml rename to stacks/argo-cd/applicationsets/commons-operator.yaml index 97fafa90..19a2bbc6 100644 --- a/stacks/argo-cd/applications/commons-operator.yaml +++ b/stacks/argo-cd/applicationsets/commons-operator.yaml @@ -25,7 +25,7 @@ spec: - .spec.versions[].additionalPrinterColumns | select(. == []) source: repoURL: https://repo.stackable.tech/repository/helm-stable/ - targetRevision: "{{ stackableVersion }}" + targetRevision: "{{ stackableReleaseVersion }}" chart: commons-operator helm: releaseName: commons-operator diff --git a/stacks/argo-cd/applicationsets/listener-operator.yaml b/stacks/argo-cd/applicationsets/listener-operator.yaml new file mode 100644 index 00000000..9053f49e --- /dev/null +++ b/stacks/argo-cd/applicationsets/listener-operator.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: listener-operator +spec: + # this interferes with stackablectl's templating and + # cannot really be used here properly but must be provided. + generators: + - list: + elements: + - cluster: development + template: + metadata: + name: listener-operator + spec: + project: stackable-operators + ignoreDifferences: + # mitigating: https://github.com/stackabletech/hdfs-operator/issues/626 + - group: "apiextensions.k8s.io" + kind: "CustomResourceDefinition" + jqPathExpressions: + - .spec.names.categories | select(. == []) + - .spec.names.shortNames | select(. == []) + - .spec.versions[].additionalPrinterColumns | select(. == []) + source: + repoURL: https://repo.stackable.tech/repository/helm-stable/ + targetRevision: "{{ stackableReleaseVersion }}" + chart: listener-operator + helm: + releaseName: listener-operator + destination: + server: https://kubernetes.default.svc + namespace: "{{ stackableOperatorNamespace }}" + syncPolicy: + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + automated: + selfHeal: true + prune: true diff --git a/stacks/argo-cd/applications/secret-operator.yaml b/stacks/argo-cd/applicationsets/secret-operator.yaml similarity index 79% rename from stacks/argo-cd/applications/secret-operator.yaml rename to stacks/argo-cd/applicationsets/secret-operator.yaml index e8e46e42..7312c0bd 100644 --- a/stacks/argo-cd/applications/secret-operator.yaml +++ b/stacks/argo-cd/applicationsets/secret-operator.yaml @@ -4,13 +4,12 @@ kind: ApplicationSet metadata: name: secret-operator spec: + # this interferes with stackablectl's templating and + # cannot really be used here properly but must be provided. generators: - list: elements: - cluster: development - url: https://kubernetes.default.svc - helmChartRevision: 25.3.0 - targetRevision: HEAD template: metadata: name: secret-operator @@ -26,13 +25,13 @@ spec: - .spec.versions[].additionalPrinterColumns | select(. == []) source: repoURL: https://repo.stackable.tech/repository/helm-stable/ - targetRevision: 25.3.0 + targetRevision: "{{ stackableReleaseVersion }}" chart: secret-operator helm: releaseName: secret-operator destination: - server: "{{url}}" - namespace: stackable-operators + server: https://kubernetes.default.svc + namespace: "{{ stackableOperatorNamespace }}" syncPolicy: syncOptions: - CreateNamespace=true diff --git a/stacks/argo-cd/projects/airflow.yaml b/stacks/argo-cd/projects/airflow.yaml new file mode 100644 index 00000000..3661434f --- /dev/null +++ b/stacks/argo-cd/projects/airflow.yaml @@ -0,0 +1,14 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: airflow +spec: + description: Project to manage Stackable Airflow via Git + sourceRepos: + - "*" + destinations: + - namespace: stackable-airflow + server: https://kubernetes.default.svc + clusterResourceWhitelist: + - group: "*" + kind: "*" diff --git a/stacks/argo-cd/projects/stackable-operators.yaml b/stacks/argo-cd/projects/stackable-operators.yaml index 3e3e82a0..3ee37fc2 100644 --- a/stacks/argo-cd/projects/stackable-operators.yaml +++ b/stacks/argo-cd/projects/stackable-operators.yaml @@ -3,16 +3,12 @@ kind: AppProject metadata: name: stackable-operators spec: - description: Stackable operators ArgoCD Project + description: Project to manage Stackable Operators via Helm sourceRepos: - "*" destinations: - - namespace: argo-cd - server: https://kubernetes.default.svc - namespace: stackable-operators server: https://kubernetes.default.svc - - namespace: stackable-products - server: https://kubernetes.default.svc # required to deploy clusterwide resources like clusteroles etc. clusterResourceWhitelist: - group: "*" diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index d48df9dc..f9bf053f 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -11,8 +11,18 @@ stacks: manifests: #- helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/argocd.yaml - helmChart: stacks/_templates/argo-cd.yaml + # projects - plainYaml: stacks/argo-cd/projects/stackable-operators.yaml - - plainYaml: stacks/argo-cd/applications/commons-operator.yaml + - plainYaml: stacks/argo-cd/projects/airflow.yaml + # operators + - plainYaml: stacks/argo-cd/applicationsets/commons-operator.yaml + # listener is currently deployed via stackablectl since it complains if no operators are deployed... + #- plainYaml: stacks/argo-cd/applicationsets/listener-operator.yaml + - plainYaml: stacks/argo-cd/applicationsets/secret-operator.yaml + - plainYaml: stacks/argo-cd/applicationsets/airflow-operator.yaml + # products + - plainYaml: stacks/argo-cd/applications/airflow-postgres.yaml + - plainYaml: stacks/argo-cd/applications/airflow.yaml supportedNamespaces: - argo-cd resourceRequests: @@ -20,9 +30,12 @@ stacks: cpu: 2000m pvc: 20Gi parameters: - - name: stackableVersion + - name: stackableReleaseVersion description: Stackable release to be installed via Argo default: 25.3.0 + - name: demoTargetRevision + description: The target revision, HEAD or e.g. release-25.3 + default: release-25.3 - name: stackableOperatorNamespace description: Stackable namespace for the operators default: stackable-operators @@ -30,6 +43,12 @@ stacks: description: Password of the ArgoCD admin user # generated via: `htpasswd -nbBC 10 "" adminadmin | tr -d ':\n'` default: $2y$10$HhJC3pGHTlk8RyBoS39N/.wC72mdWxV2X8QS1wROUwCFxl.2tGfky + - name: airflowAdminPassword + description: Password of the Airflow admin user + default: adminadmin + - name: airflowSecretKey + description: Airflow's secret key used to generate e.g. user session tokens + default: airflowSecretKey monitoring: description: Stack containing Prometheus and Grafana stackableRelease: dev From d64a4170f5e01668b74d4e19c1945dd12452457e Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 11 Apr 2025 20:23:55 +0200 Subject: [PATCH 03/37] added spark op --- .../applicationsets/spark-k8s-operator.yaml | 42 +++++++++++++++++++ stacks/stacks-v2.yaml | 18 ++++---- 2 files changed, 53 insertions(+), 7 deletions(-) create mode 100644 stacks/argo-cd/applicationsets/spark-k8s-operator.yaml diff --git a/stacks/argo-cd/applicationsets/spark-k8s-operator.yaml b/stacks/argo-cd/applicationsets/spark-k8s-operator.yaml new file mode 100644 index 00000000..b5686b60 --- /dev/null +++ b/stacks/argo-cd/applicationsets/spark-k8s-operator.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: spark-k8s-operator +spec: + # this interferes with stackablectl's templating and + # cannot really be used here properly but must be provided. + generators: + - list: + elements: + - cluster: development + template: + metadata: + name: spark-k8s-operator + spec: + project: stackable-operators + ignoreDifferences: + # mitigating: https://github.com/stackabletech/hdfs-operator/issues/626 + - group: "apiextensions.k8s.io" + kind: "CustomResourceDefinition" + jqPathExpressions: + - .spec.names.categories | select(. == []) + - .spec.names.shortNames | select(. == []) + - .spec.versions[].additionalPrinterColumns | select(. == []) + source: + repoURL: https://repo.stackable.tech/repository/helm-stable/ + targetRevision: "{{ stackableReleaseVersion }}" + chart: spark-k8s-operator + helm: + releaseName: spark-k8s-operator + destination: + server: https://kubernetes.default.svc + namespace: "{{ stackableOperatorNamespace }}" + syncPolicy: + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + automated: + selfHeal: true + prune: true diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index f9bf053f..89befdf7 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -3,31 +3,35 @@ stacks: argocd: description: TODO stackableRelease: dev - # we dont not want to install operators... + # TODO: We actually want to deploy all operators via ArgoCD, but we currently *have to* install + # operators with stackablectl. Therefore we install the internal operators via stackablectl. + # stackableOperators: [] stackableOperators: - listener + - commons + - secret labels: - argocd manifests: - #- helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/argocd.yaml - helmChart: stacks/_templates/argo-cd.yaml # projects - plainYaml: stacks/argo-cd/projects/stackable-operators.yaml - plainYaml: stacks/argo-cd/projects/airflow.yaml # operators - - plainYaml: stacks/argo-cd/applicationsets/commons-operator.yaml - # listener is currently deployed via stackablectl since it complains if no operators are deployed... + # currently deployed via stackablectl since it complains if no operators are deployed... + #- plainYaml: stacks/argo-cd/applicationsets/commons-operator.yaml #- plainYaml: stacks/argo-cd/applicationsets/listener-operator.yaml - - plainYaml: stacks/argo-cd/applicationsets/secret-operator.yaml + #- plainYaml: stacks/argo-cd/applicationsets/secret-operator.yaml - plainYaml: stacks/argo-cd/applicationsets/airflow-operator.yaml + - plainYaml: stacks/argo-cd/applicationsets/spark-k8s-operator.yaml # products - plainYaml: stacks/argo-cd/applications/airflow-postgres.yaml - plainYaml: stacks/argo-cd/applications/airflow.yaml supportedNamespaces: - argo-cd resourceRequests: - memory: 2000Mi - cpu: 2000m + memory: 10000Mi + cpu: 8000m pvc: 20Gi parameters: - name: stackableReleaseVersion From 64daa28a701a667ac240fddfdfe37c40c3f61479 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 11 Apr 2025 22:57:10 +0200 Subject: [PATCH 04/37] wip --- stacks/argo-cd/applications/airflow.yaml | 3 +- .../argo-cd/applications/sealed-secrets.yaml | 28 ++ stacks/argo-cd/manifests/airflow/airflow.yaml | 308 ++++++++++++++++++ .../airflow/sealed-airflow-credentials.yaml | 24 ++ .../argo-cd/secrets/sealed-secrets-key.yaml | 90 +++++ stacks/stacks-v2.yaml | 22 +- 6 files changed, 465 insertions(+), 10 deletions(-) create mode 100644 stacks/argo-cd/applications/sealed-secrets.yaml create mode 100644 stacks/argo-cd/manifests/airflow/airflow.yaml create mode 100644 stacks/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml create mode 100644 stacks/argo-cd/secrets/sealed-secrets-key.yaml diff --git a/stacks/argo-cd/applications/airflow.yaml b/stacks/argo-cd/applications/airflow.yaml index 4789efe9..9db9f6a7 100644 --- a/stacks/argo-cd/applications/airflow.yaml +++ b/stacks/argo-cd/applications/airflow.yaml @@ -10,8 +10,7 @@ spec: source: repoURL: https://github.com/stackabletech/demos.git targetRevision: "{{ demoTargetRevision }}" - # TODO: change to other directory - path: stacks/airflow/ + path: stacks/argo-cd/manifests/airflow/ syncPolicy: syncOptions: - CreateNamespace=true diff --git a/stacks/argo-cd/applications/sealed-secrets.yaml b/stacks/argo-cd/applications/sealed-secrets.yaml new file mode 100644 index 00000000..7faa76b4 --- /dev/null +++ b/stacks/argo-cd/applications/sealed-secrets.yaml @@ -0,0 +1,28 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: sealed-secrets +spec: + project: default + sources: + - repoURL: "registry-1.docker.io/bitnamicharts" + path: sealed-secrets + targetRevision: 2.5.9 # 0.29.0 + chart: sealed-secrets + helm: + releaseName: sealed-secrets-controller + valuesObject: + secretName: sealed-secrets-key + - repoURL: https://github.com/stackabletech/demos.git + # TODO: adapt to release + targetRevision: HEAD + path: stacks/argo-cd/secrets/ + destination: + server: https://kubernetes.default.svc + namespace: kube-system + syncPolicy: + syncOptions: + - CreateNamespace=true + automated: + selfHeal: true + prune: true diff --git a/stacks/argo-cd/manifests/airflow/airflow.yaml b/stacks/argo-cd/manifests/airflow/airflow.yaml new file mode 100644 index 00000000..703f409c --- /dev/null +++ b/stacks/argo-cd/manifests/airflow/airflow.yaml @@ -0,0 +1,308 @@ +--- +# {% raw %} +apiVersion: airflow.stackable.tech/v1alpha1 +kind: AirflowCluster +metadata: + name: airflow + namespace: stackable-airflow +spec: + image: + productVersion: 2.10.4 + clusterConfig: + listenerClass: external-unstable + loadExamples: false + exposeConfig: false + credentialsSecret: airflow-credentials + volumes: + - name: airflow-dags + configMap: + name: airflow-dags + volumeMounts: + - name: airflow-dags + mountPath: /dags/date_demo.py + subPath: date_demo.py + - name: airflow-dags + mountPath: /dags/pyspark_pi.py + subPath: pyspark_pi.py + - name: airflow-dags + mountPath: /dags/pyspark_pi.yaml + subPath: pyspark_pi.yaml + webservers: + config: + resources: + cpu: + min: 400m + max: "1" + memory: + limit: 2Gi + gracefulShutdownTimeout: 30s + roleGroups: + default: + envOverrides: + AIRFLOW__CORE__DAGS_FOLDER: "/dags" + AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" + replicas: 1 + kubernetesExecutors: + envOverrides: + AIRFLOW__CORE__DAGS_FOLDER: "/dags" + AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" + schedulers: + config: + gracefulShutdownTimeout: 30s + resources: + cpu: + min: 400m + max: "1" + memory: + limit: 1Gi + roleGroups: + default: + envOverrides: + AIRFLOW__CORE__DAGS_FOLDER: "/dags" + AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" + replicas: 1 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: airflow-dags + namespace: stackable-airflow +data: + date_demo.py: | + """Example DAG returning the current date""" + from datetime import datetime, timedelta + + from airflow import DAG + from airflow.operators.bash import BashOperator + + with DAG( + dag_id='date_demo', + schedule_interval='0-59 * * * *', + start_date=datetime(2021, 1, 1), + catchup=False, + dagrun_timeout=timedelta(minutes=5), + tags=['example'], + params={}, + ) as dag: + + run_this = BashOperator( + task_id='run_every_minute', + bash_command='date', + ) + pyspark_pi.py: | + """Example DAG demonstrating how to apply a Kubernetes Resource from Airflow running in-cluster""" + from datetime import datetime, timedelta + from airflow import DAG + from typing import TYPE_CHECKING, Optional, Sequence, Dict + from kubernetes import client + from airflow.exceptions import AirflowException + from airflow.sensors.base import BaseSensorOperator + from airflow.models import BaseOperator + from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook + import yaml + from airflow.utils import yaml + import os + + if TYPE_CHECKING: + from airflow.utils.context import Context + + class SparkKubernetesOperator(BaseOperator): + template_fields: Sequence[str] = ('application_file', 'namespace') + template_ext: Sequence[str] = ('.yaml', '.yml', '.json') + ui_color = '#f4a460' + + def __init__( + self, + *, + application_file: str, + namespace: Optional[str] = None, + kubernetes_conn_id: str = 'kubernetes_in_cluster', + api_group: str = 'spark.stackable.tech', + api_version: str = 'v1alpha1', + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.application_file = application_file + self.namespace = namespace + self.kubernetes_conn_id = kubernetes_conn_id + self.api_group = api_group + self.api_version = api_version + self.plural = "sparkapplications" + + def execute(self, context: 'Context'): + hook = KubernetesHook(conn_id=self.kubernetes_conn_id) + self.log.info("Creating SparkApplication...") + response = hook.create_custom_object( + group=self.api_group, + version=self.api_version, + plural=self.plural, + body=self.application_file, + namespace=self.namespace, + ) + return response + + + class SparkKubernetesSensor(BaseSensorOperator): + template_fields = ("application_name", "namespace") + # See https://github.com/stackabletech/spark-k8s-operator/pull/460/files#diff-d737837121132af6b60f50279a78464b05dcfd06c05d1d090f4198a5e962b5f6R371 + # Unknown is set immediately so it must be excluded from the failed states. + FAILURE_STATES = ("Failed") + SUCCESS_STATES = ("Succeeded") + + def __init__( + self, + *, + application_name: str, + attach_log: bool = False, + namespace: Optional[str] = None, + kubernetes_conn_id: str = 'kubernetes_in_cluster', + api_group: str = 'spark.stackable.tech', + api_version: str = 'v1alpha1', + poke_interval: float = 60, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.application_name = application_name + self.attach_log = attach_log + self.namespace = namespace + self.kubernetes_conn_id = kubernetes_conn_id + self.hook = KubernetesHook(conn_id=self.kubernetes_conn_id) + self.api_group = api_group + self.api_version = api_version + self.poke_interval = poke_interval + + def _log_driver(self, application_state: str, response: dict) -> None: + if not self.attach_log: + return + status_info = response["status"] + if "driverInfo" not in status_info: + return + driver_info = status_info["driverInfo"] + if "podName" not in driver_info: + return + driver_pod_name = driver_info["podName"] + namespace = response["metadata"]["namespace"] + log_method = self.log.error if application_state in self.FAILURE_STATES else self.log.info + try: + log = "" + for line in self.hook.get_pod_logs(driver_pod_name, namespace=namespace): + log += line.decode() + log_method(log) + except client.rest.ApiException as e: + self.log.warning( + "Could not read logs for pod %s. It may have been disposed.\n" + "Make sure timeToLiveSeconds is set on your SparkApplication spec.\n" + "underlying exception: %s", + driver_pod_name, + e, + ) + + def poke(self, context: Dict) -> bool: + self.log.info("Poking: %s", self.application_name) + response = self.hook.get_custom_object( + group=self.api_group, + version=self.api_version, + plural="sparkapplications", + name=self.application_name, + namespace=self.namespace, + ) + try: + application_state = response["status"]["phase"] + except KeyError: + self.log.debug(f"SparkApplication status could not be established: {response}") + return False + if self.attach_log and application_state in self.FAILURE_STATES + self.SUCCESS_STATES: + self._log_driver(application_state, response) + if application_state in self.FAILURE_STATES: + raise AirflowException(f"SparkApplication failed with state: {application_state}") + elif application_state in self.SUCCESS_STATES: + self.log.info("SparkApplication ended successfully") + return True + else: + self.log.info("SparkApplication is still in state: %s", application_state) + return False + + with DAG( + dag_id='sparkapp_dag', + schedule_interval=None, + start_date=datetime(2022, 1, 1), + catchup=False, + dagrun_timeout=timedelta(minutes=60), + tags=['example'], + params={"example_key": "example_value"}, + ) as dag: + + def load_body_to_dict(body): + try: + body_dict = yaml.safe_load(body) + except yaml.YAMLError as e: + raise AirflowException(f"Exception when loading resource definition: {e}\n") + return body_dict + + yaml_path = os.path.join(os.environ.get('AIRFLOW__CORE__DAGS_FOLDER'), 'pyspark_pi.yaml') + + with open(yaml_path, 'r') as file: + crd = file.read() + with open('/run/secrets/kubernetes.io/serviceaccount/namespace', 'r') as file: + ns = file.read() + + document=load_body_to_dict(crd) + application_name='pyspark-pi-'+datetime.utcnow().strftime('%Y%m%d%H%M%S') + document.update({'metadata': {'name': application_name, 'namespace': ns}}) + + t1 = SparkKubernetesOperator( + task_id='spark_pi_submit', + namespace=ns, + application_file=document, + do_xcom_push=True, + dag=dag, + ) + + t2 = SparkKubernetesSensor( + task_id='spark_pi_monitor', + namespace=ns, + application_name="{{ task_instance.xcom_pull(task_ids='spark_pi_submit')['metadata']['name'] }}", + poke_interval=5, + dag=dag, + ) + + t1 >> t2 + pyspark_pi.yaml: | + --- + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkApplication + metadata: + name: pyspark-pi + spec: + version: "1.0" + sparkImage: + productVersion: 3.5.2 + mode: cluster + mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py + job: + config: + resources: + cpu: + min: 500m + max: 500m + memory: + limit: 512Mi + driver: + config: + resources: + cpu: + min: 1000m + max: 1200m + memory: + limit: 1024Mi + executor: + config: + resources: + cpu: + min: 500m + max: 1000m + memory: + limit: 1024Mi + replicas: 3 + +# {% endraw %} diff --git a/stacks/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml b/stacks/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml new file mode 100644 index 00000000..bc631bc7 --- /dev/null +++ b/stacks/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: airflow-credentials + namespace: stackable-airflow +spec: + encryptedData: + adminUser.email: AgCmhs/OLHZ0j4CnTdZvfj76m+y7e62IuAeJ/LksxaTOWbHlvTJ0p8dBsp+WX3RfiuRK8M7kJDundrSKYzbkw8MISjoHF4/v4pSu6qWLBBuDPaMGh3i1bpAJ9gKUti4SGLhp7dr/Lx1oFWnY5T69WMARQqTbhHnpH70LPkFWKRvlIkCN26PbuYsE7stKXaVweEP204sfUChK5qzafE4Lcm5l+1yGFfILBBKAbBGxLX2I3O4n4LuP/iv0Ne14wGnxss80Qapt+ue8Ce16EZiJPtY+ZCXa05fYgtitVr393pYZks/fO9as76ABjWFc2lnE5GaGV58gK9V/5VRd1TBfkK3Y8UZPxqHNULlkwVh4Vy2EHHBMzhtec0m6w72aWuWFecyUNkrwIuivZ8e9tiH6aulpYqgLshnbDZCQi8yQVDNx/xLv4GH0/VvDUEIB7O13CHRL4M+OH+GYgLpJYqhTTkJNvXdrmjyK6kWeiErw8yG77akVplDGSG6LschN0DGhJVoZLLwbjvvM9em6tHu3gRSXAYhfG80bB9PMwmU9Lei+cdq6Fp7RceqxR3F6lnigP2e5gCaEpc6YYzBKEnzgcdlR1iA57D+5lBK9SwvWkppH5/dbPRkzt5+kqWaExRq8VmEnwooiBJ/4Jx4i4FKWFnx3cnDF5OYXovIKPyYPZW1HVvifrjHHZAOe1jORxYpMOwPYOLraDIO83comkVEBzCVUOh8v + adminUser.firstname: AgB3NcEnKhwMsyFPOfSTfxKFkhIg1TyeixtdqW8RSq5I6zrMX9we431QLuOSfxQlaizsm3o670wjO+4yDNpOGB4/vXaSSpA+835tiMDymJ/URR7vb1vvLV/xx6azdn5JmmHWd+yEja47TxQ1ip5KDai1Oxc0BKbgzu2Ogalq1U/56riSKSe8MvRjJ9KY8IwBpoY1rKAjTFJWavUVVt5Ms0fdSB7sL69NoKVzUQEpucTemy6k8RQFabK+KwirQ2KSDcQ5J+df/ODx1/HZKG8j81Wla7dBJ0nwIeie0StAZdZ+j9QlsWQF4zCRaEPxn9p1vken7Na/EqQdYwIV62o3mo66p06t+ObdjxKRHEO5BiY2lYqY3XcgaETBpaDfl2PiMbneLngGpBfOQSKnVZC60svqB/BpvBlMCiwcr+0ZJczFM1f+ptmJGHLJirf1kTYKvm6ezstGussB848Ca/ayhzzt9cestW9UFS/SlAo7d+N+YUKv3O+8QPIb5AdN2nPN9kcQxdh2T+ogtg3yIKsb6makP8LC6Z0Hl8C1DuzSh3Inj5VxcmkRuB9LRYymfNu+6GjkVlAWk58FUdQM5+5syMDgiHdVIMmnoTjRNm3mCm3l6g6rG33ggaAI4X1ypyjnmMlxbpRJi/+KgsQNNByYYQG1Foo3MZ7JFQojABBFMVSCPumMyjR7i8kvYCTr7acrHS3ZSjvR+Msz + adminUser.lastname: AgCYaEJ973OSMSL0vBrQcHbFtmqQls6Ids1xcPVz3/uVpxZw89Phn67Xx6dPSNHqZIYHWbZ/gV24xOJd0XbtlFINUknGgjwqjwGJHm4Q+0F1nmCVmhrRqB+Jc8I7qqb5BT6XpsgQOa0vazDc+NVKlgm2DmbIKz8VvHrhUNRb1rd33RYd3lm0Kj8lSl4QZm8tIX+BZi3J9gSLDIMzGQftg5BxS8Wt8cbYsMW8lFeZ6Dg7Fmx9r0GCeAa7kZWy8UME0saeKj13tg/oYj79MNFCu/Q510Q+YbLcQAJ22ifPgIaPdN4YK03DYL9qiIeo4im8rlgfKqHaNkfNzmXXh758T4s74Nau9KLdG+qE2+j175B9nrYjBYlT2EDoJIF6m1iiOCAc93tk4FIc3fw5RNLkOsAYdEWoECWLOdDloQsDHa+DGwHnF6d9ZpzlATnKLMOJ6F3+RAW10NzD38bWMlUUJoGZTuhVOz9tQCyFs2/SqMC8/LauMrHx2TFbKEq2kpc4MRTYwUQe9L+zQFI2Oc5HXGulXDdteShL6ayT8sXGRIsNI0lCpNhHsUauGJ7sSkW4bklGMEPzRh48+Lua2/QQKOcLWxgPx53b492/7SvWExx7rs+POSZouLdwUlngXC2D1H5yvmnPXEnnTLVGwBNDRBa+BBefocVOmq6yZGX5trkhudTr/Nu/rZQkDGUtSfiMDzd6Uo73LQ== + adminUser.password: AgA7BxW4/FggUvHMVvnYxdP6Pmju3uSskl29Lc1ywr6AbtCSqiysp29Ravyr0muT+PlEb5zmUYrct+HaffstMOudDCYw5wSm50hhelRPFgqppJgoUvXwfeUMkx73rrkddfN2RnCepUZxmkJ8PcXA38+KPwTZM4zZmOHeq7lgKDlsjgin/O0MrAeochZGJ1JmsJ0B3QUxFit4aURq4Pko67ytrWMJBO2KQRLHUm7Swf3umqjCBWuqfqhbRtKrWEOhoAEHDzKOq4fUCaV5rf2uxpvNCC0KJY6zmhPcPgZAO+Wuc+W9iNmM4ROoTFCvDS/stXcneDR7oRObgtXN5gn/0xno3xhdypg0lqfm2bEoEiokwu2GVwrzPr5iNI2HlUVhXeo9bNwjJ1alWjt0faUvwtZRSbKI1AihBF0gelbXhKT9Iyya336TTuz0riW3sFeDixHDTRoVC3ZaHzvCTXLTLtw3xgmhxM1mCebU+pxS8DV4AAIy9MfkK4ZHkjpcCdVbV5D4Mt2A9Ltm1O7qHlKsdWF0+vMmllLn7dZzAmtAiCiI9srOUh0xj+bIOqqN2Ky8a2K73JPh3kXxO5njW9N0/lFFQ4D98VpgI/DDqoZwX97yzLcthLb54xr6OUtduvX6rDakgrRZT4OCSvlGUIWzvtTcoDwuBubSuOnLh6AFNd7xz/Ik+eepInSOCMpd3HIdp5KBB3TPm9eLFcLW + adminUser.username: AgAQWpaTaFTvYsZx0p9Am3iTwoUEZ/pWiETvhVSF0NANgKe17V5b1lQj+DamMVlFUGfGgV8Qcxdyw4Lc71yzGgwEB0fQERoy54Z2lwVpYLJm2qLcXuolkzFSyZEMBBulddkTGhF/zhDZWfpOnayhfAbQyyp/T3KUuZira220DN9HiXDb5BiId1+Gzu6xTW6WD5/c+7yFiD6SWFReGpIuqHofh4y02p8A5RED2jtcr/p7ltCZ/tqjIpnZcTP5K7wauEjDiJmDbLTcdAmzdSEOI4Q952+oK3xjPAIrhhivC3DAtQ0lgV2ye9859tx+CDEEkLkjewascVNwmX1yeYPxnh8ayJfabkEpNM8KOfmrDE/AZB82FeKChmvAcpQAu3fUuxBMyUit6RkNW73fvDf8hIyOmNSZdr0f21FPhJWFYnsF4Qfm3zOo/fXuG1LJz2WB2Qy7RuTmRGW1AE4VQOvzvVDCEOq5BsVfPdlGrSfo5oeI6ctCqTL4S+2CXS3VRUkIX61QdN54CU7ZeT4cAzZGpkUP6elfsrLv0KnQc4KvvzfBNu71j3xgjjYN41aGDNi8dqWaCVebYbuQtDqbcxLxUQNX+VqBdCflMcnCOC4kAEI+B2n/rUrGLWjKUIMhZvZIKJQd437j1fvJo2AHO8TbqtORspg3/H8OuwZpwcDsWl1dVQ1B0EczZweo3I5NkytTVxNEEXjsQA== + connections.celeryBrokerUrl: AgBEl833Xp1UogzJpkpJ1zhjlw6KKAqcfWzSqQ8LIPQ96GaIUfo6G8wx7jozw7yEF0vnAuTTdzdEZ7x8/PVwWQyqNJKiS9o6mIEdkhv9mQ+EG/wzpB8dnbQR4Xl758f+dPRqbIA0RYetmcD9NsiF08obXIIzZA+G95kR6x24WiixwJL+txMNdno0Qu92fRqasv6jV22CUju/kUTRFEov5IrzGOj7gLKQ7d2yrCGZd/W9SnP9sgX1ltz78C2yiEXX7aI0n066OjJUxE7DFTJCUWNrknDcJuSjGFrWTzf8GNydrPmkVp6g5JA3fGgUTm76osischHr/0/+cwzQzEyQKyb5H/fxcAd/5yqtXhD/xibPDoVHs484PQAmgy8M3PMG8fNQ9XyrXNB/ZuxcCnt+rqPkeP0jqEsdxog2Q3U1lXfH/FU2eJr5tLHb1zNISV7vNX28VRzLoefyYFtHkp1F2yVYThmuSCPj1cqkfOpt02rYsIDIYqwryxxOvUpTAH0uNbdoIWSvXTzX2ZARUVVm0bboDgfe4RxVQofxKjk1mNPHhEZ9jYd6GX1G1SCYuSVdrRN4fLSOxlyP+vPkyzOJtQ7WmugBZFTmZ/iUN38ce4HbDIajVKtMvDqD35kkN/q4NZ3d+/YjmZA9YvJ03Hd0XNznImr8R4wqOt5oZsK2+XaMxHTzPfxV2MpLT9Xgb+CAHiGKvCaheKdRNWODhFQQXAw8l6t4z8dvYNkZPAZAD6ehFUtlvpTbHVVqtEdiTA== + connections.celeryResultBackend: AgBywQulzF0AZOPR8ypHDnsQIicOWUri9rV2sLwR8kiD+J98eT6CWvblzUv8x3/I4gJWwHbwjmACv5Ke+MD1GxqVsJYl2zc/A78SkVvNT6V+kaUFqxXMEQ3EJN7q+BHUPFphxQDS2Ur5+ssON1hth56L+HAxZLmt4XC9Fjz20cO7Dcx+vpgXG7KVtdRmLvchs0PCaMnMZDANg/rgGC9lMcgi3BROhobGapbp3+iLjiIDtNJ4SwwQpQaA5KWKXqC8Dmfvtm18lyNAK5oWSrMQI6+eWe+ML56C8bMlz3jtlZsFO6upe7eOueD1WoK1+YI+NEfvw3HDChgMaQ559x3y/pIG94F7QNtltZbfTfXptStQru+H2rxmCUBYb/BaHv3oqZsOgQe95LubQNfrpH9tIIygGZcQDZGjZsnfjmu1ahYzqiKb6UEoPxZJskIMy5BAakWQf0FiVJQa7stt7/k/NosBrUfJbyPJNWA6xVwilV7K4Auz4MuBsMk1fRofgpw3N85SeipVmU3673GIs+L1IFy8UEYAtYpv14aFOvqJrE4YBpkufr+K/IvVpYfvIBL2zAF3CKm9SEzYAQnADwb3UgOCIm8w233vogjHo5oqL8wacUeZgzrZ+T1FIR9e1tXrYxk2VLmyv5d0VOWYe0AleUk+O3rVdLpi5yUJQjvKnExVXzMMLsApTJ3dy5OHwou99Po1mlg2D6xh439hDUlAcUQz4NfGYvJtYSgAb5cjMs3bEMUcAuadGIGidlIy74HQArfXosundworwKC8 + connections.secretKey: AgBpwR5AZJUjGxJwi7rS6dLH7WJKYza4wKjPxLF+RVskwtwtkGtYSpbRZ+vXxE6pn47eJQC0OpJzreiNqYZtSR1CHv6HyHZsnqCSnLSzlUBCPp9H51nfVQ57V+S967/jEVJ3V7446gqQqdJb3IS4uwbb+G8CJsjsZiEcwx/z/OK5vu+k7TiG5jiyksxXhC7EFzYKfc/CD8Lsr6LBimUEa2i1W/PhOoDcqKa7qvDMIM/iAEilvtHXjA+EU8kxplDYy2an1cadockyyMSoHvDmSOg6BU62bK8Xqx9o0X6uaSrTufeueymtBYb/wVHB0bTxm1UXb8QdGYfDePemnP0oRNwvETrupuD0HRgEutetyxLYEEezC98ZpRM377kirX9Zqb0/fZ9YVwECBL7hznl/sp8w1hE2O76orf3iAGNMftrFtOjEyzro+41zd+bz+sy1ivGluIWwovQbgondk25AoYRC0LhGDl7kQDVX3ND/XeH2NYErAuCEyjQ1MGHLJxQGGDxoCPaDXtLMuHvXlg3H71jibzT7rV+YikAExjfUVim0kuDtKqdBwAhvRqX+Si/kCRMrIUhRowWSqx0g54U9mwC6wfbSHV90RnYtvBPbFc144WbgV7irc5bfnekidrCj8MiIPKegUByqlB26ziyexEmiRN74BH2yCJ/M2yf1IWSad0uqb5d/48BcgEP8E2BOgeeaCJ6WOMeR80qvas07hN5e + connections.sqlalchemyDatabaseUri: AgAFm5FacxXw3q9rxKT+9eTpY3FPf7NSp3wdPUUzkBpQPsEa0JvafDeIZ5/+MC9WaFBWuSt0awrIh/FyHlNW4/KRhTWzFL4h2MrDAfCEdCaOgB4wrgYJF7tGz3968eNhBn0ALX7MU6dgiGawPGvX91FgPo4dwz/5Ob5+f5MhA/paH8uBXR8W96cTTeJlcJaqLV/xMzGCInurUpOwyrLWu8zjOHNdn/11P3n5gKVlwbi6w57AKAYtqux869oh9al4kIbgIEbRYRI42Q2Xa3v+RtmFCX3CelCs6uoWtzNgt+wUbQbfi48MuHrakiJf49yrqFHdc2vfj03O8jlKLGSGcSsy5Epbb1VBuTh04jVXAO03xb6Top3gqZvxY8/gWOWPs0SejeilvubNIwj40bhzRxPj+r7TnE2dmS+KnTN1fYR1fk39C6SNjSKVncMie8PH/xfq8WOVZSGhzlIwYNMs7/WpXYLVs58Wz+pVVRcbYf/CQjRNKy4ikEf+eHwMLvmsvHnslxr9S9Bwt2An7mFkkO0VvGubCYfOK/W1SoN9NBhFmJO6Es3vfTKuS1gqgcJAzd1oz0+hz6Kno8XRpZUzdUa+P8LPo4Y+mhif+y9OH0sXFH5rYGWvLbUxEgFfdUMWQRKpGRqJiGNKFBLo0kpRt/DqG3v9wcWHbYLl+/cRLCsC46FUsPW5L73T4AEtV3TLh6Q5hZiHCWTEyxW4WyeFD6Bd7FLPNbIA8gMD0oCP3ztd+odbSDecURUzhQifICH/SCJl4VH8jvr40Rd+TjrqEj5t + template: + metadata: + creationTimestamp: null + name: airflow-credentials + namespace: stackable-airflow + type: Opaque diff --git a/stacks/argo-cd/secrets/sealed-secrets-key.yaml b/stacks/argo-cd/secrets/sealed-secrets-key.yaml new file mode 100644 index 00000000..60e7e522 --- /dev/null +++ b/stacks/argo-cd/secrets/sealed-secrets-key.yaml @@ -0,0 +1,90 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: sealed-secrets-key +stringData: + tls.crt: | + -----BEGIN CERTIFICATE----- + MIIFEzCCAvugAwIBAgIUTIeBhvXRk1w2QJRP2RpyQWJCB48wDQYJKoZIhvcNAQEL + BQAwGTEXMBUGA1UEAwwOc2VhbGVkLXNlY3JldHMwHhcNMjUwNDExMTk1NzUzWhcN + MjYwNDExMTk1NzUzWjAZMRcwFQYDVQQDDA5zZWFsZWQtc2VjcmV0czCCAiIwDQYJ + KoZIhvcNAQEBBQADggIPADCCAgoCggIBAMobRqdxhAE8Nr45tCRRI6kz2WopwnKb + 8bWGM0LbM/p04xfcLLAVzK4nC5HHBmfPycz6XyGjv7ViUSaHVk1j3OLxFEX4sd+2 + JoTQRenRc/oUHdwaGskBr5frXFAO3Tiou+pJ6Tu8ewCeCvGUzB9HtiRq5pCGPu6M + wzXbmR6GKaCJTu80LZ8fwFdSWI0gDoqeWffCTfcTgJfTa+DgMfTM3zzUNwJVfLo3 + r596wGmlxKfMcUtzw2J5iLr0dXJuJnlmA0gU31kwVcf8Wo1DAdh8hhxmUAkmGss8 + AktmB8OvvNygVVzzwmX/L2MH1zPXzgM9sYenkJwLSZhMx/uyfgX5x54/QaC7lwfk + wPnp52pK0JA2VmfFBsvlnPsVOWgwWU0jUtMGBtKlu/OonELAGaffbkhjD3eDe0B3 + X0gwtju+T4WCzQmgTFyXFdp18/S2rz19TBrq2XponRRpcXBo4qe8P59Eq++NNjAg + rYymFcTfrDZqMIt3jBv3jSnLlAnwg/H1neRfgx6kxXqxApF+vMDD7VVDaLYnLbFV + lwdkWxPrHStk05eBsLu/CkGv2ykkUAa0qqg/Dm/bkOl5lZlauVFC9wuiLh2cHgZr + tBpGuYlMFMRvx7A0+IBUTjTQswBF1+mBStUyBThOw1aJYbE2FCWj+lGrMX8qr+ad + sKsS+5DjOwCVAgMBAAGjUzBRMB0GA1UdDgQWBBRFs1lkbmy+f7TN10DGuHLa75sj + BDAfBgNVHSMEGDAWgBRFs1lkbmy+f7TN10DGuHLa75sjBDAPBgNVHRMBAf8EBTAD + AQH/MA0GCSqGSIb3DQEBCwUAA4ICAQA+6nknVdXMATXxbyk0E5fDlM/D2aceDw6D + 3kydc6wBT6k9SkhxFtrCI0IBO0f067ppFPj4ApJI0mrh3dsNiAzjqGetlGCqXjKq + pwdVxWYrmn/ELTKgvxToQG4J4PMUeo7tlhviU5HJo90DroDpoZHALubl+XsjzspA + MHH94CiF24z2nxoVolVa1qqq7WDlMWhZuit3WDo+jIgrpBgcPegNrZqrZqt/xlzM + Q91iOwwd3ykfo+xLdNuG1i8Qm1UFOEmXsustHBPDCtcI/ddLda4NbZtrNeZVmDbE + Px4Zzsjrbd6IKhVbdD5E4d9K/XMNsBIl4sidqdaUMhkNTYXkI/a5dTFVlFprLFnM + yB7A6OplQPp+LTITXW2IvD/eZetdl5073X0OQtuJrsm4mAJZ74sqXgcyVs7bGPIw + aXt1ttNH0qA5lvbp0cOIdXmpSWQfDRvlAExqX4p86549J3GMhIsVCMw5KsqduN5e + rSi0HCi24S8CeXiYV1phkSuxf0sROPuRb8SvY/6qcb73DMEMQx3zPWLV/+pG7DcG + uD327xvb3uJG0kFLi4MDnoCHLHYZeoPtwIEwG+1LjwHj+oKn12wlKJvYTxWALi4k + tKVW3QYL8beLaI3XfF+Vd3kCVpU6N9aRt3l+PpzkE1VCkayz67Sf5dJy88YIB7o4 + A08j/AH+GA== + -----END CERTIFICATE----- + tls.key: | + -----BEGIN PRIVATE KEY----- + MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDKG0ancYQBPDa+ + ObQkUSOpM9lqKcJym/G1hjNC2zP6dOMX3CywFcyuJwuRxwZnz8nM+l8ho7+1YlEm + h1ZNY9zi8RRF+LHftiaE0EXp0XP6FB3cGhrJAa+X61xQDt04qLvqSek7vHsAngrx + lMwfR7YkauaQhj7ujMM125kehimgiU7vNC2fH8BXUliNIA6Knln3wk33E4CX02vg + 4DH0zN881DcCVXy6N6+fesBppcSnzHFLc8NieYi69HVybiZ5ZgNIFN9ZMFXH/FqN + QwHYfIYcZlAJJhrLPAJLZgfDr7zcoFVc88Jl/y9jB9cz184DPbGHp5CcC0mYTMf7 + sn4F+ceeP0Ggu5cH5MD56edqStCQNlZnxQbL5Zz7FTloMFlNI1LTBgbSpbvzqJxC + wBmn325IYw93g3tAd19IMLY7vk+Fgs0JoExclxXadfP0tq89fUwa6tl6aJ0UaXFw + aOKnvD+fRKvvjTYwIK2MphXE36w2ajCLd4wb940py5QJ8IPx9Z3kX4MepMV6sQKR + frzAw+1VQ2i2Jy2xVZcHZFsT6x0rZNOXgbC7vwpBr9spJFAGtKqoPw5v25DpeZWZ + WrlRQvcLoi4dnB4Ga7QaRrmJTBTEb8ewNPiAVE400LMARdfpgUrVMgU4TsNWiWGx + NhQlo/pRqzF/Kq/mnbCrEvuQ4zsAlQIDAQABAoICAB2p4TXkWF6iCtrit0j5S8Wi + 4Y9Ob8bIkrJ07wMqDzf8ffRbkYeKu474Fh+gWZVVDyZxvRK/6PxjYMvJ42XaeWop + vjMC7ICMSJ0HwKsv3djfqP5MKpBEGhSvK85oUBnn0iTyEjR9VoRhrfOFRmx05wvz + UMH3ojb7HA/EmnW7dVKltJsxSlznmSB36p1UDi5UTZoqv6y7BQ3DrzrDTTHD/CP+ + ddB24aanU7SRnLok5XdHb32GkS/b7LCb/rz749O/oE2tGpcJnB2NMXE4W4X9yOVU + Mv0Uxav8s82DPTKAkbTEJ6NL98VmrfrYBMMMMqWRIYRqmLP+iB+9bUDJEuwnnB+g + HQ+W9MliiKfpVx3LpcwaT68SNLSoSHt+hHcd0b8iccyvpxag7LExe36NEmg5Xvn7 + NfTcsCeRrbFW7NWmJnS0yZ5vNpf1K6Y9MVPLlNLOYL61QJAVf5EKQxxT66xvFjE6 + L068mFAn1w0vreXIf/Z7eS+A5nU3EBHigdNJO+EjrJ9Myr5fCJewE6tCptTUdkep + ahC2VvWj7d5JOmGVgdvO7/lb2N1Zv8Z7M3aj2tj1xLIeGh5/T/GaGBn56CZpoRtl + i76E08ehrXoKVZM+ej4awmipqkW3x3qDXFTP0Khdrr0FvFptT4/kJ7rNFnrHbNhC + 9fQsHMzxohtlq285hehFAoIBAQDldt9FiaWJr1YTcaaCopJO3VR39M+Ap+Nc3LCm + G320DMzBV310q8zjnbpQ72wCm8zOV8ZRoGLfSEI0cJBsw/w0fkPK3FqPdHosUQte + 4TvYqhz7qNV4fMqgqeFgwPXLzfKjeJwd1GSyBt6+LtL6vIVXsbiFa04wTi3Zal0S + aiKMt2l2qLC1RjIIQC/go3+Be5n4r/DsmPuIKXydsBJbmYxwIp8zGxE5SqEle66i + kg8CXm3TzMnydaSXW69s4/E4TsNdsdoUMEVFX9TR9QaYBGbYa4uBgblxnwapmoCG + 6/Qx72Wwn6jclnIJ6O4oCxRwt4+PFQoB1iggqYXvUVka7pdrAoIBAQDhen7s+u5/ + Gmkna0Z2gxwpUleEAeoqAPEab5HEZYvUfQFpmsCvavSLVH7I72HmXwxox41a634U + t19JGiPajRVy3d7NWJwT775Rr3JvIPOxJz7z0Erthvovt+usmybjM0GUAygcL3KX + NI7NIE/20+FrE1t5KydSDpnwGDjUOP3qke2a2LH3gkxye98zUiSP1beS9ZtND0t0 + tb15H6kE0RzKzoDr3g0H96uIYN3f/u/vd0QcoeUbeZ4zAfFT13FLGkTzM9hryi+c + u2GlNx8jlxjEdP1WQ3R16F2wiNUDXEO0rIJ39yJxq4Q+HitHs1II0ku8Zsl4j2ji + 4NjDWnVlGsf/AoIBACq3PkkLnq1K3pMwDPc0nFfiC99oy1f/6OtYieniJDxoRZWY + W9pkQPf+XLjEWS8DIdUnsdtwJ/e409o/OiOFOGXtO2GAM7uQy02EqME43JMw18Rh + YZvUX0QZW8NPKHKQHCMiSK2k5Dnf3Yc54Dyc65Wvl9H+b8e+Rfq7Oig7acMYQaG9 + NWgJcTbn/AGtbgcEo10QI6FnuStXzcLtC/3RT7twNoKk/0fTBLydkZzh47llKrzK + 2q0wCTvD7Zwvsq/wuZ2vhhFMCxknBc8v9dJGv356RrYNGCB2oh7gjByRwBurnRqM + HjsyJniBzBPYPC3fffH80KgW3dF2vWAXnKbO05ECggEBAN2KBfvB8WNhB8B7XphC + 6gCPrv09BF6Q3lQjGw3k2CbHTAP+0SOJ6wya0JeM3JwE96RbrT+P8ilYon6o+GFc + DZ728FrCnVfZTx3jTIRz0/xTmz7jgTswS9Fm3GCTcPn6+ov7LgXDeMuoEEbrYKYc + OHeEZXQAOnodbpLAudWKybEYAGUMVI/jrkkt/HfoVZQdYGk8eIKWbIUXrDq+KMta + yvdZPsKBQjx51EgDJP1Y91ZJ0NpxFzKPpChp2DuQivhHr0dlwVbnKTDNy3sRVb8a + TnN4nxGK5XqT/LnfN6w9kyiJ9wb4Axe8dgqyicPBJPEV7uQF+h5CicOhOFl5H69f + SPECggEAFQUHIFV+GqiBwoJv7PESfHWiCwmCNWsA5q1lRLhIiWHJ8veIXAizzYGj + B+AWnMIccF3KLPvX++7o7qY4kkhAjob6AoulRg49C/g3YtuuxPe3CI7MV12nOHKc + ltzRO/CZbcg7LMNoPDrsmVoxYsN91pAg7hH83MNLjEiHhsJMKW5MXcnV+ZEiSukK + gYEyN7T2sI7sGQYEYERfFybOyqNw9RjojHKFEAlfrVa+rDbpU837AATcd4+E+aav + iwVBz6BCvn+ozhvmu2ZkOminlve3Bfnr7eJ5p8NHBwB9ELscKBZK2SNcKlu3wtg4 + w6IEdTFRspEvNwyrTH9geMzYmqbIsA== + -----END PRIVATE KEY----- diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 89befdf7..6fa318c9 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -1,7 +1,7 @@ --- stacks: argocd: - description: TODO + description: Deploying Stackable Demos with ArgoCD stackableRelease: dev # TODO: We actually want to deploy all operators via ArgoCD, but we currently *have to* install # operators with stackablectl. Therefore we install the internal operators via stackablectl. @@ -12,20 +12,32 @@ stacks: - secret labels: - argocd + - sealed secrets manifests: - helmChart: stacks/_templates/argo-cd.yaml + ################################ # projects + ################################ - plainYaml: stacks/argo-cd/projects/stackable-operators.yaml - plainYaml: stacks/argo-cd/projects/airflow.yaml + ################################ # operators + ################################ # currently deployed via stackablectl since it complains if no operators are deployed... #- plainYaml: stacks/argo-cd/applicationsets/commons-operator.yaml #- plainYaml: stacks/argo-cd/applicationsets/listener-operator.yaml #- plainYaml: stacks/argo-cd/applicationsets/secret-operator.yaml - plainYaml: stacks/argo-cd/applicationsets/airflow-operator.yaml - plainYaml: stacks/argo-cd/applicationsets/spark-k8s-operator.yaml - # products + ################################ + # prerequisites + ################################ + - plainYaml: stacks/argo-cd/applications/sealed-secrets.yaml - plainYaml: stacks/argo-cd/applications/airflow-postgres.yaml + ################################ + # products + ################################ + # via argo - plainYaml: stacks/argo-cd/applications/airflow.yaml supportedNamespaces: - argo-cd @@ -47,12 +59,6 @@ stacks: description: Password of the ArgoCD admin user # generated via: `htpasswd -nbBC 10 "" adminadmin | tr -d ':\n'` default: $2y$10$HhJC3pGHTlk8RyBoS39N/.wC72mdWxV2X8QS1wROUwCFxl.2tGfky - - name: airflowAdminPassword - description: Password of the Airflow admin user - default: adminadmin - - name: airflowSecretKey - description: Airflow's secret key used to generate e.g. user session tokens - default: airflowSecretKey monitoring: description: Stack containing Prometheus and Grafana stackableRelease: dev From 15612f7eb363274e932fbf168bbc31bfb6faa92c Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 11 Apr 2025 22:59:22 +0200 Subject: [PATCH 05/37] fix sealed secret location --- stacks/argo-cd/applications/sealed-secrets.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/argo-cd/applications/sealed-secrets.yaml b/stacks/argo-cd/applications/sealed-secrets.yaml index 7faa76b4..27310ea5 100644 --- a/stacks/argo-cd/applications/sealed-secrets.yaml +++ b/stacks/argo-cd/applications/sealed-secrets.yaml @@ -15,7 +15,7 @@ spec: secretName: sealed-secrets-key - repoURL: https://github.com/stackabletech/demos.git # TODO: adapt to release - targetRevision: HEAD + targetRevision: spike/argocd-demo path: stacks/argo-cd/secrets/ destination: server: https://kubernetes.default.svc From e81624d7042b0f1addbe19fe5281b39de37ec07c Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 11 Apr 2025 23:03:13 +0200 Subject: [PATCH 06/37] fix demo branches --- stacks/argo-cd/applications/sealed-secrets.yaml | 2 +- stacks/stacks-v2.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/stacks/argo-cd/applications/sealed-secrets.yaml b/stacks/argo-cd/applications/sealed-secrets.yaml index 27310ea5..aef6042a 100644 --- a/stacks/argo-cd/applications/sealed-secrets.yaml +++ b/stacks/argo-cd/applications/sealed-secrets.yaml @@ -15,7 +15,7 @@ spec: secretName: sealed-secrets-key - repoURL: https://github.com/stackabletech/demos.git # TODO: adapt to release - targetRevision: spike/argocd-demo + targetRevision: "{{ demoTargetRevision }}" path: stacks/argo-cd/secrets/ destination: server: https://kubernetes.default.svc diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 6fa318c9..47080633 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -51,7 +51,8 @@ stacks: default: 25.3.0 - name: demoTargetRevision description: The target revision, HEAD or e.g. release-25.3 - default: release-25.3 + #default: release-25.3 + default: spike/argocd-demo - name: stackableOperatorNamespace description: Stackable namespace for the operators default: stackable-operators From 822dee502614ec04d8e040665f5101da40fb7416 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 11 Apr 2025 23:05:02 +0200 Subject: [PATCH 07/37] fixes --- stacks/argo-cd/applications/airflow.yaml | 2 +- stacks/argo-cd/applications/sealed-secrets.yaml | 3 +-- stacks/stacks-v2.yaml | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/stacks/argo-cd/applications/airflow.yaml b/stacks/argo-cd/applications/airflow.yaml index 9db9f6a7..dcd20380 100644 --- a/stacks/argo-cd/applications/airflow.yaml +++ b/stacks/argo-cd/applications/airflow.yaml @@ -9,7 +9,7 @@ spec: server: https://kubernetes.default.svc source: repoURL: https://github.com/stackabletech/demos.git - targetRevision: "{{ demoTargetRevision }}" + targetRevision: "{{ demoReleaseVersion }}" path: stacks/argo-cd/manifests/airflow/ syncPolicy: syncOptions: diff --git a/stacks/argo-cd/applications/sealed-secrets.yaml b/stacks/argo-cd/applications/sealed-secrets.yaml index aef6042a..145df9de 100644 --- a/stacks/argo-cd/applications/sealed-secrets.yaml +++ b/stacks/argo-cd/applications/sealed-secrets.yaml @@ -14,8 +14,7 @@ spec: valuesObject: secretName: sealed-secrets-key - repoURL: https://github.com/stackabletech/demos.git - # TODO: adapt to release - targetRevision: "{{ demoTargetRevision }}" + targetRevision: "{{ demoReleaseVersion }}" path: stacks/argo-cd/secrets/ destination: server: https://kubernetes.default.svc diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 47080633..f8a7d0bc 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -49,7 +49,7 @@ stacks: - name: stackableReleaseVersion description: Stackable release to be installed via Argo default: 25.3.0 - - name: demoTargetRevision + - name: demoReleaseVersion description: The target revision, HEAD or e.g. release-25.3 #default: release-25.3 default: spike/argocd-demo From 4bf262686fed388a6aecfbed1d9336109b2e0bdb Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 11 Apr 2025 23:17:37 +0200 Subject: [PATCH 08/37] add role and binding for airflow / spark --- .../airflow/airflow-spark-clusterrole.yaml | 36 +++++++++++++++++++ .../airflow-spark-clusterrolebinding.yaml | 13 +++++++ 2 files changed, 49 insertions(+) create mode 100644 stacks/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml create mode 100644 stacks/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml diff --git a/stacks/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml b/stacks/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml new file mode 100644 index 00000000..66abed2f --- /dev/null +++ b/stacks/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: airflow-spark-clusterrole +rules: + - apiGroups: + - spark.stackable.tech + resources: + - sparkapplications + verbs: + - create + - get + - list + - apiGroups: + - airflow.stackable.tech + resources: + - airflowdbs + verbs: + - create + - get + - list + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - watch + - list + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - list diff --git a/stacks/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml b/stacks/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml new file mode 100644 index 00000000..1f9e1b5d --- /dev/null +++ b/stacks/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: airflow-spark-clusterrole-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: airflow-spark-clusterrole +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:serviceaccounts From b1e33bc706814e876c3be6e305971ae57d11b580 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 11 Apr 2025 23:19:02 +0200 Subject: [PATCH 09/37] remove ns --- stacks/argo-cd/manifests/airflow/airflow.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/stacks/argo-cd/manifests/airflow/airflow.yaml b/stacks/argo-cd/manifests/airflow/airflow.yaml index 703f409c..a3399671 100644 --- a/stacks/argo-cd/manifests/airflow/airflow.yaml +++ b/stacks/argo-cd/manifests/airflow/airflow.yaml @@ -4,7 +4,6 @@ apiVersion: airflow.stackable.tech/v1alpha1 kind: AirflowCluster metadata: name: airflow - namespace: stackable-airflow spec: image: productVersion: 2.10.4 @@ -66,7 +65,6 @@ apiVersion: v1 kind: ConfigMap metadata: name: airflow-dags - namespace: stackable-airflow data: date_demo.py: | """Example DAG returning the current date""" From f8ab04351e989238abe1354955d8d1f5810ca2c5 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 12 Apr 2025 01:19:00 +0200 Subject: [PATCH 10/37] test minio --- stacks/argo-cd/applications/minio.yaml | 44 +++++++++++++++++++ stacks/argo-cd/manifests/airflow/airflow.yaml | 14 +++--- stacks/argo-cd/projects/minio.yaml | 14 ++++++ stacks/stacks-v2.yaml | 4 +- 4 files changed, 68 insertions(+), 8 deletions(-) create mode 100644 stacks/argo-cd/applications/minio.yaml create mode 100644 stacks/argo-cd/projects/minio.yaml diff --git a/stacks/argo-cd/applications/minio.yaml b/stacks/argo-cd/applications/minio.yaml new file mode 100644 index 00000000..76ec47d3 --- /dev/null +++ b/stacks/argo-cd/applications/minio.yaml @@ -0,0 +1,44 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: minio +spec: + project: minio + destination: + server: https://kubernetes.default.svc + namespace: minio + source: + repoURL: https://charts.min.io/ + targetRevision: 5.4.0 # RELEASE.2024-12-18T13-15-44Z + chart: minio + helm: + releaseName: minio + valuesObject: + additionalLabels: + stackable.tech/vendor: Stackable + podLabels: + stackable.tech/vendor: Stackable + rootUser: admin + rootPassword: adminadmin + mode: standalone + persistence: + size: 10Gi + buckets: + - name: demo + policy: public + resources: + requests: + cpu: 1 + memory: 2Gi + service: + type: NodePort + nodePort: null + consoleService: + type: NodePort + nodePort: null + syncPolicy: + syncOptions: + - CreateNamespace=true + automated: + selfHeal: true + prune: true diff --git a/stacks/argo-cd/manifests/airflow/airflow.yaml b/stacks/argo-cd/manifests/airflow/airflow.yaml index a3399671..ce4975ad 100644 --- a/stacks/argo-cd/manifests/airflow/airflow.yaml +++ b/stacks/argo-cd/manifests/airflow/airflow.yaml @@ -37,14 +37,16 @@ spec: gracefulShutdownTimeout: 30s roleGroups: default: - envOverrides: + envOverrides: &envOverrides AIRFLOW__CORE__DAGS_FOLDER: "/dags" AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" + AIRFLOW_CONN_MINIO_DEFAULT: "s3://admin:adminadmin@minio.minio.svc.cluster.local:9000?extra=%7B%22aws_access_key_id%22%3A%22admin%22%2C%22aws_secret_access_key%22%3A%22adminadmin%22%2C%22endpoint_url%22%3A%22https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000%22%7D" + AIRFLOW__LOGGING__REMOTE_LOGGING: "True" + AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER: s3://demo/airflow-task-logs/ + AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio_default replicas: 1 kubernetesExecutors: - envOverrides: - AIRFLOW__CORE__DAGS_FOLDER: "/dags" - AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" + envOverrides: *envOverrides schedulers: config: gracefulShutdownTimeout: 30s @@ -56,9 +58,7 @@ spec: limit: 1Gi roleGroups: default: - envOverrides: - AIRFLOW__CORE__DAGS_FOLDER: "/dags" - AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" + envOverrides: *envOverrides replicas: 1 --- apiVersion: v1 diff --git a/stacks/argo-cd/projects/minio.yaml b/stacks/argo-cd/projects/minio.yaml new file mode 100644 index 00000000..aaa033ba --- /dev/null +++ b/stacks/argo-cd/projects/minio.yaml @@ -0,0 +1,14 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: minio +spec: + description: Project to manage Minio via Helm + sourceRepos: + - "*" + destinations: + - namespace: minio + server: https://kubernetes.default.svc + clusterResourceWhitelist: + - group: "*" + kind: "*" diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index f8a7d0bc..35e45149 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -13,6 +13,7 @@ stacks: labels: - argocd - sealed secrets + - airflow manifests: - helmChart: stacks/_templates/argo-cd.yaml ################################ @@ -20,6 +21,7 @@ stacks: ################################ - plainYaml: stacks/argo-cd/projects/stackable-operators.yaml - plainYaml: stacks/argo-cd/projects/airflow.yaml + - plainYaml: stacks/argo-cd/projects/minio.yaml ################################ # operators ################################ @@ -34,10 +36,10 @@ stacks: ################################ - plainYaml: stacks/argo-cd/applications/sealed-secrets.yaml - plainYaml: stacks/argo-cd/applications/airflow-postgres.yaml + - plainYaml: stacks/argo-cd/applications/minio.yaml ################################ # products ################################ - # via argo - plainYaml: stacks/argo-cd/applications/airflow.yaml supportedNamespaces: - argo-cd From 3d8e66429512c9e4a87e195080cc13acd78ef427 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 12 Apr 2025 01:22:46 +0200 Subject: [PATCH 11/37] fix sync policy --- stacks/argo-cd/applications/minio.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/stacks/argo-cd/applications/minio.yaml b/stacks/argo-cd/applications/minio.yaml index 76ec47d3..d78139a0 100644 --- a/stacks/argo-cd/applications/minio.yaml +++ b/stacks/argo-cd/applications/minio.yaml @@ -36,9 +36,9 @@ spec: consoleService: type: NodePort nodePort: null - syncPolicy: - syncOptions: - - CreateNamespace=true - automated: - selfHeal: true - prune: true + syncPolicy: + syncOptions: + - CreateNamespace=true + automated: + selfHeal: true + prune: true From 150fd766a3437d1c135493296689e4f0d028afad Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 12 Apr 2025 01:58:24 +0200 Subject: [PATCH 12/37] testing --- stacks/argo-cd/applications/minio.yaml | 31 +- stacks/argo-cd/manifests/airflow/airflow.yaml | 2 +- stacks/argo-cd/manifests/minio/minio.yaml | 712 ++++++++++++++++++ 3 files changed, 716 insertions(+), 29 deletions(-) create mode 100644 stacks/argo-cd/manifests/minio/minio.yaml diff --git a/stacks/argo-cd/applications/minio.yaml b/stacks/argo-cd/applications/minio.yaml index d78139a0..05d27e88 100644 --- a/stacks/argo-cd/applications/minio.yaml +++ b/stacks/argo-cd/applications/minio.yaml @@ -8,34 +8,9 @@ spec: server: https://kubernetes.default.svc namespace: minio source: - repoURL: https://charts.min.io/ - targetRevision: 5.4.0 # RELEASE.2024-12-18T13-15-44Z - chart: minio - helm: - releaseName: minio - valuesObject: - additionalLabels: - stackable.tech/vendor: Stackable - podLabels: - stackable.tech/vendor: Stackable - rootUser: admin - rootPassword: adminadmin - mode: standalone - persistence: - size: 10Gi - buckets: - - name: demo - policy: public - resources: - requests: - cpu: 1 - memory: 2Gi - service: - type: NodePort - nodePort: null - consoleService: - type: NodePort - nodePort: null + repoURL: https://github.com/stackabletech/demos.git + targetRevision: "{{ demoReleaseVersion }}" + path: stacks/argo-cd/manifests/minio/ syncPolicy: syncOptions: - CreateNamespace=true diff --git a/stacks/argo-cd/manifests/airflow/airflow.yaml b/stacks/argo-cd/manifests/airflow/airflow.yaml index ce4975ad..dc155030 100644 --- a/stacks/argo-cd/manifests/airflow/airflow.yaml +++ b/stacks/argo-cd/manifests/airflow/airflow.yaml @@ -40,7 +40,7 @@ spec: envOverrides: &envOverrides AIRFLOW__CORE__DAGS_FOLDER: "/dags" AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" - AIRFLOW_CONN_MINIO_DEFAULT: "s3://admin:adminadmin@minio.minio.svc.cluster.local:9000?extra=%7B%22aws_access_key_id%22%3A%22admin%22%2C%22aws_secret_access_key%22%3A%22adminadmin%22%2C%22endpoint_url%22%3A%22https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000%22%7D" + AIRFLOW_CONN_MINIO_DEFAULT: "s3://admin:adminadmin@minio.minio.svc.cluster.local:9000?extra=%7B%22endpoint_url%22%3A%22https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000%22%7D" AIRFLOW__LOGGING__REMOTE_LOGGING: "True" AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER: s3://demo/airflow-task-logs/ AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio_default diff --git a/stacks/argo-cd/manifests/minio/minio.yaml b/stacks/argo-cd/manifests/minio/minio.yaml new file mode 100644 index 00000000..8681086f --- /dev/null +++ b/stacks/argo-cd/manifests/minio/minio.yaml @@ -0,0 +1,712 @@ +--- +# Source: minio/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "minio-sa" +--- +# Source: minio/templates/secrets.yaml +apiVersion: v1 +kind: Secret +metadata: + name: minio + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm +type: Opaque +data: + rootUser: "YWRtaW4=" + rootPassword: "YWRtaW5hZG1pbg==" +--- +# Source: minio/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: minio + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm +data: + initialize: |- + #!/bin/sh + set -e # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # connectToMinio + # Use a check-sleep-check loop to wait for MinIO service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 + LIMIT=29 # Allow 30 attempts + set -e # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) + SECRET=$(cat /config/rootPassword) + set +e # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" + $MC_COMMAND + STATUS=$? + until [ $STATUS = 0 ]; do + ATTEMPTS=$(expr $ATTEMPTS + 1) + echo \"Failed attempts: $ATTEMPTS\" + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 + fi + sleep 2 # 1 second intervals between attempts + $MC_COMMAND + STATUS=$? + done + set -e # reset `e` as active + return 0 + } + + # checkBucketExists ($bucket) + # Check if the bucket exists, by using the exit code of `mc ls` + checkBucketExists() { + BUCKET=$1 + CMD=$(${MC} stat myminio/$BUCKET >/dev/null 2>&1) + return $? + } + + # createBucket ($bucket, $policy, $purge) + # Ensure bucket exists, purging if asked to + createBucket() { + BUCKET=$1 + POLICY=$2 + PURGE=$3 + VERSIONING=$4 + OBJECTLOCKING=$5 + + # Purge the bucket, if set & exists + # Since PURGE is user input, check explicitly for `true` + if [ $PURGE = true ]; then + if checkBucketExists $BUCKET; then + echo "Purging bucket '$BUCKET'." + set +e # don't exit if this fails + ${MC} rm -r --force myminio/$BUCKET + set -e # reset `e` as active + else + echo "Bucket '$BUCKET' does not exist, skipping purge." + fi + fi + + # Create the bucket if it does not exist and set objectlocking if enabled (NOTE: versioning will be not changed if OBJECTLOCKING is set because it enables versioning to the Buckets created) + if ! checkBucketExists $BUCKET; then + if [ ! -z $OBJECTLOCKING ]; then + if [ $OBJECTLOCKING = true ]; then + echo "Creating bucket with OBJECTLOCKING '$BUCKET'" + ${MC} mb --with-lock myminio/$BUCKET + elif [ $OBJECTLOCKING = false ]; then + echo "Creating bucket '$BUCKET'" + ${MC} mb myminio/$BUCKET + fi + elif [ -z $OBJECTLOCKING ]; then + echo "Creating bucket '$BUCKET'" + ${MC} mb myminio/$BUCKET + else + echo "Bucket '$BUCKET' already exists." + fi + fi + + # set versioning for bucket if objectlocking is disabled or not set + if [ $OBJECTLOCKING = false ]; then + if [ ! -z $VERSIONING ]; then + if [ $VERSIONING = true ]; then + echo "Enabling versioning for '$BUCKET'" + ${MC} version enable myminio/$BUCKET + elif [ $VERSIONING = false ]; then + echo "Suspending versioning for '$BUCKET'" + ${MC} version suspend myminio/$BUCKET + fi + fi + else + echo "Bucket '$BUCKET' versioning unchanged." + fi + + # At this point, the bucket should exist, skip checking for existence + # Set policy on the bucket + echo "Setting policy of bucket '$BUCKET' to '$POLICY'." + ${MC} anonymous set $POLICY myminio/$BUCKET + } + + # Try connecting to MinIO instance + scheme=https + connectToMinio $scheme + + + + # Create the buckets + createBucket demo "public" false false false + + add-user: |- + #!/bin/sh + set -e ; # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. + # Special characters for example : ',",<,>,{,} + MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_tmp" + + # connectToMinio + # Use a check-sleep-check loop to wait for MinIO service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 1 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 + } + + # checkUserExists () + # Check if the user exists, by using the exit code of `mc admin user info` + checkUserExists() { + CMD=$(${MC} admin user info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) + return $? + } + + # createUser ($policy) + createUser() { + POLICY=$1 + #check accessKey_and_secretKey_tmp file + if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then + echo "credentials file does not exist" + return 1 + fi + if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then + echo "credentials file is invalid" + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + return 1 + fi + USER=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) + # Create the user if it does not exist + if ! checkUserExists ; then + echo "Creating user '$USER'" + cat $MINIO_ACCESSKEY_SECRETKEY_TMP | ${MC} admin user add myminio + else + echo "User '$USER' already exists." + fi + #clean up credentials files. + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + + # set policy for user + if [ ! -z $POLICY -a $POLICY != " " ] ; then + echo "Adding policy '$POLICY' for '$USER'" + set +e ; # policy already attach errors out, allow it. + ${MC} admin policy attach myminio $POLICY --user=$USER + set -e + else + echo "User '$USER' has no policy attached." + fi + } + + # Try connecting to MinIO instance + scheme=https + connectToMinio $scheme + + + + # Create the users + echo console > $MINIO_ACCESSKEY_SECRETKEY_TMP + echo console123 >> $MINIO_ACCESSKEY_SECRETKEY_TMP + createUser consoleAdmin + + add-policy: |- + #!/bin/sh + set -e ; # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # connectToMinio + # Use a check-sleep-check loop to wait for MinIO service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 1 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 + } + + # checkPolicyExists ($policy) + # Check if the policy exists, by using the exit code of `mc admin policy info` + checkPolicyExists() { + POLICY=$1 + CMD=$(${MC} admin policy info myminio $POLICY > /dev/null 2>&1) + return $? + } + + # createPolicy($name, $filename) + createPolicy () { + NAME=$1 + FILENAME=$2 + + # Create the name if it does not exist + echo "Checking policy: $NAME (in /config/$FILENAME.json)" + if ! checkPolicyExists $NAME ; then + echo "Creating policy '$NAME'" + else + echo "Policy '$NAME' already exists." + fi + ${MC} admin policy create myminio $NAME /config/$FILENAME.json + + } + + # Try connecting to MinIO instance + scheme=https + connectToMinio $scheme + + add-svcacct: |- + #!/bin/sh + set -e ; # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. + # Special characters for example : ',",<,>,{,} + MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_svcacct_tmp" + + # connectToMinio + # Use a check-sleep-check loop to wait for MinIO service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 2 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 + } + + # checkSvcacctExists () + # Check if the svcacct exists, by using the exit code of `mc admin user svcacct info` + checkSvcacctExists() { + CMD=$(${MC} admin user svcacct info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) + return $? + } + + # createSvcacct ($user) + createSvcacct () { + USER=$1 + FILENAME=$2 + #check accessKey_and_secretKey_tmp file + if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then + echo "credentials file does not exist" + return 1 + fi + if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then + echo "credentials file is invalid" + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + return 1 + fi + SVCACCT=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) + # Create the svcacct if it does not exist + if ! checkSvcacctExists ; then + echo "Creating svcacct '$SVCACCT'" + # Check if policy file is define + if [ -z $FILENAME ]; then + ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) myminio $USER + else + ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --policy /config/$FILENAME.json myminio $USER + fi + else + echo "Svcacct '$SVCACCT' already exists." + fi + #clean up credentials files. + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + } + + # Try connecting to MinIO instance + scheme=https + connectToMinio $scheme + + custom-command: |- + #!/bin/sh + set -e ; # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # connectToMinio + # Use a check-sleep-check loop to wait for MinIO service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 1 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 + } + + # runCommand ($@) + # Run custom mc command + runCommand() { + ${MC} "$@" + return $? + } + + # Try connecting to MinIO instance + scheme=https + connectToMinio $scheme +--- +# Source: minio/templates/pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm +spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "10Gi" +--- +# Source: minio/templates/console-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: minio-console + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm +spec: + type: NodePort + externalTrafficPolicy: "Cluster" + ports: + - name: https + port: 9001 + protocol: TCP + targetPort: 9001 + selector: + app: minio + release: minio +--- +# Source: minio/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: minio + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm + monitoring: "true" +spec: + type: NodePort + externalTrafficPolicy: "Cluster" + ports: + - name: https + port: 9000 + protocol: TCP + targetPort: 9000 + selector: + app: minio + release: minio +--- +# Source: minio/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm + stackable.tech/vendor: Stackable +spec: + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 100% + maxUnavailable: 0 + replicas: 1 + selector: + matchLabels: + app: minio + release: minio + template: + metadata: + name: minio + labels: + app: minio + release: minio + stackable.tech/vendor: Stackable + annotations: + checksum/secrets: fa63e34a92c817c84057e2d452fa683e66462a57b0529388fb96a57e05f38e57 + checksum/config: ebea49cc4c1bfbd1b156a58bf770a776ff87fe199f642d31c2816b5515112e72 + spec: + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 1000 + runAsUser: 1000 + + serviceAccountName: minio-sa + containers: + - name: minio + image: "quay.io/minio/minio:RELEASE.2024-12-18T13-15-44Z" + imagePullPolicy: IfNotPresent + command: + - "/bin/sh" + - "-ce" + - | + # minio requires the TLS key pair to be specially named + # mkdir -p /etc/minio/certs + cp -v /etc/minio/original_certs/tls.crt /etc/minio/certs/public.crt + cp -v /etc/minio/original_certs/tls.key /etc/minio/certs/private.key + + /usr/bin/docker-entrypoint.sh minio server /export -S /etc/minio/certs/ --address :9000 --console-address :9001 + volumeMounts: + - name: minio-user + mountPath: "/tmp/credentials" + readOnly: true + - name: export + mountPath: /export + - mountPath: /etc/minio/original_certs + name: tls + - mountPath: /etc/minio/certs + name: certs + ports: + - name: https + containerPort: 9000 + - name: https-console + containerPort: 9001 + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: minio + key: rootUser + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: minio + key: rootPassword + - name: MINIO_PROMETHEUS_AUTH_TYPE + value: "public" + resources: + requests: + cpu: 1 + memory: 2Gi + securityContext: + readOnlyRootFilesystem: false + volumes: + - name: export + persistentVolumeClaim: + claimName: minio + - name: minio-user + secret: + secretName: minio + + - ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: service=minio + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1 + storageClassName: secrets.stackable.tech + name: tls + - emptyDir: + medium: Memory + sizeLimit: 5Mi + name: certs +--- +# Source: minio/templates/post-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: minio-post-job + labels: + app: minio-post-job + chart: minio-5.4.0 + release: minio + heritage: Helm + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation +spec: + template: + metadata: + labels: + app: minio-job + release: minio + stackable.tech/vendor: Stackable + spec: + restartPolicy: OnFailure + volumes: + - name: etc-path + emptyDir: {} + - name: tmp + emptyDir: {} + - name: minio-configuration + projected: + sources: + - configMap: + name: minio + - secret: + name: minio + - ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: service=minio + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1 + storageClassName: secrets.stackable.tech + name: tls + - emptyDir: + medium: Memory + sizeLimit: 5Mi + name: certs + serviceAccountName: minio-sa + containers: + - name: minio-make-bucket + image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" + imagePullPolicy: IfNotPresent + command: + - "/bin/sh" + - "-ce" + - | + # Copy the CA cert from the "tls" SecretClass + # mkdir -p /etc/minio/mc/certs/CAs + cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt + + . /config/initialize + env: + - name: MINIO_ENDPOINT + value: minio + - name: MINIO_PORT + value: "9000" + volumeMounts: + - name: etc-path + mountPath: /etc/minio/mc + - name: tmp + mountPath: /tmp + - name: minio-configuration + mountPath: /config + - name: tls + mountPath: /etc/minio/mc/original_certs + - name: certs + mountPath: /etc/minio/mc/certs/CAs + resources: + requests: + memory: 128Mi + - name: minio-make-user + image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" + imagePullPolicy: IfNotPresent + command: + - "/bin/sh" + - "-ce" + - | + # Copy the CA cert from the "tls" SecretClass + # mkdir -p /etc/minio/mc/certs/CAs + cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt + + . /config/add-user + env: + - name: MINIO_ENDPOINT + value: minio + - name: MINIO_PORT + value: "9000" + volumeMounts: + - name: etc-path + mountPath: /etc/minio/mc + - name: tmp + mountPath: /tmp + - name: minio-configuration + mountPath: /config + - name: tls + mountPath: /etc/minio/mc/original_certs + - name: certs + mountPath: /etc/minio/mc/certs/CAs + resources: + requests: + memory: 128Mi From f05edcd8efb390fe90998312f06110dd3337d11c Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 12 Apr 2025 04:09:58 +0200 Subject: [PATCH 13/37] add airflow logs minio --- stacks/argo-cd/manifests/airflow/airflow.yaml | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/stacks/argo-cd/manifests/airflow/airflow.yaml b/stacks/argo-cd/manifests/airflow/airflow.yaml index dc155030..66f8088c 100644 --- a/stacks/argo-cd/manifests/airflow/airflow.yaml +++ b/stacks/argo-cd/manifests/airflow/airflow.yaml @@ -16,6 +16,20 @@ spec: - name: airflow-dags configMap: name: airflow-dags + - name: minio-tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: pod,node + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech volumeMounts: - name: airflow-dags mountPath: /dags/date_demo.py @@ -26,6 +40,8 @@ spec: - name: airflow-dags mountPath: /dags/pyspark_pi.yaml subPath: pyspark_pi.yaml + - name: minio-tls + mountPath: /stackable/minio-tls webservers: config: resources: @@ -40,10 +56,11 @@ spec: envOverrides: &envOverrides AIRFLOW__CORE__DAGS_FOLDER: "/dags" AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" - AIRFLOW_CONN_MINIO_DEFAULT: "s3://admin:adminadmin@minio.minio.svc.cluster.local:9000?extra=%7B%22endpoint_url%22%3A%22https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000%22%7D" + AIRFLOW_CONN_MINIO: "aws://admin:adminadmin@/?endpoint_url=https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000" + AWS_CA_BUNDLE: "/stackable/minio-tls/ca.crt" AIRFLOW__LOGGING__REMOTE_LOGGING: "True" AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER: s3://demo/airflow-task-logs/ - AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio_default + AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio replicas: 1 kubernetesExecutors: envOverrides: *envOverrides From 3cedcfaa5b4fbc2fafeaf5a0f9887ac2e0063687 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 12 Apr 2025 04:15:04 +0200 Subject: [PATCH 14/37] fixes --- stacks/argo-cd/applications/airflow.yaml | 2 +- stacks/argo-cd/projects/minio.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stacks/argo-cd/applications/airflow.yaml b/stacks/argo-cd/applications/airflow.yaml index dcd20380..9d938bc9 100644 --- a/stacks/argo-cd/applications/airflow.yaml +++ b/stacks/argo-cd/applications/airflow.yaml @@ -5,8 +5,8 @@ metadata: spec: project: airflow destination: - namespace: stackable-airflow server: https://kubernetes.default.svc + namespace: stackable-airflow source: repoURL: https://github.com/stackabletech/demos.git targetRevision: "{{ demoReleaseVersion }}" diff --git a/stacks/argo-cd/projects/minio.yaml b/stacks/argo-cd/projects/minio.yaml index aaa033ba..ca58ee8a 100644 --- a/stacks/argo-cd/projects/minio.yaml +++ b/stacks/argo-cd/projects/minio.yaml @@ -3,7 +3,7 @@ kind: AppProject metadata: name: minio spec: - description: Project to manage Minio via Helm + description: Project to manage Minio sourceRepos: - "*" destinations: From 871d45912d3359dddb05740898431811cf739e57 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 09:48:17 +0200 Subject: [PATCH 15/37] extend cert expiry to 10 years --- stacks/argo-cd/manifests/airflow/airflow.yaml | 2 +- .../argo-cd/secrets/sealed-secrets-key.yaml | 159 +++++++++--------- 2 files changed, 82 insertions(+), 79 deletions(-) diff --git a/stacks/argo-cd/manifests/airflow/airflow.yaml b/stacks/argo-cd/manifests/airflow/airflow.yaml index 66f8088c..de8613d1 100644 --- a/stacks/argo-cd/manifests/airflow/airflow.yaml +++ b/stacks/argo-cd/manifests/airflow/airflow.yaml @@ -65,6 +65,7 @@ spec: kubernetesExecutors: envOverrides: *envOverrides schedulers: + envOverrides: *envOverrides config: gracefulShutdownTimeout: 30s resources: @@ -75,7 +76,6 @@ spec: limit: 1Gi roleGroups: default: - envOverrides: *envOverrides replicas: 1 --- apiVersion: v1 diff --git a/stacks/argo-cd/secrets/sealed-secrets-key.yaml b/stacks/argo-cd/secrets/sealed-secrets-key.yaml index 60e7e522..7196e082 100644 --- a/stacks/argo-cd/secrets/sealed-secrets-key.yaml +++ b/stacks/argo-cd/secrets/sealed-secrets-key.yaml @@ -4,87 +4,90 @@ kind: Secret metadata: name: sealed-secrets-key stringData: + # Generated on 2025/04/12 + # openssl req -x509 -nodes -newkey rsa:4096 -keyout tls.key -out tls.crt -subj "/CN=sealed-secrets" -days 3650 + # required for sealed secrets decryption tls.crt: | -----BEGIN CERTIFICATE----- - MIIFEzCCAvugAwIBAgIUTIeBhvXRk1w2QJRP2RpyQWJCB48wDQYJKoZIhvcNAQEL - BQAwGTEXMBUGA1UEAwwOc2VhbGVkLXNlY3JldHMwHhcNMjUwNDExMTk1NzUzWhcN - MjYwNDExMTk1NzUzWjAZMRcwFQYDVQQDDA5zZWFsZWQtc2VjcmV0czCCAiIwDQYJ - KoZIhvcNAQEBBQADggIPADCCAgoCggIBAMobRqdxhAE8Nr45tCRRI6kz2WopwnKb - 8bWGM0LbM/p04xfcLLAVzK4nC5HHBmfPycz6XyGjv7ViUSaHVk1j3OLxFEX4sd+2 - JoTQRenRc/oUHdwaGskBr5frXFAO3Tiou+pJ6Tu8ewCeCvGUzB9HtiRq5pCGPu6M - wzXbmR6GKaCJTu80LZ8fwFdSWI0gDoqeWffCTfcTgJfTa+DgMfTM3zzUNwJVfLo3 - r596wGmlxKfMcUtzw2J5iLr0dXJuJnlmA0gU31kwVcf8Wo1DAdh8hhxmUAkmGss8 - AktmB8OvvNygVVzzwmX/L2MH1zPXzgM9sYenkJwLSZhMx/uyfgX5x54/QaC7lwfk - wPnp52pK0JA2VmfFBsvlnPsVOWgwWU0jUtMGBtKlu/OonELAGaffbkhjD3eDe0B3 - X0gwtju+T4WCzQmgTFyXFdp18/S2rz19TBrq2XponRRpcXBo4qe8P59Eq++NNjAg - rYymFcTfrDZqMIt3jBv3jSnLlAnwg/H1neRfgx6kxXqxApF+vMDD7VVDaLYnLbFV - lwdkWxPrHStk05eBsLu/CkGv2ykkUAa0qqg/Dm/bkOl5lZlauVFC9wuiLh2cHgZr - tBpGuYlMFMRvx7A0+IBUTjTQswBF1+mBStUyBThOw1aJYbE2FCWj+lGrMX8qr+ad - sKsS+5DjOwCVAgMBAAGjUzBRMB0GA1UdDgQWBBRFs1lkbmy+f7TN10DGuHLa75sj - BDAfBgNVHSMEGDAWgBRFs1lkbmy+f7TN10DGuHLa75sjBDAPBgNVHRMBAf8EBTAD - AQH/MA0GCSqGSIb3DQEBCwUAA4ICAQA+6nknVdXMATXxbyk0E5fDlM/D2aceDw6D - 3kydc6wBT6k9SkhxFtrCI0IBO0f067ppFPj4ApJI0mrh3dsNiAzjqGetlGCqXjKq - pwdVxWYrmn/ELTKgvxToQG4J4PMUeo7tlhviU5HJo90DroDpoZHALubl+XsjzspA - MHH94CiF24z2nxoVolVa1qqq7WDlMWhZuit3WDo+jIgrpBgcPegNrZqrZqt/xlzM - Q91iOwwd3ykfo+xLdNuG1i8Qm1UFOEmXsustHBPDCtcI/ddLda4NbZtrNeZVmDbE - Px4Zzsjrbd6IKhVbdD5E4d9K/XMNsBIl4sidqdaUMhkNTYXkI/a5dTFVlFprLFnM - yB7A6OplQPp+LTITXW2IvD/eZetdl5073X0OQtuJrsm4mAJZ74sqXgcyVs7bGPIw - aXt1ttNH0qA5lvbp0cOIdXmpSWQfDRvlAExqX4p86549J3GMhIsVCMw5KsqduN5e - rSi0HCi24S8CeXiYV1phkSuxf0sROPuRb8SvY/6qcb73DMEMQx3zPWLV/+pG7DcG - uD327xvb3uJG0kFLi4MDnoCHLHYZeoPtwIEwG+1LjwHj+oKn12wlKJvYTxWALi4k - tKVW3QYL8beLaI3XfF+Vd3kCVpU6N9aRt3l+PpzkE1VCkayz67Sf5dJy88YIB7o4 - A08j/AH+GA== + MIIFEzCCAvugAwIBAgIUFWEYhsQ0cTg6XATOSu70TgA/BGQwDQYJKoZIhvcNAQEL + BQAwGTEXMBUGA1UEAwwOc2VhbGVkLXNlY3JldHMwHhcNMjUwNDEyMDIxODEyWhcN + MzUwNDEwMDIxODEyWjAZMRcwFQYDVQQDDA5zZWFsZWQtc2VjcmV0czCCAiIwDQYJ + KoZIhvcNAQEBBQADggIPADCCAgoCggIBAMbJgXSu4ELhULYyraB4l/fnGHf8NXNM + z/PG452XrKBGdElOScxYhkentBtGUlfsYeoanOwZikKLr3ngoMzvYIuoLCqkQbA2 + u5jlP8VUtTsMc0amYPAJeEvW/KLbH/6/BFJC9Qrt/kglvVRyDWOb2VN1Dx8RbY0Q + uWqdF460OGacStyHnRgpp43a7vb4YRKxNkb7Na1s+KYUIa6hJPgIIfLGxksiQKAr + zuRKFATImFDVzXiXLHP8c7qcQa4+bg9C194CUoWt+hC2SbrGGFEJ9M6c4TzU7/5y + aE9f4TTZfadH77XpukraI26fDnLKuspK5r3VFmEWE7d8qft6J8i9Dwl9OI7DJfUx + /dHOFAb/6oiN8TmIRtR6fnreuBv5qWq3B8wnfxmVNf22C0rS8KQaj019IQdZk1Sm + hOXfTtoZ4vZk9pw+gFdkoNF1rMqzmQnoR+IcKmWRydLx0pftQ3IpGYBqncfNnaK0 + tCag+dZbmPVtL8M5ovWX1WUoAfKSczBaxMnYO3I57R5jEEOR4BfWIdPEVVMgni2u + tO560gYMHpQHGzkaH8Doa0ZyV64rctEQhCIU4bV6JzwvDnYE8O96ej3n75NIYwKF + NmSNslExd5DP2fLlIK9mnh5q8FwFqGjqefTVUJ+0pCCMRCvID8FjcqCPEF4Xvpf7 + 6LRNSF5gerelAgMBAAGjUzBRMB0GA1UdDgQWBBTuOIW0EZNXykIIvU0L8xhOfU5z + mzAfBgNVHSMEGDAWgBTuOIW0EZNXykIIvU0L8xhOfU5zmzAPBgNVHRMBAf8EBTAD + AQH/MA0GCSqGSIb3DQEBCwUAA4ICAQBQlrQ0pfAjUSu44ZWwXpkSxNERwKI1eMUO + XroKPRSAmdB/tB7JgH3wdT/Yz7pdzvt0LKu1SlcIQXRJ5VsG9k7clhJXQ1Ogse7w + g7TvQWp49uw8OlVY/mJXlia7dgKiyREGSGjpSsUC+pHcFRmZ4N2los7agY62hZS0 + 68esldNIHLPR9PfM84jGfkk1CkFefayROL7kITVemBKHBPw5Jo8JSLnahSmfiJ1e + K7SY7u/j5PY3MobnjB0MtxaDovmJjjvZ9eQZsTSRnDsHOWZEcHZ1Eoga6NEjaGG2 + kYX8RFp8TM1xhq77/+uTWxhXyekrlxmhBY/ft/C34cm4jM0qw8db8fMXsYtPo/G1 + Cq3ry8+XpqM28TtqHy8hj9nK/WQNvJ9z892FVk7nbBcFzctomdXO9vh+p9C5PW6u + dkofuMW3EC0HuinXZoUiFMwVSGF6lm7lxgkNnn9JJa7v1OCJVqQxFGoJk2gDh+Ub + zHCuy/s4nu5MsBX3xJgp2h7/DPAr1DO04tmAUZ9OwpEyeYzWogM2cIm6yeMRNl5+ + xqhWZGh7/5s5iik1g0wB5o08IwKlTMI/b1cNbJBymgjwqGlJRRAKP2IwvFGae3sX + no3+9FB1FJjEX7sKIHFqN03w1GyrPW4qqc/9is6UaLmxTlu98QeMikel+Wjpd8bP + DAIkrU/MPw== -----END CERTIFICATE----- tls.key: | -----BEGIN PRIVATE KEY----- - MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDKG0ancYQBPDa+ - ObQkUSOpM9lqKcJym/G1hjNC2zP6dOMX3CywFcyuJwuRxwZnz8nM+l8ho7+1YlEm - h1ZNY9zi8RRF+LHftiaE0EXp0XP6FB3cGhrJAa+X61xQDt04qLvqSek7vHsAngrx - lMwfR7YkauaQhj7ujMM125kehimgiU7vNC2fH8BXUliNIA6Knln3wk33E4CX02vg - 4DH0zN881DcCVXy6N6+fesBppcSnzHFLc8NieYi69HVybiZ5ZgNIFN9ZMFXH/FqN - QwHYfIYcZlAJJhrLPAJLZgfDr7zcoFVc88Jl/y9jB9cz184DPbGHp5CcC0mYTMf7 - sn4F+ceeP0Ggu5cH5MD56edqStCQNlZnxQbL5Zz7FTloMFlNI1LTBgbSpbvzqJxC - wBmn325IYw93g3tAd19IMLY7vk+Fgs0JoExclxXadfP0tq89fUwa6tl6aJ0UaXFw - aOKnvD+fRKvvjTYwIK2MphXE36w2ajCLd4wb940py5QJ8IPx9Z3kX4MepMV6sQKR - frzAw+1VQ2i2Jy2xVZcHZFsT6x0rZNOXgbC7vwpBr9spJFAGtKqoPw5v25DpeZWZ - WrlRQvcLoi4dnB4Ga7QaRrmJTBTEb8ewNPiAVE400LMARdfpgUrVMgU4TsNWiWGx - NhQlo/pRqzF/Kq/mnbCrEvuQ4zsAlQIDAQABAoICAB2p4TXkWF6iCtrit0j5S8Wi - 4Y9Ob8bIkrJ07wMqDzf8ffRbkYeKu474Fh+gWZVVDyZxvRK/6PxjYMvJ42XaeWop - vjMC7ICMSJ0HwKsv3djfqP5MKpBEGhSvK85oUBnn0iTyEjR9VoRhrfOFRmx05wvz - UMH3ojb7HA/EmnW7dVKltJsxSlznmSB36p1UDi5UTZoqv6y7BQ3DrzrDTTHD/CP+ - ddB24aanU7SRnLok5XdHb32GkS/b7LCb/rz749O/oE2tGpcJnB2NMXE4W4X9yOVU - Mv0Uxav8s82DPTKAkbTEJ6NL98VmrfrYBMMMMqWRIYRqmLP+iB+9bUDJEuwnnB+g - HQ+W9MliiKfpVx3LpcwaT68SNLSoSHt+hHcd0b8iccyvpxag7LExe36NEmg5Xvn7 - NfTcsCeRrbFW7NWmJnS0yZ5vNpf1K6Y9MVPLlNLOYL61QJAVf5EKQxxT66xvFjE6 - L068mFAn1w0vreXIf/Z7eS+A5nU3EBHigdNJO+EjrJ9Myr5fCJewE6tCptTUdkep - ahC2VvWj7d5JOmGVgdvO7/lb2N1Zv8Z7M3aj2tj1xLIeGh5/T/GaGBn56CZpoRtl - i76E08ehrXoKVZM+ej4awmipqkW3x3qDXFTP0Khdrr0FvFptT4/kJ7rNFnrHbNhC - 9fQsHMzxohtlq285hehFAoIBAQDldt9FiaWJr1YTcaaCopJO3VR39M+Ap+Nc3LCm - G320DMzBV310q8zjnbpQ72wCm8zOV8ZRoGLfSEI0cJBsw/w0fkPK3FqPdHosUQte - 4TvYqhz7qNV4fMqgqeFgwPXLzfKjeJwd1GSyBt6+LtL6vIVXsbiFa04wTi3Zal0S - aiKMt2l2qLC1RjIIQC/go3+Be5n4r/DsmPuIKXydsBJbmYxwIp8zGxE5SqEle66i - kg8CXm3TzMnydaSXW69s4/E4TsNdsdoUMEVFX9TR9QaYBGbYa4uBgblxnwapmoCG - 6/Qx72Wwn6jclnIJ6O4oCxRwt4+PFQoB1iggqYXvUVka7pdrAoIBAQDhen7s+u5/ - Gmkna0Z2gxwpUleEAeoqAPEab5HEZYvUfQFpmsCvavSLVH7I72HmXwxox41a634U - t19JGiPajRVy3d7NWJwT775Rr3JvIPOxJz7z0Erthvovt+usmybjM0GUAygcL3KX - NI7NIE/20+FrE1t5KydSDpnwGDjUOP3qke2a2LH3gkxye98zUiSP1beS9ZtND0t0 - tb15H6kE0RzKzoDr3g0H96uIYN3f/u/vd0QcoeUbeZ4zAfFT13FLGkTzM9hryi+c - u2GlNx8jlxjEdP1WQ3R16F2wiNUDXEO0rIJ39yJxq4Q+HitHs1II0ku8Zsl4j2ji - 4NjDWnVlGsf/AoIBACq3PkkLnq1K3pMwDPc0nFfiC99oy1f/6OtYieniJDxoRZWY - W9pkQPf+XLjEWS8DIdUnsdtwJ/e409o/OiOFOGXtO2GAM7uQy02EqME43JMw18Rh - YZvUX0QZW8NPKHKQHCMiSK2k5Dnf3Yc54Dyc65Wvl9H+b8e+Rfq7Oig7acMYQaG9 - NWgJcTbn/AGtbgcEo10QI6FnuStXzcLtC/3RT7twNoKk/0fTBLydkZzh47llKrzK - 2q0wCTvD7Zwvsq/wuZ2vhhFMCxknBc8v9dJGv356RrYNGCB2oh7gjByRwBurnRqM - HjsyJniBzBPYPC3fffH80KgW3dF2vWAXnKbO05ECggEBAN2KBfvB8WNhB8B7XphC - 6gCPrv09BF6Q3lQjGw3k2CbHTAP+0SOJ6wya0JeM3JwE96RbrT+P8ilYon6o+GFc - DZ728FrCnVfZTx3jTIRz0/xTmz7jgTswS9Fm3GCTcPn6+ov7LgXDeMuoEEbrYKYc - OHeEZXQAOnodbpLAudWKybEYAGUMVI/jrkkt/HfoVZQdYGk8eIKWbIUXrDq+KMta - yvdZPsKBQjx51EgDJP1Y91ZJ0NpxFzKPpChp2DuQivhHr0dlwVbnKTDNy3sRVb8a - TnN4nxGK5XqT/LnfN6w9kyiJ9wb4Axe8dgqyicPBJPEV7uQF+h5CicOhOFl5H69f - SPECggEAFQUHIFV+GqiBwoJv7PESfHWiCwmCNWsA5q1lRLhIiWHJ8veIXAizzYGj - B+AWnMIccF3KLPvX++7o7qY4kkhAjob6AoulRg49C/g3YtuuxPe3CI7MV12nOHKc - ltzRO/CZbcg7LMNoPDrsmVoxYsN91pAg7hH83MNLjEiHhsJMKW5MXcnV+ZEiSukK - gYEyN7T2sI7sGQYEYERfFybOyqNw9RjojHKFEAlfrVa+rDbpU837AATcd4+E+aav - iwVBz6BCvn+ozhvmu2ZkOminlve3Bfnr7eJ5p8NHBwB9ELscKBZK2SNcKlu3wtg4 - w6IEdTFRspEvNwyrTH9geMzYmqbIsA== + MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDGyYF0ruBC4VC2 + Mq2geJf35xh3/DVzTM/zxuOdl6ygRnRJTknMWIZHp7QbRlJX7GHqGpzsGYpCi695 + 4KDM72CLqCwqpEGwNruY5T/FVLU7DHNGpmDwCXhL1vyi2x/+vwRSQvUK7f5IJb1U + cg1jm9lTdQ8fEW2NELlqnReOtDhmnErch50YKaeN2u72+GESsTZG+zWtbPimFCGu + oST4CCHyxsZLIkCgK87kShQEyJhQ1c14lyxz/HO6nEGuPm4PQtfeAlKFrfoQtkm6 + xhhRCfTOnOE81O/+cmhPX+E02X2nR++16bpK2iNunw5yyrrKSua91RZhFhO3fKn7 + eifIvQ8JfTiOwyX1Mf3RzhQG/+qIjfE5iEbUen563rgb+alqtwfMJ38ZlTX9tgtK + 0vCkGo9NfSEHWZNUpoTl307aGeL2ZPacPoBXZKDRdazKs5kJ6EfiHCplkcnS8dKX + 7UNyKRmAap3HzZ2itLQmoPnWW5j1bS/DOaL1l9VlKAHyknMwWsTJ2DtyOe0eYxBD + keAX1iHTxFVTIJ4trrTuetIGDB6UBxs5Gh/A6GtGcleuK3LREIQiFOG1eic8Lw52 + BPDveno95++TSGMChTZkjbJRMXeQz9ny5SCvZp4eavBcBaho6nn01VCftKQgjEQr + yA/BY3KgjxBeF76X++i0TUheYHq3pQIDAQABAoICAAi5ieTG6Ha5qxWgF6DyLLYC + TtTfUoeUZhEnQtC6ejwQ847PG0FB27zpNlK3Iqs+ElXDJBg0Y8OEHCgfgjc/Gg/v + H5HMjUnbER9GupE3j0+/k2SqWJlYdQ7xE+a1nYvMXaJwUrnPfx2yswKx5yFe0v+i + ejmCcI+PMxrOjnI35JIpBxvIQ6PD75keiZFN4/wnyn/Ri/DSwP67EKJjQLs2fNT0 + wU8aulboAqfgPMzb8rF453svjrwq0aKDCVqeM1hDS7RbCj0DSTUyvFLbWcW0HT+f + QOnZrQyQKsCxxDGSCn2NKaR56nBZbtIutCSOKNDIT6R30IhjfsCK0xluP7DuuVTn + u8jHZ/Lw3WT6YdkCC/XamQ78sz8YbsccvcIGL1IOoh41DsJ71A4De+L6awqj6d2v + n/YUx+qNrAQPC/qXU5FEb+KUOaHuCL31UjdRCjQSyFmu8OH5Z0IuYClO9c0gW3ej + QfLcMpASnQft/jbwhmK16SceEpssu7XSFbfSNDCeK+Au7jXXvl3joEr8sqfWTryR + LYRVEZH1nzpycl2Q6B7Lr8GAj3BEIpmYUiz+5Iz7n/Ocl7m0pLO0rsriL3p5nMcs + ZN3/J5vrelx+t8NJZ1idg7wUjHpRnFpvTIfdfZactZH1M22jVH2kaMsN78shIr7Z + NKCEPW3/jZPDQ1Qw58y5AoIBAQDlOsoUWwHbS2ZHCYYZ6Ef8CXpJHYpGnqfu6dEK + VOavL5HtxrrAhot/DYodATn5LdsU9iI6j+TBRgafgrcyhVSzXdpNDYCeCCroQAyl + P+p4MeC5YTKnqEOHeCEJ2hXbxL67umjykP5tAGSgNianUKO3MUm6qyJY3mpuotFj + t4zxstVxjk4xwROzSReN6VvNS8NbGZ5AkIiVaXiHk48jNOY7KwByRHh+RLk2juiK + UTLeIsVL193kF26WTFKzz+4OImk5OWmIS4bW901qsEeTMk5DEZf+YalFCs/iXjKF + BjlPkkA4U52lF8ps2RRBl97p5AuypFFFWTL6qu+5K+nkhKUZAoIBAQDeAJXjnbNQ + BZT4I1JpuUBwTQVkedDMch7aCsnxQ86I7Enu39w+5UKvgW5b4XOGGvzHpqrwNvXa + tTLjcS5o0ucL2dwAuQ2k31TxC1mHyzPsf9gFBr/qm3ej/qFM+F4qFDH5JpHRMxSR + LU4jX1Z9jk3IjDvQYgCAcQ9Q63BANoaMwIw8oitiysaM9oRe5NXGBURuegre8xGQ + mM1867z67vigG9+oWAvOiCYibq8AqS7QH4azP39H0w840SDkxPNpmyfjo/cIkKoQ + X4h8TbWHSDAWZZYzqLB9pSHfLZXZpvIaJLyqc8UnkvRbrUQuwB3nORlIMB5lc1L0 + TMnDHfb1hUxtAoIBAQC+/TiZtSna2PuifGDq/QndCNiGSsKHDHHHMbEl9o8tr2ja + wBWhi3h2p4VHnVn8LyS2Vk02leV20y15gaO5yDVGLRBR2soq4Ys+bv8aQCLPel9O + lZlq71s0Z/a/CqHjtg1kxG/kisIxd35frj6OcTW20K4phaTcfpQXEjl759bbGrGU + J4knE0OiwWeKqb//UPy3zqR4DvR96J6v91Xvc4fQDIgWrM3oBsJ6np9Jf1XulvHj + uzv5azyMpZLSlpA13p3lO9yd3q++M9ag6bb5AZULdIE7RYuPJu8GII3fH8xoa+Zu + B9z1TNkvz7582Fi2b3b/pMDCEwvSptGNavU5nPshAoIBABT5mS/YRojY/iozrCWx + CxWRMWE84S1v2tpfCpVxAqKNMK8RF44ZuyM552Nm+bNB+YAjjyMDDleLpzM2Dlx5 + ELtd6htnYJKjRzM/D0Z1PIKr0Y0oBhn0dmlYNYVAMiVnoxEXQ8+wM9FDpo5RA6r5 + NwArv7sAh2ubQwcyN3jZrfy+dIGGEPNUlU3cPMV6eXgsJzsI7N4DuTUgOsoCu+g5 + i+QUvhz5tKPe2TuHmaFGWtXvJJrq9OukVVvY5nK26QihUDOB2mVExcymOg8EiJIv + j9uN3ZMj7QjyATFE7VOTImy1691u8sG6cLv4suh33ltqWakoYrbTHpNWbfmxaxPn + dEECggEALNKGY0JSHzuwA8wL7XoiOawHJdfBx5BenpM8NpsKAFVACsVZcY4OzJHB + Dwdr4bhBcRaHrHdM1RrqIvnWnF2gaiNNifNZ9MwHberWhjqQMqdRiv7RD6wp/jPe + jeIFj/endqSWp3LhyayK8pHZDAEbx57sex0doiosyhCvqYWlcDe2XwvTxGkdpM8J + V9nqyQh7zNYvVZUby43Gl+Bdbxirolq5Jcx+Z2wxiXt2pL3JArGtnbwHFNP0oZio + 2RmFjD0ZqLQzYmWObQeYKMHcz/8kMNtNhgUax0ZXTGOGEt4bO6Iu2S/V/thVg/Wn + hnSzVDPZAGg6LcqDPW9csv6W+BCo9w== -----END PRIVATE KEY----- From e486d9cd5ba8fda35a07f117712efe26028732e3 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 10:20:47 +0200 Subject: [PATCH 16/37] split stack & demo --- .../applications/airflow-postgres.yaml | 0 .../argo-cd/applications/airflow.yaml | 3 +- .../argo-cd/applications/minio.yaml | 3 +- .../airflow/airflow-credentials.yaml | 17 +++++++++++ .../airflow/airflow-spark-clusterrole.yaml | 0 .../airflow-spark-clusterrolebinding.yaml | 0 .../argo-cd/manifests/airflow/airflow.yaml | 0 .../airflow/sealed-airflow-credentials.yaml | 24 ++++++++++++++++ .../argo-cd/manifests/minio/minio.yaml | 0 .../argo-cd/projects/airflow.yaml | 0 {stacks => demos}/argo-cd/projects/minio.yaml | 0 demos/demos-v2.yaml | 28 +++++++++++++++++++ .../argo-cd/applications/sealed-secrets.yaml | 3 +- .../airflow/sealed-airflow-credentials.yaml | 24 ---------------- stacks/stacks-v2.yaml | 12 -------- 15 files changed, 75 insertions(+), 39 deletions(-) rename {stacks => demos}/argo-cd/applications/airflow-postgres.yaml (100%) rename {stacks => demos}/argo-cd/applications/airflow.yaml (84%) rename {stacks => demos}/argo-cd/applications/minio.yaml (84%) create mode 100644 demos/argo-cd/manifests/airflow/airflow-credentials.yaml rename {stacks => demos}/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml (100%) rename {stacks => demos}/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml (100%) rename {stacks => demos}/argo-cd/manifests/airflow/airflow.yaml (100%) create mode 100644 demos/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml rename {stacks => demos}/argo-cd/manifests/minio/minio.yaml (100%) rename {stacks => demos}/argo-cd/projects/airflow.yaml (100%) rename {stacks => demos}/argo-cd/projects/minio.yaml (100%) delete mode 100644 stacks/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml diff --git a/stacks/argo-cd/applications/airflow-postgres.yaml b/demos/argo-cd/applications/airflow-postgres.yaml similarity index 100% rename from stacks/argo-cd/applications/airflow-postgres.yaml rename to demos/argo-cd/applications/airflow-postgres.yaml diff --git a/stacks/argo-cd/applications/airflow.yaml b/demos/argo-cd/applications/airflow.yaml similarity index 84% rename from stacks/argo-cd/applications/airflow.yaml rename to demos/argo-cd/applications/airflow.yaml index 9d938bc9..9196c2cf 100644 --- a/stacks/argo-cd/applications/airflow.yaml +++ b/demos/argo-cd/applications/airflow.yaml @@ -9,7 +9,8 @@ spec: namespace: stackable-airflow source: repoURL: https://github.com/stackabletech/demos.git - targetRevision: "{{ demoReleaseVersion }}" + # TODO: adapt to release-25.3 + targetRevision: "spike/argocd-demo" path: stacks/argo-cd/manifests/airflow/ syncPolicy: syncOptions: diff --git a/stacks/argo-cd/applications/minio.yaml b/demos/argo-cd/applications/minio.yaml similarity index 84% rename from stacks/argo-cd/applications/minio.yaml rename to demos/argo-cd/applications/minio.yaml index 05d27e88..bfea03c9 100644 --- a/stacks/argo-cd/applications/minio.yaml +++ b/demos/argo-cd/applications/minio.yaml @@ -9,7 +9,8 @@ spec: namespace: minio source: repoURL: https://github.com/stackabletech/demos.git - targetRevision: "{{ demoReleaseVersion }}" + # TODO: adapt to release-25.3 + targetRevision: "spike/argocd-demo" path: stacks/argo-cd/manifests/minio/ syncPolicy: syncOptions: diff --git a/demos/argo-cd/manifests/airflow/airflow-credentials.yaml b/demos/argo-cd/manifests/airflow/airflow-credentials.yaml new file mode 100644 index 00000000..aed1fd28 --- /dev/null +++ b/demos/argo-cd/manifests/airflow/airflow-credentials.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: airflow-credentials + namespace: stackable-airflow +type: Opaque +stringData: + adminUser.username: admin + adminUser.firstname: Airflow + adminUser.lastname: Admin + adminUser.email: airflow@airflow.com + adminUser.password: adminadmin + connections.secretKey: airflowSecretKey + connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@postgresql-airflow/airflow + connections.celeryResultBackend: db+postgresql://airflow:airflow@postgresql-airflow/airflow + connections.celeryBrokerUrl: redis://:airflow@redis-airflow-master:6379/0 diff --git a/stacks/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml b/demos/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml similarity index 100% rename from stacks/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml rename to demos/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml diff --git a/stacks/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml b/demos/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml similarity index 100% rename from stacks/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml rename to demos/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml diff --git a/stacks/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml similarity index 100% rename from stacks/argo-cd/manifests/airflow/airflow.yaml rename to demos/argo-cd/manifests/airflow/airflow.yaml diff --git a/demos/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml b/demos/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml new file mode 100644 index 00000000..9f763340 --- /dev/null +++ b/demos/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: airflow-credentials + namespace: stackable-airflow +spec: + encryptedData: + adminUser.email: AgBJq/dTwq0biM8b8bl2/daQSbWPRXSnOHOMnrJRjxM7A5KX68HI0Ifp4r47gSTZDQuKxNx5PaFuyU6W0tT9CvbFSPdhTXb6G5xLXITafrpd2he9LiTRzI0wzh/xa8sOChTNHK3rXBpU8n0Y7aYVDp7mibFk6CQKhMMq8s/OL5Scp2R4mbiajAtXMwQ2wt1fF3CAqR4QgxqGGr0cEugGf8zl/Z3GryUjJsqjNyhkjDxBN2E6oiP8Q1NCMV7AGGxdqHyi3PJH6hFddk07fTKu24oV1/IkdPZ7QgTXzhV2dk57KcSX8wB9gksWsCiFGlL4RnKwkmMbNzXq5m9zAk9YkN0QyMu0pRBI/NqhFTfhXIBcVsIbEa/37rGxV6oIaLqndneSG4fDTu0HP6ga98pM/nD2Nrzlj1gS+KwaBSGWq+CWs3HSah35treHxne3U2nuG98+nnIvUQUYI36b7MAP9xQ5Koykk4iU6LrVoxyck42k3VxeWNLc4W1axOBLGyjSxxUmZMWF4lmlz4Z045/SuBF8O4mmIHj/3cK5EEcAlme3lebNKht9/w3iaxDd+dG04CP8RMQGE7i3OdHpteilZMszXyjYz8m08u/wgiKG8aEkh8ad9fNeR4RnGP5CMpifrLZeCsismj4R9JN6B3lr3q2s1Ot/aedPsk4JMj0RgJAnLkSQHbYI3twJAOq4C/6HH+fSsripLIL4CjC39Hevp6Nku6ef + adminUser.firstname: AgC+TLraEdynGLvTTBkzXgBEvnK0c+97PsdcVTK3BIFD0L4APn3HM6V2WiUcyBH//YJqUiNvg5mnJZx3EA1EakPdzkXlitYNFy6XoMDTkrMZTgiD/VAiABBmPyVJesGz5xxlovqFGpbGWQbXjr+fGOINqSciQi1JMzN/jQ9EJQRnyxr7Xq3rmF4/NsWo8s+fEbQHx8SmNrS8mu9BFd+QSL5pTskZJkcaUuzduCXIheacJcRE+YwiufQVOFSo2AkgGcbwzLBRJuL6+u6MST3qr/xapsBuGSwHtEoVcI/NApTYsGZCeNN6H8e083BnRGCunfnw/R9/5Xg4uxdUEbGT16gP4H5/o8V22AmH2opSgPgxVSntAn6h5cxva2YUc5Bezn+pKFZmK0KCmC8tUSDusje3X0emMIs9+fC+ksj81UwSikC+hKbuZz+HuzS23UCy+w5R9X2SDGk802zxdLVhsXn/46dgaEqzOEa1pYJEyCRZUUv4KTdM0/XXfkid4r1uP0d6bZ7jlS9tSDQ/ajm5dAhl5W7nIYdRVAqM5KhvvY3mWpqIdscdr+bhpE46mJPstP6O4FM3/0OYPntmjxSVTFf0Uc8mHY4RRAM2sN/ViqcDGumn3Q8xlQtG4vMO2Tx+GAStzIu9AGsTFYcak3fI8kpZhs7mhYCXeigSusaIcsH5bfC7uekMrqhrKraY5hOU/xaxjydPIzN6 + adminUser.lastname: AgBzqzvmZLHUVDcG6f5cyBxUef4Ov0PNBGaqrYLKPMk4a681XpPlfJnNvMk87aV8Ts1xunoozWs4uM2Z6pQa6ZIYoR5oRkO4bLqK16frBaVypKVsKRPkol+bSyWPLGuI0izIKdAcHrtuaEffb6Sr98i6ColIqGLEYlzUalnPIuK/W7QqKXO91jHYnTv4m6LUMere1N/x1cmeZ3+W+BukWLkfiLko4wgCSauUCene9azY5ehtmlIAUo9xNC3oDA+rTGTI1SkG2Xe/ytI92sedCEMmVX7oU9PI0KGfckchc6azmOTsWN0hSnUdIHPPDn+9L0x77HwHeMfiVgO2Bai76lC5fO2b9Pw0OxID5EKY4jvIrtQ2uQANCtYupge6Z37njpzI070U/H65NDtCKAGmcuy4pFgdF60VrEPrfAN+uaVJXmJGu75QmaKqGWE0RAW9UcjiKD9YaQf7pN2tx6v2oga/OR/cDTR+LJYIcwAC2HI+v9t2cb3LqznnoixVwU83qOOqDelq/Pd1vV3Sf1R2Kj5E4UlYm/lEtSczKJyxX4Rim34kzMqZnmYP/56bs4NwXjJ8vmMHEeFkO5tL3NaVgeJ3yKvLG4yUtasXxlrSUgLdXXTu6wN7PuBUkzCN4maYHrDjJYApo1iIXhObtgRbDatosCUCJOSvE0BEEf9CKP++SkUdipWVk84QK3ryNYQOZ12Lc4rnkw== + adminUser.password: AgAXw2meNuLCnWIDeBdrUI1osShroov1kjvRJ5+mLtGHD9cOTfQlAJpGXN+QwVWXz/hsRl0zhMxI1EomJhXOOyNI67ptlaMnSV/Wtt5X92ISmHReR/1R2cXv0e/NQLbD4M3AmCpvSz0agYMBxtIFh6mVeMah2RRjVD74Xc3P3SrHjnY7CiLYe58JWVkxGAp6WO2JTpopUJI1Iagr9MHfkGzPp3ygFVxwiMjPIVCM+IeAbtHLNi2w6slrcKa3IJduLSaGOQEr/p8YcPm2Qx2TGHOpiM6Lw9api8JtN28mS5hArc7qj5kkwkfEiHW2exYMXVSdWy7glgEGFuTr7MRi0LpLqS72DT55E9uxAObnlWuiKVmk6UfExX2mMfze+Iz4sKXrqOFoLAIXx8+yjO3t5oDvJf9pxLNihl4rYyLg9v8+UkkGJLQZPOqjQoeuH9ighDkHdfkiUDdprnG9iZAQBFzKN7On+tMA09McwHkC6jV+RIv47zty1CMoEPQ2wEM+zDHU6DG2HvtHQYov1pCpXzyvMz48hcKiAwIjZQM1RZBL0tN5qd8Tp38MqtAvVt7PvllwAN2LKt9HdV5j82kyWseE6MPYkB1l8Q6ctUn5YY0kw+tWwgtmYbeRY9+kSIUphxntX+unzhCKCAY1oBOc/cdim3YQInGGe6lx3JGUQvz5LpP6+aVBPyy8HOPnUSyxdu2ZaBkG5SKhKvfH + adminUser.username: AgCVVTX2c+4moS0v6mXuNQ5v6EtxZ43hN0QYg8wgYG+GPR3qZ2Ep+FkUJq86q9dFcuX6EXXtS2h8BJr2sUrRW6QySFnhtmg7A1d4anslN3yAjLipwrreNT9cn1t4PsdIhU0q5Z2uPmMsnLF28D73AxWA59iiveIId2KSuB2lsaC8S8CeW3Df48LW3cnukHtOvc+j84YQgFdagujBtwAMCiym2pdBJG5GojTHQ/5P3AnV+iM5wzXaYIeF5UZ0wLs5WU3CqHLxsY3q7NrLKGzS0xTqqdc6H++tJQFZY5G3q4NcCx+jq9oRErE0ARz3ok2pOHPArUEbyas9+iibA9HJcCmrKguQ8QBvaIQh9L0YtHCtwvaMbR12ZmMfRtMW6pZnM/d2K648JFekuezGTOVEjLk+ZvVm45L4USZpB0MD5LXEy3m8cDdkn2wqTWvblfjgfi7opoLgpwLyf/BZuunrFWT08x9K1WmNQ7OkM4WxscLRyccFOtZQfoUtT8hW+K5/mxLjOQDpryWPCUeBDAMnBvbGG/0AABp7a+vWG9KVus2ZyfUHVFIHgJBsHXXDtONIkmRvTal5bjlRsVk4Y0O3DoJa+UjXYoIBSm1JwdUJMOnL1MoS7Ijk+iXKPMc/3fcbrnfqYxw3ymr485bL8fKh2n99MsCdR1yX/mTxw3f6bRZdN5O7IUTkkKVxx1b08FD1svSwNKHBZQ== + connections.celeryBrokerUrl: AgAD1Iu/Oggx5NbNY29aJweDDVHlnL2KnsSJwBufPSxsaQouGdIIMIsWr/y2z+qCtd+TBG7f/ZK6vvTlT8kGeq9PGguGTOs+jQObFsqOWoWMn+zbZl+H3SsXHE9ohlrq36n6b68vRrpR/zIvI/QcFg7FC8qhH7GhxzYoUfXbIVEg9ZUyg8W9GgznIM8pJD7v1blz16fXerC3D3wcY449wvQuygZucGyGNfCK9+y75/HQldYVg5lG0Gb914uT5hQ4lATlH7xnHQcxwXhT+30qphNcIJSNZW2D6S0kovtoZ8xk6P/xoO5uZNzNyFnRxIu/AhVi3T2L/dzlO9ztbxwjCu19xxDo16/vhXQ5awGcyJjHqZc8Pu4v5e7sJTElguAHooQraL/dcx9fJvVOxk+3GUMwixfzSR3R/BD2Na4l9mT3fwiW53UTYIcDm/jRwZ8EPZz6GtWVPtwoOkpe6y/OLoRtGNlJEW2fR2d4VwsrqnPV57716FuI5ETwvCSJcqaoMk1kPZBzWGjgf02owizSD7gKONBLX8G34981JsTu/vloPD9uPYQoQ9oFlKVtdmYKhL6u08ekiwLoojp51T/JB5I0RYLKSu4nR8O8SK49RH2CF/4viBs5cCezP8Cfq4qxSBHsd12PXHocUWK3v5e7xgQXi4wJzoLifNcS2FXwV+97DvHaYCM9KUogdL3K/MnPSfb6SGI5WHJkExK5Hr2WqKG+3zFIr+TfIf8V5QcrXaFEz6m0iK49x5hIvjgmnQ== + connections.celeryResultBackend: AgAEg3o3zzMI1fqVt119TiNZ2U9Uvg5ua1gSW1klHtI8P49n/ZTmYftFJcPkJfw8mjawxuSzwTvPWNirkcTyF0jpauyn2W3L7q2SwuMMmS96z4R0FBwa4a1AjP7Rovq9r4lPjtjn45IPX5VPDG2gBiscVhBFLzozFucgXVk6bLo3bu8/lXuf+5PXaTH7TtRfYuTipiY4jbvlCx5jR3omzgnLThOWaoplnfIQPQbhnLF4t9ydipLxV+Dr2ccGnT+yehHxbmROK5kRD8DgPe0oe1+qFS5zIRyG6KZBgI3+ru3wAkbREeuE/Bq9nDWmRS5JVDlOKaFqFtYU2h12LB/f+MKPhwPdvzCiWLmT+XU8WW25pjUXHDakXRm4RJI2J+24cTjue2jv9GbNF4+n+ywunUpNqEagJ3q0VFZ8o5evZWjw55qMnsE6eMpSoPygssD1sfQ5jf4W00onxjSEijR7STqi/6zBEZzKzPSij3fH5HeWmsd9MwnQ+1/J8VIighxKm2RSRn6ClRsXggMvzCCf/wg5zwYClMWage65fytKYH12lR4Dl2fNHFYexTIWvrL/36xuuKhP/Glpz6dJHKm5bHOjeNo+RMtKLTWvr3NVqMQMzK9yl0JdPJKByoqLy+xR5pKy9TacihXJXZhpEPd2IDK2/zxCI+71OIN0iNqL1KnfI5VMtFJu/jEmkgzD+Hqg4UlI3a2frgxtv5chMvIusHIxf/2fv3OoWVhfIjObD9nlnbwFQHhAfnMemeMhaxlnxQ34aI4gFZJ1m3hg + connections.secretKey: AgB9N48o16//pMoE7YPHGHQUDd6n5gQGJcsDK2kmsxM1fj4POpG8eRwxXpRlNwBcH8VhgJAWJKLWUrIy5cQ4DM77gDGcG1YOpHIXDMDzTtocLmLJmpxoFGrswv0lEB0dCZwSpOph3btBzCDuKNA/nzHD/iwoVkZiDao/CyC+wjRrudamIfC0aJJsnSo1NAzcmsP1IEaURF0LoWdAPXWCqzEHbJ6FjCdQe/5F2md30PHGt1p0x3/ro4i4gv54ZePWBAStZml/a5i219UBvGROJgBCAJYEb0wlQ5Aut+7PVzjgUfLgWpvGEl1XBXGjuR5NUOiEwcYok+QsonQcUGaNScMzzqrfWRIagAlcGnSzs4TsLUgA2opBlKClVdEqnyS1QEH062ZTrxoFJ0mGg6T1oh6rElGfzmOTMhU95nRIF3wtYh4Si20IDAbWkIJgk1j5+Vvg31KHOTQXR6WdXc1CqYuo3LVzjP4JeIvrsdyqIJC4dg2WAKDTrQ3qixDv9Yq+bT9jMHrdQPQBz1clb8g6OEqtuDeobkTJFU/vdlZRSz86/mtsujpBNHr2SPa5IwIr2ALBx61NXfalc+gp2lUDTQSZPRf/oHIyD/SbARFDCNGmf0rPO1yQ2dg85+kLLSE2+K6uPYPwqippSIyiazIzCi5FLa6B6IEqAxjdrPyBdrMu3Qgkzdhp23UaL7yRnXJkHnPtb+URhrbnqPVIDiNeF0Nt + connections.sqlalchemyDatabaseUri: AgBGW3Nva7kn/GxNKinig8hj/b0KSa6nESiU2zKPwW6ZDhwTA3Kf3jcJCfKE2ISSLY3L7qNeihVnYP3f6S00y+b7oixLxvYxUxCGaVvNON/xtK+dFYoYQfh3hXevLWYuf7Bg5et6Rr2hoDJbkq6Nix2uIEMDiRjWyll3ma3ajlVSxbt4xdboXb/OqRzlCLS433nAruu2woSvw2d4ypoDHYd0uGw/W4ushxOEIDVC61kJU3T9K7UqzUK4bTMljiYxrwMAb3cZ0dnPw2FU/fpT9SDomCWtjnktMkfbzgVWgD/oIh+2IZH0p3Y9GoIDJUhHlBFudDu/t+tQ91F2Xpy3gqmWFSHkfnCex699hx4dBqAeSlf2Frwfz0ZR0CM4aSIKlPH1eyumz8VA5z8AmK8fDcWCvwYeRdS67qPcu0SDKodAlhA0XR3CSj+357TmId8KHS+DZCisrxAuMJGMt6FbCbTRxY6lM4yJHrkP+L9Hv3474deDApZix4h64pqiH5P1hYIsiDM02EkdI5JwrRuZewcfVMDzGhve/u9nSa5dMADrDdjKZAys8hCUgi//ynfJzpOKi/WbfvNeGcYbQdvDX9dyXTbvsSYQ/+jq8DnU69RwQWF3lYIwZAWeoGAThX4jvSppuDY+FtS/1TsrymLQSpPH4lXjgSw7yomRvNx2x2n4Pa3mPpgQ2SR4qKQX4lLeE2t+R39xKQreA4FO3XItLFnZ8ddLj1mcmatGhSMmD6AWI2hmxxxJJ3Of4AnBDrCoV68fYdPk/HeY3C8JfHQnSKcG + template: + metadata: + creationTimestamp: null + name: airflow-credentials + namespace: stackable-airflow + type: Opaque diff --git a/stacks/argo-cd/manifests/minio/minio.yaml b/demos/argo-cd/manifests/minio/minio.yaml similarity index 100% rename from stacks/argo-cd/manifests/minio/minio.yaml rename to demos/argo-cd/manifests/minio/minio.yaml diff --git a/stacks/argo-cd/projects/airflow.yaml b/demos/argo-cd/projects/airflow.yaml similarity index 100% rename from stacks/argo-cd/projects/airflow.yaml rename to demos/argo-cd/projects/airflow.yaml diff --git a/stacks/argo-cd/projects/minio.yaml b/demos/argo-cd/projects/minio.yaml similarity index 100% rename from stacks/argo-cd/projects/minio.yaml rename to demos/argo-cd/projects/minio.yaml diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml index bc4d2591..00fc8d22 100644 --- a/demos/demos-v2.yaml +++ b/demos/demos-v2.yaml @@ -1,5 +1,33 @@ --- demos: + argocd: + description: Deploy Stackable operators and Airflow via ArgoCD and activate a simple Airflow DAG. + stackableStack: argocd + labels: + - argocd + - airflow + - job-scheduling + manifests: + ################################ + # projects + ################################ + - plainYaml: demos/argo-cd/projects/airflow.yaml + - plainYaml: demos/argo-cd/projects/minio.yaml + ################################ + # prerequisites + ################################ + - plainYaml: demos/argo-cd/applications/airflow-postgres.yaml + - plainYaml: demos/argo-cd/applications/minio.yaml + ################################ + # products + ################################ + - plainYaml: demos/argo-cd/applications/airflow.yaml + supportedNamespaces: + - argo-cd + resourceRequests: + cpu: 2401m + memory: 9010Mi + pvc: 24Gi airflow-scheduled-job: description: Activate a simple Airflow DAG to run continuously at a set interval stackableStack: airflow diff --git a/stacks/argo-cd/applications/sealed-secrets.yaml b/stacks/argo-cd/applications/sealed-secrets.yaml index 145df9de..73fd722c 100644 --- a/stacks/argo-cd/applications/sealed-secrets.yaml +++ b/stacks/argo-cd/applications/sealed-secrets.yaml @@ -14,7 +14,8 @@ spec: valuesObject: secretName: sealed-secrets-key - repoURL: https://github.com/stackabletech/demos.git - targetRevision: "{{ demoReleaseVersion }}" + # TODO: adapt to release-25.3 + targetRevision: "spike/argocd-demo" path: stacks/argo-cd/secrets/ destination: server: https://kubernetes.default.svc diff --git a/stacks/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml b/stacks/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml deleted file mode 100644 index bc631bc7..00000000 --- a/stacks/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -apiVersion: bitnami.com/v1alpha1 -kind: SealedSecret -metadata: - creationTimestamp: null - name: airflow-credentials - namespace: stackable-airflow -spec: - encryptedData: - adminUser.email: AgCmhs/OLHZ0j4CnTdZvfj76m+y7e62IuAeJ/LksxaTOWbHlvTJ0p8dBsp+WX3RfiuRK8M7kJDundrSKYzbkw8MISjoHF4/v4pSu6qWLBBuDPaMGh3i1bpAJ9gKUti4SGLhp7dr/Lx1oFWnY5T69WMARQqTbhHnpH70LPkFWKRvlIkCN26PbuYsE7stKXaVweEP204sfUChK5qzafE4Lcm5l+1yGFfILBBKAbBGxLX2I3O4n4LuP/iv0Ne14wGnxss80Qapt+ue8Ce16EZiJPtY+ZCXa05fYgtitVr393pYZks/fO9as76ABjWFc2lnE5GaGV58gK9V/5VRd1TBfkK3Y8UZPxqHNULlkwVh4Vy2EHHBMzhtec0m6w72aWuWFecyUNkrwIuivZ8e9tiH6aulpYqgLshnbDZCQi8yQVDNx/xLv4GH0/VvDUEIB7O13CHRL4M+OH+GYgLpJYqhTTkJNvXdrmjyK6kWeiErw8yG77akVplDGSG6LschN0DGhJVoZLLwbjvvM9em6tHu3gRSXAYhfG80bB9PMwmU9Lei+cdq6Fp7RceqxR3F6lnigP2e5gCaEpc6YYzBKEnzgcdlR1iA57D+5lBK9SwvWkppH5/dbPRkzt5+kqWaExRq8VmEnwooiBJ/4Jx4i4FKWFnx3cnDF5OYXovIKPyYPZW1HVvifrjHHZAOe1jORxYpMOwPYOLraDIO83comkVEBzCVUOh8v - adminUser.firstname: AgB3NcEnKhwMsyFPOfSTfxKFkhIg1TyeixtdqW8RSq5I6zrMX9we431QLuOSfxQlaizsm3o670wjO+4yDNpOGB4/vXaSSpA+835tiMDymJ/URR7vb1vvLV/xx6azdn5JmmHWd+yEja47TxQ1ip5KDai1Oxc0BKbgzu2Ogalq1U/56riSKSe8MvRjJ9KY8IwBpoY1rKAjTFJWavUVVt5Ms0fdSB7sL69NoKVzUQEpucTemy6k8RQFabK+KwirQ2KSDcQ5J+df/ODx1/HZKG8j81Wla7dBJ0nwIeie0StAZdZ+j9QlsWQF4zCRaEPxn9p1vken7Na/EqQdYwIV62o3mo66p06t+ObdjxKRHEO5BiY2lYqY3XcgaETBpaDfl2PiMbneLngGpBfOQSKnVZC60svqB/BpvBlMCiwcr+0ZJczFM1f+ptmJGHLJirf1kTYKvm6ezstGussB848Ca/ayhzzt9cestW9UFS/SlAo7d+N+YUKv3O+8QPIb5AdN2nPN9kcQxdh2T+ogtg3yIKsb6makP8LC6Z0Hl8C1DuzSh3Inj5VxcmkRuB9LRYymfNu+6GjkVlAWk58FUdQM5+5syMDgiHdVIMmnoTjRNm3mCm3l6g6rG33ggaAI4X1ypyjnmMlxbpRJi/+KgsQNNByYYQG1Foo3MZ7JFQojABBFMVSCPumMyjR7i8kvYCTr7acrHS3ZSjvR+Msz - adminUser.lastname: AgCYaEJ973OSMSL0vBrQcHbFtmqQls6Ids1xcPVz3/uVpxZw89Phn67Xx6dPSNHqZIYHWbZ/gV24xOJd0XbtlFINUknGgjwqjwGJHm4Q+0F1nmCVmhrRqB+Jc8I7qqb5BT6XpsgQOa0vazDc+NVKlgm2DmbIKz8VvHrhUNRb1rd33RYd3lm0Kj8lSl4QZm8tIX+BZi3J9gSLDIMzGQftg5BxS8Wt8cbYsMW8lFeZ6Dg7Fmx9r0GCeAa7kZWy8UME0saeKj13tg/oYj79MNFCu/Q510Q+YbLcQAJ22ifPgIaPdN4YK03DYL9qiIeo4im8rlgfKqHaNkfNzmXXh758T4s74Nau9KLdG+qE2+j175B9nrYjBYlT2EDoJIF6m1iiOCAc93tk4FIc3fw5RNLkOsAYdEWoECWLOdDloQsDHa+DGwHnF6d9ZpzlATnKLMOJ6F3+RAW10NzD38bWMlUUJoGZTuhVOz9tQCyFs2/SqMC8/LauMrHx2TFbKEq2kpc4MRTYwUQe9L+zQFI2Oc5HXGulXDdteShL6ayT8sXGRIsNI0lCpNhHsUauGJ7sSkW4bklGMEPzRh48+Lua2/QQKOcLWxgPx53b492/7SvWExx7rs+POSZouLdwUlngXC2D1H5yvmnPXEnnTLVGwBNDRBa+BBefocVOmq6yZGX5trkhudTr/Nu/rZQkDGUtSfiMDzd6Uo73LQ== - adminUser.password: AgA7BxW4/FggUvHMVvnYxdP6Pmju3uSskl29Lc1ywr6AbtCSqiysp29Ravyr0muT+PlEb5zmUYrct+HaffstMOudDCYw5wSm50hhelRPFgqppJgoUvXwfeUMkx73rrkddfN2RnCepUZxmkJ8PcXA38+KPwTZM4zZmOHeq7lgKDlsjgin/O0MrAeochZGJ1JmsJ0B3QUxFit4aURq4Pko67ytrWMJBO2KQRLHUm7Swf3umqjCBWuqfqhbRtKrWEOhoAEHDzKOq4fUCaV5rf2uxpvNCC0KJY6zmhPcPgZAO+Wuc+W9iNmM4ROoTFCvDS/stXcneDR7oRObgtXN5gn/0xno3xhdypg0lqfm2bEoEiokwu2GVwrzPr5iNI2HlUVhXeo9bNwjJ1alWjt0faUvwtZRSbKI1AihBF0gelbXhKT9Iyya336TTuz0riW3sFeDixHDTRoVC3ZaHzvCTXLTLtw3xgmhxM1mCebU+pxS8DV4AAIy9MfkK4ZHkjpcCdVbV5D4Mt2A9Ltm1O7qHlKsdWF0+vMmllLn7dZzAmtAiCiI9srOUh0xj+bIOqqN2Ky8a2K73JPh3kXxO5njW9N0/lFFQ4D98VpgI/DDqoZwX97yzLcthLb54xr6OUtduvX6rDakgrRZT4OCSvlGUIWzvtTcoDwuBubSuOnLh6AFNd7xz/Ik+eepInSOCMpd3HIdp5KBB3TPm9eLFcLW - adminUser.username: AgAQWpaTaFTvYsZx0p9Am3iTwoUEZ/pWiETvhVSF0NANgKe17V5b1lQj+DamMVlFUGfGgV8Qcxdyw4Lc71yzGgwEB0fQERoy54Z2lwVpYLJm2qLcXuolkzFSyZEMBBulddkTGhF/zhDZWfpOnayhfAbQyyp/T3KUuZira220DN9HiXDb5BiId1+Gzu6xTW6WD5/c+7yFiD6SWFReGpIuqHofh4y02p8A5RED2jtcr/p7ltCZ/tqjIpnZcTP5K7wauEjDiJmDbLTcdAmzdSEOI4Q952+oK3xjPAIrhhivC3DAtQ0lgV2ye9859tx+CDEEkLkjewascVNwmX1yeYPxnh8ayJfabkEpNM8KOfmrDE/AZB82FeKChmvAcpQAu3fUuxBMyUit6RkNW73fvDf8hIyOmNSZdr0f21FPhJWFYnsF4Qfm3zOo/fXuG1LJz2WB2Qy7RuTmRGW1AE4VQOvzvVDCEOq5BsVfPdlGrSfo5oeI6ctCqTL4S+2CXS3VRUkIX61QdN54CU7ZeT4cAzZGpkUP6elfsrLv0KnQc4KvvzfBNu71j3xgjjYN41aGDNi8dqWaCVebYbuQtDqbcxLxUQNX+VqBdCflMcnCOC4kAEI+B2n/rUrGLWjKUIMhZvZIKJQd437j1fvJo2AHO8TbqtORspg3/H8OuwZpwcDsWl1dVQ1B0EczZweo3I5NkytTVxNEEXjsQA== - connections.celeryBrokerUrl: AgBEl833Xp1UogzJpkpJ1zhjlw6KKAqcfWzSqQ8LIPQ96GaIUfo6G8wx7jozw7yEF0vnAuTTdzdEZ7x8/PVwWQyqNJKiS9o6mIEdkhv9mQ+EG/wzpB8dnbQR4Xl758f+dPRqbIA0RYetmcD9NsiF08obXIIzZA+G95kR6x24WiixwJL+txMNdno0Qu92fRqasv6jV22CUju/kUTRFEov5IrzGOj7gLKQ7d2yrCGZd/W9SnP9sgX1ltz78C2yiEXX7aI0n066OjJUxE7DFTJCUWNrknDcJuSjGFrWTzf8GNydrPmkVp6g5JA3fGgUTm76osischHr/0/+cwzQzEyQKyb5H/fxcAd/5yqtXhD/xibPDoVHs484PQAmgy8M3PMG8fNQ9XyrXNB/ZuxcCnt+rqPkeP0jqEsdxog2Q3U1lXfH/FU2eJr5tLHb1zNISV7vNX28VRzLoefyYFtHkp1F2yVYThmuSCPj1cqkfOpt02rYsIDIYqwryxxOvUpTAH0uNbdoIWSvXTzX2ZARUVVm0bboDgfe4RxVQofxKjk1mNPHhEZ9jYd6GX1G1SCYuSVdrRN4fLSOxlyP+vPkyzOJtQ7WmugBZFTmZ/iUN38ce4HbDIajVKtMvDqD35kkN/q4NZ3d+/YjmZA9YvJ03Hd0XNznImr8R4wqOt5oZsK2+XaMxHTzPfxV2MpLT9Xgb+CAHiGKvCaheKdRNWODhFQQXAw8l6t4z8dvYNkZPAZAD6ehFUtlvpTbHVVqtEdiTA== - connections.celeryResultBackend: AgBywQulzF0AZOPR8ypHDnsQIicOWUri9rV2sLwR8kiD+J98eT6CWvblzUv8x3/I4gJWwHbwjmACv5Ke+MD1GxqVsJYl2zc/A78SkVvNT6V+kaUFqxXMEQ3EJN7q+BHUPFphxQDS2Ur5+ssON1hth56L+HAxZLmt4XC9Fjz20cO7Dcx+vpgXG7KVtdRmLvchs0PCaMnMZDANg/rgGC9lMcgi3BROhobGapbp3+iLjiIDtNJ4SwwQpQaA5KWKXqC8Dmfvtm18lyNAK5oWSrMQI6+eWe+ML56C8bMlz3jtlZsFO6upe7eOueD1WoK1+YI+NEfvw3HDChgMaQ559x3y/pIG94F7QNtltZbfTfXptStQru+H2rxmCUBYb/BaHv3oqZsOgQe95LubQNfrpH9tIIygGZcQDZGjZsnfjmu1ahYzqiKb6UEoPxZJskIMy5BAakWQf0FiVJQa7stt7/k/NosBrUfJbyPJNWA6xVwilV7K4Auz4MuBsMk1fRofgpw3N85SeipVmU3673GIs+L1IFy8UEYAtYpv14aFOvqJrE4YBpkufr+K/IvVpYfvIBL2zAF3CKm9SEzYAQnADwb3UgOCIm8w233vogjHo5oqL8wacUeZgzrZ+T1FIR9e1tXrYxk2VLmyv5d0VOWYe0AleUk+O3rVdLpi5yUJQjvKnExVXzMMLsApTJ3dy5OHwou99Po1mlg2D6xh439hDUlAcUQz4NfGYvJtYSgAb5cjMs3bEMUcAuadGIGidlIy74HQArfXosundworwKC8 - connections.secretKey: AgBpwR5AZJUjGxJwi7rS6dLH7WJKYza4wKjPxLF+RVskwtwtkGtYSpbRZ+vXxE6pn47eJQC0OpJzreiNqYZtSR1CHv6HyHZsnqCSnLSzlUBCPp9H51nfVQ57V+S967/jEVJ3V7446gqQqdJb3IS4uwbb+G8CJsjsZiEcwx/z/OK5vu+k7TiG5jiyksxXhC7EFzYKfc/CD8Lsr6LBimUEa2i1W/PhOoDcqKa7qvDMIM/iAEilvtHXjA+EU8kxplDYy2an1cadockyyMSoHvDmSOg6BU62bK8Xqx9o0X6uaSrTufeueymtBYb/wVHB0bTxm1UXb8QdGYfDePemnP0oRNwvETrupuD0HRgEutetyxLYEEezC98ZpRM377kirX9Zqb0/fZ9YVwECBL7hznl/sp8w1hE2O76orf3iAGNMftrFtOjEyzro+41zd+bz+sy1ivGluIWwovQbgondk25AoYRC0LhGDl7kQDVX3ND/XeH2NYErAuCEyjQ1MGHLJxQGGDxoCPaDXtLMuHvXlg3H71jibzT7rV+YikAExjfUVim0kuDtKqdBwAhvRqX+Si/kCRMrIUhRowWSqx0g54U9mwC6wfbSHV90RnYtvBPbFc144WbgV7irc5bfnekidrCj8MiIPKegUByqlB26ziyexEmiRN74BH2yCJ/M2yf1IWSad0uqb5d/48BcgEP8E2BOgeeaCJ6WOMeR80qvas07hN5e - connections.sqlalchemyDatabaseUri: AgAFm5FacxXw3q9rxKT+9eTpY3FPf7NSp3wdPUUzkBpQPsEa0JvafDeIZ5/+MC9WaFBWuSt0awrIh/FyHlNW4/KRhTWzFL4h2MrDAfCEdCaOgB4wrgYJF7tGz3968eNhBn0ALX7MU6dgiGawPGvX91FgPo4dwz/5Ob5+f5MhA/paH8uBXR8W96cTTeJlcJaqLV/xMzGCInurUpOwyrLWu8zjOHNdn/11P3n5gKVlwbi6w57AKAYtqux869oh9al4kIbgIEbRYRI42Q2Xa3v+RtmFCX3CelCs6uoWtzNgt+wUbQbfi48MuHrakiJf49yrqFHdc2vfj03O8jlKLGSGcSsy5Epbb1VBuTh04jVXAO03xb6Top3gqZvxY8/gWOWPs0SejeilvubNIwj40bhzRxPj+r7TnE2dmS+KnTN1fYR1fk39C6SNjSKVncMie8PH/xfq8WOVZSGhzlIwYNMs7/WpXYLVs58Wz+pVVRcbYf/CQjRNKy4ikEf+eHwMLvmsvHnslxr9S9Bwt2An7mFkkO0VvGubCYfOK/W1SoN9NBhFmJO6Es3vfTKuS1gqgcJAzd1oz0+hz6Kno8XRpZUzdUa+P8LPo4Y+mhif+y9OH0sXFH5rYGWvLbUxEgFfdUMWQRKpGRqJiGNKFBLo0kpRt/DqG3v9wcWHbYLl+/cRLCsC46FUsPW5L73T4AEtV3TLh6Q5hZiHCWTEyxW4WyeFD6Bd7FLPNbIA8gMD0oCP3ztd+odbSDecURUzhQifICH/SCJl4VH8jvr40Rd+TjrqEj5t - template: - metadata: - creationTimestamp: null - name: airflow-credentials - namespace: stackable-airflow - type: Opaque diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 35e45149..ec207d46 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -20,8 +20,6 @@ stacks: # projects ################################ - plainYaml: stacks/argo-cd/projects/stackable-operators.yaml - - plainYaml: stacks/argo-cd/projects/airflow.yaml - - plainYaml: stacks/argo-cd/projects/minio.yaml ################################ # operators ################################ @@ -35,12 +33,6 @@ stacks: # prerequisites ################################ - plainYaml: stacks/argo-cd/applications/sealed-secrets.yaml - - plainYaml: stacks/argo-cd/applications/airflow-postgres.yaml - - plainYaml: stacks/argo-cd/applications/minio.yaml - ################################ - # products - ################################ - - plainYaml: stacks/argo-cd/applications/airflow.yaml supportedNamespaces: - argo-cd resourceRequests: @@ -51,10 +43,6 @@ stacks: - name: stackableReleaseVersion description: Stackable release to be installed via Argo default: 25.3.0 - - name: demoReleaseVersion - description: The target revision, HEAD or e.g. release-25.3 - #default: release-25.3 - default: spike/argocd-demo - name: stackableOperatorNamespace description: Stackable namespace for the operators default: stackable-operators From 46f48b44b0e714503529f52940d603b1c9f3f432 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 10:31:52 +0200 Subject: [PATCH 17/37] install all operators --- .../applicationsets/commons-operator.yaml | 42 ------------------- .../applicationsets/listener-operator.yaml | 42 ------------------- .../applicationsets/secret-operator.yaml | 42 ------------------- .../applicationsets/spark-k8s-operator.yaml | 42 ------------------- ...operator.yaml => stackable-operators.yaml} | 36 ++++++++++++---- stacks/stacks-v2.yaml | 10 +---- 6 files changed, 28 insertions(+), 186 deletions(-) delete mode 100644 stacks/argo-cd/applicationsets/commons-operator.yaml delete mode 100644 stacks/argo-cd/applicationsets/listener-operator.yaml delete mode 100644 stacks/argo-cd/applicationsets/secret-operator.yaml delete mode 100644 stacks/argo-cd/applicationsets/spark-k8s-operator.yaml rename stacks/argo-cd/applicationsets/{airflow-operator.yaml => stackable-operators.yaml} (54%) diff --git a/stacks/argo-cd/applicationsets/commons-operator.yaml b/stacks/argo-cd/applicationsets/commons-operator.yaml deleted file mode 100644 index 19a2bbc6..00000000 --- a/stacks/argo-cd/applicationsets/commons-operator.yaml +++ /dev/null @@ -1,42 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: ApplicationSet -metadata: - name: commons-operator -spec: - # this interferes with stackablectl's templating and - # cannot really be used here properly but must be provided. - generators: - - list: - elements: - - cluster: development - template: - metadata: - name: commons-operator - spec: - project: stackable-operators - ignoreDifferences: - # mitigating: https://github.com/stackabletech/hdfs-operator/issues/626 - - group: "apiextensions.k8s.io" - kind: "CustomResourceDefinition" - jqPathExpressions: - - .spec.names.categories | select(. == []) - - .spec.names.shortNames | select(. == []) - - .spec.versions[].additionalPrinterColumns | select(. == []) - source: - repoURL: https://repo.stackable.tech/repository/helm-stable/ - targetRevision: "{{ stackableReleaseVersion }}" - chart: commons-operator - helm: - releaseName: commons-operator - destination: - server: https://kubernetes.default.svc - namespace: "{{ stackableOperatorNamespace }}" - syncPolicy: - syncOptions: - - CreateNamespace=true - - ServerSideApply=true - - RespectIgnoreDifferences=true - automated: - selfHeal: true - prune: true diff --git a/stacks/argo-cd/applicationsets/listener-operator.yaml b/stacks/argo-cd/applicationsets/listener-operator.yaml deleted file mode 100644 index 9053f49e..00000000 --- a/stacks/argo-cd/applicationsets/listener-operator.yaml +++ /dev/null @@ -1,42 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: ApplicationSet -metadata: - name: listener-operator -spec: - # this interferes with stackablectl's templating and - # cannot really be used here properly but must be provided. - generators: - - list: - elements: - - cluster: development - template: - metadata: - name: listener-operator - spec: - project: stackable-operators - ignoreDifferences: - # mitigating: https://github.com/stackabletech/hdfs-operator/issues/626 - - group: "apiextensions.k8s.io" - kind: "CustomResourceDefinition" - jqPathExpressions: - - .spec.names.categories | select(. == []) - - .spec.names.shortNames | select(. == []) - - .spec.versions[].additionalPrinterColumns | select(. == []) - source: - repoURL: https://repo.stackable.tech/repository/helm-stable/ - targetRevision: "{{ stackableReleaseVersion }}" - chart: listener-operator - helm: - releaseName: listener-operator - destination: - server: https://kubernetes.default.svc - namespace: "{{ stackableOperatorNamespace }}" - syncPolicy: - syncOptions: - - CreateNamespace=true - - ServerSideApply=true - - RespectIgnoreDifferences=true - automated: - selfHeal: true - prune: true diff --git a/stacks/argo-cd/applicationsets/secret-operator.yaml b/stacks/argo-cd/applicationsets/secret-operator.yaml deleted file mode 100644 index 7312c0bd..00000000 --- a/stacks/argo-cd/applicationsets/secret-operator.yaml +++ /dev/null @@ -1,42 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: ApplicationSet -metadata: - name: secret-operator -spec: - # this interferes with stackablectl's templating and - # cannot really be used here properly but must be provided. - generators: - - list: - elements: - - cluster: development - template: - metadata: - name: secret-operator - spec: - project: stackable-operators - ignoreDifferences: - # mitigating: https://github.com/stackabletech/hdfs-operator/issues/626 - - group: "apiextensions.k8s.io" - kind: "CustomResourceDefinition" - jqPathExpressions: - - .spec.names.categories | select(. == []) - - .spec.names.shortNames | select(. == []) - - .spec.versions[].additionalPrinterColumns | select(. == []) - source: - repoURL: https://repo.stackable.tech/repository/helm-stable/ - targetRevision: "{{ stackableReleaseVersion }}" - chart: secret-operator - helm: - releaseName: secret-operator - destination: - server: https://kubernetes.default.svc - namespace: "{{ stackableOperatorNamespace }}" - syncPolicy: - syncOptions: - - CreateNamespace=true - - ServerSideApply=true - - RespectIgnoreDifferences=true - automated: - selfHeal: true - prune: true diff --git a/stacks/argo-cd/applicationsets/spark-k8s-operator.yaml b/stacks/argo-cd/applicationsets/spark-k8s-operator.yaml deleted file mode 100644 index b5686b60..00000000 --- a/stacks/argo-cd/applicationsets/spark-k8s-operator.yaml +++ /dev/null @@ -1,42 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: ApplicationSet -metadata: - name: spark-k8s-operator -spec: - # this interferes with stackablectl's templating and - # cannot really be used here properly but must be provided. - generators: - - list: - elements: - - cluster: development - template: - metadata: - name: spark-k8s-operator - spec: - project: stackable-operators - ignoreDifferences: - # mitigating: https://github.com/stackabletech/hdfs-operator/issues/626 - - group: "apiextensions.k8s.io" - kind: "CustomResourceDefinition" - jqPathExpressions: - - .spec.names.categories | select(. == []) - - .spec.names.shortNames | select(. == []) - - .spec.versions[].additionalPrinterColumns | select(. == []) - source: - repoURL: https://repo.stackable.tech/repository/helm-stable/ - targetRevision: "{{ stackableReleaseVersion }}" - chart: spark-k8s-operator - helm: - releaseName: spark-k8s-operator - destination: - server: https://kubernetes.default.svc - namespace: "{{ stackableOperatorNamespace }}" - syncPolicy: - syncOptions: - - CreateNamespace=true - - ServerSideApply=true - - RespectIgnoreDifferences=true - automated: - selfHeal: true - prune: true diff --git a/stacks/argo-cd/applicationsets/airflow-operator.yaml b/stacks/argo-cd/applicationsets/stackable-operators.yaml similarity index 54% rename from stacks/argo-cd/applicationsets/airflow-operator.yaml rename to stacks/argo-cd/applicationsets/stackable-operators.yaml index 109ecc94..46a08902 100644 --- a/stacks/argo-cd/applicationsets/airflow-operator.yaml +++ b/stacks/argo-cd/applicationsets/stackable-operators.yaml @@ -2,17 +2,31 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: airflow-operator + name: stackable-operators spec: - # this interferes with stackablectl's templating and - # cannot really be used here properly but must be provided. generators: - list: elements: - - cluster: development + # Left out as currently installed via stackablectl + # operator: commons + # operator: listener + # operator: secret + - operator: airflow + - operator: druid + - operator: hbase + - operator: hdfs + - operator: hive + - operator: kafka + - operator: nifi + - operator: opa + - operator: spark-k8s + - operator: superset + - operator: trino template: metadata: - name: airflow-operator + # {% raw %} + name: "{{ operator }}-operator" + # {% endraw %} spec: project: stackable-operators ignoreDifferences: @@ -25,13 +39,17 @@ spec: - .spec.versions[].additionalPrinterColumns | select(. == []) source: repoURL: https://repo.stackable.tech/repository/helm-stable/ - targetRevision: "{{ stackableReleaseVersion }}" - chart: airflow-operator + targetRevision: "25.3.0" + # {% raw %} + chart: "{{ operator }}-operator" + # {% endraw %} helm: - releaseName: airflow-operator + # {% raw %} + releaseName: "{{ operator }}-operator" + # {% endraw %} destination: server: https://kubernetes.default.svc - namespace: "{{ stackableOperatorNamespace }}" + namespace: stackable-operators syncPolicy: syncOptions: - CreateNamespace=true diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index ec207d46..6355b5bd 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -23,12 +23,7 @@ stacks: ################################ # operators ################################ - # currently deployed via stackablectl since it complains if no operators are deployed... - #- plainYaml: stacks/argo-cd/applicationsets/commons-operator.yaml - #- plainYaml: stacks/argo-cd/applicationsets/listener-operator.yaml - #- plainYaml: stacks/argo-cd/applicationsets/secret-operator.yaml - - plainYaml: stacks/argo-cd/applicationsets/airflow-operator.yaml - - plainYaml: stacks/argo-cd/applicationsets/spark-k8s-operator.yaml + - plainYaml: stacks/argo-cd/applicationsets/stackable-operators.yaml ################################ # prerequisites ################################ @@ -43,9 +38,6 @@ stacks: - name: stackableReleaseVersion description: Stackable release to be installed via Argo default: 25.3.0 - - name: stackableOperatorNamespace - description: Stackable namespace for the operators - default: stackable-operators - name: argocdAdminPassword description: Password of the ArgoCD admin user # generated via: `htpasswd -nbBC 10 "" adminadmin | tr -d ':\n'` From 5dca1f82cef9749447491d54b05b01422d4c16dc Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 10:34:36 +0200 Subject: [PATCH 18/37] fixes --- demos/argo-cd/applications/airflow.yaml | 2 +- demos/argo-cd/applications/minio.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/demos/argo-cd/applications/airflow.yaml b/demos/argo-cd/applications/airflow.yaml index 9196c2cf..edf5ac43 100644 --- a/demos/argo-cd/applications/airflow.yaml +++ b/demos/argo-cd/applications/airflow.yaml @@ -11,7 +11,7 @@ spec: repoURL: https://github.com/stackabletech/demos.git # TODO: adapt to release-25.3 targetRevision: "spike/argocd-demo" - path: stacks/argo-cd/manifests/airflow/ + path: demos/argo-cd/manifests/airflow/ syncPolicy: syncOptions: - CreateNamespace=true diff --git a/demos/argo-cd/applications/minio.yaml b/demos/argo-cd/applications/minio.yaml index bfea03c9..d3eaf0a5 100644 --- a/demos/argo-cd/applications/minio.yaml +++ b/demos/argo-cd/applications/minio.yaml @@ -11,7 +11,7 @@ spec: repoURL: https://github.com/stackabletech/demos.git # TODO: adapt to release-25.3 targetRevision: "spike/argocd-demo" - path: stacks/argo-cd/manifests/minio/ + path: demos/argo-cd/manifests/minio/ syncPolicy: syncOptions: - CreateNamespace=true From 95f9e5a6b2679cf245d34828907baf0369d3dbaa Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 13:41:28 +0200 Subject: [PATCH 19/37] use sealed secrets for minio / postgres --- .../applications/airflow-postgres.yaml | 35 ++++++++++--------- .../sealed-airflow-postgres-credentials.yaml | 17 +++++++++ .../airflow/airflow-credentials.yaml | 17 --------- demos/argo-cd/manifests/minio/minio.yaml | 15 -------- .../minio/sealed-minio-credentials.yaml | 17 +++++++++ 5 files changed, 53 insertions(+), 48 deletions(-) create mode 100644 demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml delete mode 100644 demos/argo-cd/manifests/airflow/airflow-credentials.yaml create mode 100644 demos/argo-cd/manifests/minio/sealed-minio-credentials.yaml diff --git a/demos/argo-cd/applications/airflow-postgres.yaml b/demos/argo-cd/applications/airflow-postgres.yaml index d59f9961..2d06423f 100644 --- a/demos/argo-cd/applications/airflow-postgres.yaml +++ b/demos/argo-cd/applications/airflow-postgres.yaml @@ -7,22 +7,25 @@ spec: destination: server: https://kubernetes.default.svc namespace: stackable-airflow - source: - repoURL: "registry-1.docker.io/bitnamicharts" - path: postgresql - # helm inspect chart oci://registry-1.docker.io/bitnamicharts/postgresql - targetRevision: 16.6.3 # 17.4.0 - chart: postgresql - helm: - # TODO this breaks naming as long as we use the airflow stack yaml which needs this svc name - releaseName: postgresql-airflow - valuesObject: - commonLabels: - stackable.tech/vendor: Stackable - auth: - username: airflow - password: airflow - database: airflow + sources: + - repoURL: "registry-1.docker.io/bitnamicharts" + path: postgresql + # helm inspect chart oci://registry-1.docker.io/bitnamicharts/postgresql + targetRevision: 16.6.3 # 17.4.0 + chart: postgresql + helm: + # TODO this breaks naming as long as we use the airflow stack yaml which needs this svc name + releaseName: postgresql-airflow + valuesObject: + commonLabels: + stackable.tech/vendor: Stackable + auth: + database: airflow + existingSecret: postgres-credentials + - repoURL: https://github.com/stackabletech/demos.git + # TODO: adapt to release-25.3 + targetRevision: "spike/argocd-demo" + path: demo/argo-cd/manifests/airflow-postgres syncPolicy: syncOptions: - CreateNamespace=true diff --git a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml new file mode 100644 index 00000000..3af276ae --- /dev/null +++ b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: postgresql-credentials + namespace: stackable-airflow +spec: + encryptedData: + password: AgAFfYyDQzfNHmkIuwkQjm8Z25AdLgelTPx43vKNggCrfHBjcBoA/5FKzwrfQxQldVFmuQtn7PpVagu6aZZfl+JBmzLwjYFA6kQPKYtLoskYvZpocfCAsWQJIp5yusI1780lT7b4JTz7LDkzVdy5aK2wm6/Nz5y5NzpEZNxThdc8tnzz0VtZHCpMdgi9q3CmhEPeWn0pOroocbCTkBg9pMiOOTH4YuCv1ca0/xtSSbFbFvN0luXhLIBBPWx7GVlJZGggVtiHNvvvCYVAGXogSXKWTAJv9HQvsV9ruHIa94UeWXnAXV9fhqq1nD700Nhmy0uDjBjntvvNTsKqhhVwOBKuvDYkmPXzSA98A+okty3+3Mv9D/fbAic20636tpzfEIAVGvnMkAhVhY4jth7QpTi4MMk18IiVOm58kSJz/oXDEpRB3CN6a7aB0HHf1xTqxKuqQ2FuZku+kriMAdgrBxCFGJPAcfi9PoqPjRWAawKrW3B9T8TJVYxnXmDvGUIaEMli9++J4laTI2ClER3udi+ZmcVZ3YZYrTdr7dhh7vyMJVLYIde9vh01FGBQZ0EOCPDp3ch0TtLKAAnH5jI0CYPvOgyLo+vUJufDIrIgRY7jwTZD9wmulQaFxH2V8CumCH5LHqOVLbY49oQZtAVrqIWUVyEejKgPkHGJiPQ63jHmz976RTdddAoi0BZ9LrLY/bqY67tctJtm + postgres-password: AgB7V06AGcL09k9vpmHOT417G5QY/sCmyN7OONfzEup898UpjD58SailnyZRSGwkhxo5Mhz9PlVP7AaFnMf+ZY/kVxEgjZ0kjBROB32e7zKwsITnp6ix7WJ85TYItde7P8YgyphmP4xxGx1crkFNjNDwJP0HtmIt9j0RN6EX+jDUupP9l6fH0LHFnJVE9NU6o2xvJa9gQs+iJ9peLzL8Nc4uXJiXrrdXl5ptrYti5w/cJpkKzlVoWpAeph390eog5QPU0dardhc/9WnmVfomjkJE4axwZl/yHbmcQTR0FgxXh24ICHRFBjPv9PdFhs7LsXqffkkt7RILEflUBBrSU2lCusn2Jscg6F4DBDR/6aa4eDu1tsk0UL2PVQzErErDt16qKPLaAygdevSsRCjXEcYlMnj+eFFAHiplEVMNc32yc70L13wf1rnndFAfXLNI2v1Aq6OoLnZ2HHFtpNxW8FKE+5vOyxykuUtU/E/alR5o3OrgZBgvmYQQLUap+NMAXjbbk/ZnVIAhPANOIs7rM2b5RLJmrEmIZkOZDPYt8/aRZezPdVb2THo2H3N1daOI6liA3WjdzFPNsawFRkDh2cBfQ308GzM49PhlGV5DDu99qDXWhy1a6nMdexZRiyv1cZ+GoBT0AKollpx6xh5XqF19d4BIpRFZlxHQ5b5heIQDNE639g3QsSTFqW9nS0NBVQLlajOJFwQ3 + template: + metadata: + creationTimestamp: null + name: postgresql-credentials + namespace: stackable-airflow + type: Opaque diff --git a/demos/argo-cd/manifests/airflow/airflow-credentials.yaml b/demos/argo-cd/manifests/airflow/airflow-credentials.yaml deleted file mode 100644 index aed1fd28..00000000 --- a/demos/argo-cd/manifests/airflow/airflow-credentials.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: airflow-credentials - namespace: stackable-airflow -type: Opaque -stringData: - adminUser.username: admin - adminUser.firstname: Airflow - adminUser.lastname: Admin - adminUser.email: airflow@airflow.com - adminUser.password: adminadmin - connections.secretKey: airflowSecretKey - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@postgresql-airflow/airflow - connections.celeryResultBackend: db+postgresql://airflow:airflow@postgresql-airflow/airflow - connections.celeryBrokerUrl: redis://:airflow@redis-airflow-master:6379/0 diff --git a/demos/argo-cd/manifests/minio/minio.yaml b/demos/argo-cd/manifests/minio/minio.yaml index 8681086f..687e37a7 100644 --- a/demos/argo-cd/manifests/minio/minio.yaml +++ b/demos/argo-cd/manifests/minio/minio.yaml @@ -5,21 +5,6 @@ kind: ServiceAccount metadata: name: "minio-sa" --- -# Source: minio/templates/secrets.yaml -apiVersion: v1 -kind: Secret -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -type: Opaque -data: - rootUser: "YWRtaW4=" - rootPassword: "YWRtaW5hZG1pbg==" ---- # Source: minio/templates/configmap.yaml apiVersion: v1 kind: ConfigMap diff --git a/demos/argo-cd/manifests/minio/sealed-minio-credentials.yaml b/demos/argo-cd/manifests/minio/sealed-minio-credentials.yaml new file mode 100644 index 00000000..5a030480 --- /dev/null +++ b/demos/argo-cd/manifests/minio/sealed-minio-credentials.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: minio + namespace: minio +spec: + encryptedData: + rootPassword: AgBSBE4tAAJHV28O13Z+lg4CAzk/snvC+2WAM/ErBUeg4JiQZBIT+9o0MAIwM/IPcT7TpjrIpjx9UUiM5o8b/xjBIkxYBkpr8dNPE57IX+yVyWSxe5CPmBqazxEHU6cdlVT+s2NrydxyEsreSd6gs3+sHRd88AHQUhf3QJiD+hV2UE33jRe/so1if3OG9om5mNa/8EOqXeBnNBTCynaMDDuGyUcCUyzh5MEdaLG1YUu9flYCqlXNEsIkDR+1vgmbX1gcYdGcAIF7C71g8KMnGSt5dtAzxyuDhtDTCaq4+z3FbwEoJ4JQpmiDZzqtR5FDVUD8yS9b01Weg2cSlIEeNq83qN40wn8PkDQ6GEkl8Dou6yBpVklgnNJ+zgr+CxmFjrE6tGJnKQK1eOZmVCW43QFOOBvhyqXE0gpIlTO6rTzwfawHyWT1Wl83qCCcslb+/YylQx4tl/7fdFda5aNRN+1KQrfhkO9kc1zqeaJgO2nI31OpZOxu2U86WF9cQ/acFtOcF4/DDfYOu2dvvzyfzazfj6NIP0P2DjIpqIt7lTZ5t0SYgWB+cf5afsR8Y2TrO4K5B4Ys1Q9RTojqTuL/lyjuhNxYm2b/sIaQbBoJthntNrwaQO/KJ2nWngeRA+lFZTgbWigR6eQaLZXMIA+GioKKLiQxDiLeIvqfVMO6fwQWpeqznBruuQjVcB1aH18wLejD89ze9ztBI9JB + rootUser: AgAydeD6HzjMO94+dVqBLipjRRMLqHBDnAgqvDc+GUwT7dUuenMl2Qjx6D4G9Lk6HIuoqGpW1ZNVs65w+IEIETUGBR23I+E4drp2DFkGamXng+flkCJIvvvnmVoHYSDvO1mGEaqiF109vPOqj8vHtNHDjM89DdVWvdMkLQzM/XWK8pjIY/9g+K+a2te9yM3MzzlQBh7DPw5WdL5Nn97Ip5dUMA9D6fSLNidSqz24TLBbS9cLvRHERy9BLow7MqxyK0017iNH8TnpIzWphE+6iH9qPXGLNVlmeq8Ex4L74x3BIyas+yC2ErXPlvBNb/PG+L/788d4ZHW8JRNnQRhI2u8tvrf/Gg55/9a6hu2DQ+oqgT5jSJTPUE0meHIwAIVtLfxHozHYPD0IIdsgZgj89yfzaM3uIv02TnqAaDU113DVq5r7yZhh/THz9euw82/lPJYpuYwuSOuarddz18mOYWgQrhfBgOINxOr/OsFBl2HAfxnJyE2J4N50o4g9P5idr8+Px8P2bIKmrIbYI/YWVkHuGrslyXJFvdQvo3VJVXp7qGVJLOu3BTujxHc13CfMMDQGWXP3LvUFtDYH2fn1LHl0TlSlzyNzFdRir/b75tNKWRRuyOQ4qUJGSES1sDCmK7C8BA7d4oD0Sn0HZt7n4xp+3+b/8Rmicm/w8Uh1syTaxgW/2KQA9iXmHT8Un6gET71NF9E8mg== + template: + metadata: + creationTimestamp: null + name: minio + namespace: minio + type: Opaque From 74b893c63ac457ed8486b640daaf013107ebb7e4 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 13:44:27 +0200 Subject: [PATCH 20/37] add zookeeper --- stacks/argo-cd/applicationsets/stackable-operators.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/stacks/argo-cd/applicationsets/stackable-operators.yaml b/stacks/argo-cd/applicationsets/stackable-operators.yaml index 46a08902..4ef1670a 100644 --- a/stacks/argo-cd/applicationsets/stackable-operators.yaml +++ b/stacks/argo-cd/applicationsets/stackable-operators.yaml @@ -22,6 +22,7 @@ spec: - operator: spark-k8s - operator: superset - operator: trino + - operator: zookeeper template: metadata: # {% raw %} From 57f70cc0a631a0f2e3580abf6b3280fbdca73998 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 13:45:52 +0200 Subject: [PATCH 21/37] fix path --- demos/argo-cd/applications/airflow-postgres.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/argo-cd/applications/airflow-postgres.yaml b/demos/argo-cd/applications/airflow-postgres.yaml index 2d06423f..3c6c3cec 100644 --- a/demos/argo-cd/applications/airflow-postgres.yaml +++ b/demos/argo-cd/applications/airflow-postgres.yaml @@ -25,7 +25,7 @@ spec: - repoURL: https://github.com/stackabletech/demos.git # TODO: adapt to release-25.3 targetRevision: "spike/argocd-demo" - path: demo/argo-cd/manifests/airflow-postgres + path: demos/argo-cd/manifests/airflow-postgres syncPolicy: syncOptions: - CreateNamespace=true From cbdb400ed57925c68d0a39f2241c109db2325a91 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 13:46:08 +0200 Subject: [PATCH 22/37] fix path 2 --- demos/argo-cd/applications/airflow-postgres.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/argo-cd/applications/airflow-postgres.yaml b/demos/argo-cd/applications/airflow-postgres.yaml index 3c6c3cec..a7c06b18 100644 --- a/demos/argo-cd/applications/airflow-postgres.yaml +++ b/demos/argo-cd/applications/airflow-postgres.yaml @@ -25,7 +25,7 @@ spec: - repoURL: https://github.com/stackabletech/demos.git # TODO: adapt to release-25.3 targetRevision: "spike/argocd-demo" - path: demos/argo-cd/manifests/airflow-postgres + path: demos/argo-cd/manifests/airflow-postgres/ syncPolicy: syncOptions: - CreateNamespace=true From 0b436835690d3e02a50978043598dcb6d9923ae6 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 13:51:14 +0200 Subject: [PATCH 23/37] fix secret name --- demos/argo-cd/applications/airflow-postgres.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/argo-cd/applications/airflow-postgres.yaml b/demos/argo-cd/applications/airflow-postgres.yaml index a7c06b18..faf49c9e 100644 --- a/demos/argo-cd/applications/airflow-postgres.yaml +++ b/demos/argo-cd/applications/airflow-postgres.yaml @@ -21,7 +21,7 @@ spec: stackable.tech/vendor: Stackable auth: database: airflow - existingSecret: postgres-credentials + existingSecret: postgresql-credentials - repoURL: https://github.com/stackabletech/demos.git # TODO: adapt to release-25.3 targetRevision: "spike/argocd-demo" From 3fa06919cc001b8a092ab4843314797cd85bf389 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 13:59:10 +0200 Subject: [PATCH 24/37] fix credentials --- .../airflow-postgres/sealed-airflow-postgres-credentials.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml index 3af276ae..a0d76fe7 100644 --- a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml +++ b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml @@ -7,8 +7,8 @@ metadata: namespace: stackable-airflow spec: encryptedData: - password: AgAFfYyDQzfNHmkIuwkQjm8Z25AdLgelTPx43vKNggCrfHBjcBoA/5FKzwrfQxQldVFmuQtn7PpVagu6aZZfl+JBmzLwjYFA6kQPKYtLoskYvZpocfCAsWQJIp5yusI1780lT7b4JTz7LDkzVdy5aK2wm6/Nz5y5NzpEZNxThdc8tnzz0VtZHCpMdgi9q3CmhEPeWn0pOroocbCTkBg9pMiOOTH4YuCv1ca0/xtSSbFbFvN0luXhLIBBPWx7GVlJZGggVtiHNvvvCYVAGXogSXKWTAJv9HQvsV9ruHIa94UeWXnAXV9fhqq1nD700Nhmy0uDjBjntvvNTsKqhhVwOBKuvDYkmPXzSA98A+okty3+3Mv9D/fbAic20636tpzfEIAVGvnMkAhVhY4jth7QpTi4MMk18IiVOm58kSJz/oXDEpRB3CN6a7aB0HHf1xTqxKuqQ2FuZku+kriMAdgrBxCFGJPAcfi9PoqPjRWAawKrW3B9T8TJVYxnXmDvGUIaEMli9++J4laTI2ClER3udi+ZmcVZ3YZYrTdr7dhh7vyMJVLYIde9vh01FGBQZ0EOCPDp3ch0TtLKAAnH5jI0CYPvOgyLo+vUJufDIrIgRY7jwTZD9wmulQaFxH2V8CumCH5LHqOVLbY49oQZtAVrqIWUVyEejKgPkHGJiPQ63jHmz976RTdddAoi0BZ9LrLY/bqY67tctJtm - postgres-password: AgB7V06AGcL09k9vpmHOT417G5QY/sCmyN7OONfzEup898UpjD58SailnyZRSGwkhxo5Mhz9PlVP7AaFnMf+ZY/kVxEgjZ0kjBROB32e7zKwsITnp6ix7WJ85TYItde7P8YgyphmP4xxGx1crkFNjNDwJP0HtmIt9j0RN6EX+jDUupP9l6fH0LHFnJVE9NU6o2xvJa9gQs+iJ9peLzL8Nc4uXJiXrrdXl5ptrYti5w/cJpkKzlVoWpAeph390eog5QPU0dardhc/9WnmVfomjkJE4axwZl/yHbmcQTR0FgxXh24ICHRFBjPv9PdFhs7LsXqffkkt7RILEflUBBrSU2lCusn2Jscg6F4DBDR/6aa4eDu1tsk0UL2PVQzErErDt16qKPLaAygdevSsRCjXEcYlMnj+eFFAHiplEVMNc32yc70L13wf1rnndFAfXLNI2v1Aq6OoLnZ2HHFtpNxW8FKE+5vOyxykuUtU/E/alR5o3OrgZBgvmYQQLUap+NMAXjbbk/ZnVIAhPANOIs7rM2b5RLJmrEmIZkOZDPYt8/aRZezPdVb2THo2H3N1daOI6liA3WjdzFPNsawFRkDh2cBfQ308GzM49PhlGV5DDu99qDXWhy1a6nMdexZRiyv1cZ+GoBT0AKollpx6xh5XqF19d4BIpRFZlxHQ5b5heIQDNE639g3QsSTFqW9nS0NBVQLlajOJFwQ3 + password: AgCt9AmvCSFrTO8mzuI2j3hmK8ue/f1Jqa/hXrIrH1Gjk5cMlELzHslhh/WVwNU5N37PhmAZLrepAoPXSis/pGt4BvwRav/+qTVW3Y7gYYtEKT5XAFBoqWmOy8xd+fwIXurzTLdDom09xaywm5uWnYjt8qk45HIpcP/eNXT1+QwSnmRx4OlHrEG6yUv/qKHFfVhngZuOnC51iH6Cpa3kna+8UYr1DZW2kyoLB+334fadOH6r0VW4Kl8FcZb15Re+fGjO5+/aQ3oBQiX+l/5GDB46C19buwvk8Ak4dTCSNXiQAHBUGXHAYR8L6iYRh1rwSr0f5LVZS5RRgco20nGMJWwTd2/hUr1prI/GOSH9kjDqQRJabxbeQKjIkYv9muEEH5qUGEN1j9m8rBUi/z9NQSx9apjB0YWX9nqXi6C1RmQ/GMYyEbWGgSXx+76UKOjmdAOnWPrriaQSR7YR8QjD5qBWYFzSbp/81O4Ua6BoFIQyfot7Owl4Z4B8Ce7T9kqFW5DBUf6uidP4hWdHvxmbxhhzwAqeotHEDCDJDINJ6zOg82ZSOaJHO6GRhTAMeERiv8WEVmUG4wgr6CaxXipShVjAdbHfnXPQoXYAhAAySQ9BgxvluAJxoY6cLQLAv34N23LND+bPdulY+ylLIdkR+eSp2uA7nPGTf5gofL1pQx4w3+ncGF6kc7lFRdLLM4kgRvUx8WYeDzqt + postgres-password: AgAFZH+PLIC4MCIE8dp+U/Y+DAu9B7Te2aZZjr7Tu1j7FjZSBXDSfu1XBC4d9a/Brwhlolw/HhBsKie6gQXWOxhIzI4wH9FN+mnkWwfnh9xXD2TU4XIRivARKZ10dpxsyRFTZzbKJNxDjnaTZxJMYQRQ24kX9WUHQYNjDIh8r2SVEjIeE5NMP7nPPJJVkqYqIeu5LyhDtyX80rsHQNI65d2Uot/9RIG7BEi4EF2MDHiEK0/O65pP+nH2VOJqxiHjp1P+GppE5MotFmLVOsL2q+z2nBSlNHT9ocBE17ozUPmWaIdvjmPQ5WkQxSHkqea/pD53geCB+Tr5jTOrP5KgulCFJOKIJ/77Ey9zEMlOl3cmqoD5k90S+AVJ+1WLVtdpHmkSJkdiwSAClxzH3LVgOcAxK5BzhCU3Z2AblzYmA4UiIxEHSdmdJleG2mBKyU1w5fjV77S6rn+vzjqzPsTF9l/siuvAxQn4wxHYGzerVe+N8FdPhTclTzmJ0usbbZZhfhvjLl85EW2LCAHpWDYx4D9ZU6RayhfxLqlTbFHj2stW8LCHogRkpktWT/Z9DoEy+9KQkdNxwV3PSR0wsV9A9ntGDBmdJafD3s0aDPsLbIxaONAOeAWxOpNY7POwBnH+uWLIrT1T3RNNrFaS8mjUU/1pMgZYBaMd9xMWLJ0q5JObNG3Oy6AWJ9gYupvZiv/nLprMgXwHyBFaILvSXLz9YldH template: metadata: creationTimestamp: null From d2b82f78722a95ed89be07ecc295845e116c84d7 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 14:08:25 +0200 Subject: [PATCH 25/37] attempt to fix secret --- .../airflow-postgres/sealed-airflow-postgres-credentials.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml index a0d76fe7..ef5896d3 100644 --- a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml +++ b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml @@ -7,8 +7,8 @@ metadata: namespace: stackable-airflow spec: encryptedData: - password: AgCt9AmvCSFrTO8mzuI2j3hmK8ue/f1Jqa/hXrIrH1Gjk5cMlELzHslhh/WVwNU5N37PhmAZLrepAoPXSis/pGt4BvwRav/+qTVW3Y7gYYtEKT5XAFBoqWmOy8xd+fwIXurzTLdDom09xaywm5uWnYjt8qk45HIpcP/eNXT1+QwSnmRx4OlHrEG6yUv/qKHFfVhngZuOnC51iH6Cpa3kna+8UYr1DZW2kyoLB+334fadOH6r0VW4Kl8FcZb15Re+fGjO5+/aQ3oBQiX+l/5GDB46C19buwvk8Ak4dTCSNXiQAHBUGXHAYR8L6iYRh1rwSr0f5LVZS5RRgco20nGMJWwTd2/hUr1prI/GOSH9kjDqQRJabxbeQKjIkYv9muEEH5qUGEN1j9m8rBUi/z9NQSx9apjB0YWX9nqXi6C1RmQ/GMYyEbWGgSXx+76UKOjmdAOnWPrriaQSR7YR8QjD5qBWYFzSbp/81O4Ua6BoFIQyfot7Owl4Z4B8Ce7T9kqFW5DBUf6uidP4hWdHvxmbxhhzwAqeotHEDCDJDINJ6zOg82ZSOaJHO6GRhTAMeERiv8WEVmUG4wgr6CaxXipShVjAdbHfnXPQoXYAhAAySQ9BgxvluAJxoY6cLQLAv34N23LND+bPdulY+ylLIdkR+eSp2uA7nPGTf5gofL1pQx4w3+ncGF6kc7lFRdLLM4kgRvUx8WYeDzqt - postgres-password: AgAFZH+PLIC4MCIE8dp+U/Y+DAu9B7Te2aZZjr7Tu1j7FjZSBXDSfu1XBC4d9a/Brwhlolw/HhBsKie6gQXWOxhIzI4wH9FN+mnkWwfnh9xXD2TU4XIRivARKZ10dpxsyRFTZzbKJNxDjnaTZxJMYQRQ24kX9WUHQYNjDIh8r2SVEjIeE5NMP7nPPJJVkqYqIeu5LyhDtyX80rsHQNI65d2Uot/9RIG7BEi4EF2MDHiEK0/O65pP+nH2VOJqxiHjp1P+GppE5MotFmLVOsL2q+z2nBSlNHT9ocBE17ozUPmWaIdvjmPQ5WkQxSHkqea/pD53geCB+Tr5jTOrP5KgulCFJOKIJ/77Ey9zEMlOl3cmqoD5k90S+AVJ+1WLVtdpHmkSJkdiwSAClxzH3LVgOcAxK5BzhCU3Z2AblzYmA4UiIxEHSdmdJleG2mBKyU1w5fjV77S6rn+vzjqzPsTF9l/siuvAxQn4wxHYGzerVe+N8FdPhTclTzmJ0usbbZZhfhvjLl85EW2LCAHpWDYx4D9ZU6RayhfxLqlTbFHj2stW8LCHogRkpktWT/Z9DoEy+9KQkdNxwV3PSR0wsV9A9ntGDBmdJafD3s0aDPsLbIxaONAOeAWxOpNY7POwBnH+uWLIrT1T3RNNrFaS8mjUU/1pMgZYBaMd9xMWLJ0q5JObNG3Oy6AWJ9gYupvZiv/nLprMgXwHyBFaILvSXLz9YldH + password: AgAwuhroRPQeCx9j4lS/+pdIfnKzb9OiOutLfN80HnRK8+4nAACpwv+pXUyMDpp4IgY913//lvGHL9EYEUxN8V2LCI6ev+cd2MnbCL1zNE0RzFkeVEQgNlej+o+I/iS2KIHw8Z9q9vKmsfRFY1eOI+j86v5SA6sGSNFn4oRaYPapCZimqmi1BY9gV46mJC91hEJTu2HRZzr19JaZPb/CFudL6lX8oM5MmgKBJdlHCxX+G95h2zd2RiMpUhUpOSwESQ4cZFnRtTvTQiQaUBo72893ZGJGXP9AH+pqIp9p+UhAn56HV2StDaiC9weIz6RcfXCj/jFJiBWLzixqXGyQUhBepviE2DTY8Kguv/7c+bJ58Lp4qRMxVJrk/HxCJ+yiqDOfr+G6oxdhAJmQpqOp6u95ZOqPjvU0X9Pg3vwdw2KTdmxD6RVrsl3ZVv84Gey2HCkSe02JSO/9/JFJjA3fBnlzMyp+Bli/4/2ZjkDL0cxDdI7AOIPyEKitp1gJqJDGz7frKffKsWdHkrsRxPQhfk40BwbBG8z/V/gX6IEi9yhWTD1oP+r6tRA9ZxeJP4CNp6DC+9de/ACLt2UjxfKUP3No4fr+1ZuRWzwJa4ExTR3vWIVPLru8WAi4xoJvQclf6GIuwR3qmqj3oBDJJp1v2VAQgDc7U2oD3QdayfxPCuqJHt4uV2gsUFcfVVvePvoIvz4aTwRDtD79 + postgres-password: AgCo/nYBzH2rLBsSZjYX47OI/NCblDtXSvHvhH2IA8FKoZCcxh+ZV0Mg3YaZpaeSsT59FnwnbApFfVrCxXEFdubpWNvwWt+wE0UFZazUdst4x+OdUQlG2b4V6hs9pbt94h7G+OXDsgMblAACitqRaJufOaemgBh6ZU8bhU2flgMn02hFWaooilhQKoKWybDRNEr5oMzVRaxAAVoWhGzjoCRX0J73ZwCHUKvTZ/CH6vgoiKq4q2HpMEzxbZFTfOAsbWgEGxmk7230WQTIBn9ynERPAEgo5jRyug1cVtH3+t7dgH+HthkgI+m5tfgc7bEbBPtbaW53O2M31FZGMVTwlUl5AyqHZyUWU8DT2PlrUx/vQYFip8BtpBdIaKKWLQBv97YKaGSGLy1YoWuaTXAJaQhbcfeAa32+929P0DYD/8aVV/7HfbyIyrS4ZzJr5z4xa9942fxSnxwAHfNDGxrgSig1M6q2vNFpXppC/HP2TN9OGTh34HlsE4k5a4a66Ue94kA+zZtis2t1kWmrI3MrJ4p6wo6LOYqOVrb5g6D9HVwG+0nyPoWUtnDK6g/X07zQNAo8i6O36WspS05UGpAWX6idgVkPTBMq3TePKJweRhLFHUnwLyeylqQDJP0shhGrItvbfFDtmLBaxnEdAWoG3BU1qfw2gWMXR+zQqMvhXF5P8roZPL/FsYHgqH/WUTWiwW2YNBDFG8EPSOby8qo= template: metadata: creationTimestamp: null From a7a95fcbef28f9f59536dd0f96253baf3c9cdf47 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 14:30:11 +0200 Subject: [PATCH 26/37] seal minio connection --- .../applications/airflow-postgres.yaml | 12 ++++---- .../sealed-airflow-postgres-credentials.yaml | 4 +-- demos/argo-cd/manifests/airflow/airflow.yaml | 29 ++++++++++++++----- .../sealed-airflow-minio-connection.yaml | 16 ++++++++++ 4 files changed, 46 insertions(+), 15 deletions(-) create mode 100644 demos/argo-cd/manifests/airflow/sealed-airflow-minio-connection.yaml diff --git a/demos/argo-cd/applications/airflow-postgres.yaml b/demos/argo-cd/applications/airflow-postgres.yaml index faf49c9e..8a550d1d 100644 --- a/demos/argo-cd/applications/airflow-postgres.yaml +++ b/demos/argo-cd/applications/airflow-postgres.yaml @@ -20,12 +20,14 @@ spec: commonLabels: stackable.tech/vendor: Stackable auth: + username: airflow + password: airflow database: airflow - existingSecret: postgresql-credentials - - repoURL: https://github.com/stackabletech/demos.git - # TODO: adapt to release-25.3 - targetRevision: "spike/argocd-demo" - path: demos/argo-cd/manifests/airflow-postgres/ + # existingSecret: postgresql-credentials + # - repoURL: https://github.com/stackabletech/demos.git + # # TODO: adapt to release-25.3 + # targetRevision: "spike/argocd-demo" + # path: demos/argo-cd/manifests/airflow-postgres/ syncPolicy: syncOptions: - CreateNamespace=true diff --git a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml index ef5896d3..9a4bbca6 100644 --- a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml +++ b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml @@ -7,8 +7,8 @@ metadata: namespace: stackable-airflow spec: encryptedData: - password: AgAwuhroRPQeCx9j4lS/+pdIfnKzb9OiOutLfN80HnRK8+4nAACpwv+pXUyMDpp4IgY913//lvGHL9EYEUxN8V2LCI6ev+cd2MnbCL1zNE0RzFkeVEQgNlej+o+I/iS2KIHw8Z9q9vKmsfRFY1eOI+j86v5SA6sGSNFn4oRaYPapCZimqmi1BY9gV46mJC91hEJTu2HRZzr19JaZPb/CFudL6lX8oM5MmgKBJdlHCxX+G95h2zd2RiMpUhUpOSwESQ4cZFnRtTvTQiQaUBo72893ZGJGXP9AH+pqIp9p+UhAn56HV2StDaiC9weIz6RcfXCj/jFJiBWLzixqXGyQUhBepviE2DTY8Kguv/7c+bJ58Lp4qRMxVJrk/HxCJ+yiqDOfr+G6oxdhAJmQpqOp6u95ZOqPjvU0X9Pg3vwdw2KTdmxD6RVrsl3ZVv84Gey2HCkSe02JSO/9/JFJjA3fBnlzMyp+Bli/4/2ZjkDL0cxDdI7AOIPyEKitp1gJqJDGz7frKffKsWdHkrsRxPQhfk40BwbBG8z/V/gX6IEi9yhWTD1oP+r6tRA9ZxeJP4CNp6DC+9de/ACLt2UjxfKUP3No4fr+1ZuRWzwJa4ExTR3vWIVPLru8WAi4xoJvQclf6GIuwR3qmqj3oBDJJp1v2VAQgDc7U2oD3QdayfxPCuqJHt4uV2gsUFcfVVvePvoIvz4aTwRDtD79 - postgres-password: AgCo/nYBzH2rLBsSZjYX47OI/NCblDtXSvHvhH2IA8FKoZCcxh+ZV0Mg3YaZpaeSsT59FnwnbApFfVrCxXEFdubpWNvwWt+wE0UFZazUdst4x+OdUQlG2b4V6hs9pbt94h7G+OXDsgMblAACitqRaJufOaemgBh6ZU8bhU2flgMn02hFWaooilhQKoKWybDRNEr5oMzVRaxAAVoWhGzjoCRX0J73ZwCHUKvTZ/CH6vgoiKq4q2HpMEzxbZFTfOAsbWgEGxmk7230WQTIBn9ynERPAEgo5jRyug1cVtH3+t7dgH+HthkgI+m5tfgc7bEbBPtbaW53O2M31FZGMVTwlUl5AyqHZyUWU8DT2PlrUx/vQYFip8BtpBdIaKKWLQBv97YKaGSGLy1YoWuaTXAJaQhbcfeAa32+929P0DYD/8aVV/7HfbyIyrS4ZzJr5z4xa9942fxSnxwAHfNDGxrgSig1M6q2vNFpXppC/HP2TN9OGTh34HlsE4k5a4a66Ue94kA+zZtis2t1kWmrI3MrJ4p6wo6LOYqOVrb5g6D9HVwG+0nyPoWUtnDK6g/X07zQNAo8i6O36WspS05UGpAWX6idgVkPTBMq3TePKJweRhLFHUnwLyeylqQDJP0shhGrItvbfFDtmLBaxnEdAWoG3BU1qfw2gWMXR+zQqMvhXF5P8roZPL/FsYHgqH/WUTWiwW2YNBDFG8EPSOby8qo= + password: AgAiudvaW+Vk7SUbXFSNuHOqfLeZzfNTg2mVDv7+YaZicFhO8JjspjjGEzBsxZqiXEVAkx4Ui2IvLe+SeTAM3NNE62VpG6JTipzF1QEC8seh6CpvlitkCWcmu4izhUY/30qQVRmcuWDbiGOJMCQfd9fQNr9BGPNf4v0Bm4/JRBQlAzZHjNuZvlrQXDXh8VJpaNix4+h7AWKbhADUwex37Qm3reumYGyB2s9WDQ1wzpZxo4N6lxn4CMOmEBt6ROZF/ZJlhYH5FOfB5531Mv2ohs1F6aR1ms+KnJUJOw27LVH9rOqbLv22LN63KfgHHTlh7xY4Tg4sMDa2rbJE68BtYtdvFD4cTp6Q3G/6sWlfEbW4OTDghrW++Ah/y1Oyaz/2Fl6Gwg6iP8naKqqX6dznuuDeRFgZEgQGgaJYjbe/ql+9HdIjlbk07Oe04XK6y5COzncN+RP9QbgsvSjXhohjmOsPLAZdSgwZRh7wcwdxSxmvP6oZCm9bxBMmWQm3TNFp3j3X556sAN81zhjbG4uMqlzcFyfFUxzDstiDlUGISMgGCrzGZaRL2esJpZA4SH5awISsr+su8uzMV66byvj53CuTzFqFM/rdh4H/ZQPasdsXHjBJRF9f2Mbve4KI8A8Sf0ci1HTVRWkS3rtQfqNbtd+wLWU30A2NHbYLEuHtN3K3R6FV54LA4872JyMlNJkkbT9Etpa3eNZO + postgres-password: AgCWp/TsvOCQIpg29hQosIGxR2T1bCWP4fT7NfBz5k1jmsI7i3VQNDE7/mO4mfEZV+EJLuHGfeSggAq1wxbJclMm2CzgarEstOtrduG/M1Ak4o3JJCmHcMLbJMghVSMoBIOPoYfSLB6UQUyr8FsvutIpXRcet41pFN6G4n/YZ1xDBnfiE6d8EDnJeEvFB5zTkQ9cwsDMUf2PKEZMH4SY+pumDjqPZXEi4uW+6JqL79u4KlcUqSDmB3gQ3PSlovMm9ls3zDokbH1TchR+4ycG+qen8NpD5zplcjF5RsM3mx++5KjPykcO0bzxJbygUylU52oMPv6Yo129k4KI1JNeDSqug3ZxgmgQlEs5V5aAlZYrObd1PRJZaZO+9vlDeoFrc++4Zysu0ydCWApjZqOZdXPZgIe8yNXT41b2XcEZvUgZT7rrr6n6KrTD31X2z4GHWw+sdYfIDDGVyn9YHfUjKOhds18MDi4iKLPRhWrRq+w7Rhyrxve4MnZlWoNQ8/YwN2Qkhb8EWMFEb/+EcpElUTwC8yVSd9V0dDkwlEnENb079i404qzKs7R3hP67fxr6s8A2F6Hfdty8XV8HRbMV/b7fzMriO8R3KRwJZUR93EesqkfQQrTgppwSu4+llMCTeO4GQf2Y+wRBnXE5j2YZmwgL/+vkpiXeLn2Kfc+vFsUJS7ANthf7RRRwchmlJhPAWOhv9XFbA9DwYrYXUSU= template: metadata: creationTimestamp: null diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index de8613d1..8aee92a5 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -43,6 +43,25 @@ spec: - name: minio-tls mountPath: /stackable/minio-tls webservers: + envOverrides: &envOverrides + AIRFLOW__CORE__DAGS_FOLDER: "/dags" + AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" + # Via sealed secrets, just kept for reference here + #AIRFLOW_CONN_MINIO: "aws://admin:adminadmin@/?endpoint_url=https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000" + AWS_CA_BUNDLE: "/stackable/minio-tls/ca.crt" + AIRFLOW__LOGGING__REMOTE_LOGGING: "True" + AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER: s3://demo/airflow-task-logs/ + AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio + podOverrides: &podOverrides + spec: + container: + - name: airflow + env: + - name: AIRFLOW_CONN_MINIO + valueFrom: + secretKeyRef: + name: airflow-minio-connection + key: airflow-minio-connection config: resources: cpu: @@ -53,19 +72,13 @@ spec: gracefulShutdownTimeout: 30s roleGroups: default: - envOverrides: &envOverrides - AIRFLOW__CORE__DAGS_FOLDER: "/dags" - AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" - AIRFLOW_CONN_MINIO: "aws://admin:adminadmin@/?endpoint_url=https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000" - AWS_CA_BUNDLE: "/stackable/minio-tls/ca.crt" - AIRFLOW__LOGGING__REMOTE_LOGGING: "True" - AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER: s3://demo/airflow-task-logs/ - AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio replicas: 1 kubernetesExecutors: envOverrides: *envOverrides + podOverrides: *podOverrides schedulers: envOverrides: *envOverrides + podOverrides: *podOverrides config: gracefulShutdownTimeout: 30s resources: diff --git a/demos/argo-cd/manifests/airflow/sealed-airflow-minio-connection.yaml b/demos/argo-cd/manifests/airflow/sealed-airflow-minio-connection.yaml new file mode 100644 index 00000000..8e9f4f8d --- /dev/null +++ b/demos/argo-cd/manifests/airflow/sealed-airflow-minio-connection.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: airflow-minio-connection + namespace: stackable-airflow +spec: + encryptedData: + airflow-minio-connection: AgAHQL1NIeZiWH1opQYqHINixw0muWaNsYKMhtgJqqdVMLeKIU/sckZ7Z6aokaEYXGVQRe+Tp+D9AaYaF5CdKeMQKQ3MmhxXYZh0EuAcNRyvz4u4K696KEIa4XpaT5k6oRbIwRUGCc4hiRVApmMOgGYl4iEwVQ/1Twd5ffEAohMsySgZQBn17zB+hKC3u2AcIrZ1btHvBrVhC4pil/kyytBwjTkMq1iAkF8lwt7gatLukD9ldLaVdYTe21R+nP/UT7DwHRnBlI1dzkfrMdplFud2XIEmSrFF7ihzc5r+Lsm2vsnjaV5OgzDjIPI3RRA7g8/06Lebpru/E9h6HB5RIu59ga72g02QIHh5HffGUCnbEsLTDDbBcuAQB2jSodotfsofVKOhZChOIJDchvohhrQ86wqlkQyjfNMq2Gh46Nc7ZjomlpL/p0afmak34GFoPYMCD7IJ2DafEsmKA8dYfjTI1kVzLhYuoeDB3f82tu8xL9cXCEUsw7Y7EosZjeITeLcV1oMm+jeHzNHGaLmCI9k1sY5bL6xjbG59sOvuEgI302tS9ua592u9uqOOKKjE4eZM7CAmEjgMmDPbeteXGnZcOqfj8FWmlp0f4gm9s+68KH5PDtkKdtBckNjDeLj3QhMC2gstXnPJR3Hnm/bFQ4i/6XpnkuOFCvQT/Baar5scw4vhJ0tN8TGeMd651YXU7N2MEB+2WtiO0StbzhcWIoiGmvN+SRLawPZ87Sqlt26/bs5+jnuEv4Fy1oXhxbZwuf/YsZjEaKFI+HbFVvwM0K2d00/z9DYAB44oY2PmrZwsk0KqP2hkuumq + template: + metadata: + creationTimestamp: null + name: airflow-minio-connection + namespace: stackable-airflow + type: Opaque From 450775e81ec06e3430defdbc8466d1a3230cab69 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 14:35:26 +0200 Subject: [PATCH 27/37] fix secret --- demos/argo-cd/applications/airflow-postgres.yaml | 12 +++++------- .../sealed-airflow-postgres-credentials.yaml | 4 ++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/demos/argo-cd/applications/airflow-postgres.yaml b/demos/argo-cd/applications/airflow-postgres.yaml index 8a550d1d..faf49c9e 100644 --- a/demos/argo-cd/applications/airflow-postgres.yaml +++ b/demos/argo-cd/applications/airflow-postgres.yaml @@ -20,14 +20,12 @@ spec: commonLabels: stackable.tech/vendor: Stackable auth: - username: airflow - password: airflow database: airflow - # existingSecret: postgresql-credentials - # - repoURL: https://github.com/stackabletech/demos.git - # # TODO: adapt to release-25.3 - # targetRevision: "spike/argocd-demo" - # path: demos/argo-cd/manifests/airflow-postgres/ + existingSecret: postgresql-credentials + - repoURL: https://github.com/stackabletech/demos.git + # TODO: adapt to release-25.3 + targetRevision: "spike/argocd-demo" + path: demos/argo-cd/manifests/airflow-postgres/ syncPolicy: syncOptions: - CreateNamespace=true diff --git a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml index 9a4bbca6..bd47fe64 100644 --- a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml +++ b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml @@ -7,8 +7,8 @@ metadata: namespace: stackable-airflow spec: encryptedData: - password: AgAiudvaW+Vk7SUbXFSNuHOqfLeZzfNTg2mVDv7+YaZicFhO8JjspjjGEzBsxZqiXEVAkx4Ui2IvLe+SeTAM3NNE62VpG6JTipzF1QEC8seh6CpvlitkCWcmu4izhUY/30qQVRmcuWDbiGOJMCQfd9fQNr9BGPNf4v0Bm4/JRBQlAzZHjNuZvlrQXDXh8VJpaNix4+h7AWKbhADUwex37Qm3reumYGyB2s9WDQ1wzpZxo4N6lxn4CMOmEBt6ROZF/ZJlhYH5FOfB5531Mv2ohs1F6aR1ms+KnJUJOw27LVH9rOqbLv22LN63KfgHHTlh7xY4Tg4sMDa2rbJE68BtYtdvFD4cTp6Q3G/6sWlfEbW4OTDghrW++Ah/y1Oyaz/2Fl6Gwg6iP8naKqqX6dznuuDeRFgZEgQGgaJYjbe/ql+9HdIjlbk07Oe04XK6y5COzncN+RP9QbgsvSjXhohjmOsPLAZdSgwZRh7wcwdxSxmvP6oZCm9bxBMmWQm3TNFp3j3X556sAN81zhjbG4uMqlzcFyfFUxzDstiDlUGISMgGCrzGZaRL2esJpZA4SH5awISsr+su8uzMV66byvj53CuTzFqFM/rdh4H/ZQPasdsXHjBJRF9f2Mbve4KI8A8Sf0ci1HTVRWkS3rtQfqNbtd+wLWU30A2NHbYLEuHtN3K3R6FV54LA4872JyMlNJkkbT9Etpa3eNZO - postgres-password: AgCWp/TsvOCQIpg29hQosIGxR2T1bCWP4fT7NfBz5k1jmsI7i3VQNDE7/mO4mfEZV+EJLuHGfeSggAq1wxbJclMm2CzgarEstOtrduG/M1Ak4o3JJCmHcMLbJMghVSMoBIOPoYfSLB6UQUyr8FsvutIpXRcet41pFN6G4n/YZ1xDBnfiE6d8EDnJeEvFB5zTkQ9cwsDMUf2PKEZMH4SY+pumDjqPZXEi4uW+6JqL79u4KlcUqSDmB3gQ3PSlovMm9ls3zDokbH1TchR+4ycG+qen8NpD5zplcjF5RsM3mx++5KjPykcO0bzxJbygUylU52oMPv6Yo129k4KI1JNeDSqug3ZxgmgQlEs5V5aAlZYrObd1PRJZaZO+9vlDeoFrc++4Zysu0ydCWApjZqOZdXPZgIe8yNXT41b2XcEZvUgZT7rrr6n6KrTD31X2z4GHWw+sdYfIDDGVyn9YHfUjKOhds18MDi4iKLPRhWrRq+w7Rhyrxve4MnZlWoNQ8/YwN2Qkhb8EWMFEb/+EcpElUTwC8yVSd9V0dDkwlEnENb079i404qzKs7R3hP67fxr6s8A2F6Hfdty8XV8HRbMV/b7fzMriO8R3KRwJZUR93EesqkfQQrTgppwSu4+llMCTeO4GQf2Y+wRBnXE5j2YZmwgL/+vkpiXeLn2Kfc+vFsUJS7ANthf7RRRwchmlJhPAWOhv9XFbA9DwYrYXUSU= + password: AgAW0KA8iN9DeYjbGhYkM9xP4NwWK78YhZR43WI20igr7ZZucfHhANCWdCcNZFaakS73B/XK4wgN/gg9N+XJQY8MsDsfYrvVNVmrEBP1Hs7ryMbbDEGWuc8WTB7oq57O7HEtbI/anitpKLQHvciTpfz7m2gxpxd/DmZdMgKVER2CHqIH6r1pkREQWDKx2rBhPForFd8nXs5J5ALgD32XMCUE7P/nqgk0yr5b3OxFUflweAFMaQcTRpwu3AEeoQSOFquPXDaqNclww8FOPrE2EeitPq0YV1O5smFHlazYMqr8wk+WQuRK48Hn3/8SVHZEmKjH6it85//+VVaVhNmuUQlaxSBTZqY4aXI47Vfn92i3ICmUp7pGcD2AfNJ2+Xp2krXUQW+LVyVHI/0JpJsiGAz9I/CHr5Gs23R6GmLedzZJR3SVIvEWkxd2Y6iXow5Yvz2v5UgRXugn5ai2gPBqwrGUf5msuCV83f8LgcAVgZbpL2izfyuXLVvrE1Snv1/IJbPpPxTI9gb5xOS7KHQbveiq+QL5rRcVemLKUkzKn9dXLynCxyUo/QYRDdl83kaOacwFZN38yke01wGe0aJ9G9935DCheDKijEXCTbjzbjr5LuQYMxEBNmNiAz1Iq64E3W0O9PG0Nyhqd445gA/XWqk6r9ExCDhQ+TyGp2ZNMgNtlDuzc2eb/bqw3LTyvDqiWINwCcwJKeRA + postgres-password: AgAL+Go2p0sE5fkrFiInfUqQ2XTvaAT9B/exQI+LXI1ESzfANIx6nBC3GZv7ywIzBojScr3n32QFnyjZXsP2nG03+XQVdHXrVFWaSYwvyBlFmv5Zgsc26OK3jsptO0OsSxVN6PW4Y+oeTJVjLSjzCnFpQRSaN1G6IXUuEC8PMgRSPkX417xkX76xBIsK5UhGpiE5URo/0su8tacR+ncSP+jKSXO8TokyBuDHyrAlbEROdZLqtJZJsS5RWuvr66Du6qW8d6MR4ox/+mx585/lMgsQq/jHjSTlxJYQJUAW18wvDzKc6o7vW3O5jTN6Q30va/0KEN+1e6pMOqLXKdvfmdR70UxuBTR2NfeBCXCl6mr/oWEKUn4smRsmSWWC6DpvGMaIslmH3IK0SyNFSn81xpnKR9O81cvLIaN65MAfr+oUqNCUPSkroHWyFEDjiMJmOLx8pqaH+QHP0ibB27CI97xDaYJbXK+eHnjfBa4f7ierYX6HBbgfuqL0E34VyArFbPOmDjsVBsMcKzpC3NPAAcy6WxVqiBHhHdTRnN5U3p883TFGxsK2xH/Bkls2sgXVNUWKt4bWiP8jpnYhuekVrljIBOTaDB2ikemguXrPgxwc/QVDcuoOCDB7d1/jRRUCsVPzmL524ik1M5hw1qShyIMdEy6ZCwTwTUsaGwfwWyyGkIDqC236osYbCyemRf/W1F72fX3yRpHGdRgU template: metadata: creationTimestamp: null From 9319fcf18f7cab1ed8cbe5ea865b7363e4e24a83 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 14:46:14 +0200 Subject: [PATCH 28/37] try fix postgres secret --- demos/argo-cd/applications/airflow-postgres.yaml | 1 + .../sealed-airflow-postgres-credentials.yaml | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/demos/argo-cd/applications/airflow-postgres.yaml b/demos/argo-cd/applications/airflow-postgres.yaml index faf49c9e..df12dbc8 100644 --- a/demos/argo-cd/applications/airflow-postgres.yaml +++ b/demos/argo-cd/applications/airflow-postgres.yaml @@ -21,6 +21,7 @@ spec: stackable.tech/vendor: Stackable auth: database: airflow + username: airflow existingSecret: postgresql-credentials - repoURL: https://github.com/stackabletech/demos.git # TODO: adapt to release-25.3 diff --git a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml index bd47fe64..51bd342f 100644 --- a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml +++ b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml @@ -7,8 +7,10 @@ metadata: namespace: stackable-airflow spec: encryptedData: - password: AgAW0KA8iN9DeYjbGhYkM9xP4NwWK78YhZR43WI20igr7ZZucfHhANCWdCcNZFaakS73B/XK4wgN/gg9N+XJQY8MsDsfYrvVNVmrEBP1Hs7ryMbbDEGWuc8WTB7oq57O7HEtbI/anitpKLQHvciTpfz7m2gxpxd/DmZdMgKVER2CHqIH6r1pkREQWDKx2rBhPForFd8nXs5J5ALgD32XMCUE7P/nqgk0yr5b3OxFUflweAFMaQcTRpwu3AEeoQSOFquPXDaqNclww8FOPrE2EeitPq0YV1O5smFHlazYMqr8wk+WQuRK48Hn3/8SVHZEmKjH6it85//+VVaVhNmuUQlaxSBTZqY4aXI47Vfn92i3ICmUp7pGcD2AfNJ2+Xp2krXUQW+LVyVHI/0JpJsiGAz9I/CHr5Gs23R6GmLedzZJR3SVIvEWkxd2Y6iXow5Yvz2v5UgRXugn5ai2gPBqwrGUf5msuCV83f8LgcAVgZbpL2izfyuXLVvrE1Snv1/IJbPpPxTI9gb5xOS7KHQbveiq+QL5rRcVemLKUkzKn9dXLynCxyUo/QYRDdl83kaOacwFZN38yke01wGe0aJ9G9935DCheDKijEXCTbjzbjr5LuQYMxEBNmNiAz1Iq64E3W0O9PG0Nyhqd445gA/XWqk6r9ExCDhQ+TyGp2ZNMgNtlDuzc2eb/bqw3LTyvDqiWINwCcwJKeRA - postgres-password: AgAL+Go2p0sE5fkrFiInfUqQ2XTvaAT9B/exQI+LXI1ESzfANIx6nBC3GZv7ywIzBojScr3n32QFnyjZXsP2nG03+XQVdHXrVFWaSYwvyBlFmv5Zgsc26OK3jsptO0OsSxVN6PW4Y+oeTJVjLSjzCnFpQRSaN1G6IXUuEC8PMgRSPkX417xkX76xBIsK5UhGpiE5URo/0su8tacR+ncSP+jKSXO8TokyBuDHyrAlbEROdZLqtJZJsS5RWuvr66Du6qW8d6MR4ox/+mx585/lMgsQq/jHjSTlxJYQJUAW18wvDzKc6o7vW3O5jTN6Q30va/0KEN+1e6pMOqLXKdvfmdR70UxuBTR2NfeBCXCl6mr/oWEKUn4smRsmSWWC6DpvGMaIslmH3IK0SyNFSn81xpnKR9O81cvLIaN65MAfr+oUqNCUPSkroHWyFEDjiMJmOLx8pqaH+QHP0ibB27CI97xDaYJbXK+eHnjfBa4f7ierYX6HBbgfuqL0E34VyArFbPOmDjsVBsMcKzpC3NPAAcy6WxVqiBHhHdTRnN5U3p883TFGxsK2xH/Bkls2sgXVNUWKt4bWiP8jpnYhuekVrljIBOTaDB2ikemguXrPgxwc/QVDcuoOCDB7d1/jRRUCsVPzmL524ik1M5hw1qShyIMdEy6ZCwTwTUsaGwfwWyyGkIDqC236osYbCyemRf/W1F72fX3yRpHGdRgU + database: AgAlOcdpjYAaeiHbIwmGM3lb7okU4DROr+4bs7oQBPBY924XdamqlSw4Y7V8svvHRqD5Hz8SRx0SY2dkLaBQZBaXKV8qaMaVdqX0zK8zvoIbw/bQFXMTFpPq/kUD/kBdTwGJdAd/VpdY1m4Dcn247AhsQuNbqcZ8xibH2IW1e41f1/5nidHY+0a0O2qrSjAU8RxBAuBlLS6ZZiT2vmgahe6CUarJ3Dzr8InZRmtgwT7F8Zzli6cqgOPDsUg768KwDOIxpO+tnOT9b57SAbLUn5SOedFbAodWgQAg913URTiPOCrt5Loyp9B4tQXv1s74WFmgPlEkQmavkIyQg3R2Yki0QhkFysOFtp41ilRFTvgauxV3irryTv/frgA+XJDwfYexG2whgWH/VbT/HHIMoHBuRD864lpWZ/qDdLhFx7A5Eflli+YO+xuOQP4eFEsWeVYtcBNEi8Epy86LBiDCttCYWJ96XEmriqTpBebjxpaihxfCnPEbq5bkLv9RhO9vgzgYPgceLvPJ9GSpSJFv2s+ohvvJYyeAiuNLRkPR++tHFaofCVnuIfbeFgS1WVi/F4TrNsq2JlNfUnrzGTkc1CIkx/u02cE7K+Smfi/Dz30iDE7A+TLxDz7pguozMHX0eQCYDJ4ehehkArfH/XXe/ZH7Hb5MV6nMV32ZxkNi4x5iMdIsuLUdXJHWqSi0Ue9cZFBsP2rqmr5S + password: AgB/x1/DNC9uYWCTH1xu17qSr9GrnTcQ4LD5BCTu/5wUd/TK2rWM8zHce0mF97e6km+FyT+G96utiRThutNkAOpzT7xrZwkp45F0HqF0aD+NOYMzMn4duVLaK7JXLm6u7187Ci6kFMkmZ+Npb94wX+iUa+kgTQTkMP1HsdgQj4c8gj1ldpnoUDQt8/wsTF/gEukXD14E0BbEMkRTPh8PzkiirlLVWCff3YauQVRKXL2yWa79Fo47zkZ/WMJXyB8PVBQot2OIDL1ygddTBklBoP6ts2MOo3G7blmQWu5cgS/+IAw837pEWpDg5+UnUkM1+OQ+52PWLRXPbiSMFFlcbZIHwuASx7lRRm482q+kfJBb4oALd7EPHmjcBg1lV1iR74NetJ2wkMMcqnvkITxw8U8To1ZKseBpNH7mBWB9u2lvpjT5EWMOveVLCq6NwU2IcdgHhUaMByrxq5I3u1k1Y1hZvM0txQp8Mr/Zas/EEJ/w3sN+i9O/JWIonRx84Y9OJxIQEXManz7LzzzIxXikCvenm/8oQMfPHa7w5Scdb6sEQKsFXZ+aMQMcUTp7xuZbYzuX/7CmVdPHlFOhAXGAbRE2J739uWmyMVQkJ9cHSKzbCJdXDD3+oJB42Xz37tJg08xvMGBTPJjJoKnXfHIPS2yQISg1VgfyWNg/T2LZLuqt5htpUbQsWsL6K9/z7qtsama5pUiVTsmu + postgres-password: AgBpGdazaNMQ2j2jI1sK5D1PCFvhXn/kH7Xhj2P2GgHesCxml2WN8B99dtk2O81BDt7IYNqadhwdXe+pmvJU48qkbfcmgE+Kbjbbdo8bv9GxMQcYPPH0ahqIfFAxNgMBHnbQ0Zxy+a3vzNiXLeD4FAwU0Pxa7kSwED0/EiyfmZcvph2J5JTi4FwhU2ITycMwsjQnt7KVw1SGWMZkro6HoJ9wGSj7GRSeHX7HEMBCehf8TOMEo1Emy6IqYHAywdcNgfBEoLvgUUFElBxYLshCP2NTTCU2o374Xw2fiErjiR7Q075Mshpxl4bqGf2HgGT44Lsn5KDupPNnV7FPx0xTk3OmmE5Ja5gUAeLYU3bF8B7DSm61ltQnkL+FQ5sy3tjCTsH3wyYbg4zVoQlJuyRXJdG48Y0aHkQHwMpZlZEkB8IWGZZ9Kd9UKTTFnS3odca8OhdWYQ28ARz4Mj0Jio0dEO/63EyCJZiL4/Ck4rj8I49Wz832GjHVkMNFqthK6jN+HmlGcKXRKRMFLyIrvH7mUv1EuOsvh2+qtrsGGWSYwfETmb1J/iUp7X2+KhPnn6GP/poXLKeeDtiRfRSXTP70vRDllOMgSRTorRxS44FD/ac3KVA4RlpA+Ho+XU4QhYnNx8Q6haTQINY8Yypw/JRAPO4l379EFZtFdCSrKZ9XmRPsSZkZPZZJSVUe2ByGs/SScOR1xf1pwdPjWoOd + username: AgC716fxKUrMXCsRsWeiSTliHKP8NrwBr0TD7/w2F1HO93pqjSVU5/6hKOiFdNllbyPRudSHAeYWqJhQxxl5tS9kSaEp29zkxAUnmNwfLL3MKdCohxz2LZzC7saaOdjDkX2x8jcy68STPupsdg5bX40D4tO/s8qXnWWZJrW0ht/J+orgjbnfHk51JBMRBh6WD43yKLOBTHGO5U/TqRKg20kn+jDm2hcdec8axcjn2E57B3wrL8tvUcVRElO0XOcLHHBrav/BumRkwS4bMGqa+Vcarrr8U9MuQXVUzbCuwt326q/Kg862ozYY9uWAVtUy+Yfb4SAcEUC9Y+Ioo776H3YIr6q8P6RI+lFNmz4Q8AFDN9D5IRNjQ/D73llHw944dYRJl8CVjoJJYvRXv8z+4GzL7O+eF+smx4EhcY3sFxjkNCv+s+EDbUXvQR0ULqTEc5WZlBqp9TQEvzBwN3CVYO0k76FQYdjoU6xb/32gALe6zMxOLTliR9axqDvCL3OCj97/HJy8RLL2sYH82NZ1sGHqfTkTKYGk24Zfzd+a+p0PDbU14ZBuXKWBDkzNaVdpLvWhtdBczarfVp4Z2b6IutbT2C43fcmi7WYJOtV5em1sGhKOtbEy+wC+r5GKr6qRV58jm88cwtqZgm6IirDt/bzJDqhPndBzW5iQZzkko/1Tr4y/6oaTleTcmkDlSg2Cn8q/R7MNYUjT template: metadata: creationTimestamp: null From 84ee9a555b86b2f409285550a2d6550e4cc63fec Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 14:57:29 +0200 Subject: [PATCH 29/37] fix env override --- demos/argo-cd/manifests/airflow/airflow.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index 8aee92a5..b4ce9dd0 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -56,12 +56,12 @@ spec: spec: container: - name: airflow - env: - - name: AIRFLOW_CONN_MINIO - valueFrom: - secretKeyRef: - name: airflow-minio-connection - key: airflow-minio-connection + env: + - name: AIRFLOW_CONN_MINIO + valueFrom: + secretKeyRef: + name: airflow-minio-connection + key: airflow-minio-connection config: resources: cpu: From 4846f005b4f60569f0bb940f3e589d4930716bed Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 15:03:36 +0200 Subject: [PATCH 30/37] fix overrides --- demos/argo-cd/manifests/airflow/airflow.yaml | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index b4ce9dd0..29e6a071 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -54,7 +54,7 @@ spec: AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio podOverrides: &podOverrides spec: - container: + containers: - name: airflow env: - name: AIRFLOW_CONN_MINIO @@ -62,14 +62,6 @@ spec: secretKeyRef: name: airflow-minio-connection key: airflow-minio-connection - config: - resources: - cpu: - min: 400m - max: "1" - memory: - limit: 2Gi - gracefulShutdownTimeout: 30s roleGroups: default: replicas: 1 @@ -79,14 +71,6 @@ spec: schedulers: envOverrides: *envOverrides podOverrides: *podOverrides - config: - gracefulShutdownTimeout: 30s - resources: - cpu: - min: 400m - max: "1" - memory: - limit: 1Gi roleGroups: default: replicas: 1 From 8f7d766f3fa31f7edb133c4815ac95f6c904b021 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 15:16:51 +0200 Subject: [PATCH 31/37] fix container name --- demos/argo-cd/manifests/airflow/airflow.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index 29e6a071..3eca242e 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -67,7 +67,16 @@ spec: replicas: 1 kubernetesExecutors: envOverrides: *envOverrides - podOverrides: *podOverrides + podOverrides: &podOverrides + spec: + containers: + - name: base + env: + - name: AIRFLOW_CONN_MINIO + valueFrom: + secretKeyRef: + name: airflow-minio-connection + key: airflow-minio-connection schedulers: envOverrides: *envOverrides podOverrides: *podOverrides From 1c8b5a5ca36537ed52b8d110ddf2c0fdbc3de49b Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 15:24:28 +0200 Subject: [PATCH 32/37] fix overrides --- demos/argo-cd/manifests/airflow/airflow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index 3eca242e..75ace4c6 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -67,7 +67,7 @@ spec: replicas: 1 kubernetesExecutors: envOverrides: *envOverrides - podOverrides: &podOverrides + podOverrides: spec: containers: - name: base From 4fcc2a1468b0b9cd3aa0ba9ef18596793bcc2063 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 15:36:14 +0200 Subject: [PATCH 33/37] enable gitsync --- demos/argo-cd/dags/date_demo.py | 20 ++ demos/argo-cd/dags/pyspark_pi.py | 177 +++++++++++++ demos/argo-cd/dags/pyspark_pi.yaml | 36 +++ demos/argo-cd/manifests/airflow/airflow.yaml | 263 +------------------ 4 files changed, 238 insertions(+), 258 deletions(-) create mode 100644 demos/argo-cd/dags/date_demo.py create mode 100644 demos/argo-cd/dags/pyspark_pi.py create mode 100644 demos/argo-cd/dags/pyspark_pi.yaml diff --git a/demos/argo-cd/dags/date_demo.py b/demos/argo-cd/dags/date_demo.py new file mode 100644 index 00000000..704aac51 --- /dev/null +++ b/demos/argo-cd/dags/date_demo.py @@ -0,0 +1,20 @@ +"""Example DAG returning the current date""" +from datetime import datetime, timedelta + +from airflow import DAG +from airflow.operators.bash import BashOperator + +with DAG( + dag_id='date_demo', + schedule_interval='0-59 * * * *', + start_date=datetime(2021, 1, 1), + catchup=False, + dagrun_timeout=timedelta(minutes=5), + tags=['example'], + params={}, +) as dag: + + run_this = BashOperator( + task_id='run_every_minute', + bash_command='date', + ) diff --git a/demos/argo-cd/dags/pyspark_pi.py b/demos/argo-cd/dags/pyspark_pi.py new file mode 100644 index 00000000..16c81442 --- /dev/null +++ b/demos/argo-cd/dags/pyspark_pi.py @@ -0,0 +1,177 @@ +"""Example DAG demonstrating how to apply a Kubernetes Resource from Airflow running in-cluster""" +from datetime import datetime, timedelta +from airflow import DAG +from typing import TYPE_CHECKING, Optional, Sequence, Dict +from kubernetes import client +from airflow.exceptions import AirflowException +from airflow.sensors.base import BaseSensorOperator +from airflow.models import BaseOperator +from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook +import yaml +from airflow.utils import yaml +import os + +if TYPE_CHECKING: + from airflow.utils.context import Context + +class SparkKubernetesOperator(BaseOperator): + template_fields: Sequence[str] = ('application_file', 'namespace') + template_ext: Sequence[str] = ('.yaml', '.yml', '.json') + ui_color = '#f4a460' + + def __init__( + self, + *, + application_file: str, + namespace: Optional[str] = None, + kubernetes_conn_id: str = 'kubernetes_in_cluster', + api_group: str = 'spark.stackable.tech', + api_version: str = 'v1alpha1', + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.application_file = application_file + self.namespace = namespace + self.kubernetes_conn_id = kubernetes_conn_id + self.api_group = api_group + self.api_version = api_version + self.plural = "sparkapplications" + + def execute(self, context: 'Context'): + hook = KubernetesHook(conn_id=self.kubernetes_conn_id) + self.log.info("Creating SparkApplication...") + response = hook.create_custom_object( + group=self.api_group, + version=self.api_version, + plural=self.plural, + body=self.application_file, + namespace=self.namespace, + ) + return response + + +class SparkKubernetesSensor(BaseSensorOperator): + template_fields = ("application_name", "namespace") + # See https://github.com/stackabletech/spark-k8s-operator/pull/460/files#diff-d737837121132af6b60f50279a78464b05dcfd06c05d1d090f4198a5e962b5f6R371 + # Unknown is set immediately so it must be excluded from the failed states. + FAILURE_STATES = ("Failed") + SUCCESS_STATES = ("Succeeded") + + def __init__( + self, + *, + application_name: str, + attach_log: bool = False, + namespace: Optional[str] = None, + kubernetes_conn_id: str = 'kubernetes_in_cluster', + api_group: str = 'spark.stackable.tech', + api_version: str = 'v1alpha1', + poke_interval: float = 60, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.application_name = application_name + self.attach_log = attach_log + self.namespace = namespace + self.kubernetes_conn_id = kubernetes_conn_id + self.hook = KubernetesHook(conn_id=self.kubernetes_conn_id) + self.api_group = api_group + self.api_version = api_version + self.poke_interval = poke_interval + + def _log_driver(self, application_state: str, response: dict) -> None: + if not self.attach_log: + return + status_info = response["status"] + if "driverInfo" not in status_info: + return + driver_info = status_info["driverInfo"] + if "podName" not in driver_info: + return + driver_pod_name = driver_info["podName"] + namespace = response["metadata"]["namespace"] + log_method = self.log.error if application_state in self.FAILURE_STATES else self.log.info + try: + log = "" + for line in self.hook.get_pod_logs(driver_pod_name, namespace=namespace): + log += line.decode() + log_method(log) + except client.rest.ApiException as e: + self.log.warning( + "Could not read logs for pod %s. It may have been disposed.\n" + "Make sure timeToLiveSeconds is set on your SparkApplication spec.\n" + "underlying exception: %s", + driver_pod_name, + e, + ) + + def poke(self, context: Dict) -> bool: + self.log.info("Poking: %s", self.application_name) + response = self.hook.get_custom_object( + group=self.api_group, + version=self.api_version, + plural="sparkapplications", + name=self.application_name, + namespace=self.namespace, + ) + try: + application_state = response["status"]["phase"] + except KeyError: + self.log.debug(f"SparkApplication status could not be established: {response}") + return False + if self.attach_log and application_state in self.FAILURE_STATES + self.SUCCESS_STATES: + self._log_driver(application_state, response) + if application_state in self.FAILURE_STATES: + raise AirflowException(f"SparkApplication failed with state: {application_state}") + elif application_state in self.SUCCESS_STATES: + self.log.info("SparkApplication ended successfully") + return True + else: + self.log.info("SparkApplication is still in state: %s", application_state) + return False + +with DAG( + dag_id='sparkapp_dag', + schedule_interval=None, + start_date=datetime(2022, 1, 1), + catchup=False, + dagrun_timeout=timedelta(minutes=60), + tags=['example'], + params={"example_key": "example_value"}, +) as dag: + + def load_body_to_dict(body): + try: + body_dict = yaml.safe_load(body) + except yaml.YAMLError as e: + raise AirflowException(f"Exception when loading resource definition: {e}\n") + return body_dict + + yaml_path = os.path.join(os.environ.get('AIRFLOW__CORE__DAGS_FOLDER'), 'pyspark_pi.yaml') + + with open(yaml_path, 'r') as file: + crd = file.read() + with open('/run/secrets/kubernetes.io/serviceaccount/namespace', 'r') as file: + ns = file.read() + + document=load_body_to_dict(crd) + application_name='pyspark-pi-'+datetime.utcnow().strftime('%Y%m%d%H%M%S') + document.update({'metadata': {'name': application_name, 'namespace': ns}}) + + t1 = SparkKubernetesOperator( + task_id='spark_pi_submit', + namespace=ns, + application_file=document, + do_xcom_push=True, + dag=dag, + ) + + t2 = SparkKubernetesSensor( + task_id='spark_pi_monitor', + namespace=ns, + application_name="{{ task_instance.xcom_pull(task_ids='spark_pi_submit')['metadata']['name'] }}", + poke_interval=5, + dag=dag, + ) + + t1 >> t2 diff --git a/demos/argo-cd/dags/pyspark_pi.yaml b/demos/argo-cd/dags/pyspark_pi.yaml new file mode 100644 index 00000000..559f4822 --- /dev/null +++ b/demos/argo-cd/dags/pyspark_pi.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: spark.stackable.tech/v1alpha1 +kind: SparkApplication +metadata: + name: pyspark-pi +spec: + version: "1.0" + sparkImage: + productVersion: 3.5.2 + mode: cluster + mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py + job: + config: + resources: + cpu: + min: 500m + max: 500m + memory: + limit: 512Mi + driver: + config: + resources: + cpu: + min: 1000m + max: 1200m + memory: + limit: 1024Mi + executor: + config: + resources: + cpu: + min: 500m + max: 1000m + memory: + limit: 1024Mi + replicas: 3 diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index 75ace4c6..206555ae 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -1,5 +1,4 @@ --- -# {% raw %} apiVersion: airflow.stackable.tech/v1alpha1 kind: AirflowCluster metadata: @@ -12,10 +11,12 @@ spec: loadExamples: false exposeConfig: false credentialsSecret: airflow-credentials + dagsGitSync: + - repo: https://github.com/stackabletech/demos/ + branch: spike/argocd-demo + gitFolder: "demos/argo-cd/dags" + depth: 2 volumes: - - name: airflow-dags - configMap: - name: airflow-dags - name: minio-tls ephemeral: volumeClaimTemplate: @@ -30,16 +31,6 @@ spec: requests: storage: "1" storageClassName: secrets.stackable.tech - volumeMounts: - - name: airflow-dags - mountPath: /dags/date_demo.py - subPath: date_demo.py - - name: airflow-dags - mountPath: /dags/pyspark_pi.py - subPath: pyspark_pi.py - - name: airflow-dags - mountPath: /dags/pyspark_pi.yaml - subPath: pyspark_pi.yaml - name: minio-tls mountPath: /stackable/minio-tls webservers: @@ -83,247 +74,3 @@ spec: roleGroups: default: replicas: 1 ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: airflow-dags -data: - date_demo.py: | - """Example DAG returning the current date""" - from datetime import datetime, timedelta - - from airflow import DAG - from airflow.operators.bash import BashOperator - - with DAG( - dag_id='date_demo', - schedule_interval='0-59 * * * *', - start_date=datetime(2021, 1, 1), - catchup=False, - dagrun_timeout=timedelta(minutes=5), - tags=['example'], - params={}, - ) as dag: - - run_this = BashOperator( - task_id='run_every_minute', - bash_command='date', - ) - pyspark_pi.py: | - """Example DAG demonstrating how to apply a Kubernetes Resource from Airflow running in-cluster""" - from datetime import datetime, timedelta - from airflow import DAG - from typing import TYPE_CHECKING, Optional, Sequence, Dict - from kubernetes import client - from airflow.exceptions import AirflowException - from airflow.sensors.base import BaseSensorOperator - from airflow.models import BaseOperator - from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook - import yaml - from airflow.utils import yaml - import os - - if TYPE_CHECKING: - from airflow.utils.context import Context - - class SparkKubernetesOperator(BaseOperator): - template_fields: Sequence[str] = ('application_file', 'namespace') - template_ext: Sequence[str] = ('.yaml', '.yml', '.json') - ui_color = '#f4a460' - - def __init__( - self, - *, - application_file: str, - namespace: Optional[str] = None, - kubernetes_conn_id: str = 'kubernetes_in_cluster', - api_group: str = 'spark.stackable.tech', - api_version: str = 'v1alpha1', - **kwargs, - ) -> None: - super().__init__(**kwargs) - self.application_file = application_file - self.namespace = namespace - self.kubernetes_conn_id = kubernetes_conn_id - self.api_group = api_group - self.api_version = api_version - self.plural = "sparkapplications" - - def execute(self, context: 'Context'): - hook = KubernetesHook(conn_id=self.kubernetes_conn_id) - self.log.info("Creating SparkApplication...") - response = hook.create_custom_object( - group=self.api_group, - version=self.api_version, - plural=self.plural, - body=self.application_file, - namespace=self.namespace, - ) - return response - - - class SparkKubernetesSensor(BaseSensorOperator): - template_fields = ("application_name", "namespace") - # See https://github.com/stackabletech/spark-k8s-operator/pull/460/files#diff-d737837121132af6b60f50279a78464b05dcfd06c05d1d090f4198a5e962b5f6R371 - # Unknown is set immediately so it must be excluded from the failed states. - FAILURE_STATES = ("Failed") - SUCCESS_STATES = ("Succeeded") - - def __init__( - self, - *, - application_name: str, - attach_log: bool = False, - namespace: Optional[str] = None, - kubernetes_conn_id: str = 'kubernetes_in_cluster', - api_group: str = 'spark.stackable.tech', - api_version: str = 'v1alpha1', - poke_interval: float = 60, - **kwargs, - ) -> None: - super().__init__(**kwargs) - self.application_name = application_name - self.attach_log = attach_log - self.namespace = namespace - self.kubernetes_conn_id = kubernetes_conn_id - self.hook = KubernetesHook(conn_id=self.kubernetes_conn_id) - self.api_group = api_group - self.api_version = api_version - self.poke_interval = poke_interval - - def _log_driver(self, application_state: str, response: dict) -> None: - if not self.attach_log: - return - status_info = response["status"] - if "driverInfo" not in status_info: - return - driver_info = status_info["driverInfo"] - if "podName" not in driver_info: - return - driver_pod_name = driver_info["podName"] - namespace = response["metadata"]["namespace"] - log_method = self.log.error if application_state in self.FAILURE_STATES else self.log.info - try: - log = "" - for line in self.hook.get_pod_logs(driver_pod_name, namespace=namespace): - log += line.decode() - log_method(log) - except client.rest.ApiException as e: - self.log.warning( - "Could not read logs for pod %s. It may have been disposed.\n" - "Make sure timeToLiveSeconds is set on your SparkApplication spec.\n" - "underlying exception: %s", - driver_pod_name, - e, - ) - - def poke(self, context: Dict) -> bool: - self.log.info("Poking: %s", self.application_name) - response = self.hook.get_custom_object( - group=self.api_group, - version=self.api_version, - plural="sparkapplications", - name=self.application_name, - namespace=self.namespace, - ) - try: - application_state = response["status"]["phase"] - except KeyError: - self.log.debug(f"SparkApplication status could not be established: {response}") - return False - if self.attach_log and application_state in self.FAILURE_STATES + self.SUCCESS_STATES: - self._log_driver(application_state, response) - if application_state in self.FAILURE_STATES: - raise AirflowException(f"SparkApplication failed with state: {application_state}") - elif application_state in self.SUCCESS_STATES: - self.log.info("SparkApplication ended successfully") - return True - else: - self.log.info("SparkApplication is still in state: %s", application_state) - return False - - with DAG( - dag_id='sparkapp_dag', - schedule_interval=None, - start_date=datetime(2022, 1, 1), - catchup=False, - dagrun_timeout=timedelta(minutes=60), - tags=['example'], - params={"example_key": "example_value"}, - ) as dag: - - def load_body_to_dict(body): - try: - body_dict = yaml.safe_load(body) - except yaml.YAMLError as e: - raise AirflowException(f"Exception when loading resource definition: {e}\n") - return body_dict - - yaml_path = os.path.join(os.environ.get('AIRFLOW__CORE__DAGS_FOLDER'), 'pyspark_pi.yaml') - - with open(yaml_path, 'r') as file: - crd = file.read() - with open('/run/secrets/kubernetes.io/serviceaccount/namespace', 'r') as file: - ns = file.read() - - document=load_body_to_dict(crd) - application_name='pyspark-pi-'+datetime.utcnow().strftime('%Y%m%d%H%M%S') - document.update({'metadata': {'name': application_name, 'namespace': ns}}) - - t1 = SparkKubernetesOperator( - task_id='spark_pi_submit', - namespace=ns, - application_file=document, - do_xcom_push=True, - dag=dag, - ) - - t2 = SparkKubernetesSensor( - task_id='spark_pi_monitor', - namespace=ns, - application_name="{{ task_instance.xcom_pull(task_ids='spark_pi_submit')['metadata']['name'] }}", - poke_interval=5, - dag=dag, - ) - - t1 >> t2 - pyspark_pi.yaml: | - --- - apiVersion: spark.stackable.tech/v1alpha1 - kind: SparkApplication - metadata: - name: pyspark-pi - spec: - version: "1.0" - sparkImage: - productVersion: 3.5.2 - mode: cluster - mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py - job: - config: - resources: - cpu: - min: 500m - max: 500m - memory: - limit: 512Mi - driver: - config: - resources: - cpu: - min: 1000m - max: 1200m - memory: - limit: 1024Mi - executor: - config: - resources: - cpu: - min: 500m - max: 1000m - memory: - limit: 1024Mi - replicas: 3 - -# {% endraw %} From f52cb08de68f3cc86632c386a711729ba3a12603 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 15:39:43 +0200 Subject: [PATCH 34/37] fix git sync --- demos/argo-cd/manifests/airflow/airflow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index 206555ae..114021a4 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -31,11 +31,11 @@ spec: requests: storage: "1" storageClassName: secrets.stackable.tech + volumeMounts: - name: minio-tls mountPath: /stackable/minio-tls webservers: envOverrides: &envOverrides - AIRFLOW__CORE__DAGS_FOLDER: "/dags" AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" # Via sealed secrets, just kept for reference here #AIRFLOW_CONN_MINIO: "aws://admin:adminadmin@/?endpoint_url=https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000" From 32011e805da338703f65b0492618010d7b0f3ab6 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 15:50:51 +0200 Subject: [PATCH 35/37] move yaml out of dags git sync --- demos/argo-cd/dags/pyspark_pi.py | 2 +- demos/argo-cd/{dags => manifests/spark-k8s}/pyspark_pi.yaml | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename demos/argo-cd/{dags => manifests/spark-k8s}/pyspark_pi.yaml (100%) diff --git a/demos/argo-cd/dags/pyspark_pi.py b/demos/argo-cd/dags/pyspark_pi.py index 16c81442..35bd4c38 100644 --- a/demos/argo-cd/dags/pyspark_pi.py +++ b/demos/argo-cd/dags/pyspark_pi.py @@ -147,7 +147,7 @@ def load_body_to_dict(body): raise AirflowException(f"Exception when loading resource definition: {e}\n") return body_dict - yaml_path = os.path.join(os.environ.get('AIRFLOW__CORE__DAGS_FOLDER'), 'pyspark_pi.yaml') + yaml_path = os.path.join(os.environ.get('AIRFLOW__CORE__DAGS_FOLDER'), '../manifests/spark-k8s/pyspark_pi.yaml') with open(yaml_path, 'r') as file: crd = file.read() diff --git a/demos/argo-cd/dags/pyspark_pi.yaml b/demos/argo-cd/manifests/spark-k8s/pyspark_pi.yaml similarity index 100% rename from demos/argo-cd/dags/pyspark_pi.yaml rename to demos/argo-cd/manifests/spark-k8s/pyspark_pi.yaml From 2efbab2b580bd777a1b885c2d165c0ce1bb6cd2e Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 15:55:35 +0200 Subject: [PATCH 36/37] set resources --- demos/argo-cd/manifests/airflow/airflow.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index 114021a4..58e45727 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -53,6 +53,14 @@ spec: secretKeyRef: name: airflow-minio-connection key: airflow-minio-connection + config: + gracefulShutdownTimeout: 30s + resources: + cpu: + min: 400m + max: "1" + memory: + limit: 1Gi roleGroups: default: replicas: 1 @@ -71,6 +79,14 @@ spec: schedulers: envOverrides: *envOverrides podOverrides: *podOverrides + config: + gracefulShutdownTimeout: 30s + resources: + cpu: + min: 400m + max: "1" + memory: + limit: 1Gi roleGroups: default: replicas: 1 From 945cbf28218fedb7e8f63b8ccf22a7702b7266d6 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 16:28:41 +0200 Subject: [PATCH 37/37] linter --- demos/argo-cd/applications/airflow-postgres.yaml | 1 + demos/argo-cd/applications/airflow.yaml | 1 + demos/argo-cd/applications/minio.yaml | 1 + demos/argo-cd/manifests/airflow/airflow.yaml | 2 +- demos/argo-cd/projects/airflow.yaml | 1 + demos/argo-cd/projects/minio.yaml | 1 + stacks/argo-cd/applications/sealed-secrets.yaml | 1 + stacks/argo-cd/projects/stackable-operators.yaml | 1 + 8 files changed, 8 insertions(+), 1 deletion(-) diff --git a/demos/argo-cd/applications/airflow-postgres.yaml b/demos/argo-cd/applications/airflow-postgres.yaml index df12dbc8..7052b64e 100644 --- a/demos/argo-cd/applications/airflow-postgres.yaml +++ b/demos/argo-cd/applications/airflow-postgres.yaml @@ -1,3 +1,4 @@ +--- apiVersion: argoproj.io/v1alpha1 kind: Application metadata: diff --git a/demos/argo-cd/applications/airflow.yaml b/demos/argo-cd/applications/airflow.yaml index edf5ac43..82c1b71c 100644 --- a/demos/argo-cd/applications/airflow.yaml +++ b/demos/argo-cd/applications/airflow.yaml @@ -1,3 +1,4 @@ +--- apiVersion: argoproj.io/v1alpha1 kind: Application metadata: diff --git a/demos/argo-cd/applications/minio.yaml b/demos/argo-cd/applications/minio.yaml index d3eaf0a5..42ca5407 100644 --- a/demos/argo-cd/applications/minio.yaml +++ b/demos/argo-cd/applications/minio.yaml @@ -1,3 +1,4 @@ +--- apiVersion: argoproj.io/v1alpha1 kind: Application metadata: diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index 58e45727..8ebbb9bf 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -38,7 +38,7 @@ spec: envOverrides: &envOverrides AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" # Via sealed secrets, just kept for reference here - #AIRFLOW_CONN_MINIO: "aws://admin:adminadmin@/?endpoint_url=https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000" + # AIRFLOW_CONN_MINIO: "aws://admin:adminadmin@/?endpoint_url=https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000" AWS_CA_BUNDLE: "/stackable/minio-tls/ca.crt" AIRFLOW__LOGGING__REMOTE_LOGGING: "True" AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER: s3://demo/airflow-task-logs/ diff --git a/demos/argo-cd/projects/airflow.yaml b/demos/argo-cd/projects/airflow.yaml index 3661434f..f518fd55 100644 --- a/demos/argo-cd/projects/airflow.yaml +++ b/demos/argo-cd/projects/airflow.yaml @@ -1,3 +1,4 @@ +--- apiVersion: argoproj.io/v1alpha1 kind: AppProject metadata: diff --git a/demos/argo-cd/projects/minio.yaml b/demos/argo-cd/projects/minio.yaml index ca58ee8a..38fee1df 100644 --- a/demos/argo-cd/projects/minio.yaml +++ b/demos/argo-cd/projects/minio.yaml @@ -1,3 +1,4 @@ +--- apiVersion: argoproj.io/v1alpha1 kind: AppProject metadata: diff --git a/stacks/argo-cd/applications/sealed-secrets.yaml b/stacks/argo-cd/applications/sealed-secrets.yaml index 73fd722c..450e7225 100644 --- a/stacks/argo-cd/applications/sealed-secrets.yaml +++ b/stacks/argo-cd/applications/sealed-secrets.yaml @@ -1,3 +1,4 @@ +--- apiVersion: argoproj.io/v1alpha1 kind: Application metadata: diff --git a/stacks/argo-cd/projects/stackable-operators.yaml b/stacks/argo-cd/projects/stackable-operators.yaml index 3ee37fc2..4acfc6cd 100644 --- a/stacks/argo-cd/projects/stackable-operators.yaml +++ b/stacks/argo-cd/projects/stackable-operators.yaml @@ -1,3 +1,4 @@ +--- apiVersion: argoproj.io/v1alpha1 kind: AppProject metadata: