Repository for dip
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

532 lines
25 KiB

{{- $artifactCommandPrefix := "default-artifact-root" }}
{{- $artifactCommand := printf "--%s=%s" $artifactCommandPrefix .Values.artifactRoot.defaultArtifactRoot }}
{{- if .Values.artifactRoot.proxiedArtifactStorage }}
{{- $artifactCommandPrefix = "artifacts-destination" }}
{{- $artifactCommand = printf "--%s=%s" $artifactCommandPrefix .Values.artifactRoot.defaultArtifactsDestination }}
{{- end }}
{{- if .Values.artifactRoot.azureBlob.enabled }}
{{- $artifactCommand = printf "--%s=wasbs://%s@%s.blob.core.windows.net/%s" $artifactCommandPrefix .Values.artifactRoot.azureBlob.container .Values.artifactRoot.azureBlob.storageAccount .Values.artifactRoot.azureBlob.path }}
{{- else if .Values.artifactRoot.s3.enabled }}
{{- $artifactCommand = printf "--%s=s3://%s/%s" $artifactCommandPrefix .Values.artifactRoot.s3.bucket .Values.artifactRoot.s3.path }}
{{- else if .Values.artifactRoot.gcs.enabled }}
{{- $artifactCommand = printf "--%s=gs://%s/%s" $artifactCommandPrefix .Values.artifactRoot.gcs.bucket .Values.artifactRoot.gcs.path }}
{{- end }}
{{- $dbConnectionDriver := "" }}
{{- if and (or .Values.backendStore.postgres.enabled .Values.postgresql.enabled) .Values.backendStore.postgres.driver }}
{{- $dbConnectionDriver = printf "+%s" .Values.backendStore.postgres.driver }}
{{- else if and (or .Values.backendStore.mysql.enabled .Values.mysql.enabled) .Values.backendStore.mysql.driver }}
{{- $dbConnectionDriver = printf "+%s" .Values.backendStore.mysql.driver }}
{{- else if and .Values.backendStore.mssql.enabled .Values.backendStore.mssql.driver }}
{{- $dbConnectionDriver = printf "+%s" .Values.backendStore.mssql.driver }}
{{- end }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "mlflow.fullname" . }}
labels:
{{- include "mlflow.labels" . | nindent 4 }}
spec:
{{- if not .Values.autoscaling.enabled }}
replicas: {{ .Values.replicaCount }}
{{- end }}
strategy:
type: {{ .Values.strategy.type }}
{{- if eq .Values.strategy.type "RollingUpdate" }}
rollingUpdate:
maxUnavailable: {{ .Values.strategy.rollingUpdate.maxUnavailable }}
maxSurge: {{ .Values.strategy.rollingUpdate.maxSurge }}
{{- end }}
selector:
matchLabels:
{{- include "mlflow.selectorLabels" . | nindent 6 }}
template:
metadata:
annotations:
checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "mlflow.selectorLabels" . | nindent 8 }}
{{- with .Values.extraPodLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "mlflow.serviceAccountName" . }}
{{- with .Values.podSecurityContext }}
securityContext:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- if or (and (or .Values.backendStore.postgres.enabled .Values.backendStore.mysql.enabled .Values.backendStore.mssql.enabled .Values.postgresql.enabled .Values.mysql.enabled) (or .Values.backendStore.databaseConnectionCheck .Values.backendStore.databaseMigration) ) .Values.auth.enabled .Values.ldapAuth.enabled .Values.initContainers }}
initContainers:
{{- end }}
{{- if and (or .Values.backendStore.postgres.enabled .Values.backendStore.mysql.enabled .Values.backendStore.mssql.enabled .Values.postgresql.enabled .Values.mysql.enabled) .Values.backendStore.databaseConnectionCheck }}
- name: dbchecker
image: "{{ .Values.initImages.dbchecker.repository }}:{{ .Values.initImages.dbchecker.tag }}"
imagePullPolicy: {{ .Values.initImages.dbchecker.pullPolicy }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
command: [ "/script/dbchecker.sh" ]
{{- if or .Values.postgresql.enabled .Values.mysql.enabled .Values.backendStore.existingDatabaseSecret.name }}
env:
{{- end }}
{{- if .Values.postgresql.enabled }}
- name: PGPASSWORD
valueFrom:
secretKeyRef:
name: {{ (include "mlflow.postgresql.fullname" .) }}
{{- if .Values.postgresql.auth.username }}
key: password
{{- else }}
key: postgres-password
{{- end }}
optional: true
{{- end }}
{{- if .Values.mysql.enabled }}
- name: MYSQL_PWD
valueFrom:
secretKeyRef:
name: {{ (include "mlflow.mysql.fullname" .) }}
{{- if .Values.mysql.auth.username }}
key: password
{{- else }}
key: mysql-root-password
{{- end }}
optional: true
{{- end }}
{{- if .Values.backendStore.existingDatabaseSecret.name }}
{{- if .Values.backendStore.postgres.enabled }}
- name: PGPASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.backendStore.existingDatabaseSecret.name }}
key: {{ .Values.backendStore.existingDatabaseSecret.passwordKey }}
- name: PGUSER
valueFrom:
secretKeyRef:
name: {{ .Values.backendStore.existingDatabaseSecret.name }}
key: {{ .Values.backendStore.existingDatabaseSecret.usernameKey }}
{{- end }}
{{- if .Values.backendStore.mysql.enabled }}
- name: MYSQL_PWD
valueFrom:
secretKeyRef:
name: {{ .Values.backendStore.existingDatabaseSecret.name }}
key: {{ .Values.backendStore.existingDatabaseSecret.passwordKey }}
- name: MYSQL_USERNAME
valueFrom:
secretKeyRef:
name: {{ .Values.backendStore.existingDatabaseSecret.name }}
key: {{ .Values.backendStore.existingDatabaseSecret.usernameKey }}
{{- end }}
{{- if .Values.backendStore.mssql.enabled }}
- name: MSSQL_PWD
valueFrom:
secretKeyRef:
name: {{ .Values.backendStore.existingDatabaseSecret.name }}
key: {{ .Values.backendStore.existingDatabaseSecret.passwordKey }}
- name: MSSQL_USERNAME
valueFrom:
secretKeyRef:
name: {{ .Values.backendStore.existingDatabaseSecret.name }}
key: {{ .Values.backendStore.existingDatabaseSecret.usernameKey }}
{{- end }}
{{- end }}
envFrom:
- configMapRef:
name: {{ template "mlflow.fullname" . }}-env-configmap
volumeMounts:
- name: dbchecker
mountPath: /script/dbchecker.sh
subPath: dbchecker.sh
{{- end }}
{{- if and (or .Values.backendStore.postgres.enabled .Values.backendStore.mysql.enabled .Values.backendStore.mssql.enabled .Values.postgresql.enabled .Values.mysql.enabled) .Values.backendStore.databaseMigration }}
- name: "mlflow-db-migration"
{{- with .Values.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
image: "{{ .Values.initImages.mlflowDbMigration.repository }}:{{ .Values.initImages.mlflowDbMigration.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.initImages.mlflowDbMigration.pullPolicy }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
{{- if or .Values.postgresql.enabled .Values.mysql.enabled .Values.backendStore.existingDatabaseSecret.name }}
env:
{{- end }}
{{- if .Values.postgresql.enabled }}
- name: PGPASSWORD
valueFrom:
secretKeyRef:
name: {{ (include "mlflow.postgresql.fullname" .) }}
{{- if .Values.postgresql.auth.username }}
key: password
{{- else }}
key: postgres-password
{{- end }}
optional: true
{{- end }}
{{- if .Values.mysql.enabled }}
- name: MYSQL_PWD
valueFrom:
secretKeyRef:
name: {{ (include "mlflow.mysql.fullname" .) }}
{{- if .Values.mysql.auth.username }}
key: password
{{- else }}
key: mysql-root-password
{{- end }}
optional: true
{{- end }}
{{- if .Values.backendStore.existingDatabaseSecret.name }}
{{- if .Values.backendStore.postgres.enabled }}
- name: PGPASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.backendStore.existingDatabaseSecret.name }}
key: {{ .Values.backendStore.existingDatabaseSecret.passwordKey }}
- name: PGUSER
valueFrom:
secretKeyRef:
name: {{ .Values.backendStore.existingDatabaseSecret.name }}
key: {{ .Values.backendStore.existingDatabaseSecret.usernameKey }}
{{- end }}
{{- if .Values.backendStore.mysql.enabled }}
- name: MYSQL_PWD
valueFrom:
secretKeyRef:
name: {{ .Values.backendStore.existingDatabaseSecret.name }}
key: {{ .Values.backendStore.existingDatabaseSecret.passwordKey }}
- name: MYSQL_USERNAME
valueFrom:
secretKeyRef:
name: {{ .Values.backendStore.existingDatabaseSecret.name }}
key: {{ .Values.backendStore.existingDatabaseSecret.usernameKey }}
{{- end }}
{{- if .Values.backendStore.mssql.enabled }}
- name: MSSQL_PWD
valueFrom:
secretKeyRef:
name: {{ .Values.backendStore.existingDatabaseSecret.name }}
key: {{ .Values.backendStore.existingDatabaseSecret.passwordKey }}
- name: MSSQL_USERNAME
valueFrom:
secretKeyRef:
name: {{ .Values.backendStore.existingDatabaseSecret.name }}
key: {{ .Values.backendStore.existingDatabaseSecret.usernameKey }}
{{- end }}
{{- end }}
envFrom:
- configMapRef:
name: {{ template "mlflow.fullname" . }}-env-configmap
- secretRef:
name: {{ template "mlflow.fullname" . }}-env-secret
- secretRef:
name: {{ template "mlflow.fullname" . }}-flask-server-secret-key
command: ["python"]
args:
- /opt/mlflow/migrations.py
volumeMounts:
- name: migrations-config
mountPath: /opt/mlflow/migrations.py
subPath: migrations.py
readOnly: true
{{- end }}
{{- if or .Values.auth.enabled .Values.ldapAuth.enabled }}
- name: ini-file-initializer
image: "{{ .Values.initImages.iniFileInitializer.repository }}:{{ .Values.initImages.iniFileInitializer.tag }}"
imagePullPolicy: {{ .Values.initImages.iniFileInitializer.pullPolicy }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
command: ['sh', '-c']
args:
- |
sed -e "s/\$(ADMIN_USERNAME_PLACEHOLDER)/${ADMIN_USERNAME}/g" \
-e "s/\$(ADMIN_PASSWORD_PLACEHOLDER)/${ADMIN_PASSWORD}/g" \
{{- if .Values.auth.postgres.enabled }}
-e "s/\$(POSTGRES_USERNAME_PLACEHOLDER)/${DB_USERNAME}/g" \
-e "s/\$(POSTGRES_PASSWORD_PLACEHOLDER)/${DB_PASSWORD}/g" \
{{- end }}
/template/auth_template.ini > /config/auth_result.ini
env:
{{- if .Values.auth.postgres.enabled }}
- name: DB_USERNAME
valueFrom:
secretKeyRef:
name: {{ default (printf "%s-auth-pg-secret" (include "mlflow.fullname" .)) .Values.auth.postgres.existingSecret.name }}
key: {{ ternary "username" .Values.auth.postgres.existingSecret.usernameKey (eq .Values.auth.postgres.existingSecret.name "") }}
- name: DB_PASSWORD
valueFrom:
secretKeyRef:
name: {{ default (printf "%s-auth-pg-secret" (include "mlflow.fullname" .)) .Values.auth.postgres.existingSecret.name }}
key: {{ ternary "password" .Values.auth.postgres.existingSecret.passwordKey (eq .Values.auth.postgres.existingSecret.name "") }}
{{- end }}
- name: ADMIN_USERNAME
valueFrom:
secretKeyRef:
name: {{ default (printf "%s-auth-admin-secret" (include "mlflow.fullname" .)) .Values.auth.existingAdminSecret.name }}
key: {{ ternary "username" .Values.auth.existingAdminSecret.usernameKey (eq .Values.auth.existingAdminSecret.name "") }}
- name: ADMIN_PASSWORD
valueFrom:
secretKeyRef:
name: {{ default (printf "%s-auth-admin-secret" (include "mlflow.fullname" .)) .Values.auth.existingAdminSecret.name }}
key: {{ ternary "password" .Values.auth.existingAdminSecret.passwordKey (eq .Values.auth.existingAdminSecret.name "") }}
volumeMounts:
- name: auth-ini-template-file
mountPath: /template
readOnly: true
- name: ini-file
mountPath: /config
{{- end }}
{{- with .Values.initContainers }}
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: {{ .Chart.Name }}
{{- with .Values.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
command: ["mlflow"]
args:
- server
- --host=0.0.0.0
- --port={{ .Values.service.containerPort }}
{{- if or .Values.backendStore.postgres.enabled .Values.postgresql.enabled }}
- --backend-store-uri=postgresql{{ $dbConnectionDriver }}://
{{- else if or .Values.backendStore.mysql.enabled .Values.mysql.enabled }}
- --backend-store-uri=mysql{{ $dbConnectionDriver }}://$(MYSQL_USERNAME):$(MYSQL_PWD)@$(MYSQL_HOST):$(MYSQL_TCP_PORT)/$(MYSQL_DATABASE)
{{- else if or .Values.backendStore.mssql.enabled }}
- --backend-store-uri=mssql{{ $dbConnectionDriver }}://$(MSSQL_USERNAME):$(MSSQL_PWD)@$(MSSQL_HOST):$(MSSQL_TCP_PORT)/$(MSSQL_DATABASE)
{{- else }}
- "--backend-store-uri=sqlite:///{{ .Values.backendStore.defaultSqlitePath }}"
{{- end }}
- {{ $artifactCommand }}
{{- if .Values.artifactRoot.proxiedArtifactStorage }}
- --serve-artifacts
{{- end }}
{{- if .Values.serviceMonitor.enabled }}
- --expose-prometheus=/mlflow/metrics
{{- end }}
{{- if .Values.auth.enabled }}
- --app-name={{ .Values.auth.appName }}
{{- else if .Values.ldapAuth.enabled }}
- --app-name=basic-auth
{{- end }}
{{- if and .Values.log.enabled (not (hasKey .Values.extraArgs "gunicornOpts")) }}
- {{ printf "--gunicorn-opts='--log-level=%s'" .Values.log.level }}
{{- end }}
{{- range $key, $value := .Values.extraArgs }}
{{- if and $.Values.log.enabled (eq $key "gunicornOpts") (contains "--log-level" $value) }}
- {{ printf "--gunicorn-opts='%s'" (regexReplaceAll "--log-level=[^\\s]*" $value (printf "--log-level=%s" $.Values.log.level)) }}
{{- else if and $.Values.log.enabled (eq $key "gunicornOpts") (not (contains "--log-level" $value)) }}
- {{ printf "--gunicorn-opts='--log-level=%s %s'" $.Values.log.level $value }}
{{- else }}
- {{ printf "--%s=%s" (kebabcase $key) $value }}
{{- end }}
{{- end }}
{{- range .Values.extraFlags }}
- --{{ kebabcase . }}
{{- end }}
ports:
- name: {{ .Values.service.containerPortName }}
containerPort: {{ .Values.service.containerPort }}
protocol: TCP
livenessProbe:
httpGet:
path: {{ include "mlflow.health" . }}
port: {{ .Values.service.containerPortName }}
{{- with .Values.livenessProbe }}
{{- toYaml . | nindent 12 }}
{{- end }}
readinessProbe:
httpGet:
path: {{ include "mlflow.health" . }}
port: {{ .Values.service.containerPortName }}
{{- with .Values.readinessProbe }}
{{- toYaml . | nindent 12 }}
{{- end }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
env:
- name: MLFLOW_VERSION
value: {{ .Values.image.tag | default .Chart.AppVersion | quote }}
{{- if .Values.auth.enabled }}
- name: MLFLOW_AUTH_CONFIG_PATH
value: {{ printf "%s/%s" (.Values.auth.configPath | trimSuffix "/") (default "basic_auth.ini" (.Values.auth.configFile | trimPrefix "/")) }}
{{- else if .Values.ldapAuth.enabled }}
- name: MLFLOW_AUTH_CONFIG_PATH
value: "/etc/mlflow/auth/ldap_basic_auth.ini"
{{- if or .Values.ldapAuth.encodedTrustedCACertificate .Values.ldapAuth.externalSecretForTrustedCACertificate }}
- name: LDAP_CA
value: "/certs/ca.crt"
{{- end }}
{{- end }}
{{- if .Values.postgresql.enabled }}
- name: PGPASSWORD
valueFrom:
secretKeyRef:
name: {{ (include "mlflow.postgresql.fullname" .) }}
{{- if .Values.postgresql.auth.username }}
key: password
{{- else }}
key: postgres-password
{{- end }}
optional: true
{{- end }}
{{- if .Values.mysql.enabled }}
- name: MYSQL_PWD
valueFrom:
secretKeyRef:
name: {{ (include "mlflow.mysql.fullname" .) }}
{{- if .Values.mysql.auth.username }}
key: password
{{- else }}
key: mysql-root-password
{{- end }}
optional: true
{{- end }}
{{- if .Values.backendStore.existingDatabaseSecret.name }}
{{- if .Values.backendStore.postgres.enabled }}
- name: PGPASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.backendStore.existingDatabaseSecret.name }}
key: {{ .Values.backendStore.existingDatabaseSecret.passwordKey }}
- name: PGUSER
valueFrom:
secretKeyRef:
name: {{ .Values.backendStore.existingDatabaseSecret.name }}
key: {{ .Values.backendStore.existingDatabaseSecret.usernameKey }}
{{- end }}
{{- if .Values.backendStore.mysql.enabled }}
- name: MYSQL_PWD
valueFrom:
secretKeyRef:
name: {{ .Values.backendStore.existingDatabaseSecret.name }}
key: {{ .Values.backendStore.existingDatabaseSecret.passwordKey }}
- name: MYSQL_USERNAME
valueFrom:
secretKeyRef:
name: {{ .Values.backendStore.existingDatabaseSecret.name }}
key: {{ .Values.backendStore.existingDatabaseSecret.usernameKey }}
{{- end }}
{{- if .Values.backendStore.mssql.enabled }}
- name: MSSQL_PWD
valueFrom:
secretKeyRef:
name: {{ .Values.backendStore.existingDatabaseSecret.name }}
key: {{ .Values.backendStore.existingDatabaseSecret.passwordKey }}
- name: MSSQL_USERNAME
valueFrom:
secretKeyRef:
name: {{ .Values.backendStore.existingDatabaseSecret.name }}
key: {{ .Values.backendStore.existingDatabaseSecret.usernameKey }}
{{- end }}
{{- end }}
{{- if and .Values.artifactRoot.s3.enabled .Values.artifactRoot.s3.existingSecret.name }}
{{- if .Values.artifactRoot.s3.existingSecret.keyOfAccessKeyId }}
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: {{ .Values.artifactRoot.s3.existingSecret.name }}
key: {{ .Values.artifactRoot.s3.existingSecret.keyOfAccessKeyId }}
{{- end }}
{{- if .Values.artifactRoot.s3.existingSecret.keyOfSecretAccessKey }}
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: {{ .Values.artifactRoot.s3.existingSecret.name }}
key: {{ .Values.artifactRoot.s3.existingSecret.keyOfSecretAccessKey }}
{{- end }}
{{- end }}
{{- range $key, $value := .Values.extraEnvVars }}
- name: {{ upper $key }}
value: {{ $value | quote }}
{{- end }}
envFrom:
- configMapRef:
name: {{ template "mlflow.fullname" . }}-env-configmap
- secretRef:
name: {{ template "mlflow.fullname" . }}-env-secret
- secretRef:
name: {{ template "mlflow.fullname" . }}-flask-server-secret-key
{{- range .Values.extraSecretNamesForEnvFrom }}
- secretRef:
name: {{ . }}
{{- end }}
{{- if or .Values.extraVolumeMounts .Values.auth.enabled .Values.ldapAuth.enabled }}
volumeMounts:
{{- if .Values.auth.enabled }}
- name: ini-file
mountPath: {{ printf "%s/%s" (.Values.auth.configPath | trimSuffix "/") (default "basic_auth.ini" (.Values.auth.configFile | trimPrefix "/")) }}
subPath: auth_result.ini
readOnly: true
{{- else if .Values.ldapAuth.enabled }}
- name: ini-file
mountPath: /etc/mlflow/auth/ldap_basic_auth.ini
subPath: auth_result.ini
readOnly: true
{{- if or .Values.ldapAuth.encodedTrustedCACertificate .Values.ldapAuth.externalSecretForTrustedCACertificate }}
- name: trusted-ca-cert-secret
mountPath: /certs/ca.crt
subPath: ca.crt
readOnly: true
{{- end }}
{{- end }}
{{- with .Values.extraVolumeMounts }}
{{ toYaml . | nindent 12 }}
{{- end }}
{{- end }}
{{- with .Values.extraContainers }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- if or (and (or .Values.backendStore.postgres.enabled .Values.backendStore.mysql.enabled .Values.backendStore.mssql.enabled .Values.postgresql.enabled .Values.mysql.enabled) (or .Values.backendStore.databaseConnectionCheck .Values.backendStore.databaseMigration) ) .Values.extraVolumes .Values.auth.enabled .Values.ldapAuth.enabled }}
volumes:
{{- if and (or .Values.backendStore.postgres.enabled .Values.backendStore.mysql.enabled .Values.backendStore.mssql.enabled .Values.postgresql.enabled .Values.mysql.enabled) .Values.backendStore.databaseConnectionCheck }}
- name: dbchecker
configMap:
name: {{ template "mlflow.fullname" . }}-dbchecker
defaultMode: 0777
{{- end }}
{{- if and (or .Values.backendStore.postgres.enabled .Values.backendStore.mysql.enabled .Values.backendStore.mssql.enabled .Values.postgresql.enabled .Values.mysql.enabled) .Values.backendStore.databaseMigration }}
- name: migrations-config
configMap:
name: {{ template "mlflow.fullname" . }}-migrations
{{- end }}
{{- if or .Values.auth.enabled .Values.ldapAuth.enabled }}
- name: ini-file
emptyDir: {}
- name: auth-ini-template-file
configMap:
name: {{ template "mlflow.fullname" . }}-auth-ini-template
{{- if or .Values.ldapAuth.encodedTrustedCACertificate .Values.ldapAuth.externalSecretForTrustedCACertificate }}
- name: trusted-ca-cert-secret
secret:
secretName: {{ default (printf "%s-trusted-ca-cert-secret" (include "mlflow.fullname" .)) .Values.ldapAuth.externalSecretForTrustedCACertificate }}
{{- end }}
{{- end }}
{{- with .Values.extraVolumes }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}