Add airflow chart

This commit is contained in:
wbsong111
2025-04-14 08:27:00 +09:00
parent c86a229ab2
commit 3984453577
171 changed files with 33849 additions and 0 deletions
+206
View File
@@ -0,0 +1,206 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
Thank you for installing Apache {{ title .Chart.Name }} {{ .Values.airflowVersion }}!
Your release is named {{ .Release.Name }}.
{{- if or .Values.ingress.web.enabled .Values.ingress.flower.enabled .Values.ingress.enabled }}
You can now access your service(s) by following defined Ingress urls:
{{- if .Values.ingress.web.host }}
DEPRECATION WARNING:
`ingress.web.host` has been renamed to `ingress.web.hosts` and is now an array.
Please change your values as support for the old name will be dropped in a future release.
{{- end }}
{{- if .Values.ingress.web.tls }}
DEPRECATION WARNING:
`ingress.web.tls` has been renamed to `ingress.web.hosts[*].tls` and can be set per host.
Please change your values as support for the old name will be dropped in a future release.
{{- end }}
{{- if .Values.ingress.flower.host }}
DEPRECATION WARNING:
`ingress.flower.host` has been renamed to `ingress.flower.hosts` and is now an array.
Please change your values as support for the old name will be dropped in a future release.
{{- end }}
{{- if .Values.ingress.flower.tls }}
DEPRECATION WARNING:
`ingress.flower.tls` has been renamed to `ingress.flower.hosts[*].tls` and can be set per host.
Please change your values as support for the old name will be dropped in a future release.
{{- end }}
{{- if .Values.ingress.enabled }}
DEPRECATION WARNING:
`ingress.enabled` has been deprecated. There are now separate flags to control the webserver and
flower individually, ``ingress.web.enabled`` and ``ingress.flower.enabled``.
Please change your values as support for the old name will be dropped in a future release.
{{- end }}
{{- if or .Values.ingress.web.enabled .Values.ingress.enabled }}
Airflow Webserver:
{{- range .Values.ingress.web.hosts | default (list .Values.ingress.web.host) }}
{{- $tlsEnabled := $.Values.ingress.web.tls.enabled -}}
{{- $hostname := $.Values.ingress.web.host -}}
{{- if . | kindIs "string" | not }}
{{- if .tls }}
{{- $tlsEnabled = .tls.enabled -}}
{{- $hostname = .name -}}
{{- end }}
{{- end }}
http{{ if $tlsEnabled }}s{{ end }}://{{ (tpl $hostname $) }}{{ $.Values.ingress.web.path }}/
{{- end }}
{{- end }}
{{- if and (or .Values.ingress.flower.enabled .Values.ingress.enabled) (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }}
Flower dashboard:
{{- range .Values.ingress.flower.hosts | default (list .Values.ingress.flower.host) }}
{{- $tlsEnabled := $.Values.ingress.flower.tls.enabled -}}
{{- $hostname := $.Values.ingress.flower.host -}}
{{- if . | kindIs "string" | not }}
{{- if .tls }}
{{- $tlsEnabled = .tls.enabled -}}
{{- $hostname = .name -}}
{{- end }}
{{- end }}
http{{ if $tlsEnabled }}s{{ end }}://{{ (tpl $hostname $) }}{{ $.Values.ingress.flower.path }}/
{{- end }}
{{- end }}
{{- else }}
You can now access your dashboard(s) by executing the following command(s) and visiting the corresponding port at localhost in your browser:
Airflow Webserver: kubectl port-forward svc/{{ include "airflow.fullname" . }}-webserver {{ .Values.ports.airflowUI }}:{{ .Values.ports.airflowUI }} --namespace {{ .Release.Namespace }}
{{- if .Values.flower.enabled }}
{{- if or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)}}
Flower dashboard: kubectl port-forward svc/{{ include "airflow.fullname" . }}-flower {{ .Values.ports.flowerUI }}:{{ .Values.ports.flowerUI }} --namespace {{ .Release.Namespace }}
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.webserver.defaultUser.enabled}}
Default Webserver (Airflow UI) Login credentials:
username: {{ .Values.webserver.defaultUser.username }}
password: {{ .Values.webserver.defaultUser.password }}
{{- end }}
{{- if .Values.postgresql.enabled }}
Default Postgres connection credentials:
username: {{ .Values.data.metadataConnection.user }}
password: {{ .Values.data.metadataConnection.pass }}
port: {{ .Values.data.metadataConnection.port }}
{{- end }}
{{- if not .Values.fernetKeySecretName }}
You can get Fernet Key value by running the following:
echo Fernet Key: $(kubectl get secret --namespace {{ .Release.Namespace }} {{ .Release.Name }}-fernet-key -o jsonpath="{.data.fernet-key}" | base64 --decode)
{{- end }}
{{- if or (contains "KubernetesExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) }}
{{- if and (not .Values.logs.persistence.enabled) (eq (lower (tpl .Values.config.logging.remote_logging .)) "false") }}
WARNING:
Kubernetes workers task logs may not persist unless you configure log persistence or remote logging!
Logging options can be found at: https://airflow.apache.org/docs/helm-chart/stable/manage-logs.html
(This warning can be ignored if logging is configured with environment variables or secrets backend)
{{- end }}
{{- end }}
{{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.sshKeySecret (not .Values.dags.gitSync.knownHosts)}}
#####################################################
# WARNING: You should set dags.gitSync.knownHosts #
#####################################################
You are using ssh authentication for your gitsync repo, however you currently have SSH known_hosts verification disabled,
making you susceptible to man-in-the-middle attacks!
Information on how to set knownHosts can be found here:
https://airflow.apache.org/docs/helm-chart/stable/production-guide.html#knownhosts
{{- end }}
{{- if .Values.flower.extraNetworkPolicies }}
DEPRECATION WARNING:
`flower.extraNetworkPolicies` has been renamed to `flower.networkPolicy.peers`.
Please change your values as support for the old name will be dropped in a future release.
{{- end }}
{{- if .Values.webserver.extraNetworkPolicies }}
DEPRECATION WARNING:
`webserver.extraNetworkPolicies` has been renamed to `webserver.networkPolicy.peers`.
Please change your values as support for the old name will be dropped in a future release.
{{- end }}
{{- if not (or .Values.webserverSecretKey .Values.webserverSecretKeySecretName) }}
{{- if .Values.securityContext }}
DEPRECATION WARNING:
`securityContext` has been renamed to `securityContexts`, to be enabled on container and pod level.
Please change your values as support for the old name will be dropped in a future release.
{{- end }}
###########################################################
# WARNING: You should set a static webserver secret key #
###########################################################
You are using a dynamically generated webserver secret key, which can lead to
unnecessary restarts of your Airflow components.
Information on how to set a static webserver secret key can be found here:
https://airflow.apache.org/docs/helm-chart/stable/production-guide.html#webserver-secret-key
{{- end }}
{{- if or .Values.postgresql.postgresqlUsername .Values.postgresql.postgresqlPassword }}
{{ fail "postgresql.postgresqlUsername and postgresql.postgresqlPassword are no longer supported. If you wish to use the 'postgres' user, set its password with postgresql.auth.postgresPassword. If you wish to create a different user, do so with postgresql.auth.username and postgresql.auth.password." }}
{{- end }}
{{- if ne .Values.executor (tpl .Values.config.core.executor $) }}
{{ fail "Please configure the executor with `executor`, not `config.core.executor`." }}
{{- end }}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,241 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow API Server Deployment
#################################
{{- if semverCompare ">=3.0.0" .Values.airflowVersion }}
{{- $nodeSelector := or .Values.apiServer.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.apiServer.affinity .Values.affinity }}
{{- $tolerations := or .Values.apiServer.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.apiServer.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $revisionHistoryLimit := or .Values.apiServer.revisionHistoryLimit .Values.revisionHistoryLimit }}
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.apiServer) }}
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.apiServer) }}
{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list . .Values.apiServer.waitForMigrations) }}
{{- $containerLifecycleHooks := or .Values.apiServer.containerLifecycleHooks .Values.containerLifecycleHooks }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "airflow.fullname" . }}-api-server
labels:
tier: airflow
component: api-server
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if .Values.apiServer.annotations }}
annotations: {{- toYaml .Values.apiServer.annotations | nindent 4 }}
{{- end }}
spec:
replicas: {{ .Values.apiServer.replicas }}
{{- if $revisionHistoryLimit }}
revisionHistoryLimit: {{ $revisionHistoryLimit }}
{{- end }}
strategy:
{{- if .Values.apiServer.strategy }}
{{- toYaml .Values.apiServer.strategy | nindent 4 }}
{{- else }}
# Here we define the rolling update strategy
# - maxSurge define how many pod we can add at a time
# - maxUnavailable define how many pod can be unavailable
# during the rolling update
# Setting maxUnavailable to 0 would make sure we have the appropriate
# capacity during the rolling update.
# You can also use percentage based value instead of integer.
type: RollingUpdate
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
{{- end }}
selector:
matchLabels:
tier: airflow
component: api-server
release: {{ .Release.Name }}
template:
metadata:
labels:
tier: airflow
component: api-server
release: {{ .Release.Name }}
{{- if or (.Values.labels) (.Values.apiServer.labels) }}
{{- mustMerge .Values.apiServer.labels .Values.labels | toYaml | nindent 8 }}
{{- end }}
annotations:
checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }}
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }}
checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }}
checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }}
{{- if .Values.airflowPodAnnotations }}
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
{{- end }}
{{- if .Values.apiServer.podAnnotations }}
{{- toYaml .Values.apiServer.podAnnotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.apiServer.hostAliases }}
hostAliases: {{- toYaml .Values.apiServer.hostAliases | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "apiServer.serviceAccountName" . }}
{{- if .Values.apiServer.priorityClassName }}
priorityClassName: {{ .Values.apiServer.priorityClassName }}
{{- end }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity:
{{- if $affinity }}
{{- toYaml $affinity | nindent 8 }}
{{- else }}
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
component: api-server
topologyKey: kubernetes.io/hostname
weight: 100
{{- end }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
restartPolicy: Always
securityContext: {{ $securityContext | nindent 8 }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
initContainers:
{{- if .Values.apiServer.waitForMigrations.enabled }}
- name: wait-for-airflow-migrations
resources: {{- toYaml .Values.apiServer.resources | nindent 12 }}
image: {{ template "airflow_image_for_migrations" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }}
volumeMounts:
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.apiServer.extraVolumeMounts }}
{{- tpl (toYaml .Values.apiServer.extraVolumeMounts) . | nindent 12 }}
{{- end }}
args: {{- include "wait-for-migrations-command" . | indent 10 }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- if .Values.apiServer.waitForMigrations.env }}
{{- tpl (toYaml .Values.apiServer.waitForMigrations.env) $ | nindent 12 }}
{{- end }}
{{- end }}
{{- if .Values.apiServer.extraInitContainers }}
{{- toYaml .Values.apiServer.extraInitContainers | nindent 8 }}
{{- end }}
containers:
- name: api-server
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
{{- if .Values.apiServer.command }}
command: {{ tpl (toYaml .Values.apiServer.command) . | nindent 12 }}
{{- end }}
{{- if .Values.apiServer.args }}
args: {{- tpl (toYaml .Values.apiServer.args) . | nindent 12 }}
{{- end }}
resources: {{- toYaml .Values.apiServer.resources | nindent 12 }}
volumeMounts:
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if .Values.logs.persistence.enabled }}
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- end }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.apiServer.extraVolumeMounts }}
{{- tpl (toYaml .Values.apiServer.extraVolumeMounts) . | nindent 12 }}
{{- end }}
ports:
- name: api-server
containerPort: {{ .Values.ports.apiServer }}
livenessProbe:
httpGet:
path: /api/v2/version
port: {{ .Values.ports.apiServer }}
scheme: {{ .Values.apiServer.livenessProbe.scheme | default "http" }}
initialDelaySeconds: {{ .Values.apiServer.livenessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.apiServer.livenessProbe.timeoutSeconds }}
failureThreshold: {{ .Values.apiServer.livenessProbe.failureThreshold }}
periodSeconds: {{ .Values.apiServer.livenessProbe.periodSeconds }}
readinessProbe:
httpGet:
path: /api/v2/version
port: {{ .Values.ports.apiServer }}
scheme: {{ .Values.apiServer.readinessProbe.scheme | default "http" }}
initialDelaySeconds: {{ .Values.apiServer.readinessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.apiServer.readinessProbe.timeoutSeconds }}
failureThreshold: {{ .Values.apiServer.readinessProbe.failureThreshold }}
periodSeconds: {{ .Values.apiServer.readinessProbe.periodSeconds }}
startupProbe:
httpGet:
path: /api/v2/version
port: {{ .Values.ports.apiServer }}
scheme: {{ .Values.apiServer.startupProbe.scheme | default "http" }}
initialDelaySeconds: {{ .Values.apiServer.startupProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.apiServer.startupProbe.timeoutSeconds }}
failureThreshold: {{ .Values.apiServer.startupProbe.failureThreshold }}
periodSeconds: {{ .Values.apiServer.startupProbe.periodSeconds }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- include "container_extra_envs" (list . .Values.apiServer.env) | indent 10 }}
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) (semverCompare "<2.0.0" .Values.airflowVersion) }}
{{- include "git_sync_container" . | nindent 8 }}
{{- end }}
{{- if .Values.apiServer.extraContainers }}
{{- tpl (toYaml .Values.apiServer.extraContainers) . | nindent 8 }}
{{- end }}
volumes:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if (semverCompare "<2.0.0" .Values.airflowVersion) }}
{{- end }}
{{- if .Values.logs.persistence.enabled }}
- name: logs
persistentVolumeClaim:
claimName: {{ template "airflow_logs_volume_claim" . }}
{{- end }}
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 8 }}
{{- end }}
{{- if .Values.apiServer.extraVolumes }}
{{- tpl (toYaml .Values.apiServer.extraVolumes) . | nindent 8 }}
{{- end }}
{{- end }}
@@ -0,0 +1,58 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow API server NetworkPolicy
#################################
{{- if semverCompare ">=3.0.0" .Values.airflowVersion }}
{{- if .Values.networkPolicies.enabled }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ include "airflow.fullname" . }}-api-server-policy
labels:
tier: airflow
component: airflow-api-server-policy
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.apiServer.labels) }}
{{- mustMerge .Values.apiServer.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
podSelector:
matchLabels:
tier: airflow
component: api-server
release: {{ .Release.Name }}
policyTypes:
- Ingress
{{- if .Values.apiServer.networkPolicy.ingress.from }}
ingress:
- from: {{- toYaml .Values.apiServer.networkPolicy.ingress.from | nindent 6 }}
ports:
{{ range .Values.apiServer.networkPolicy.ingress.ports }}
-
{{- range $key, $val := . }}
{{ $key }}: {{ tpl (toString $val) $ }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,46 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow api-server PodDisruptionBudget
#################################
{{- if semverCompare ">=3.0.0" .Values.airflowVersion }}
{{- if .Values.apiServer.podDisruptionBudget.enabled }}
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: {{ include "airflow.fullname" . }}-api-server-pdb
labels:
tier: airflow
component: api-server
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.apiServer.labels) }}
{{- mustMerge .Values.apiServer.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
selector:
matchLabels:
tier: airflow
component: api-server
release: {{ .Release.Name }}
{{- toYaml .Values.apiServer.podDisruptionBudget.config | nindent 2 }}
{{- end }}
{{- end }}
@@ -0,0 +1,59 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow api-server Service
#################################
{{- if semverCompare ">=3.0.0" .Values.airflowVersion }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "airflow.fullname" . }}-api-server
labels:
tier: airflow
component: api-server
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.apiServer.labels) }}
{{- mustMerge .Values.apiServer.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.apiServer.service.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
spec:
type: {{ .Values.apiServer.service.type }}
selector:
tier: airflow
component: api-server
release: {{ .Release.Name }}
ports:
{{ range .Values.apiServer.service.ports }}
-
{{- range $key, $val := . }}
{{ $key }}: {{ tpl (toString $val) $ }}
{{- end }}
{{- end }}
{{- if .Values.apiServer.service.loadBalancerIP }}
loadBalancerIP: {{ .Values.apiServer.service.loadBalancerIP }}
{{- end }}
{{- if .Values.apiServer.service.loadBalancerSourceRanges }}
loadBalancerSourceRanges: {{- toYaml .Values.apiServer.service.loadBalancerSourceRanges | nindent 4 }}
{{- end }}
{{- end }}
@@ -0,0 +1,41 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
######################################
## Airflow api-server ServiceAccount
######################################
{{- if and .Values.apiServer.serviceAccount.create (semverCompare ">=3.0.0" .Values.airflowVersion) }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.apiServer.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "apiServer.serviceAccountName" . }}
labels:
tier: airflow
component: api-server
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.apiServer.labels) }}
{{- mustMerge .Values.apiServer.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.apiServer.serviceAccount.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
@@ -0,0 +1,89 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
{{- /*
The sole purpose of this yaml file is it to check the values file is consistent for some complexe combinations.
*/ -}}
{{- /*
##############################
Redis related checks
#############################
*/ -}}
{{- if or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) }}
{{- if .Values.redis.enabled }}
{{- if .Values.redis.passwordSecretName }}
{{- $existedBrokerUrlCmd := false }}
{{- range .Values.env }}
{{- if eq .name "AIRFLOW__CELERY__BROKER_URL_CMD" }}
{{- $existedBrokerUrlCmd = true }}
{{- break -}}
{{- end }}
{{- end }}
{{- if not (or .Values.data.brokerUrlSecretName $existedBrokerUrlCmd) }}
{{ required "When using the internal redis of the chart and setting the value redis.passwordSecretName, you must also set the value data.brokerUrlSecretName or AIRFLOW__CELERY__BROKER_URL_CMD in env." nil }}
{{- end }}
{{- end }}
{{- if and .Values.redis.passwordSecretName .Values.redis.password }}
{{ required "You must not set both values redis.passwordSecretName and redis.password" nil }}
{{- end }}
{{- else }}
{{- if not (or .Values.data.brokerUrlSecretName .Values.data.brokerUrl) }}
{{ required "You must set one of the values data.brokerUrlSecretName or data.brokerUrl when using a Celery based executor with redis.enabled set to false (we need the url to the redis instance)." nil }}
{{- end }}
{{- end }}
{{- if and .Values.data.brokerUrlSecretName .Values.data.brokerUrl }}
{{ required "You must not set both values data.brokerUrlSecretName and data.brokerUrl" nil }}
{{- end }}
{{- end }}
{{- if and .Values.elasticsearch.enabled .Values.opensearch.enabled }}
{{ required "You must not set both values elasticsearch.enabled and opensearch.enabled" nil }}
{{- end }}
{{- if .Values.elasticsearch.enabled }}
{{- if and .Values.elasticsearch.secretName .Values.elasticsearch.connection }}
{{ required "You must not set both values elasticsearch.secretName and elasticsearch.connection" nil }}
{{- end }}
{{- if not (or .Values.elasticsearch.secretName .Values.elasticsearch.connection) }}
{{ required "You must set one of the values elasticsearch.secretName or elasticsearch.connection when using a Elasticsearch" nil }}
{{- end }}
{{- end }}
{{- if .Values.opensearch.enabled }}
{{- if and .Values.opensearch.secretName .Values.opensearch.connection }}
{{ required "You must not set both values opensearch.secretName and opensearch.connection" nil }}
{{- end }}
{{- if not (or .Values.opensearch.secretName .Values.opensearch.connection) }}
{{ required "You must set one of the values opensearch.secretName or opensearch.connection when using OpenSearch" nil }}
{{- end }}
{{- end }}
@@ -0,0 +1,121 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Cleanup Pods CronJob
#################################
{{- if .Values.cleanup.enabled }}
{{- $nodeSelector := or .Values.cleanup.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.cleanup.affinity .Values.affinity }}
{{- $tolerations := or .Values.cleanup.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.cleanup.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.cleanup) }}
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.cleanup) }}
apiVersion: batch/v1
kind: CronJob
metadata:
name: {{ include "airflow.fullname" . }}-cleanup
labels:
tier: airflow
component: airflow-cleanup-pods
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.cleanup.jobAnnotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
spec:
schedule: "{{ tpl .Values.cleanup.schedule . }}"
# The cron job does not allow concurrent runs; if it is time for a new job run and the previous job run hasn't finished yet, the cron job skips the new job run
concurrencyPolicy: Forbid
{{- if not ( eq .Values.cleanup.failedJobsHistoryLimit nil) }}
failedJobsHistoryLimit: {{ .Values.cleanup.failedJobsHistoryLimit }}
{{- end }}
{{- if not (eq .Values.cleanup.successfulJobsHistoryLimit nil) }}
successfulJobsHistoryLimit: {{ .Values.cleanup.successfulJobsHistoryLimit }}
{{- end }}
jobTemplate:
spec:
backoffLimit: 1
template:
metadata:
labels:
tier: airflow
component: airflow-cleanup-pods
release: {{ .Release.Name }}
{{- if or (.Values.labels) (.Values.cleanup.labels) }}
{{- mustMerge .Values.cleanup.labels .Values.labels | toYaml | nindent 12 }}
{{- end }}
annotations:
sidecar.istio.io/inject: "false"
{{- if .Values.airflowPodAnnotations }}
{{- toYaml .Values.airflowPodAnnotations | nindent 12 }}
{{- end }}
{{- if .Values.cleanup.podAnnotations }}
{{- toYaml .Values.cleanup.podAnnotations | nindent 12 }}
{{- end }}
spec:
restartPolicy: Never
{{- if .Values.cleanup.priorityClassName }}
priorityClassName: {{ .Values.cleanup.priorityClassName }}
{{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 12 }}
affinity: {{- toYaml $affinity | nindent 12 }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
tolerations: {{- toYaml $tolerations | nindent 12 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 12 }}
serviceAccountName: {{ include "cleanup.serviceAccountName" . }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
securityContext: {{ $securityContext | nindent 12 }}
containers:
- name: airflow-cleanup-pods
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 16 }}
{{- if .Values.cleanup.command }}
command: {{ tpl (toYaml .Values.cleanup.command) . | nindent 16 }}
{{- end }}
{{- if .Values.cleanup.args }}
args: {{ tpl (toYaml .Values.cleanup.args) . | nindent 16 }}
{{- end }}
env:
{{- include "standard_airflow_environment" . | indent 12 }}
{{- include "container_extra_envs" (list . .Values.cleanup.env) | indent 12 }}
volumeMounts:
{{- include "airflow_config_mount" . | nindent 16 }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 16 }}
{{- end }}
resources: {{- toYaml .Values.cleanup.resources | nindent 16 }}
volumes:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 12 }}
{{- end }}
{{- end }}
@@ -0,0 +1,40 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Cleanup ServiceAccount
#################################
{{- if and .Values.cleanup.serviceAccount.create .Values.cleanup.enabled }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.cleanup.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "cleanup.serviceAccountName" . }}
labels:
tier: airflow
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.cleanup.labels) }}
{{- mustMerge .Values.cleanup.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.cleanup.serviceAccount.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
@@ -0,0 +1,81 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow ConfigMap
#################################
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ template "airflow_config" . }}
labels:
tier: airflow
component: config
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end -}}
{{- if .Values.airflowConfigAnnotations }}
annotations: {{- toYaml .Values.airflowConfigAnnotations | nindent 4 }}
{{- end }}
{{- $Global := . }}
data:
{{/*- Set a default for core.execution_api_server_url pointing to the api-server service if it's not set -*/}}
{{- if semverCompare ">=3.0.0" .Values.airflowVersion -}}
{{- $config := merge .Values.config ( dict "core" dict )}}
{{- if not (hasKey $config.core "execution_api_server_url") -}}
{{- $_ := set $config.core "execution_api_server_url" (printf "http://%s-api-server:%d/execution/" (include "airflow.fullname" .) (int .Values.ports.apiServer)) -}}
{{- end -}}
{{- end -}}
# These are system-specified config overrides.
airflow.cfg: |-
{{- range $section, $settings := .Values.config }}
[{{ $section }}]
{{- range $key, $val := $settings }}
{{ $key }} = {{ tpl ($val | toString) $Global }}
{{- end }}
{{ end }}
{{- if .Values.airflowLocalSettings }}
airflow_local_settings.py: |-
{{- tpl .Values.airflowLocalSettings . | nindent 4 }}
{{- end }}
{{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.knownHosts }}
known_hosts: |-
{{- .Values.dags.gitSync.knownHosts | nindent 4 }}
{{- end }}
{{- if or (contains "LocalKubernetesExecutor" $.Values.executor) (contains "KubernetesExecutor" $.Values.executor) (contains "CeleryKubernetesExecutor" $.Values.executor) }}
{{- if semverCompare ">=1.10.12" .Values.airflowVersion }}
pod_template_file.yaml: |-
{{- if .Values.podTemplate }}
{{- tpl .Values.podTemplate . | nindent 4 }}
{{- else }}
{{- tpl (.Files.Get "files/pod-template-file.kubernetes-helm-yaml") . | nindent 4 }}
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.kerberos.enabled }}
krb5.conf: |-
{{- tpl .Values.kerberos.config . | nindent 4 }}
{{- end }}
@@ -0,0 +1,55 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
####################################################
## Extra ConfigMaps provisioned via the chart values
####################################################
{{- $Global := . }}
{{- range $configMapName, $configMapContent := .Values.extraConfigMaps }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ tpl $configMapName $Global | quote }}
labels:
release: {{ $Global.Release.Name }}
chart: "{{ $Global.Chart.Name }}-{{ $Global.Chart.Version }}"
heritage: {{ $Global.Release.Service }}
{{- with $Global.Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if $configMapContent.labels }}
{{- toYaml $configMapContent.labels | nindent 4 }}
{{- end }}
{{- $annotations := dict }}
{{- if or $configMapContent.useHelmHooks (not (hasKey $configMapContent "useHelmHooks")) }}
{{- $_ := set $annotations "helm.sh/hook" "pre-install,pre-upgrade" }}
{{- $_ := set $annotations "helm.sh/hook-weight" "0" }}
{{- $_ := set $annotations "helm.sh/hook-delete-policy" "before-hook-creation" }}
{{- end }}
{{- with $annotations := merge $annotations ($configMapContent.annotations | default dict) }}
annotations: {{- $annotations | toYaml | nindent 4 }}
{{- end }}
{{- if $configMapContent.data }}
data:
{{- with $configMapContent.data }}
{{- tpl . $Global | nindent 2 }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,52 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow StatsD ConfigMap
#################################
{{- if and .Values.statsd.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "airflow.fullname" . }}-statsd
labels:
tier: airflow
component: config
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.statsd.configMapAnnotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
data:
mappings.yml: |-
{{- if .Values.statsd.overrideMappings }}
mappings:
{{- toYaml .Values.statsd.overrideMappings | nindent 6 }}
{{- else }}
{{- tpl (.Files.Get "files/statsd-mappings.yml") . | nindent 4 }}
{{- if .Values.statsd.extraMappings }}
{{- toYaml .Values.statsd.extraMappings | nindent 6 }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,44 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow ConfigMap
#################################
{{- if and .Values.webserver.webserverConfig (not .Values.webserver.webserverConfigConfigMapName) }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ template "airflow_webserver_config_configmap_name" . }}
labels:
tier: airflow
component: config
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.webserver.configMapAnnotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
data:
webserver_config.py: |-
{{- tpl .Values.webserver.webserverConfig . | nindent 4 }}
{{- end }}
@@ -0,0 +1,278 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Dag Processor Deployment
#################################
{{- if semverCompare ">=2.3.0" .Values.airflowVersion }}
{{- $enabled := .Values.dagProcessor.enabled }}
{{- if eq $enabled nil}}
{{ $enabled = ternary true false (semverCompare ">=3.0.0" .Values.airflowVersion) }}
{{- end }}
{{- if $enabled }}
{{- $nodeSelector := or .Values.dagProcessor.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.dagProcessor.affinity .Values.affinity }}
{{- $tolerations := or .Values.dagProcessor.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.dagProcessor.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $revisionHistoryLimit := or .Values.dagProcessor.revisionHistoryLimit .Values.revisionHistoryLimit }}
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.dagProcessor) }}
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.dagProcessor) }}
{{- $containerSecurityContextLogGroomerSidecar := include "containerSecurityContext" (list . .Values.dagProcessor.logGroomerSidecar) }}
{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list . .Values.dagProcessor.waitForMigrations) }}
{{- $containerLifecycleHooks := or .Values.dagProcessor.containerLifecycleHooks .Values.containerLifecycleHooks }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "airflow.fullname" . }}-dag-processor
labels:
tier: airflow
component: dag-processor
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if .Values.dagProcessor.annotations }}
annotations: {{- toYaml .Values.dagProcessor.annotations | nindent 4 }}
{{- end }}
spec:
replicas: {{ .Values.dagProcessor.replicas }}
{{- if $revisionHistoryLimit }}
revisionHistoryLimit: {{ $revisionHistoryLimit }}
{{- end }}
selector:
matchLabels:
tier: airflow
component: dag-processor
release: {{ .Release.Name }}
{{- if .Values.dagProcessor.strategy }}
strategy: {{- toYaml .Values.dagProcessor.strategy | nindent 4 }}
{{- end }}
template:
metadata:
labels:
tier: airflow
component: dag-processor
release: {{ .Release.Name }}
{{- with .Values.labels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }}
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }}
checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }}
checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }}
{{- if .Values.dagProcessor.safeToEvict }}
cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
{{- end }}
{{- if .Values.airflowPodAnnotations }}
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
{{- end }}
{{- if .Values.dagProcessor.podAnnotations }}
{{- toYaml .Values.dagProcessor.podAnnotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.dagProcessor.priorityClassName }}
priorityClassName: {{ .Values.dagProcessor.priorityClassName }}
{{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity:
{{- if $affinity }}
{{- toYaml $affinity | nindent 8 }}
{{- else }}
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
component: dag-processor
topologyKey: kubernetes.io/hostname
weight: 100
{{- end }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
terminationGracePeriodSeconds: {{ .Values.dagProcessor.terminationGracePeriodSeconds }}
restartPolicy: Always
serviceAccountName: {{ include "dagProcessor.serviceAccountName" . }}
securityContext: {{ $securityContext | nindent 8 }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
initContainers:
{{- if .Values.dagProcessor.waitForMigrations.enabled }}
- name: wait-for-airflow-migrations
resources: {{- toYaml .Values.dagProcessor.resources | nindent 12 }}
image: {{ template "airflow_image_for_migrations" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }}
volumeMounts:
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.dagProcessor.extraVolumeMounts }}
{{- tpl (toYaml .Values.dagProcessor.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- include "airflow_config_mount" . | nindent 12 }}
args: {{- include "wait-for-migrations-command" . | indent 10 }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- if .Values.dagProcessor.waitForMigrations.env }}
{{- tpl (toYaml .Values.dagProcessor.waitForMigrations.env) $ | nindent 12 }}
{{- end }}
{{- end }}
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
{{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }}
{{- end }}
{{- if .Values.dagProcessor.extraInitContainers }}
{{- tpl (toYaml .Values.dagProcessor.extraInitContainers) . | nindent 8 }}
{{- end }}
containers:
- name: dag-processor
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
{{- if .Values.dagProcessor.command }}
command: {{ tpl (toYaml .Values.dagProcessor.command) . | nindent 12 }}
{{- end }}
{{- if .Values.dagProcessor.args }}
args: {{ tpl (toYaml .Values.dagProcessor.args) . | nindent 12 }}
{{- end }}
resources: {{- toYaml .Values.dagProcessor.resources | nindent 12 }}
volumeMounts:
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.dagProcessor.extraVolumeMounts }}
{{- tpl (toYaml .Values.dagProcessor.extraVolumeMounts) . | nindent 12 }}
{{- end }}
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }}
{{- include "airflow_dags_mount" . | nindent 12 }}
{{- end }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- include "container_extra_envs" (list . .Values.dagProcessor.env) | indent 10 }}
livenessProbe:
initialDelaySeconds: {{ .Values.dagProcessor.livenessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.dagProcessor.livenessProbe.timeoutSeconds }}
failureThreshold: {{ .Values.dagProcessor.livenessProbe.failureThreshold }}
periodSeconds: {{ .Values.dagProcessor.livenessProbe.periodSeconds }}
exec:
command:
{{- if .Values.dagProcessor.livenessProbe.command }}
{{- toYaml .Values.dagProcessor.livenessProbe.command | nindent 16 }}
{{- else }}
{{- include "dag_processor_liveness_check_command" . | indent 14 }}
{{- end }}
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
{{- include "git_sync_container" . | indent 8 }}
{{- end }}
{{- if .Values.dagProcessor.logGroomerSidecar.enabled }}
- name: dag-processor-log-groomer
resources: {{- toYaml .Values.dagProcessor.logGroomerSidecar.resources | nindent 12 }}
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContextLogGroomerSidecar | nindent 12 }}
{{- if .Values.dagProcessor.logGroomerSidecar.command }}
command: {{ tpl (toYaml .Values.dagProcessor.logGroomerSidecar.command) . | nindent 12 }}
{{- end }}
{{- if .Values.dagProcessor.logGroomerSidecar.args }}
args: {{- tpl (toYaml .Values.dagProcessor.logGroomerSidecar.args) . | nindent 12 }}
{{- end }}
env:
{{- if .Values.dagProcessor.logGroomerSidecar.retentionDays }}
- name: AIRFLOW__LOG_RETENTION_DAYS
value: "{{ .Values.dagProcessor.logGroomerSidecar.retentionDays }}"
{{- end }}
{{- if .Values.dagProcessor.logGroomerSidecar.frequencyMinutes }}
- name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES
value: "{{ .Values.dagProcessor.logGroomerSidecar.frequencyMinutes }}"
{{- end }}
- name: AIRFLOW_HOME
value: "{{ .Values.airflowHome }}"
{{- if .Values.dagProcessor.logGroomerSidecar.env }}
{{- tpl (toYaml .Values.dagProcessor.logGroomerSidecar.env) $ | nindent 12 }}
{{- end }}
volumeMounts:
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.dagProcessor.extraVolumeMounts }}
{{- tpl (toYaml .Values.dagProcessor.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
{{- end }}
{{- if .Values.dagProcessor.extraContainers }}
{{- tpl (toYaml .Values.dagProcessor.extraContainers) . | nindent 8 }}
{{- end }}
volumes:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
- name: webserver-config
configMap:
name: {{ template "airflow_webserver_config_configmap_name" . }}
{{- end }}
{{- if .Values.dags.persistence.enabled }}
- name: dags
persistentVolumeClaim:
claimName: {{ template "airflow_dags_volume_claim" . }}
{{- else if .Values.dags.gitSync.enabled }}
- name: dags
emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 12 }}
{{- end }}
{{- if and .Values.dags.gitSync.enabled (or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey) }}
{{- include "git_sync_ssh_key_volume" . | indent 8 }}
{{- end }}
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 8 }}
{{- end }}
{{- if .Values.dagProcessor.extraVolumes }}
{{- tpl (toYaml .Values.dagProcessor.extraVolumes) . | nindent 8 }}
{{- end }}
{{- if .Values.logs.persistence.enabled }}
- name: logs
persistentVolumeClaim:
claimName: {{ template "airflow_logs_volume_claim" . }}
{{- else }}
- name: logs
emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) | nindent 12 }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,47 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Dag Processor ServiceAccount
#################################
{{- if semverCompare ">=2.3.0" .Values.airflowVersion }}
{{- $enabled := .Values.dagProcessor.enabled }}
{{- if eq $enabled nil}}
{{ $enabled = ternary true false (semverCompare ">=3.0.0" .Values.airflowVersion) }}
{{- end }}
{{- if and .Values.dagProcessor.serviceAccount.create $enabled }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.dagProcessor.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "dagProcessor.serviceAccountName" . }}
labels:
tier: airflow
component: dag-processor
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.dagProcessor.serviceAccount.annotations}}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,52 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
######################################
## Airflow DAGs PersistentVolumeClaim
######################################
{{- if and (not .Values.dags.persistence.existingClaim ) .Values.dags.persistence.enabled }}
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: {{ template "airflow_dags_volume_claim" . }}
labels:
tier: airflow
component: dags-pvc
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.dags.persistence.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
spec:
accessModes: [{{ .Values.dags.persistence.accessMode | quote }}]
resources:
requests:
storage: {{ .Values.dags.persistence.size | quote }}
{{- if .Values.dags.persistence.storageClassName }}
{{- if (eq "-" .Values.dags.persistence.storageClassName) }}
storageClassName: ""
{{- else }}
storageClassName: {{ tpl .Values.dags.persistence.storageClassName . | quote }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,184 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Flower Deployment
#################################
{{- if .Values.flower.enabled }}
{{- if or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) }}
{{- $nodeSelector := or .Values.flower.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.flower.affinity .Values.affinity }}
{{- $tolerations := or .Values.flower.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.flower.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $revisionHistoryLimit := or .Values.flower.revisionHistoryLimit .Values.revisionHistoryLimit }}
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.flower) }}
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.flower) }}
{{- $containerLifecycleHooks := or .Values.flower.containerLifecycleHooks .Values.containerLifecycleHooks }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "airflow.fullname" . }}-flower
labels:
tier: airflow
component: flower
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if .Values.flower.annotations }}
annotations: {{- toYaml .Values.flower.annotations | nindent 4 }}
{{- end }}
spec:
replicas: 1
{{- if $revisionHistoryLimit }}
revisionHistoryLimit: {{ $revisionHistoryLimit }}
{{- end }}
selector:
matchLabels:
tier: airflow
component: flower
release: {{ .Release.Name }}
template:
metadata:
labels:
tier: airflow
component: flower
release: {{ .Release.Name }}
{{- if or (.Values.labels) (.Values.flower.labels) }}
{{- mustMerge .Values.flower.labels .Values.labels | toYaml | nindent 8 }}
{{- end }}
annotations:
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }}
checksum/flower-secret: {{ include (print $.Template.BasePath "/secrets/flower-secret.yaml") . | sha256sum }}
{{- if or (.Values.airflowPodAnnotations) (.Values.flower.podAnnotations) }}
{{- mustMerge .Values.flower.podAnnotations .Values.airflowPodAnnotations | toYaml | nindent 8 }}
{{- end }}
spec:
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity: {{- toYaml $affinity | nindent 8 }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
serviceAccountName: {{ include "flower.serviceAccountName" . }}
{{- if .Values.flower.priorityClassName }}
priorityClassName: {{ .Values.flower.priorityClassName }}
{{- end }}
restartPolicy: Always
securityContext: {{ $securityContext | nindent 8 }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
containers:
- name: flower
image: {{ template "flower_image" . }}
imagePullPolicy: {{ .Values.images.flower.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
{{- if .Values.flower.command }}
command: {{ tpl (toYaml .Values.flower.command) . | nindent 12 }}
{{- end }}
{{- if .Values.flower.args }}
args: {{ tpl (toYaml .Values.flower.args) . | nindent 12 }}
{{- end }}
resources: {{- toYaml .Values.flower.resources | nindent 12 }}
volumeMounts:
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.flower.extraVolumeMounts }}
{{- tpl (toYaml .Values.flower.extraVolumeMounts) . | nindent 12 }}
{{- end }}
ports:
- name: flower-ui
containerPort: {{ .Values.ports.flowerUI }}
livenessProbe:
failureThreshold: {{ .Values.flower.livenessProbe.failureThreshold }}
exec:
command:
- curl
{{- if (or .Values.flower.secretName (and .Values.flower.username .Values.flower.password))}}
- "--user"
- $AIRFLOW__CELERY__FLOWER_BASIC_AUTH
{{- end }}
- {{ printf "localhost:%s" (.Values.ports.flowerUI | toString) }}
initialDelaySeconds: {{ .Values.flower.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.flower.livenessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.flower.livenessProbe.timeoutSeconds }}
readinessProbe:
failureThreshold: {{ .Values.flower.readinessProbe.failureThreshold }}
exec:
command:
- curl
{{- if (or .Values.flower.secretName (and .Values.flower.username .Values.flower.password))}}
- "--user"
- $AIRFLOW__CELERY__FLOWER_BASIC_AUTH
{{- end }}
- {{ printf "localhost:%s" (.Values.ports.flowerUI | toString) }}
initialDelaySeconds: {{ .Values.flower.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.flower.readinessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.flower.readinessProbe.timeoutSeconds }}
startupProbe:
failureThreshold: {{ .Values.flower.startupProbe.failureThreshold }}
exec:
command:
- curl
{{- if (or .Values.flower.secretName (and .Values.flower.username .Values.flower.password))}}
- "--user"
- $AIRFLOW__CELERY__FLOWER_BASIC_AUTH
{{- end }}
- {{ printf "localhost:%s" (.Values.ports.flowerUI | toString) }}
periodSeconds: {{ .Values.flower.startupProbe.periodSeconds }}
initialDelaySeconds: {{ .Values.flower.startupProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.flower.startupProbe.timeoutSeconds }}
envFrom:
{{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- if (or .Values.flower.secretName (and .Values.flower.username .Values.flower.password))}}
- name: AIRFLOW__CELERY__FLOWER_BASIC_AUTH
valueFrom:
secretKeyRef:
name: {{ template "flower_secret" . }}
key: basicAuth
{{- end }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "container_extra_envs" (list . .Values.flower.env) | indent 10 }}
{{- if .Values.flower.extraContainers }}
{{- tpl (toYaml .Values.flower.extraContainers) . | nindent 8 }}
{{- end }}
volumes:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 8 }}
{{- end }}
{{- if .Values.flower.extraVolumes }}
{{- tpl (toYaml .Values.flower.extraVolumes) . | nindent 8 }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,95 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Flower Ingress
#################################
{{- if .Values.flower.enabled }}
{{- if and (or .Values.ingress.flower.enabled .Values.ingress.enabled) (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }}
{{- $fullname := (include "airflow.fullname" .) }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ $fullname }}-flower-ingress
labels:
tier: airflow
component: flower-ingress
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.flower.labels) }}
{{- mustMerge .Values.flower.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.ingress.flower.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if and .Values.ingress.flower.hosts (.Values.ingress.flower.hosts | first | kindIs "string" | not) }}
{{- $anyTlsHosts := false -}}
{{- range .Values.ingress.flower.hosts }}
{{- if .tls }}
{{- if .tls.enabled }}
{{- $anyTlsHosts = true -}}
{{- end }}
{{- end }}
{{- end }}
{{- if $anyTlsHosts }}
tls:
{{- range .Values.ingress.flower.hosts }}
{{- if .tls }}
{{- if .tls.enabled }}
- hosts:
- {{ .name | quote }}
secretName: {{ .tls.secretName }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- else if .Values.ingress.flower.tls.enabled }}
tls:
- hosts:
{{- .Values.ingress.flower.hosts | default (list .Values.ingress.flower.host) | toYaml | nindent 8 }}
secretName: {{ .Values.ingress.flower.tls.secretName }}
{{- end }}
rules:
{{- range .Values.ingress.flower.hosts | default (list .Values.ingress.flower.host) }}
- http:
paths:
- backend:
service:
name: {{ $fullname }}-flower
port:
name: flower-ui
{{- if $.Values.ingress.flower.path }}
path: {{ $.Values.ingress.flower.path }}
pathType: {{ $.Values.ingress.flower.pathType }}
{{- end }}
{{- $hostname := . -}}
{{- if . | kindIs "string" | not }}
{{- $hostname = .name -}}
{{- end }}
{{- if $hostname }}
host: {{ tpl $hostname $ | quote }}
{{- end }}
{{- end }}
{{- if .Values.ingress.flower.ingressClassName }}
ingressClassName: {{ .Values.ingress.flower.ingressClassName }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,60 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Flower NetworkPolicy
#################################
{{- if .Values.flower.enabled }}
{{- $celery_executors := list "CeleryExecutor" "CeleryKubernetesExecutor"}}
{{- if and .Values.networkPolicies.enabled (has .Values.executor $celery_executors) }}
{{- $from := or .Values.flower.networkPolicy.ingress.from .Values.flower.extraNetworkPolicies }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ include "airflow.fullname" . }}-flower-policy
labels:
tier: airflow
component: airflow-flower-policy
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.flower.labels) }}
{{- mustMerge .Values.flower.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
podSelector:
matchLabels:
tier: airflow
component: flower
release: {{ .Release.Name }}
policyTypes:
- Ingress
{{- if $from }}
ingress:
- from: {{- toYaml $from | nindent 6 }}
ports:
{{ range .Values.flower.networkPolicy.ingress.ports }}
-
{{- range $key, $val := . }}
{{ $key }}: {{ tpl (toString $val) $ }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,61 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Flower Service Component
#################################
{{- if .Values.flower.enabled }}
{{- if or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "airflow.fullname" . }}-flower
labels:
tier: airflow
component: flower
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.flower.labels) }}
{{- mustMerge .Values.flower.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.flower.service.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
spec:
type: {{ .Values.flower.service.type }}
selector:
tier: airflow
component: flower
release: {{ .Release.Name }}
ports:
{{ range .Values.flower.service.ports }}
-
{{- range $key, $val := . }}
{{ $key }}: {{ tpl (toString $val) $ }}
{{- end }}
{{- end }}
{{- if .Values.flower.service.loadBalancerIP }}
loadBalancerIP: {{ .Values.flower.service.loadBalancerIP }}
{{- end }}
{{- if .Values.flower.service.loadBalancerSourceRanges }}
loadBalancerSourceRanges: {{- toYaml .Values.flower.service.loadBalancerSourceRanges | nindent 4 }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,41 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
######################################
## Airflow Flower ServiceAccount
######################################
{{- if and .Values.flower.enabled (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) .Values.flower.serviceAccount.create }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.flower.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "flower.serviceAccountName" . }}
labels:
tier: airflow
component: flower
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.flower.labels) }}
{{- mustMerge .Values.flower.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.flower.serviceAccount.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
@@ -0,0 +1,41 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
###########################################
## Airflow Create User Job ServiceAccount
###########################################
{{- if and .Values.createUserJob.serviceAccount.create .Values.webserver.defaultUser.enabled }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.createUserJob.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "createUserJob.serviceAccountName" . }}
labels:
tier: airflow
component: create-user-job
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.createUserJob.labels) }}
{{- mustMerge .Values.createUserJob.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.createUserJob.serviceAccount.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
@@ -0,0 +1,141 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Create User Job
#################################
{{- if .Values.webserver.defaultUser.enabled }}
{{- $nodeSelector := or .Values.createUserJob.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.createUserJob.affinity .Values.affinity }}
{{- $tolerations := or .Values.createUserJob.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.createUserJob.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.createUserJob) }}
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.createUserJob) }}
{{- $containerLifecycleHooks := or .Values.createUserJob.containerLifecycleHooks .Values.containerLifecycleHooks }}
apiVersion: batch/v1
kind: Job
metadata:
name: {{ include "airflow.fullname" . }}-create-user
labels:
tier: airflow
component: create-user-job
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- $annotations := dict }}
{{- if .Values.createUserJob.useHelmHooks }}
{{- $_ := set $annotations "helm.sh/hook" "post-install,post-upgrade" }}
{{- $_ := set $annotations "helm.sh/hook-weight" "2" }}
{{- $_ := set $annotations "helm.sh/hook-delete-policy" "before-hook-creation,hook-succeeded" }}
{{- end }}
{{- with $annotations := merge $annotations .Values.createUserJob.jobAnnotations }}
annotations: {{- $annotations | toYaml | nindent 4 }}
{{- end }}
spec:
{{- if not (kindIs "invalid" .Values.createUserJob.ttlSecondsAfterFinished) }}
ttlSecondsAfterFinished: {{ .Values.createUserJob.ttlSecondsAfterFinished }}
{{- end }}
template:
metadata:
labels:
tier: airflow
component: create-user-job
release: {{ .Release.Name }}
{{- if or (.Values.labels) (.Values.createUserJob.labels) }}
{{- mustMerge .Values.createUserJob.labels .Values.labels | toYaml | nindent 8 }}
{{- end }}
{{- if or .Values.airflowPodAnnotations .Values.createUserJob.annotations }}
annotations:
{{- if .Values.airflowPodAnnotations }}
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
{{- end }}
{{- if .Values.createUserJob.annotations }}
{{- toYaml .Values.createUserJob.annotations | nindent 8 }}
{{- end }}
{{- end }}
spec:
securityContext: {{ $securityContext | nindent 8 }}
restartPolicy: OnFailure
{{- if .Values.createUserJob.priorityClassName }}
priorityClassName: {{ .Values.createUserJob.priorityClassName }}
{{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity: {{- toYaml $affinity | nindent 8 }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
serviceAccountName: {{ include "createUserJob.serviceAccountName" . }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
{{- if .Values.createUserJob.extraInitContainers }}
initContainers:
{{- tpl (toYaml .Values.createUserJob.extraInitContainers) . | nindent 8 }}
{{- end }}
containers:
- name: create-user
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
{{- if .Values.createUserJob.command }}
command: {{ tpl (toYaml .Values.createUserJob.command) . | nindent 12 }}
{{- end }}
{{- if .Values.createUserJob.args }}
args: {{ tpl (toYaml .Values.createUserJob.args) . | nindent 12 }}
{{- end }}
{{- if .Values.createUserJob.applyCustomEnv }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env: {{- include "custom_airflow_environment" . | indent 10 }}
{{- else }}
env:
{{- end }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- include "container_extra_envs" (list . .Values.createUserJob.env) | indent 10 }}
resources: {{- toYaml .Values.createUserJob.resources | nindent 12 }}
volumeMounts:
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.createUserJob.extraVolumeMounts }}
{{- tpl (toYaml .Values.createUserJob.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if .Values.createUserJob.extraContainers }}
{{- tpl (toYaml .Values.createUserJob.extraContainers) . | nindent 8 }}
{{- end }}
volumes:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 8 }}
{{- end }}
{{- if .Values.createUserJob.extraVolumes }}
{{- tpl (toYaml .Values.createUserJob.extraVolumes) . | nindent 8 }}
{{- end }}
{{- end }}
@@ -0,0 +1,41 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
#############################################
## Airflow Migrate Database Job ServiceAccount
##############################################
{{- if .Values.migrateDatabaseJob.serviceAccount.create }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.migrateDatabaseJob.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "migrateDatabaseJob.serviceAccountName" . }}
labels:
tier: airflow
component: run-airflow-migrations
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.migrateDatabaseJob.labels) }}
{{- mustMerge .Values.migrateDatabaseJob.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.migrateDatabaseJob.serviceAccount.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
@@ -0,0 +1,145 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Run Migrations
#################################
{{- if .Values.migrateDatabaseJob.enabled }}
{{- $nodeSelector := or .Values.migrateDatabaseJob.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.migrateDatabaseJob.affinity .Values.affinity }}
{{- $tolerations := or .Values.migrateDatabaseJob.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.migrateDatabaseJob.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.migrateDatabaseJob) }}
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.migrateDatabaseJob) }}
{{- $containerLifecycleHooks := or .Values.migrateDatabaseJob.containerLifecycleHooks .Values.containerLifecycleHooks }}
apiVersion: batch/v1
kind: Job
metadata:
name: {{ include "airflow.fullname" . }}-run-airflow-migrations
labels:
tier: airflow
component: run-airflow-migrations
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- $annotations := dict }}
{{- if .Values.migrateDatabaseJob.useHelmHooks }}
{{- $_ := set $annotations "helm.sh/hook" "post-install,post-upgrade" }}
{{- $_ := set $annotations "helm.sh/hook-weight" "1" }}
{{- $_ := set $annotations "helm.sh/hook-delete-policy" "before-hook-creation,hook-succeeded" }}
{{- end }}
{{- with $annotations := merge $annotations .Values.migrateDatabaseJob.jobAnnotations }}
annotations: {{- $annotations | toYaml | nindent 4 }}
{{- end }}
spec:
{{- if not (kindIs "invalid" .Values.migrateDatabaseJob.ttlSecondsAfterFinished) }}
ttlSecondsAfterFinished: {{ .Values.migrateDatabaseJob.ttlSecondsAfterFinished }}
{{- end }}
template:
metadata:
labels:
tier: airflow
component: run-airflow-migrations
release: {{ .Release.Name }}
{{- if or (.Values.labels) (.Values.migrateDatabaseJob.labels) }}
{{- mustMerge .Values.migrateDatabaseJob.labels .Values.labels | toYaml | nindent 8 }}
{{- end }}
{{- if or .Values.airflowPodAnnotations .Values.migrateDatabaseJob.annotations }}
annotations:
{{- if .Values.airflowPodAnnotations }}
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
{{- end }}
{{- if .Values.migrateDatabaseJob.annotations }}
{{- toYaml .Values.migrateDatabaseJob.annotations | nindent 8 }}
{{- end }}
{{- end }}
spec:
securityContext: {{ $securityContext | nindent 8 }}
restartPolicy: OnFailure
{{- if .Values.migrateDatabaseJob.priorityClassName }}
priorityClassName: {{ .Values.migrateDatabaseJob.priorityClassName }}
{{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity: {{- toYaml $affinity | nindent 8 }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
serviceAccountName: {{ include "migrateDatabaseJob.serviceAccountName" . }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
{{- if .Values.migrateDatabaseJob.extraInitContainers }}
initContainers:
{{- tpl (toYaml .Values.migrateDatabaseJob.extraInitContainers) . | nindent 8 }}
{{- end }}
containers:
- name: run-airflow-migrations
image: {{ template "airflow_image_for_migrations" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
{{- if .Values.migrateDatabaseJob.command }}
command: {{- tpl (toYaml .Values.migrateDatabaseJob.command) . | nindent 12 }}
{{- end }}
{{- if .Values.migrateDatabaseJob.args }}
args: {{- tpl (toYaml .Values.migrateDatabaseJob.args) . | nindent 12 }}
{{- end }}
{{- if .Values.migrateDatabaseJob.applyCustomEnv }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env: {{- include "custom_airflow_environment" . | indent 10 }}
{{- else }}
env:
{{- end }}
- name: PYTHONUNBUFFERED
value: "1"
{{- include "standard_airflow_environment" . | indent 10 }}
{{- if .Values.migrateDatabaseJob.env }}
{{- tpl (toYaml .Values.migrateDatabaseJob.env) $ | nindent 12 }}
{{- end }}
resources: {{- toYaml .Values.migrateDatabaseJob.resources | nindent 12 }}
volumeMounts:
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.migrateDatabaseJob.extraVolumeMounts }}
{{- tpl (toYaml .Values.migrateDatabaseJob.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if .Values.migrateDatabaseJob.extraContainers }}
{{- tpl (toYaml .Values.migrateDatabaseJob.extraContainers) . | nindent 8 }}
{{- end }}
volumes:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 8 }}
{{- end }}
{{- if .Values.migrateDatabaseJob.extraVolumes }}
{{- tpl (toYaml .Values.migrateDatabaseJob.extraVolumes) . | nindent 8 }}
{{- end }}
{{- end }}
+39
View File
@@ -0,0 +1,39 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Namespace LimitRange
#################################
{{- if .Values.limits }}
apiVersion: v1
kind: LimitRange
metadata:
name: {{ .Release.Name }}-limit-range
labels:
tier: resources
component: limitrange
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
limits: {{- toYaml .Values.limits | nindent 4 }}
{{- end }}
@@ -0,0 +1,52 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
######################################
## Airflow LOGs PersistentVolumeClaim
######################################
{{- if and (not .Values.logs.persistence.existingClaim ) .Values.logs.persistence.enabled }}
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: {{ template "airflow_logs_volume_claim" . }}
labels:
tier: airflow
component: logs-pvc
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.logs.persistence.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
spec:
accessModes: ["ReadWriteMany"]
resources:
requests:
storage: {{ .Values.logs.persistence.size | quote }}
{{- if .Values.logs.persistence.storageClassName }}
{{- if (eq "-" .Values.logs.persistence.storageClassName) }}
storageClassName: ""
{{- else }}
storageClassName: {{ tpl .Values.logs.persistence.storageClassName . | quote }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,224 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Pgbouncer Deployment
#################################
{{- if .Values.pgbouncer.enabled }}
{{- $nodeSelector := or .Values.pgbouncer.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.pgbouncer.affinity .Values.affinity }}
{{- $tolerations := or .Values.pgbouncer.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.pgbouncer.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $revisionHistoryLimit := or .Values.pgbouncer.revisionHistoryLimit .Values.revisionHistoryLimit }}
{{- $securityContext := include "localPodSecurityContext" .Values.pgbouncer }}
{{- $containerSecurityContext := include "externalContainerSecurityContext" .Values.pgbouncer }}
{{- $containerSecurityContextMetricsExporter := include "externalContainerSecurityContext" .Values.pgbouncer.metricsExporterSidecar }}
{{- $containerLifecycleHooks := .Values.pgbouncer.containerLifecycleHooks }}
{{- $containerLifecycleHooksMetricsExporter := .Values.pgbouncer.metricsExporterSidecar.containerLifecycleHooks }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "airflow.fullname" . }}-pgbouncer
labels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if .Values.pgbouncer.annotations }}
annotations: {{- toYaml .Values.pgbouncer.annotations | nindent 4 }}
{{- end }}
spec:
replicas: {{ .Values.pgbouncer.replicas | default "1" }}
{{- if $revisionHistoryLimit }}
revisionHistoryLimit: {{ $revisionHistoryLimit }}
{{- end }}
strategy:
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
selector:
matchLabels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
template:
metadata:
labels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
{{- if or (.Values.labels) (.Values.pgbouncer.labels) }}
{{- mustMerge .Values.pgbouncer.labels .Values.labels | toYaml | nindent 8 }}
{{- end }}
annotations:
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
checksum/pgbouncer-certificates-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-certificates-secret.yaml") . | sha256sum }}
{{- if .Values.pgbouncer.podAnnotations }}
{{- toYaml .Values.pgbouncer.podAnnotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.pgbouncer.priorityClassName }}
priorityClassName: {{ .Values.pgbouncer.priorityClassName }}
{{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity: {{- toYaml $affinity | nindent 8 }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
serviceAccountName: {{ include "pgbouncer.serviceAccountName" . }}
securityContext: {{ $securityContext | nindent 8 }}
restartPolicy: Always
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
containers:
- name: pgbouncer
image: {{ template "pgbouncer_image" . }}
imagePullPolicy: {{ .Values.images.pgbouncer.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if .Values.pgbouncer.command }}
command: {{ tpl (toYaml .Values.pgbouncer.command) . | nindent 12 }}
{{- end }}
{{- if .Values.pgbouncer.args }}
args: {{ tpl (toYaml .Values.pgbouncer.args) . | nindent 12 }}
{{- end }}
resources: {{- toYaml .Values.pgbouncer.resources | nindent 12 }}
{{- with .Values.pgbouncer.env }}
env: {{- toYaml . | nindent 12 }}
{{- end }}
ports:
- name: pgbouncer
containerPort: {{ .Values.ports.pgbouncer }}
livenessProbe:
tcpSocket:
port: {{ .Values.ports.pgbouncer }}
readinessProbe:
tcpSocket:
port: {{ .Values.ports.pgbouncer }}
{{- if or .Values.pgbouncer.mountConfigSecret .Values.pgbouncer.ssl.ca .Values.pgbouncer.ssl.cert .Values.pgbouncer.ssl.key .Values.volumeMounts .Values.pgbouncer.extraVolumeMounts }}
volumeMounts:
{{- if .Values.pgbouncer.mountConfigSecret }}
- name: pgbouncer-config
subPath: pgbouncer.ini
mountPath: /etc/pgbouncer/pgbouncer.ini
readOnly: true
- name: pgbouncer-config
subPath: users.txt
mountPath: /etc/pgbouncer/users.txt
readOnly: true
{{- end}}
{{- if .Values.pgbouncer.ssl.ca }}
- name: pgbouncer-certificates
subPath: root.crt
mountPath: /etc/pgbouncer/root.crt
readOnly: true
{{- end }}
{{- if .Values.pgbouncer.ssl.cert }}
- name: pgbouncer-certificates
subPath: server.crt
mountPath: /etc/pgbouncer/server.crt
readOnly: true
{{- end }}
{{- if .Values.pgbouncer.ssl.key }}
- name: pgbouncer-certificates
subPath: server.key
mountPath: /etc/pgbouncer/server.key
readOnly: true
{{- end }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.pgbouncer.extraVolumeMounts }}
{{- tpl (toYaml .Values.pgbouncer.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- end}}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
- name: metrics-exporter
resources: {{- toYaml .Values.pgbouncer.metricsExporterSidecar.resources | nindent 12 }}
image: {{ template "pgbouncer_exporter_image" . }}
imagePullPolicy: {{ .Values.images.pgbouncerExporter.pullPolicy }}
securityContext: {{ $containerSecurityContextMetricsExporter | nindent 12 }}
env:
- name: DATABASE_URL
valueFrom:
secretKeyRef:
name: {{ template "pgbouncer_stats_secret" . }}
{{- if (and .Values.pgbouncer.metricsExporterSidecar.statsSecretName .Values.pgbouncer.metricsExporterSidecar.statsSecretKey) }}
key: {{ .Values.pgbouncer.metricsExporterSidecar.statsSecretKey }}
{{- else }}
key: "connection"
{{- end }}
ports:
- name: metrics
containerPort: {{ .Values.ports.pgbouncerScrape }}
livenessProbe:
exec:
command:
- pgbouncer_exporter
- health
initialDelaySeconds: {{ .Values.pgbouncer.metricsExporterSidecar.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.pgbouncer.metricsExporterSidecar.livenessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.pgbouncer.metricsExporterSidecar.livenessProbe.timeoutSeconds }}
readinessProbe:
exec:
command:
- pgbouncer_exporter
- health
initialDelaySeconds: {{ .Values.pgbouncer.metricsExporterSidecar.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.pgbouncer.metricsExporterSidecar.readinessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.pgbouncer.metricsExporterSidecar.readinessProbe.timeoutSeconds }}
{{- if $containerLifecycleHooksMetricsExporter }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooksMetricsExporter) . | nindent 12 }}
{{- end }}
{{- if .Values.pgbouncer.metricsExporterSidecar.extraVolumeMounts }}
volumeMounts:
{{- tpl (toYaml .Values.pgbouncer.metricsExporterSidecar.extraVolumeMounts) . | nindent 12 }}
{{- end}}
{{- if .Values.pgbouncer.extraContainers }}
{{- tpl (toYaml .Values.pgbouncer.extraContainers) . | nindent 8 }}
{{- end }}
{{- if or .Values.pgbouncer.mountConfigSecret .Values.pgbouncer.ssl.ca .Values.pgbouncer.ssl.cert .Values.pgbouncer.ssl.key .Values.volumes .Values.pgbouncer.extraVolumes }}
volumes:
{{- if .Values.pgbouncer.mountConfigSecret }}
- name: pgbouncer-config
secret:
secretName: {{ template "pgbouncer_config_secret" . }}
{{- end}}
{{- if or .Values.pgbouncer.ssl.ca .Values.pgbouncer.ssl.cert .Values.pgbouncer.ssl.key }}
- name: pgbouncer-certificates
secret:
secretName: {{ template "pgbouncer_certificates_secret" . }}
{{- end }}
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 8 }}
{{- end }}
{{- if .Values.pgbouncer.extraVolumes }}
{{- tpl (toYaml .Values.pgbouncer.extraVolumes) . | nindent 8 }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,87 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Pgbouncer Ingress
#################################
{{- if and .Values.pgbouncer.enabled .Values.ingress.pgbouncer.enabled }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ include "airflow.fullname" . }}-pgbouncer-ingress
labels:
tier: airflow
component: pgbouncer-ingress
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.pgbouncer.labels) }}
{{- mustMerge .Values.pgbouncer.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.ingress.pgbouncer.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if and .Values.ingress.pgbouncer.hosts (.Values.ingress.pgbouncer.hosts | first | kindIs "string" | not) }}
{{- $anyTlsHosts := false -}}
{{- range .Values.ingress.pgbouncer.hosts }}
{{- if .tls }}
{{- if .tls.enabled }}
{{- $anyTlsHosts = true -}}
{{- end }}
{{- end }}
{{- end }}
{{- if $anyTlsHosts }}
tls:
{{- range .Values.ingress.pgbouncer.hosts }}
{{- if .tls }}
{{- if .tls.enabled }}
- hosts:
- {{ .name | quote }}
secretName: {{ .tls.secretName }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
rules:
{{- range .Values.ingress.pgbouncer.hosts | default (list .Values.ingress.pgbouncer.host) }}
- http:
paths:
- backend:
service:
name: {{ $.Release.Name }}-pgbouncer
port:
name: pgb-metrics
{{- if $.Values.ingress.pgbouncer.path }}
path: {{ $.Values.ingress.pgbouncer.path }}
pathType: {{ $.Values.ingress.pgbouncer.pathType }}
{{- end }}
{{- $hostname := . -}}
{{- if . | kindIs "string" | not }}
{{- $hostname = .name -}}
{{- end }}
{{- if $hostname }}
host: {{ tpl $hostname $ | quote }}
{{- end }}
{{- end }}
{{- if .Values.ingress.pgbouncer.ingressClassName }}
ingressClassName: {{ .Values.ingress.pgbouncer.ingressClassName }}
{{- end }}
{{- end }}
@@ -0,0 +1,77 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Pgbouncer NetworkPolicy
#################################
{{- $workersKedaEnabled := and .Values.workers.keda.enabled (has .Values.executor (list "CeleryExecutor" "CeleryKubernetesExecutor")) }}
{{- $triggererEnabled := and (semverCompare ">=2.2.0" .Values.airflowVersion) .Values.triggerer.enabled }}
{{- $triggererKedaEnabled := and $triggererEnabled .Values.triggerer.keda.enabled }}
{{- if and .Values.pgbouncer.enabled .Values.networkPolicies.enabled }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ include "airflow.fullname" . }}-pgbouncer-policy
labels:
tier: airflow
component: airflow-pgbouncer-policy
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.pgbouncer.labels) }}
{{- mustMerge .Values.pgbouncer.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
podSelector:
matchLabels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
policyTypes:
- Ingress
ingress:
- from:
- podSelector:
matchLabels:
tier: airflow
release: {{ .Release.Name }}
{{- if or $workersKedaEnabled $triggererKedaEnabled }}
{{- if and $workersKedaEnabled .Values.workers.keda.namespaceLabels }}
- namespaceSelector:
matchLabels: {{- toYaml .Values.workers.keda.namespaceLabels | nindent 10 }}
podSelector:
{{- else if and $triggererEnabled .Values.triggerer.keda.namespaceLabels }}
- namespaceSelector:
matchLabels: {{- toYaml .Values.triggerer.keda.namespaceLabels | nindent 10 }}
podSelector:
{{- else }}
- podSelector:
{{- end }}
matchLabels:
app: keda-operator
{{- end }}
{{- if .Values.pgbouncer.extraNetworkPolicies}}
{{- toYaml .Values.pgbouncer.extraNetworkPolicies | nindent 4 }}
{{- end }}
ports:
- protocol: TCP
port: {{ .Values.ports.pgbouncer }}
- protocol: TCP
port: {{ .Values.ports.pgbouncerScrape }}
{{- end }}
@@ -0,0 +1,44 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Pgbouncer PodDisruptionBudget
#################################
{{- if and .Values.pgbouncer.enabled .Values.pgbouncer.podDisruptionBudget.enabled }}
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: {{ include "airflow.fullname" . }}-pgbouncer-pdb
labels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.pgbouncer.labels) }}
{{- mustMerge .Values.pgbouncer.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
selector:
matchLabels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
{{- toYaml .Values.pgbouncer.podDisruptionBudget.config | nindent 2 }}
{{- end }}
@@ -0,0 +1,59 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Pgbouncer Service
#################################
{{- if .Values.pgbouncer.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "airflow.fullname" . }}-pgbouncer
labels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.pgbouncer.labels) }}
{{- mustMerge .Values.pgbouncer.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: {{ .Values.ports.pgbouncerScrape | quote }}
{{- if .Values.pgbouncer.service.extraAnnotations }}
{{- toYaml .Values.pgbouncer.service.extraAnnotations | nindent 4 }}
{{- end }}
spec:
type: ClusterIP
{{- if .Values.pgbouncer.service.clusterIp }}
clusterIP: {{ .Values.pgbouncer.service.clusterIp }}
{{- end }}
selector:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
ports:
- name: pgbouncer
protocol: TCP
port: {{ .Values.ports.pgbouncer }}
- name: pgb-metrics
protocol: TCP
port: {{ .Values.ports.pgbouncerScrape }}
{{- end }}
@@ -0,0 +1,41 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
######################################
## Airflow Pgbouncer ServiceAccount
######################################
{{- if and .Values.pgbouncer.serviceAccount.create .Values.pgbouncer.enabled }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.pgbouncer.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "pgbouncer.serviceAccountName" . }}
labels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.pgbouncer.labels) }}
{{- mustMerge .Values.pgbouncer.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.pgbouncer.serviceAccount.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
@@ -0,0 +1,35 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
#################################################
## Priority classes provisioned via the chart values
#################################################
{{- $Global := . }}
{{- range $e := .Values.priorityClasses }}
---
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
name: {{ $Global.Release.Name }}-{{ $e.name }}
labels:
release: {{ $Global.Release.Name }}
preemptionPolicy: {{ default "PreemptLowerPriority" $e.preemptionPolicy }}
value: {{ $e.value | required "value is required for priority classes" }}
description: "This priority class will not cause other pods to be preempted."
{{- end }}
@@ -0,0 +1,44 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Cleanup Role
#################################
{{- if and .Values.rbac.create .Values.cleanup.enabled }}
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ include "airflow.fullname" . }}-cleanup-role
labels:
tier: airflow
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
rules:
- apiGroups:
- ""
resources:
- "pods"
verbs:
- "list"
- "delete"
{{- end }}
@@ -0,0 +1,44 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Cleanup Role Binding
#################################
{{- if and .Values.rbac.create .Values.cleanup.enabled }}
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ include "airflow.fullname" . }}-cleanup-rolebinding
labels:
tier: airflow
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: {{ include "airflow.fullname" . }}-cleanup-role
subjects:
- kind: ServiceAccount
name: {{ include "cleanup.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
@@ -0,0 +1,79 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Pod Launcher Role
#################################
{{- if and .Values.rbac.create .Values.allowPodLaunching }}
apiVersion: rbac.authorization.k8s.io/v1
{{- if .Values.multiNamespaceMode }}
kind: ClusterRole
{{- else }}
kind: Role
{{- end }}
metadata:
{{- if not .Values.multiNamespaceMode }}
name: {{ include "airflow.fullname" . }}-pod-launcher-role
namespace: "{{ .Release.Namespace }}"
{{- else }}
name: {{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-pod-launcher-role
{{- end }}
labels:
tier: airflow
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if .Values.multiNamespaceMode }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
rules:
- apiGroups:
- ""
resources:
- "pods"
verbs:
- "create"
- "list"
- "get"
- "patch"
- "watch"
- "delete"
- apiGroups:
- ""
resources:
- "pods/log"
verbs:
- "get"
- apiGroups:
- ""
resources:
- "pods/exec"
verbs:
- "create"
- "get"
- apiGroups:
- ""
resources:
- "events"
verbs:
- "list"
{{- end }}
@@ -0,0 +1,77 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Pod Launcher Role Binding
#################################
{{- if and .Values.rbac.create .Values.allowPodLaunching }}
{{- $schedulerLaunchExecutors := list "LocalExecutor" "LocalKubernetesExecutor" "KubernetesExecutor" "CeleryKubernetesExecutor" }}
{{- $workerLaunchExecutors := list "CeleryExecutor" "LocalKubernetesExecutor" "KubernetesExecutor" "CeleryKubernetesExecutor" }}
{{- $executors := split "," .Values.executor }}
apiVersion: rbac.authorization.k8s.io/v1
{{- if .Values.multiNamespaceMode }}
kind: ClusterRoleBinding
{{- else }}
kind: RoleBinding
{{- end }}
metadata:
{{- if not .Values.multiNamespaceMode }}
namespace: "{{ .Release.Namespace }}"
name: {{ include "airflow.fullname" . }}-pod-launcher-rolebinding
{{- else }}
name: {{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-pod-launcher-rolebinding
{{- end }}
labels:
tier: airflow
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if .Values.multiNamespaceMode }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
roleRef:
apiGroup: rbac.authorization.k8s.io
{{- if .Values.multiNamespaceMode }}
kind: ClusterRole
name: {{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-pod-launcher-role
{{- else }}
kind: Role
name: {{ include "airflow.fullname" . }}-pod-launcher-role
{{- end }}
subjects:
{{- range $executor := $executors }}
{{- if has $executor $schedulerLaunchExecutors }}
- kind: ServiceAccount
name: {{ include "scheduler.serviceAccountName" $ }}
namespace: "{{ $.Release.Namespace }}"
{{- break -}}
{{- end }}
{{- end }}
{{- range $executor := $executors }}
{{- if has $executor $workerLaunchExecutors }}
- kind: ServiceAccount
name: {{ include "worker.serviceAccountName" $ }}
namespace: "{{ $.Release.Namespace }}"
{{- break -}}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,64 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Pod Reader Role
#################################
{{- if and .Values.rbac.create (or .Values.webserver.allowPodLogReading .Values.triggerer.enabled) }}
apiVersion: rbac.authorization.k8s.io/v1
{{- if .Values.multiNamespaceMode }}
kind: ClusterRole
{{- else }}
kind: Role
{{- end }}
metadata:
{{- if not .Values.multiNamespaceMode }}
name: {{ include "airflow.fullname" . }}-pod-log-reader-role
namespace: "{{ .Release.Namespace }}"
{{- else }}
name: {{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-pod-log-reader-role
{{- end }}
labels:
tier: airflow
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if .Values.multiNamespaceMode }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
rules:
- apiGroups:
- ""
resources:
- "pods"
verbs:
- "list"
- "get"
- "watch"
- apiGroups:
- ""
resources:
- "pods/log"
verbs:
- "get"
- "list"
{{- end }}
@@ -0,0 +1,73 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Pod Reader Role Binding
#################################
{{- if and .Values.rbac.create (or (and .Values.webserver.allowPodLogReading (semverCompare "<3.0.0" .Values.airflowVersion)) (and .Values.apiServer.allowPodLogReading (semverCompare ">=3.0.0" .Values.airflowVersion)) .Values.triggerer.enabled) }}
apiVersion: rbac.authorization.k8s.io/v1
{{- if .Values.multiNamespaceMode }}
kind: ClusterRoleBinding
{{- else }}
kind: RoleBinding
{{- end }}
metadata:
{{- if not .Values.multiNamespaceMode }}
namespace: "{{ .Release.Namespace }}"
name: {{ include "airflow.fullname" . }}-pod-log-reader-rolebinding
{{- else }}
name: {{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-pod-log-reader-rolebinding
{{- end }}
labels:
tier: airflow
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if .Values.multiNamespaceMode }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
roleRef:
apiGroup: rbac.authorization.k8s.io
{{- if .Values.multiNamespaceMode }}
kind: ClusterRole
name: {{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-pod-log-reader-role
{{- else }}
kind: Role
name: {{ include "airflow.fullname" . }}-pod-log-reader-role
{{- end }}
subjects:
{{- if and .Values.webserver.allowPodLogReading (semverCompare "<3.0.0" .Values.airflowVersion) }}
- kind: ServiceAccount
name: {{ include "webserver.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
{{- if and .Values.apiServer.allowPodLogReading (semverCompare ">=3.0.0" .Values.airflowVersion) }}
- kind: ServiceAccount
name: {{ include "apiServer.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
{{- if .Values.triggerer.enabled }}
- kind: ServiceAccount
name: {{ include "triggerer.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
{{- end }}
@@ -0,0 +1,93 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow SCC Role Binding
#################################
{{- if and .Values.rbac.create .Values.rbac.createSCCRoleBinding }}
{{- $hasWorkers := has .Values.executor (list "CeleryExecutor" "LocalKubernetesExecutor" "KubernetesExecutor" "CeleryKubernetesExecutor") }}
apiVersion: rbac.authorization.k8s.io/v1
{{- if .Values.multiNamespaceMode }}
kind: ClusterRoleBinding
{{- else }}
kind: RoleBinding
{{- end }}
metadata:
{{- if not .Values.multiNamespaceMode }}
name: {{ include "airflow.fullname" . }}-scc-rolebinding
namespace: "{{ .Release.Namespace }}"
{{- else }}
name: {{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-scc-rolebinding
{{- end }}
labels:
tier: airflow
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if .Values.multiNamespaceMode }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:openshift:scc:anyuid
subjects:
- kind: ServiceAccount
name: {{ include "webserver.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- if $hasWorkers }}
- kind: ServiceAccount
name: {{ include "worker.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
- kind: ServiceAccount
name: {{ include "scheduler.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- if and .Values.statsd.enabled }}
- kind: ServiceAccount
name: {{ include "statsd.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
{{- if and .Values.flower.enabled (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }}
- kind: ServiceAccount
name: {{ include "flower.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
{{- if and (semverCompare ">=2.2.0" .Values.airflowVersion) }}
- kind: ServiceAccount
name: {{ include "triggerer.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
- kind: ServiceAccount
name: {{ include "migrateDatabaseJob.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- if .Values.webserver.defaultUser.enabled }}
- kind: ServiceAccount
name: {{ include "createUserJob.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
{{- if and .Values.cleanup.enabled }}
- kind: ServiceAccount
name: {{ include "cleanup.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
{{- end }}
@@ -0,0 +1,65 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Redis NetworkPolicy
#################################
{{- if and .Values.redis.enabled .Values.networkPolicies.enabled (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ include "airflow.fullname" . }}-redis-policy
labels:
tier: airflow
component: redis-policy
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
podSelector:
matchLabels:
tier: airflow
component: redis
release: {{ .Release.Name }}
policyTypes:
- Ingress
ingress:
- from:
- podSelector:
matchLabels:
tier: airflow
component: worker
release: {{ .Release.Name }}
- podSelector:
matchLabels:
tier: airflow
component: scheduler
release: {{ .Release.Name }}
- podSelector:
matchLabels:
tier: airflow
component: flower
release: {{ .Release.Name }}
ports:
- protocol: TCP
port: {{ .Values.ports.redisDB }}
{{- end }}
@@ -0,0 +1,58 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Redis Service
#################################
{{- if and .Values.redis.enabled (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "airflow.fullname" . }}-redis
labels:
tier: airflow
component: redis
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if eq .Values.redis.service.type "ClusterIP" }}
type: ClusterIP
{{- if .Values.redis.service.clusterIP }}
clusterIP: {{ .Values.redis.service.clusterIP }}
{{- end }}
{{- else }}
type: {{ .Values.redis.service.type }}
{{- end }}
selector:
tier: airflow
component: redis
release: {{ .Release.Name }}
ports:
- name: redis-db
protocol: TCP
port: {{ .Values.ports.redisDB }}
targetPort: {{ .Values.ports.redisDB }}
{{- if (and (eq .Values.redis.service.type "NodePort") (not (empty .Values.redis.service.nodePort))) }}
nodePort: {{ .Values.redis.service.nodePort }}
{{- end }}
{{- end }}
@@ -0,0 +1,41 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
######################################
## Airflow Redis ServiceAccount
######################################
{{- if and .Values.redis.enabled .Values.redis.serviceAccount.create (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.redis.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "redis.serviceAccountName" . }}
labels:
tier: airflow
component: redis
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.redis.serviceAccount.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
@@ -0,0 +1,140 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Redis StatefulSet
#################################
{{- if and .Values.redis.enabled (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }}
{{- $nodeSelector := or .Values.redis.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.redis.affinity .Values.affinity }}
{{- $tolerations := or .Values.redis.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.redis.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $securityContext := include "localPodSecurityContext" .Values.redis }}
{{- $containerSecurityContext := include "externalContainerSecurityContext" .Values.redis }}
{{- $containerLifecycleHooks := .Values.redis.containerLifecycleHooks }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: {{ include "airflow.fullname" . }}-redis
labels:
tier: airflow
component: redis
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if .Values.redis.annotations }}
annotations: {{- toYaml .Values.redis.annotations | nindent 4 }}
{{- end }}
spec:
serviceName: {{ include "airflow.fullname" . }}-redis
selector:
matchLabels:
tier: airflow
component: redis
release: {{ .Release.Name }}
template:
metadata:
labels:
tier: airflow
component: redis
release: {{ .Release.Name }}
{{- with .Values.labels }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- if or .Values.redis.safeToEvict .Values.redis.podAnnotations }}
annotations:
{{- if .Values.redis.podAnnotations }}
{{- toYaml .Values.redis.podAnnotations | nindent 8 }}
{{- end }}
{{- if .Values.redis.safeToEvict }}
cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
{{- end }}
{{- end }}
spec:
{{- if .Values.redis.priorityClassName }}
priorityClassName: {{ .Values.redis.priorityClassName }}
{{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity: {{- toYaml $affinity | nindent 8 }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
terminationGracePeriodSeconds: {{ .Values.redis.terminationGracePeriodSeconds }}
serviceAccountName: {{ include "redis.serviceAccountName" . }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
securityContext: {{ $securityContext | nindent 8 }}
containers:
- name: redis
image: {{ template "redis_image" . }}
imagePullPolicy: {{ .Values.images.redis.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
command: ["/bin/sh"]
resources: {{- toYaml .Values.redis.resources | nindent 12 }}
args: ["-c", "redis-server --requirepass ${REDIS_PASSWORD}"]
ports:
- name: redis-db
containerPort: {{ .Values.ports.redisDB }}
volumeMounts:
- name: redis-db
mountPath: /data
env:
- name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: {{ template "redis_password_secret" . }}
key: password
{{- if not .Values.redis.persistence.enabled }}
volumes:
- name: redis-db
emptyDir: {{- toYaml (default (dict) .Values.redis.emptyDirConfig) | nindent 12 }}
{{- else if .Values.redis.persistence.existingClaim }}
volumes:
- name: redis-db
persistentVolumeClaim:
claimName: {{ .Values.redis.persistence.existingClaim }}
{{- else }}
volumeClaimTemplates:
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: redis-db
{{- if .Values.redis.persistence.annotations }}
annotations: {{- toYaml .Values.redis.persistence.annotations | nindent 10 }}
{{- end }}
spec:
{{- if .Values.redis.persistence.storageClassName }}
storageClassName: {{ tpl .Values.redis.persistence.storageClassName . | quote }}
{{- end }}
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: {{ .Values.redis.persistence.size }}
{{- end }}
{{- end }}
@@ -0,0 +1,39 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Namespace ResourceQuota
#################################
{{- if .Values.quotas }}
apiVersion: v1
kind: ResourceQuota
metadata:
name: {{ .Release.Name }}-resource-quota
labels:
tier: resources
component: resourcequota
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
hard: {{- toYaml .Values.quotas | nindent 4 }}
{{- end }}
@@ -0,0 +1,355 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Scheduler Deployment/StatefulSet
#################################
{{- if .Values.scheduler.enabled }}
# Are we using a local executor?
{{- $local := contains "Local" .Values.executor }}
# Is persistence enabled on the _workers_?
# This is important because in $local mode, the scheduler assumes the role of the worker
{{- $persistence := .Values.workers.persistence.enabled }}
# If we're using a StatefulSet
{{- $stateful := and $local $persistence }}
# We can skip DAGs mounts on scheduler if dagProcessor is enabled, except with $local mode
{{- $dagProcessorEnabled := .Values.dagProcessor.enabled }}
{{- if eq $dagProcessorEnabled nil}}
{{ $dagProcessorEnabled = ternary true false (semverCompare ">=3.0.0" .Values.airflowVersion) }}
{{- end }}
{{- $localOrDagProcessorDisabled := or (not $dagProcessorEnabled) $local }}
# If we're using elasticsearch or opensearch logging
{{- $remoteLogging := or .Values.elasticsearch.enabled .Values.opensearch.enabled }}
{{- $nodeSelector := or .Values.scheduler.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.scheduler.affinity .Values.affinity }}
{{- $tolerations := or .Values.scheduler.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.scheduler.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $revisionHistoryLimit := or .Values.scheduler.revisionHistoryLimit .Values.revisionHistoryLimit }}
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.scheduler) }}
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.scheduler) }}
{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list . .Values.scheduler.waitForMigrations) }}
{{- $containerSecurityContextLogGroomerSidecar := include "containerSecurityContext" (list . .Values.scheduler.logGroomerSidecar) }}
{{- $containerLifecycleHooks := or .Values.scheduler.containerLifecycleHooks .Values.containerLifecycleHooks }}
{{- $containerLifecycleHooksLogGroomerSidecar := or .Values.scheduler.logGroomerSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }}
apiVersion: apps/v1
kind: {{ if $stateful }}StatefulSet{{ else }}Deployment{{ end }}
metadata:
name: {{ include "airflow.fullname" . }}-scheduler
labels:
tier: airflow
component: scheduler
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
executor: {{ .Values.executor | replace "," "-" | quote }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if .Values.scheduler.annotations }}
annotations: {{- toYaml .Values.scheduler.annotations | nindent 4 }}
{{- end }}
spec:
{{- if $stateful }}
serviceName: {{ include "airflow.fullname" . }}-scheduler
{{- end }}
replicas: {{ .Values.scheduler.replicas }}
{{- if $revisionHistoryLimit }}
revisionHistoryLimit: {{ $revisionHistoryLimit }}
{{- end }}
{{- if and $stateful .Values.scheduler.updateStrategy }}
updateStrategy: {{- toYaml .Values.scheduler.updateStrategy | nindent 4 }}
{{- end }}
{{- if and $stateful .Values.workers.persistence.persistentVolumeClaimRetentionPolicy }}
persistentVolumeClaimRetentionPolicy: {{- toYaml .Values.workers.persistence.persistentVolumeClaimRetentionPolicy | nindent 4 }}
{{- end }}
{{- if and (not $stateful) .Values.scheduler.strategy }}
strategy: {{- toYaml .Values.scheduler.strategy | nindent 4 }}
{{- end }}
selector:
matchLabels:
tier: airflow
component: scheduler
release: {{ .Release.Name }}
template:
metadata:
labels:
tier: airflow
component: scheduler
release: {{ .Release.Name }}
{{- if or (.Values.labels) (.Values.scheduler.labels) }}
{{- mustMerge .Values.scheduler.labels .Values.labels | toYaml | nindent 8 }}
{{- end }}
annotations:
checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }}
checksum/result-backend-secret: {{ include (print $.Template.BasePath "/secrets/result-backend-connection-secret.yaml") . | sha256sum }}
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }}
checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }}
checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }}
{{- if .Values.scheduler.safeToEvict }}
cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
{{- end }}
{{- if .Values.airflowPodAnnotations }}
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
{{- end }}
{{- if .Values.scheduler.podAnnotations }}
{{- toYaml .Values.scheduler.podAnnotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.scheduler.priorityClassName }}
priorityClassName: {{ .Values.scheduler.priorityClassName }}
{{- end }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity:
{{- if $affinity }}
{{- toYaml $affinity | nindent 8 }}
{{- else }}
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
component: scheduler
topologyKey: kubernetes.io/hostname
weight: 100
{{- end }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
restartPolicy: Always
terminationGracePeriodSeconds: {{ .Values.scheduler.terminationGracePeriodSeconds }}
serviceAccountName: {{ include "scheduler.serviceAccountName" . }}
securityContext: {{ $securityContext | nindent 8 }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
{{- if .Values.scheduler.hostAliases }}
hostAliases: {{- toYaml .Values.scheduler.hostAliases | nindent 8 }}
{{- end }}
initContainers:
{{- if .Values.scheduler.waitForMigrations.enabled }}
- name: wait-for-airflow-migrations
resources: {{- toYaml .Values.scheduler.resources | nindent 12 }}
image: {{ template "airflow_image_for_migrations" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }}
volumeMounts:
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.scheduler.extraVolumeMounts }}
{{- tpl (toYaml .Values.scheduler.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
args: {{- include "wait-for-migrations-command" . | indent 10 }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- if .Values.scheduler.waitForMigrations.env }}
{{- tpl (toYaml .Values.scheduler.waitForMigrations.env) $ | nindent 12 }}
{{- end }}
{{- end }}
{{- if and $localOrDagProcessorDisabled .Values.dags.gitSync.enabled }}
{{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }}
{{- end }}
{{- if .Values.scheduler.extraInitContainers }}
{{- tpl (toYaml .Values.scheduler.extraInitContainers) . | nindent 8 }}
{{- end }}
containers:
# Always run the main scheduler container.
- name: scheduler
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
{{- if .Values.scheduler.command }}
command: {{ tpl (toYaml .Values.scheduler.command) . | nindent 12 }}
{{- end }}
{{- if .Values.scheduler.args }}
args: {{ tpl (toYaml .Values.scheduler.args) . | nindent 12 }}
{{- end }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- include "container_extra_envs" (list . .Values.scheduler.env) | indent 10 }}
livenessProbe:
initialDelaySeconds: {{ .Values.scheduler.livenessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.scheduler.livenessProbe.timeoutSeconds }}
failureThreshold: {{ .Values.scheduler.livenessProbe.failureThreshold }}
periodSeconds: {{ .Values.scheduler.livenessProbe.periodSeconds }}
exec:
command:
{{- if .Values.scheduler.livenessProbe.command }}
{{- toYaml .Values.scheduler.livenessProbe.command | nindent 16 }}
{{- else }}
{{- include "scheduler_liveness_check_command" . | indent 14 }}
{{- end }}
startupProbe:
initialDelaySeconds: {{ .Values.scheduler.startupProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.scheduler.startupProbe.timeoutSeconds }}
failureThreshold: {{ .Values.scheduler.startupProbe.failureThreshold }}
periodSeconds: {{ .Values.scheduler.startupProbe.periodSeconds }}
exec:
command:
{{- if .Values.scheduler.startupProbe.command }}
{{- toYaml .Values.scheduler.startupProbe.command | nindent 16 }}
{{- else }}
{{- include "scheduler_startup_check_command" . | indent 14 }}
{{- end }}
{{- if and $local (not $remoteLogging) }}
# Serve logs if we're in local mode and we have neither elasticsearch nor opensearch enabled.
ports:
- name: worker-logs
containerPort: {{ .Values.ports.workerLogs }}
{{- end }}
resources: {{- toYaml .Values.scheduler.resources | nindent 12 }}
volumeMounts:
{{- if semverCompare ">=1.10.12" .Values.airflowVersion }}
- name: config
mountPath: {{ include "airflow_pod_template_file" . }}/pod_template_file.yaml
subPath: pod_template_file.yaml
readOnly: true
{{- end }}
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
{{- if and $localOrDagProcessorDisabled (or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled) }}
{{- include "airflow_dags_mount" . | nindent 12 }}
{{- end }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.scheduler.extraVolumeMounts }}
{{- tpl (toYaml .Values.scheduler.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if and $localOrDagProcessorDisabled .Values.dags.gitSync.enabled }}
{{- include "git_sync_container" . | indent 8 }}
{{- end }}
{{- if .Values.scheduler.logGroomerSidecar.enabled }}
- name: scheduler-log-groomer
resources: {{- toYaml .Values.scheduler.logGroomerSidecar.resources | nindent 12 }}
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContextLogGroomerSidecar | nindent 12 }}
{{- if $containerLifecycleHooksLogGroomerSidecar }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooksLogGroomerSidecar) . | nindent 12 }}
{{- end }}
{{- if .Values.scheduler.logGroomerSidecar.command }}
command: {{ tpl (toYaml .Values.scheduler.logGroomerSidecar.command) . | nindent 12 }}
{{- end }}
{{- if .Values.scheduler.logGroomerSidecar.args }}
args: {{- tpl (toYaml .Values.scheduler.logGroomerSidecar.args) . | nindent 12 }}
{{- end }}
env:
{{- if .Values.scheduler.logGroomerSidecar.retentionDays }}
- name: AIRFLOW__LOG_RETENTION_DAYS
value: "{{ .Values.scheduler.logGroomerSidecar.retentionDays }}"
{{- end }}
{{- if .Values.scheduler.logGroomerSidecar.frequencyMinutes }}
- name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES
value: "{{ .Values.scheduler.logGroomerSidecar.frequencyMinutes }}"
{{- end }}
- name: AIRFLOW_HOME
value: "{{ .Values.airflowHome }}"
{{- if .Values.scheduler.logGroomerSidecar.env }}
{{- tpl (toYaml .Values.scheduler.logGroomerSidecar.env) $ | nindent 12 }}
{{- end }}
volumeMounts:
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.scheduler.extraVolumeMounts }}
{{- tpl (toYaml .Values.scheduler.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
{{- end }}
{{- if .Values.scheduler.extraContainers }}
{{- tpl (toYaml .Values.scheduler.extraContainers) . | nindent 8 }}
{{- end }}
volumes:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
- name: webserver-config
configMap:
name: {{ template "airflow_webserver_config_configmap_name" . }}
{{- end }}
{{- if $localOrDagProcessorDisabled }}
{{- if .Values.dags.persistence.enabled }}
- name: dags
persistentVolumeClaim:
claimName: {{ template "airflow_dags_volume_claim" . }}
{{- else if .Values.dags.gitSync.enabled }}
- name: dags
emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 12 }}
{{- if or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey}}
{{- include "git_sync_ssh_key_volume" . | indent 8 }}
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 8 }}
{{- end }}
{{- if .Values.scheduler.extraVolumes }}
{{- tpl (toYaml .Values.scheduler.extraVolumes) . | nindent 8 }}
{{- end }}
{{- if .Values.logs.persistence.enabled }}
- name: logs
persistentVolumeClaim:
claimName: {{ template "airflow_logs_volume_claim" . }}
{{- else if not $stateful }}
- name: logs
emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) | nindent 12 }}
{{- else }}
volumeClaimTemplates:
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: logs
{{- if .Values.workers.persistence.annotations }}
annotations: {{- toYaml .Values.workers.persistence.annotations | nindent 10 }}
{{- end }}
spec:
{{- if .Values.workers.persistence.storageClassName }}
storageClassName: {{ tpl .Values.workers.persistence.storageClassName . | quote }}
{{- end }}
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: {{ .Values.workers.persistence.size }}
{{- end }}
{{- end }}
@@ -0,0 +1,59 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Scheduler NetworkPolicy
#################################
{{- if .Values.scheduler.enabled }}
{{- if .Values.networkPolicies.enabled }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ include "airflow.fullname" . }}-scheduler-policy
labels:
tier: airflow
component: airflow-scheduler-policy
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.scheduler.labels) }}
{{- mustMerge .Values.scheduler.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
podSelector:
matchLabels:
tier: airflow
component: scheduler
release: {{ .Release.Name }}
policyTypes:
- Ingress
{{- if contains "LocalExecutor" .Values.executor }}
ingress:
- from:
- podSelector:
matchLabels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
ports:
- protocol: TCP
port: {{ .Values.ports.workerLogs }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,46 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Scheduler PodDisruptionBudget
#################################
{{- if .Values.scheduler.enabled }}
{{- if .Values.scheduler.podDisruptionBudget.enabled }}
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: {{ include "airflow.fullname" . }}-scheduler-pdb
labels:
tier: airflow
component: scheduler
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.scheduler.labels) }}
{{- mustMerge .Values.scheduler.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
selector:
matchLabels:
tier: airflow
component: scheduler
release: {{ .Release.Name }}
{{- toYaml .Values.scheduler.podDisruptionBudget.config | nindent 2 }}
{{- end }}
{{- end }}
@@ -0,0 +1,50 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Scheduler Service
#################################
{{- if .Values.scheduler.enabled }}
{{- if or (contains "LocalExecutor" .Values.executor) (contains "LocalKubernetesExecutor" .Values.executor) }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "airflow.fullname" . }}-scheduler
labels:
tier: airflow
component: scheduler
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.scheduler.labels) }}
{{- mustMerge .Values.scheduler.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
clusterIP: None
selector:
tier: airflow
component: scheduler
release: {{ .Release.Name }}
ports:
- name: task-logs
protocol: TCP
port: {{ .Values.ports.workerLogs }}
targetPort: {{ .Values.ports.workerLogs }}
{{- end }}
{{- end }}
@@ -0,0 +1,46 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Scheduler ServiceAccount
#################################
{{- if and .Values.scheduler.enabled .Values.scheduler.serviceAccount.create }}
apiVersion: v1
kind: ServiceAccount
{{- if eq .Values.executor "CeleryExecutor" }}
automountServiceAccountToken: {{ .Values.scheduler.serviceAccount.automountServiceAccountToken }}
{{- end }}
metadata:
name: {{ include "scheduler.serviceAccountName" . }}
labels:
tier: airflow
component: scheduler
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.scheduler.labels) }}
{{- mustMerge .Values.scheduler.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.scheduler.serviceAccount.annotations }}
annotations:
{{- range $key, $value := . }}
{{- printf "%s: %s" $key (tpl $value $ | quote) | nindent 4 }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,44 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Elasticsearch Secret
#################################
{{- if (and .Values.elasticsearch.enabled (not .Values.elasticsearch.secretName)) }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "airflow.fullname" . }}-elasticsearch
labels:
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
type: Opaque
data:
{{- with .Values.elasticsearch.connection }}
{{- if and .user .pass }}
connection: {{ urlJoin (dict "scheme" (default "http" .scheme) "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery)) "host" (printf "%s:%s" .host ((default 9200 .port) | toString) ) ) | b64enc | quote }}
{{- else }}
connection: {{ urlJoin (dict "scheme" (default "http" .scheme) "host" (printf "%s:%s" .host ((default 9200 .port) | toString))) | b64enc | quote }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,64 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
#################################################
## Extra Secrets provisioned via the chart values
#################################################
{{- $Global := . }}
{{- range $secretName, $secretContent := .Values.extraSecrets }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ tpl $secretName $Global | quote }}
labels:
release: {{ $Global.Release.Name }}
chart: "{{ $Global.Chart.Name }}-{{ $Global.Chart.Version }}"
heritage: {{ $Global.Release.Service }}
{{- with $Global.Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if $secretContent.labels }}
{{- toYaml $secretContent.labels | nindent 4 }}
{{- end }}
{{- $annotations := dict }}
{{- if or $secretContent.useHelmHooks (not (hasKey $secretContent "useHelmHooks")) }}
{{- $_ := set $annotations "helm.sh/hook" "pre-install,pre-upgrade" }}
{{- $_ := set $annotations "helm.sh/hook-weight" "0" }}
{{- $_ := set $annotations "helm.sh/hook-delete-policy" "before-hook-creation" }}
{{- end }}
{{- with $annotations := merge $annotations ($secretContent.annotations | default dict) }}
annotations: {{- $annotations | toYaml | nindent 4 }}
{{- end }}
{{- if $secretContent.type }}
type: {{ $secretContent.type }}
{{- end }}
{{- if $secretContent.data }}
data:
{{- with $secretContent.data }}
{{- tpl . $Global | nindent 2 }}
{{- end }}
{{- end }}
{{- if $secretContent.stringData }}
stringData:
{{- with $secretContent.stringData }}
{{- tpl . $Global | nindent 2 }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,44 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Fernet Key Secret
#################################
{{- if not .Values.fernetKeySecretName }}
{{- $generated_fernet_key := (randAlphaNum 32 | b64enc) }}
apiVersion: v1
kind: Secret
metadata:
name: {{ .Release.Name }}-fernet-key
labels:
tier: airflow
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
annotations:
"helm.sh/hook": "pre-install"
"helm.sh/hook-delete-policy": "before-hook-creation"
"helm.sh/hook-weight": "0"
type: Opaque
data:
fernet-key: {{ (default $generated_fernet_key .Values.fernetKey) | b64enc | quote }}
{{- end }}
@@ -0,0 +1,38 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Flower Secret
#################################
{{- if (and (not .Values.flower.secretName) .Values.flower.username .Values.flower.password) }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "airflow.fullname" . }}-flower
labels:
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
type: Opaque
data:
basicAuth: {{ (printf "%s:%s" .Values.flower.username .Values.flower.password) | b64enc | quote }}
{{- end }}
@@ -0,0 +1,34 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
{{- if and .Values.dags.gitSync.sshKey .Values.dags.gitSync.enabled}}
apiVersion: v1
kind: Secret
metadata:
labels:
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
name: {{ template "git_sync_ssh_key" . }}
data:
gitSshKey: {{ .Values.dags.gitSync.sshKey | b64enc | quote }}
{{- end }}
@@ -0,0 +1,40 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Kerberos Secret
#################################
{{- if .Values.kerberos.keytabBase64Content }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "kerberos_keytab_secret" . | quote }}
labels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
type: Opaque
data:
kerberos.keytab: {{ .Values.kerberos.keytabBase64Content }}
{{- end }}
@@ -0,0 +1,59 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Metadata Secret
#################################
{{- if not .Values.data.metadataSecretName }}
{{- $defaultMetadataHost := .Values.postgresql.nameOverride | default (printf "%s-%s.%s" .Release.Name "postgresql" .Release.Namespace) }}
{{- $metadataHost := .Values.data.metadataConnection.host | default $defaultMetadataHost }}
{{- $pgbouncerHost := (printf "%s-%s.%s" ( include "airflow.fullname" . ) "pgbouncer" .Release.Namespace) }}
{{- $host := ternary $pgbouncerHost $metadataHost .Values.pgbouncer.enabled }}
{{- $metadataPort := .Values.data.metadataConnection.port | toString }}
{{- $port := ((ternary .Values.ports.pgbouncer $metadataPort .Values.pgbouncer.enabled) | toString) }}
{{- $metadataDatabase := .Values.data.metadataConnection.db }}
{{- $database := (ternary (printf "%s-%s" .Release.Name "metadata") $metadataDatabase .Values.pgbouncer.enabled) }}
{{- $query := ternary (printf "sslmode=%s" .Values.data.metadataConnection.sslmode) "" (eq .Values.data.metadataConnection.protocol "postgresql") }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "airflow.fullname" . }}-metadata
labels:
tier: airflow
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
type: Opaque
data:
{{- with .Values.data.metadataConnection }}
connection: {{ urlJoin (dict "scheme" .protocol "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery) ) "host" (printf "%s:%s" $host $port) "path" (printf "/%s" $database) "query" $query) | b64enc | quote }}
{{- end }}
{{- if and .Values.workers.keda.enabled .Values.pgbouncer.enabled (not .Values.workers.keda.usePgbouncer) }}
{{- with .Values.data.metadataConnection }}
kedaConnection: {{ urlJoin (dict "scheme" .protocol "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery) ) "host" (printf "%s:%s" $metadataHost $metadataPort) "path" (printf "/%s" $metadataDatabase) "query" $query) | b64enc | quote }}
{{- end }}
{{- else if and (or .Values.workers.keda.enabled .Values.triggerer.keda.enabled) (eq .Values.data.metadataConnection.protocol "mysql") }}
{{- with .Values.data.metadataConnection }}
kedaConnection: {{ urlJoin (dict "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery) ) "host" (printf "tcp(%s:%s)" $metadataHost $metadataPort) "path" (printf "/%s" $metadataDatabase) "query" $query) | trimPrefix "//" | b64enc | quote }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,44 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## OpenSearch Secret
#################################
{{- if (and .Values.opensearch.enabled (not .Values.opensearch.secretName)) }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "airflow.fullname" . }}-opensearch
labels:
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
type: Opaque
data:
{{- with .Values.opensearch.connection }}
{{- if and .user .pass }}
connection: {{ urlJoin (dict "scheme" (default "http" .scheme) "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery)) "host" (printf "%s:%s" .host ((default 9200 .port) | toString) ) ) | b64enc | quote }}
{{- else }}
connection: {{ urlJoin (dict "scheme" (default "http" .scheme) "host" (printf "%s:%s" .host ((default 9200 .port) | toString))) | b64enc | quote }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,46 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Pgbouncer Certificate Secret
#################################
{{- if or .Values.pgbouncer.ssl.ca .Values.pgbouncer.ssl.cert .Values.pgbouncer.ssl.key }}
apiVersion: v1
kind: Secret
metadata:
name: {{ template "pgbouncer_certificates_secret" . }}
labels:
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
type: Opaque
data:
{{- if .Values.pgbouncer.ssl.ca }}
root.crt: {{ .Values.pgbouncer.ssl.ca | b64enc }}
{{- end }}
{{- if .Values.pgbouncer.ssl.cert }}
server.crt: {{ .Values.pgbouncer.ssl.cert | b64enc }}
{{- end }}
{{- if .Values.pgbouncer.ssl.key }}
server.key: {{ .Values.pgbouncer.ssl.key | b64enc }}
{{- end }}
{{- end }}
@@ -0,0 +1,41 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Pgbouncer Config Secret
#################################
{{- if (and .Values.pgbouncer.enabled (not .Values.pgbouncer.configSecretName)) }}
apiVersion: v1
kind: Secret
metadata:
name: {{ template "pgbouncer_config_secret" . }}
labels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
type: Opaque
data:
pgbouncer.ini: {{ include "pgbouncer_config" . | b64enc }}
users.txt: {{ include "pgbouncer_users" . | b64enc }}
{{- end }}
@@ -0,0 +1,40 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Pgbouncer Stats Secret
#################################
{{- if (and .Values.pgbouncer.enabled (not .Values.pgbouncer.metricsExporterSidecar.statsSecretName)) }}
apiVersion: v1
kind: Secret
metadata:
name: {{ template "pgbouncer_stats_secret" . }}
labels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
type: Opaque
data:
connection: {{ urlJoin (dict "scheme" "postgresql" "userinfo" (printf "%s:%s" (.Values.data.metadataConnection.user | urlquery) (.Values.data.metadataConnection.pass | urlquery) ) "host" (printf "127.0.0.1:%s" (.Values.ports.pgbouncer | toString)) "path" "/pgbouncer" "query" (printf "sslmode=%s" (.Values.pgbouncer.metricsExporterSidecar.sslmode | toString ))) | b64enc | quote }}
{{- end }}
@@ -0,0 +1,83 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
# We will create these secrets (if necessary) _even if_ we aren't
# currently using CeleryExecutor or CeleryKubernetesExecutor. As we are
# relying on the "pre-install" hack to prevent changing randomly generated passwords,
# updating the executor later doesn't give us the opportunity to deploy them
# when we need them. We will always deploy them defensively to make the executor
# update path actually work.
################################
## Airflow Redis Password Secret
#################################
{{- $random_redis_password := randAlphaNum 10 }}
{{- if and .Values.redis.enabled (not .Values.redis.passwordSecretName) }}
# If passwordSecretName is not set, we will either use the set password, or use the generated one
apiVersion: v1
kind: Secret
metadata:
name: {{ .Release.Name }}-redis-password
labels:
tier: airflow
component: redis
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
annotations:
"helm.sh/hook": "pre-install"
"helm.sh/hook-delete-policy": "before-hook-creation"
"helm.sh/hook-weight": "0"
type: Opaque
data:
password: {{ (default $random_redis_password .Values.redis.password) | b64enc | quote }}
---
{{- end }}
{{- if not .Values.data.brokerUrlSecretName }}
##################################
## Airflow Redis Connection Secret
##################################
apiVersion: v1
kind: Secret
metadata:
name: {{ .Release.Name }}-broker-url
labels:
tier: airflow
component: redis
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
annotations:
"helm.sh/hook": "pre-install"
"helm.sh/hook-delete-policy": "before-hook-creation"
"helm.sh/hook-weight": "0"
type: Opaque
data:
{{- if .Values.redis.enabled }}
connection: {{ urlJoin (dict "scheme" "redis" "userinfo" (printf ":%s" ((default $random_redis_password .Values.redis.password) | urlquery)) "host" (printf "%s-redis:6379" (include "airflow.fullname" .) ) "path" "/0") | b64enc | quote }}
{{- else }}
connection: {{ (printf "%s" .Values.data.brokerUrl) | b64enc | quote }}
{{- end }}
{{- end }}
@@ -0,0 +1,38 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Registry Secret
#################################
{{- if (and .Values.registry.connection (not .Values.registry.secretName)) }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "airflow.fullname" . }}-registry
labels:
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
type: kubernetes.io/dockerconfigjson
data:
.dockerconfigjson: {{ include "registry_docker_config" . | b64enc }}
{{- end }}
@@ -0,0 +1,50 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Result Backend Secret
#################################
{{- if not .Values.data.resultBackendSecretName }}
{{- if or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) }}
{{- if or (semverCompare "<2.4.0" .Values.airflowVersion) (and (semverCompare ">=2.4.0" .Values.airflowVersion) .Values.data.resultBackendConnection) }}
{{- $connection := .Values.data.resultBackendConnection | default .Values.data.metadataConnection }}
{{- $resultBackendHost := $connection.host | default (printf "%s-%s" .Release.Name "postgresql") }}
{{- $pgbouncerHost := printf "%s-%s" .Release.Name "pgbouncer" }}
{{- $host := ternary $pgbouncerHost $resultBackendHost .Values.pgbouncer.enabled }}
{{- $port := (ternary .Values.ports.pgbouncer $connection.port .Values.pgbouncer.enabled) | toString }}
{{- $database := ternary (printf "%s-%s" .Release.Name "result-backend") $connection.db .Values.pgbouncer.enabled }}
{{- $query := ternary (printf "sslmode=%s" $connection.sslmode) "" (eq $connection.protocol "postgresql") }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "airflow.fullname" . }}-result-backend
labels:
tier: airflow
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
type: Opaque
data:
connection: {{ urlJoin (dict "scheme" (printf "db+%s" $connection.protocol) "userinfo" (printf "%s:%s" ($connection.user|urlquery) ($connection.pass | urlquery)) "host" (printf "%s:%s" $host $port) "path" (printf "/%s" $database) "query" $query) | b64enc | quote }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,41 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
############################################
## Airflow Webserver Flask Secret Key Secret
############################################
{{- if not .Values.webserverSecretKeySecretName }}
{{ $generated_secret_key := (randAlphaNum 32 | b64enc) }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "airflow.fullname" . }}-webserver-secret-key
labels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
type: Opaque
data:
webserver-secret-key: {{ (default $generated_secret_key .Values.webserverSecretKey) | b64enc | quote }}
{{- end }}
@@ -0,0 +1,139 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow StatsD Deployment
#################################
{{- if .Values.statsd.enabled }}
{{- $nodeSelector := or .Values.statsd.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.statsd.affinity .Values.affinity }}
{{- $tolerations := or .Values.statsd.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.statsd.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $revisionHistoryLimit := or .Values.statsd.revisionHistoryLimit .Values.revisionHistoryLimit }}
{{- $securityContext := include "localPodSecurityContext" .Values.statsd }}
{{- $containerSecurityContext := include "externalContainerSecurityContext" .Values.statsd }}
{{- $containerLifecycleHooks := .Values.statsd.containerLifecycleHooks }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "airflow.fullname" . }}-statsd
labels:
tier: airflow
component: statsd
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.statsd.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
spec:
replicas: 1
{{- if $revisionHistoryLimit }}
revisionHistoryLimit: {{ $revisionHistoryLimit }}
{{- end }}
selector:
matchLabels:
tier: airflow
component: statsd
release: {{ .Release.Name }}
template:
metadata:
labels:
tier: airflow
component: statsd
release: {{ .Release.Name }}
{{- with .Values.labels }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- if or .Values.statsd.extraMappings .Values.statsd.podAnnotations }}
annotations:
checksum/statsd-config: {{ include (print $.Template.BasePath "/configmaps/statsd-configmap.yaml") . | sha256sum }}
{{- if .Values.statsd.podAnnotations }}
{{- toYaml .Values.statsd.podAnnotations | nindent 8 }}
{{- end }}
{{- end }}
spec:
{{- if .Values.statsd.priorityClassName }}
priorityClassName: {{ .Values.statsd.priorityClassName }}
{{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity: {{- toYaml $affinity | nindent 8 }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
terminationGracePeriodSeconds: {{ .Values.statsd.terminationGracePeriodSeconds }}
serviceAccountName: {{ include "statsd.serviceAccountName" . }}
securityContext: {{ $securityContext | nindent 8 }}
restartPolicy: Always
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
containers:
- name: statsd
image: {{ template "statsd_image" . }}
imagePullPolicy: {{ .Values.images.statsd.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
{{- if .Values.statsd.args }}
args: {{ tpl (toYaml .Values.statsd.args) . | nindent 12 }}
{{- else}}
args:
- "--statsd.mapping-config=/etc/statsd-exporter/mappings.yml"
{{- end }}
resources: {{- toYaml .Values.statsd.resources | nindent 12 }}
{{- with .Values.statsd.env }}
env: {{- toYaml . | nindent 12 }}
{{- end }}
ports:
- name: statsd-ingest
protocol: UDP
containerPort: {{ .Values.ports.statsdIngest }}
- name: statsd-scrape
containerPort: {{ .Values.ports.statsdScrape }}
livenessProbe:
httpGet:
path: /metrics
port: {{ .Values.ports.statsdScrape }}
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
readinessProbe:
httpGet:
path: /metrics
port: {{ .Values.ports.statsdScrape }}
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
volumeMounts:
- name: config
mountPath: /etc/statsd-exporter/mappings.yml
subPath: mappings.yml
volumes:
- name: config
configMap:
name: {{ include "airflow.fullname" . }}-statsd
{{- end }}
@@ -0,0 +1,87 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Statsd Ingress
#################################
{{- if and .Values.statsd.enabled .Values.ingress.statsd.enabled }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ include "airflow.fullname" . }}-statsd-ingress
labels:
tier: airflow
component: statsd-ingress
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.ingress.statsd.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if and .Values.ingress.statsd.hosts (.Values.ingress.statsd.hosts | first | kindIs "string" | not) }}
{{- $anyTlsHosts := false -}}
{{- range .Values.ingress.statsd.hosts }}
{{- if .tls }}
{{- if .tls.enabled }}
{{- $anyTlsHosts = true -}}
{{- end }}
{{- end }}
{{- end }}
{{- if $anyTlsHosts }}
tls:
{{- range .Values.ingress.statsd.hosts }}
{{- if .tls }}
{{- if .tls.enabled }}
- hosts:
- {{ .name | quote }}
secretName: {{ .tls.secretName }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
rules:
{{- range .Values.ingress.statsd.hosts | default (list .Values.ingress.statsd.host) }}
- http:
paths:
- backend:
service:
name: {{ $.Release.Name }}-statsd
port:
name: statsd-scrape
{{- if $.Values.ingress.statsd.path }}
path: {{ $.Values.ingress.statsd.path }}
pathType: {{ $.Values.ingress.statsd.pathType }}
{{- end }}
{{- $hostname := . -}}
{{- if . | kindIs "string" | not }}
{{- $hostname = .name -}}
{{- end }}
{{- if $hostname }}
host: {{ tpl $hostname $ | quote }}
{{- end }}
{{- end }}
{{- if .Values.ingress.statsd.ingressClassName }}
ingressClassName: {{ .Values.ingress.statsd.ingressClassName }}
{{- end }}
{{- end }}
@@ -0,0 +1,59 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow StatsD NetworkPolicy
#################################
{{- if and .Values.networkPolicies.enabled .Values.statsd.enabled }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ include "airflow.fullname" . }}-statsd-policy
labels:
tier: airflow
component: statsd-policy
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
podSelector:
matchLabels:
tier: airflow
component: statsd
release: {{ .Release.Name }}
policyTypes:
- Ingress
ingress:
- from:
- podSelector:
matchLabels:
tier: airflow
release: {{ .Release.Name }}
{{- if .Values.statsd.extraNetworkPolicies }}
{{- toYaml .Values.statsd.extraNetworkPolicies | nindent 4 }}
{{- end }}
ports:
- protocol: UDP
port: {{ .Values.ports.statsdIngest }}
- protocol: TCP
port: {{ .Values.ports.statsdScrape }}
{{- end }}
@@ -0,0 +1,58 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow StatsD Service
#################################
{{- if .Values.statsd.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "airflow.fullname" . }}-statsd
labels:
tier: airflow
component: statsd
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: {{ .Values.ports.statsdScrape | quote }}
{{- if .Values.statsd.service.extraAnnotations }}
{{- toYaml .Values.statsd.service.extraAnnotations | nindent 4 }}
{{- end }}
spec:
type: ClusterIP
selector:
tier: airflow
component: statsd
release: {{ .Release.Name }}
ports:
- name: statsd-ingest
protocol: UDP
port: {{ .Values.ports.statsdIngest }}
targetPort: {{ .Values.ports.statsdIngest }}
- name: statsd-scrape
protocol: TCP
port: {{ .Values.ports.statsdScrape }}
targetPort: {{ .Values.ports.statsdScrape }}
{{- end }}
@@ -0,0 +1,41 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
######################################
## Airflow StatsD ServiceAccount
######################################
{{- if and .Values.statsd.enabled .Values.statsd.serviceAccount.create }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.statsd.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "statsd.serviceAccountName" . }}
labels:
tier: airflow
component: statsd
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.statsd.serviceAccount.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
@@ -0,0 +1,325 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Triggerer Deployment
#################################
{{- if semverCompare ">=2.2.0" .Values.airflowVersion }}
{{- if .Values.triggerer.enabled }}
{{- /* Airflow version 2.6.0 is when triggerer logs serve introduced */ -}}
{{- $persistence := and .Values.triggerer.persistence.enabled (semverCompare ">=2.6.0" .Values.airflowVersion) }}
{{- $keda := .Values.triggerer.keda.enabled }}
{{- $nodeSelector := or .Values.triggerer.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.triggerer.affinity .Values.affinity }}
{{- $tolerations := or .Values.triggerer.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.triggerer.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $revisionHistoryLimit := or .Values.triggerer.revisionHistoryLimit .Values.revisionHistoryLimit }}
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.triggerer) }}
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.triggerer) }}
{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list . .Values.triggerer.waitForMigrations) }}
{{- $containerSecurityContextLogGroomer := include "containerSecurityContext" (list . .Values.triggerer.logGroomerSidecar) }}
{{- $containerLifecycleHooks := or .Values.triggerer.containerLifecycleHooks .Values.containerLifecycleHooks }}
{{- $containerLifecycleHooksLogGroomerSidecar := or .Values.triggerer.logGroomerSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }}
apiVersion: apps/v1
kind: {{ if $persistence }}StatefulSet{{ else }}Deployment{{ end }}
metadata:
name: {{ include "airflow.fullname" . }}-triggerer
labels:
tier: airflow
component: triggerer
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if .Values.triggerer.annotations }}
annotations: {{- toYaml .Values.triggerer.annotations | nindent 4 }}
{{- end }}
spec:
{{- if $persistence }}
serviceName: {{ .Release.Name }}-triggerer
{{- end }}
{{- if not $keda }}
replicas: {{ .Values.triggerer.replicas }}
{{- end }}
{{- if $revisionHistoryLimit }}
revisionHistoryLimit: {{ $revisionHistoryLimit }}
{{- end }}
selector:
matchLabels:
tier: airflow
component: triggerer
release: {{ .Release.Name }}
{{- if and $persistence .Values.triggerer.updateStrategy }}
updateStrategy: {{- toYaml .Values.triggerer.updateStrategy | nindent 4 }}
{{- end }}
{{- if and (not $persistence) (.Values.triggerer.strategy) }}
strategy: {{- toYaml .Values.triggerer.strategy | nindent 4 }}
{{- end }}
{{- if and $persistence .Values.triggerer.persistence.persistentVolumeClaimRetentionPolicy }}
persistentVolumeClaimRetentionPolicy: {{- toYaml .Values.triggerer.persistence.persistentVolumeClaimRetentionPolicy | nindent 4 }}
{{- end }}
template:
metadata:
labels:
tier: airflow
component: triggerer
release: {{ .Release.Name }}
{{- if or (.Values.labels) (.Values.triggerer.labels) }}
{{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 8 }}
{{- end }}
annotations:
checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }}
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }}
checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }}
checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }}
{{- if .Values.triggerer.safeToEvict }}
cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
{{- end }}
{{- if .Values.airflowPodAnnotations }}
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
{{- end }}
{{- if .Values.triggerer.podAnnotations }}
{{- toYaml .Values.triggerer.podAnnotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.triggerer.priorityClassName }}
priorityClassName: {{ .Values.triggerer.priorityClassName }}
{{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
affinity:
{{- if $affinity }}
{{- toYaml $affinity | nindent 8 }}
{{- else }}
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
component: triggerer
topologyKey: kubernetes.io/hostname
weight: 100
{{- end }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
{{- if .Values.triggerer.hostAliases }}
hostAliases: {{- toYaml .Values.triggerer.hostAliases | nindent 8 }}
{{- end }}
terminationGracePeriodSeconds: {{ .Values.triggerer.terminationGracePeriodSeconds }}
restartPolicy: Always
serviceAccountName: {{ include "triggerer.serviceAccountName" . }}
securityContext: {{ $securityContext | nindent 8 }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
initContainers:
{{- if .Values.triggerer.waitForMigrations.enabled }}
- name: wait-for-airflow-migrations
resources:
{{- toYaml .Values.triggerer.resources | nindent 12 }}
image: {{ template "airflow_image_for_migrations" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }}
volumeMounts:
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.triggerer.extraVolumeMounts }}
{{- tpl (toYaml .Values.triggerer.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
args: {{- include "wait-for-migrations-command" . | indent 10 }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- if .Values.triggerer.waitForMigrations.env }}
{{- tpl (toYaml .Values.triggerer.waitForMigrations.env) $ | nindent 12 }}
{{- end }}
{{- end }}
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
{{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }}
{{- end }}
{{- if .Values.triggerer.extraInitContainers }}
{{- tpl (toYaml .Values.triggerer.extraInitContainers) . | nindent 8 }}
{{- end }}
containers:
- name: triggerer
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
{{- if .Values.triggerer.command }}
command: {{ tpl (toYaml .Values.triggerer.command) . | nindent 12 }}
{{- end }}
{{- if .Values.triggerer.args }}
args: {{ tpl (toYaml .Values.triggerer.args) . | nindent 12 }}
{{- end }}
resources: {{- toYaml .Values.triggerer.resources | nindent 12 }}
volumeMounts:
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.triggerer.extraVolumeMounts }}
{{- tpl (toYaml .Values.triggerer.extraVolumeMounts) . | nindent 12 }}
{{- end }}
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
{{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }}
{{- include "airflow_dags_mount" . | nindent 12 }}
{{- end }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- include "container_extra_envs" (list . .Values.triggerer.env) | nindent 10 }}
livenessProbe:
initialDelaySeconds: {{ .Values.triggerer.livenessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.triggerer.livenessProbe.timeoutSeconds }}
failureThreshold: {{ .Values.triggerer.livenessProbe.failureThreshold }}
periodSeconds: {{ .Values.triggerer.livenessProbe.periodSeconds }}
exec:
command:
{{- if .Values.triggerer.livenessProbe.command }}
{{- toYaml .Values.triggerer.livenessProbe.command | nindent 16 }}
{{- else }}
{{- include "triggerer_liveness_check_command" . | indent 14 }}
{{- end }}
{{- /* Airflow version 2.6.0 is when triggerer logs serve introduced */ -}}
{{- if semverCompare ">=2.6.0" .Values.airflowVersion }}
ports:
- name: triggerer-logs
containerPort: {{ .Values.ports.triggererLogs }}
{{- end }}
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
{{- include "git_sync_container" . | nindent 8 }}
{{- end }}
{{- if .Values.triggerer.logGroomerSidecar.enabled }}
- name: triggerer-log-groomer
resources: {{- toYaml .Values.triggerer.logGroomerSidecar.resources | nindent 12 }}
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContextLogGroomer | nindent 12 }}
{{- if $containerLifecycleHooksLogGroomerSidecar }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooksLogGroomerSidecar) . | nindent 12 }}
{{- end }}
{{- if .Values.triggerer.logGroomerSidecar.command }}
command: {{ tpl (toYaml .Values.triggerer.logGroomerSidecar.command) . | nindent 12 }}
{{- end }}
{{- if .Values.triggerer.logGroomerSidecar.args }}
args: {{- tpl (toYaml .Values.triggerer.logGroomerSidecar.args) . | nindent 12 }}
{{- end }}
env:
{{- if .Values.triggerer.logGroomerSidecar.retentionDays }}
- name: AIRFLOW__LOG_RETENTION_DAYS
value: "{{ .Values.triggerer.logGroomerSidecar.retentionDays }}"
{{- end }}
{{- if .Values.triggerer.logGroomerSidecar.frequencyMinutes }}
- name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES
value: "{{ .Values.triggerer.logGroomerSidecar.frequencyMinutes }}"
{{- end }}
- name: AIRFLOW_HOME
value: "{{ .Values.airflowHome }}"
{{- if .Values.triggerer.logGroomerSidecar.env }}
{{- tpl (toYaml .Values.triggerer.logGroomerSidecar.env) $ | nindent 12 }}
{{- end }}
volumeMounts:
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.triggerer.extraVolumeMounts }}
{{- tpl (toYaml .Values.triggerer.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
{{- end }}
{{- if .Values.triggerer.extraContainers }}
{{- tpl (toYaml .Values.triggerer.extraContainers) . | nindent 8 }}
{{- end }}
volumes:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
- name: webserver-config
configMap:
name: {{ template "airflow_webserver_config_configmap_name" . }}
{{- end }}
{{- if .Values.dags.persistence.enabled }}
- name: dags
persistentVolumeClaim:
claimName: {{ template "airflow_dags_volume_claim" . }}
{{- else if .Values.dags.gitSync.enabled }}
- name: dags
emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 12 }}
{{- if or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey}}
{{- include "git_sync_ssh_key_volume" . | nindent 8 }}
{{- end }}
{{- end }}
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 8 }}
{{- end }}
{{- if .Values.triggerer.extraVolumes }}
{{- tpl (toYaml .Values.triggerer.extraVolumes) . | nindent 8 }}
{{- end }}
{{- if .Values.logs.persistence.enabled }}
- name: logs
persistentVolumeClaim:
claimName: {{ template "airflow_logs_volume_claim" . }}
{{- else if not $persistence }}
- name: logs
emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) | nindent 12 }}
{{- else }}
volumeClaimTemplates:
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: logs
{{- if .Values.triggerer.persistence.annotations }}
annotations: {{- toYaml .Values.triggerer.persistence.annotations | nindent 10 }}
{{- end }}
spec:
{{- if .Values.triggerer.persistence.storageClassName }}
storageClassName: {{ tpl .Values.triggerer.persistence.storageClassName . | quote }}
{{- end }}
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: {{ .Values.triggerer.persistence.size }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,70 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Triggerer KEDA Scaler
#################################
{{- if semverCompare ">=2.2.0" .Values.airflowVersion }}
{{- if and .Values.triggerer.enabled .Values.triggerer.keda.enabled }}
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
name: {{ .Release.Name }}-triggerer
labels:
tier: airflow
component: triggerer-horizontalpodautoscaler
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
deploymentName: {{ .Release.Name }}-triggerer
{{- if or (.Values.labels) (.Values.triggerer.labels) }}
{{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
scaleTargetRef:
kind: {{ ternary "StatefulSet" "Deployment" .Values.triggerer.persistence.enabled }}
name: {{ .Release.Name }}-triggerer
envSourceContainerName: triggerer
pollingInterval: {{ .Values.triggerer.keda.pollingInterval }}
cooldownPeriod: {{ .Values.triggerer.keda.cooldownPeriod }}
minReplicaCount: {{ .Values.triggerer.keda.minReplicaCount }}
maxReplicaCount: {{ .Values.triggerer.keda.maxReplicaCount }}
{{- if .Values.triggerer.keda.advanced }}
advanced: {{- toYaml .Values.triggerer.keda.advanced | nindent 4 }}
{{- end }}
triggers:
{{- if eq .Values.data.metadataConnection.protocol "mysql" }}
- type: "mysql"
metadata:
queryValue: "1"
connectionStringFromEnv: KEDA_DB_CONN
query: {{ tpl .Values.triggerer.keda.query . | quote }}
{{- else }}
- type: postgresql
metadata:
targetQueryValue: "1"
{{- if and .Values.pgbouncer.enabled (not .Values.triggerer.keda.usePgbouncer) }}
connectionFromEnv: KEDA_DB_CONN
{{- else }}
connectionFromEnv: AIRFLOW_CONN_AIRFLOW_DB
{{- end }}
query: {{ tpl .Values.triggerer.keda.query . | quote }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,60 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
##################################
## Airflow triggerer NetworkPolicy
##################################
{{- /* Airflow version 2.6.0 is when triggerer logs serve introduced */ -}}
{{- if semverCompare ">=2.6.0" .Values.airflowVersion }}
{{- if .Values.networkPolicies.enabled }}
{{- if .Values.triggerer.enabled }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ include "airflow.fullname" . }}-triggerer-policy
labels:
tier: airflow
component: airflow-triggerer-policy
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.triggerer.labels) }}
{{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
podSelector:
matchLabels:
tier: airflow
component: triggerer
release: {{ .Release.Name }}
policyTypes:
- Ingress
ingress:
- from:
- podSelector:
matchLabels:
tier: airflow
release: {{ .Release.Name }}
component: webserver
ports:
- protocol: TCP
port: {{ .Values.ports.triggererLogs }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,51 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow triggerer Service
#################################
{{- /* Airflow version 2.6.0 is when triggerer logs serve introduced */ -}}
{{- if semverCompare ">=2.6.0" .Values.airflowVersion }}
{{- if .Values.triggerer.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "airflow.fullname" . }}-triggerer
labels:
tier: airflow
component: triggerer
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.triggerer.labels) }}
{{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
clusterIP: None
selector:
tier: airflow
component: triggerer
release: {{ .Release.Name }}
ports:
- name: triggerer-logs
protocol: TCP
port: {{ .Values.ports.triggererLogs }}
targetPort: {{ .Values.ports.triggererLogs }}
{{- end }}
{{- end }}
@@ -0,0 +1,43 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Triggerer ServiceAccount
#################################
{{- if semverCompare ">=2.2.0" .Values.airflowVersion }}
{{- if and .Values.triggerer.serviceAccount.create .Values.triggerer.enabled }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.triggerer.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "triggerer.serviceAccountName" . }}
labels:
tier: airflow
component: triggerer
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.triggerer.labels) }}
{{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.triggerer.serviceAccount.annotations}}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,299 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Webserver Deployment
#################################
{{- if and .Values.webserver.enabled (semverCompare "<3.0.0" .Values.airflowVersion) }}
{{- $nodeSelector := or .Values.webserver.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.webserver.affinity .Values.affinity }}
{{- $tolerations := or .Values.webserver.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.webserver.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $revisionHistoryLimit := or .Values.webserver.revisionHistoryLimit .Values.revisionHistoryLimit }}
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.webserver) }}
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.webserver) }}
{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list . .Values.webserver.waitForMigrations) }}
{{- $containerLifecycleHooks := or .Values.webserver.containerLifecycleHooks .Values.containerLifecycleHooks }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "airflow.fullname" . }}-webserver
labels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if .Values.webserver.annotations }}
annotations: {{- toYaml .Values.webserver.annotations | nindent 4 }}
{{- end }}
spec:
{{- if not .Values.webserver.hpa.enabled }}
replicas: {{ .Values.webserver.replicas }}
{{- end}}
{{- if $revisionHistoryLimit }}
revisionHistoryLimit: {{ $revisionHistoryLimit }}
{{- end }}
strategy:
{{- if .Values.webserver.strategy }}
{{- toYaml .Values.webserver.strategy | nindent 4 }}
{{- else }}
{{- if semverCompare ">=2.0.0" .Values.airflowVersion }}
# Here we define the rolling update strategy
# - maxSurge define how many pod we can add at a time
# - maxUnavailable define how many pod can be unavailable
# during the rolling update
# Setting maxUnavailable to 0 would make sure we have the appropriate
# capacity during the rolling update.
# You can also use percentage based value instead of integer.
type: RollingUpdate
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
{{- else }}
type: Recreate
{{- end }}
{{- end }}
selector:
matchLabels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
template:
metadata:
labels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
{{- if or (.Values.labels) (.Values.webserver.labels) }}
{{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 8 }}
{{- end }}
annotations:
checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }}
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
checksum/webserver-secret-key: {{ include (print $.Template.BasePath "/secrets/webserver-secret-key-secret.yaml") . | sha256sum }}
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }}
checksum/webserver-config: {{ include (print $.Template.BasePath "/configmaps/webserver-configmap.yaml") . | sha256sum }}
checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }}
checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }}
{{- if .Values.airflowPodAnnotations }}
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
{{- end }}
{{- if .Values.webserver.podAnnotations }}
{{- toYaml .Values.webserver.podAnnotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.webserver.hostAliases }}
hostAliases: {{- toYaml .Values.webserver.hostAliases | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "webserver.serviceAccountName" . }}
{{- if .Values.webserver.priorityClassName }}
priorityClassName: {{ .Values.webserver.priorityClassName }}
{{- end }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity:
{{- if $affinity }}
{{- toYaml $affinity | nindent 8 }}
{{- else }}
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
component: webserver
topologyKey: kubernetes.io/hostname
weight: 100
{{- end }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
restartPolicy: Always
terminationGracePeriodSeconds: {{ .Values.webserver.terminationGracePeriodSeconds }}
securityContext: {{ $securityContext | nindent 8 }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
initContainers:
{{- if .Values.webserver.waitForMigrations.enabled }}
- name: wait-for-airflow-migrations
resources: {{- toYaml .Values.webserver.resources | nindent 12 }}
image: {{ template "airflow_image_for_migrations" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }}
volumeMounts:
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.webserver.extraVolumeMounts }}
{{- tpl (toYaml .Values.webserver.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
args: {{- include "wait-for-migrations-command" . | indent 10 }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- if .Values.webserver.waitForMigrations.env }}
{{- tpl (toYaml .Values.webserver.waitForMigrations.env) $ | nindent 12 }}
{{- end }}
{{- end }}
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) (semverCompare "<2.0.0" .Values.airflowVersion) }}
{{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }}
{{- end }}
{{- if .Values.webserver.extraInitContainers }}
{{- tpl (toYaml .Values.webserver.extraInitContainers) . | nindent 8 }}
{{- end }}
containers:
- name: webserver
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
{{- if .Values.webserver.command }}
command: {{ tpl (toYaml .Values.webserver.command) . | nindent 12 }}
{{- end }}
{{- if .Values.webserver.args }}
args: {{- tpl (toYaml .Values.webserver.args) . | nindent 12 }}
{{- end }}
resources: {{- toYaml .Values.webserver.resources | nindent 12 }}
volumeMounts:
{{- if semverCompare ">=1.10.12" .Values.airflowVersion }}
- name: config
mountPath: {{ include "airflow_pod_template_file" . }}/pod_template_file.yaml
subPath: pod_template_file.yaml
readOnly: true
{{- end }}
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
{{- if and (semverCompare "<2.0.0" .Values.airflowVersion) (or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled) }}
{{- include "airflow_dags_mount" . | nindent 12 }}
{{- end }}
{{- if .Values.logs.persistence.enabled }}
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- end }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.webserver.extraVolumeMounts }}
{{- tpl (toYaml .Values.webserver.extraVolumeMounts) . | nindent 12 }}
{{- end }}
ports:
- name: airflow-ui
containerPort: {{ .Values.ports.airflowUI }}
livenessProbe:
httpGet:
path: {{ if .Values.config.webserver.base_url }}{{- with urlParse (tpl .Values.config.webserver.base_url .) }}{{ .path }}{{ end }}{{ end }}/health
port: {{ .Values.ports.airflowUI }}
{{- if .Values.config.webserver.base_url}}
httpHeaders:
- name: Host
value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values.config.webserver.base_url .)).host "" }}
{{- end }}
scheme: {{ .Values.webserver.livenessProbe.scheme | default "http" }}
initialDelaySeconds: {{ .Values.webserver.livenessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.webserver.livenessProbe.timeoutSeconds }}
failureThreshold: {{ .Values.webserver.livenessProbe.failureThreshold }}
periodSeconds: {{ .Values.webserver.livenessProbe.periodSeconds }}
readinessProbe:
httpGet:
path: {{ if .Values.config.webserver.base_url }}{{- with urlParse (tpl .Values.config.webserver.base_url .) }}{{ .path }}{{ end }}{{ end }}/health
port: {{ .Values.ports.airflowUI }}
{{- if .Values.config.webserver.base_url }}
httpHeaders:
- name: Host
value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values.config.webserver.base_url .)).host "" }}
{{- end }}
scheme: {{ .Values.webserver.readinessProbe.scheme | default "http" }}
initialDelaySeconds: {{ .Values.webserver.readinessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.webserver.readinessProbe.timeoutSeconds }}
failureThreshold: {{ .Values.webserver.readinessProbe.failureThreshold }}
periodSeconds: {{ .Values.webserver.readinessProbe.periodSeconds }}
startupProbe:
httpGet:
path: {{ if .Values.config.webserver.base_url }}{{- with urlParse (tpl .Values.config.webserver.base_url .) }}{{ .path }}{{ end }}{{ end }}/health
port: {{ .Values.ports.airflowUI }}
{{- if .Values.config.webserver.base_url}}
httpHeaders:
- name: Host
value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values.config.webserver.base_url .)).host "" }}
{{- end }}
scheme: {{ .Values.webserver.startupProbe.scheme | default "http" }}
initialDelaySeconds: {{ .Values.webserver.startupProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.webserver.startupProbe.timeoutSeconds }}
failureThreshold: {{ .Values.webserver.startupProbe.failureThreshold }}
periodSeconds: {{ .Values.webserver.startupProbe.periodSeconds }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- include "container_extra_envs" (list . .Values.webserver.env) | indent 10 }}
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) (semverCompare "<2.0.0" .Values.airflowVersion) }}
{{- include "git_sync_container" . | nindent 8 }}
{{- end }}
{{- if .Values.webserver.extraContainers }}
{{- tpl (toYaml .Values.webserver.extraContainers) . | nindent 8 }}
{{- end }}
volumes:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
- name: webserver-config
configMap:
name: {{ template "airflow_webserver_config_configmap_name" . }}
{{- end }}
{{- if (semverCompare "<2.0.0" .Values.airflowVersion) }}
{{- if .Values.dags.persistence.enabled }}
- name: dags
persistentVolumeClaim:
claimName: {{ template "airflow_dags_volume_claim" . }}
{{- else if .Values.dags.gitSync.enabled }}
- name: dags
emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 12 }}
{{- if or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey}}
{{- include "git_sync_ssh_key_volume" . | indent 8 }}
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.logs.persistence.enabled }}
- name: logs
persistentVolumeClaim:
claimName: {{ template "airflow_logs_volume_claim" . }}
{{- end }}
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 8 }}
{{- end }}
{{- if .Values.webserver.extraVolumes }}
{{- tpl (toYaml .Values.webserver.extraVolumes) . | nindent 8 }}
{{- end }}
{{- end }}
@@ -0,0 +1,51 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Webserver HPA
#################################
{{- if semverCompare "<3.0.0" .Values.airflowVersion }}
{{- if and .Values.webserver.enabled .Values.webserver.hpa.enabled }}
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "airflow.fullname" . }}-webserver
labels:
tier: airflow
component: webserver-horizontalpodautoscaler
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
deploymentName: {{ .Release.Name }}-webserver
{{- if or (.Values.labels) (.Values.webserver.labels) }}
{{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "airflow.fullname" . }}-webserver
minReplicas: {{ .Values.webserver.hpa.minReplicaCount }}
maxReplicas: {{ .Values.webserver.hpa.maxReplicaCount }}
metrics: {{- toYaml .Values.webserver.hpa.metrics | nindent 4 }}
{{- with .Values.webserver.hpa.behavior }}
behavior: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,113 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Webserver Ingress
#################################
{{- if and .Values.webserver.enabled (semverCompare "<3.0.0" .Values.airflowVersion) }}
{{- if or .Values.ingress.web.enabled .Values.ingress.enabled }}
{{- $fullname := (include "airflow.fullname" .) }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ $fullname }}-ingress
labels:
tier: airflow
component: airflow-ingress
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.webserver.labels) }}
{{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.ingress.web.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if and .Values.ingress.web.hosts (.Values.ingress.web.hosts | first | kindIs "string" | not) }}
{{- $anyTlsHosts := false -}}
{{- range .Values.ingress.web.hosts }}
{{- if .tls }}
{{- if .tls.enabled }}
{{- $anyTlsHosts = true -}}
{{- end }}
{{- end }}
{{- end }}
{{- if $anyTlsHosts }}
tls:
{{- range .Values.ingress.web.hosts }}
{{- if .tls }}
{{- if .tls.enabled }}
- hosts:
- {{ .name | quote }}
secretName: {{ .tls.secretName }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- else if .Values.ingress.web.tls.enabled }}
tls:
- hosts:
{{- .Values.ingress.web.hosts | default (list .Values.ingress.web.host) | toYaml | nindent 8 }}
secretName: {{ .Values.ingress.web.tls.secretName }}
{{- end }}
rules:
{{- range .Values.ingress.web.hosts | default (list .Values.ingress.web.host) }}
- http:
paths:
{{- range $.Values.ingress.web.precedingPaths }}
- path: {{ .path }}
pathType: {{ .pathType }}
backend:
service:
name: {{ .serviceName }}
port:
name: {{ .servicePort }}
{{- end }}
- backend:
service:
name: {{ $fullname }}-webserver
port:
name: airflow-ui
{{- if $.Values.ingress.web.path }}
path: {{ $.Values.ingress.web.path }}
pathType: {{ $.Values.ingress.web.pathType }}
{{- end }}
{{- range $.Values.ingress.web.succeedingPaths }}
- path: {{ .path }}
pathType: {{ .pathType }}
backend:
service:
name: {{ .serviceName }}
port:
name: {{ .servicePort }}
{{- end }}
{{- $hostname := . -}}
{{- if . | kindIs "string" | not }}
{{- $hostname = .name -}}
{{- end }}
{{- if $hostname }}
host: {{ tpl $hostname $ | quote }}
{{- end }}
{{- end }}
{{- if .Values.ingress.web.ingressClassName }}
ingressClassName: {{ .Values.ingress.web.ingressClassName }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,59 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Webserver NetworkPolicy
#################################
{{- if and .Values.webserver.enabled (semverCompare "<3.0.0" .Values.airflowVersion) }}
{{- if .Values.networkPolicies.enabled }}
{{- $from := or .Values.webserver.networkPolicy.ingress.from .Values.webserver.extraNetworkPolicies }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ include "airflow.fullname" . }}-webserver-policy
labels:
tier: airflow
component: airflow-webserver-policy
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.webserver.labels) }}
{{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
podSelector:
matchLabels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
policyTypes:
- Ingress
{{- if $from }}
ingress:
- from: {{- toYaml $from | nindent 6 }}
ports:
{{ range .Values.webserver.networkPolicy.ingress.ports }}
-
{{- range $key, $val := . }}
{{ $key }}: {{ tpl (toString $val) $ }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,46 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Webserver PodDisruptionBudget
#################################
{{- if and .Values.webserver.enabled (semverCompare "<3.0.0" .Values.airflowVersion) }}
{{- if .Values.webserver.podDisruptionBudget.enabled }}
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: {{ include "airflow.fullname" . }}-webserver-pdb
labels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.webserver.labels) }}
{{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
selector:
matchLabels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
{{- toYaml .Values.webserver.podDisruptionBudget.config | nindent 2 }}
{{- end }}
{{- end }}
@@ -0,0 +1,59 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Webserver Service
#################################
{{- if and .Values.webserver.enabled (semverCompare "<3.0.0" .Values.airflowVersion) }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "airflow.fullname" . }}-webserver
labels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.webserver.labels) }}
{{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.webserver.service.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
spec:
type: {{ .Values.webserver.service.type }}
selector:
tier: airflow
component: webserver
release: {{ .Release.Name }}
ports:
{{ range .Values.webserver.service.ports }}
-
{{- range $key, $val := . }}
{{ $key }}: {{ tpl (toString $val) $ }}
{{- end }}
{{- end }}
{{- if .Values.webserver.service.loadBalancerIP }}
loadBalancerIP: {{ .Values.webserver.service.loadBalancerIP }}
{{- end }}
{{- if .Values.webserver.service.loadBalancerSourceRanges }}
loadBalancerSourceRanges: {{- toYaml .Values.webserver.service.loadBalancerSourceRanges | nindent 4 }}
{{- end }}
{{- end }}
@@ -0,0 +1,43 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
######################################
## Airflow Webserver ServiceAccount
######################################
{{- if semverCompare "<3.0.0" .Values.airflowVersion }}
{{- if and .Values.webserver.enabled .Values.webserver.serviceAccount.create }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.webserver.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "webserver.serviceAccountName" . }}
labels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.webserver.labels) }}
{{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.webserver.serviceAccount.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,463 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Worker Deployment
#################################
{{- $persistence := .Values.workers.persistence.enabled }}
{{- $keda := .Values.workers.keda.enabled }}
{{- $hpa := and .Values.workers.hpa.enabled (not .Values.workers.keda.enabled) }}
{{- if or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) }}
{{- $nodeSelector := or .Values.workers.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.workers.affinity .Values.affinity }}
{{- $tolerations := or .Values.workers.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.workers.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $revisionHistoryLimit := or .Values.workers.revisionHistoryLimit .Values.revisionHistoryLimit }}
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.workers) }}
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.workers) }}
{{- $containerSecurityContextPersistence := include "containerSecurityContext" (list . .Values.workers.persistence) }}
{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list . .Values.workers.waitForMigrations) }}
{{- $containerSecurityContextLogGroomerSidecar := include "containerSecurityContext" (list . .Values.workers.logGroomerSidecar) }}
{{- $containerSecurityContextKerberosSidecar := include "containerSecurityContext" (list . .Values.workers.kerberosSidecar) }}
{{- $containerLifecycleHooks := or .Values.workers.containerLifecycleHooks .Values.containerLifecycleHooks }}
{{- $containerLifecycleHooksLogGroomerSidecar := or .Values.workers.logGroomerSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }}
{{- $containerLifecycleHooksKerberosSidecar := or .Values.workers.kerberosSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }}
{{- $safeToEvict := dict "cluster-autoscaler.kubernetes.io/safe-to-evict" (.Values.workers.safeToEvict | toString) }}
{{- $podAnnotations := mergeOverwrite (deepCopy .Values.airflowPodAnnotations) $safeToEvict .Values.workers.podAnnotations }}
apiVersion: apps/v1
kind: {{ if $persistence }}StatefulSet{{ else }}Deployment{{ end }}
metadata:
name: {{ include "airflow.fullname" . }}-worker
labels:
tier: airflow
component: worker
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if .Values.workers.annotations }}
annotations: {{- toYaml .Values.workers.annotations | nindent 4 }}
{{- end }}
spec:
{{- if $persistence }}
serviceName: {{ include "airflow.fullname" . }}-worker
{{- end }}
{{- if and (not $keda) (not $hpa) }}
replicas: {{ .Values.workers.replicas }}
{{- end }}
{{- if $revisionHistoryLimit }}
revisionHistoryLimit: {{ $revisionHistoryLimit }}
{{- end }}
{{- if and $persistence .Values.workers.persistence.persistentVolumeClaimRetentionPolicy }}
persistentVolumeClaimRetentionPolicy: {{- toYaml .Values.workers.persistence.persistentVolumeClaimRetentionPolicy | nindent 4 }}
{{- end }}
selector:
matchLabels:
tier: airflow
component: worker
release: {{ .Release.Name }}
{{- if and $persistence .Values.workers.podManagementPolicy }}
podManagementPolicy: {{ .Values.workers.podManagementPolicy }}
{{- end }}
{{- if and $persistence .Values.workers.updateStrategy }}
updateStrategy: {{- toYaml .Values.workers.updateStrategy | nindent 4 }}
{{- end }}
{{- if and (not $persistence) (.Values.workers.strategy) }}
strategy: {{- toYaml .Values.workers.strategy | nindent 4 }}
{{- end }}
template:
metadata:
labels:
tier: airflow
component: worker
release: {{ .Release.Name }}
{{- if or (.Values.labels) (.Values.workers.labels) }}
{{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 8 }}
{{- end }}
annotations:
checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }}
checksum/result-backend-secret: {{ include (print $.Template.BasePath "/secrets/result-backend-connection-secret.yaml") . | sha256sum }}
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
checksum/webserver-secret-key: {{ include (print $.Template.BasePath "/secrets/webserver-secret-key-secret.yaml") . | sha256sum }}
checksum/kerberos-keytab: {{ include (print $.Template.BasePath "/secrets/kerberos-keytab-secret.yaml") . | sha256sum }}
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }}
checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }}
checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }}
{{- if $podAnnotations }}
{{- toYaml $podAnnotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.workers.runtimeClassName }}
runtimeClassName: {{ .Values.workers.runtimeClassName }}
{{- end }}
{{- if .Values.workers.priorityClassName }}
priorityClassName: {{ .Values.workers.priorityClassName }}
{{- end }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity:
{{- if $affinity }}
{{- toYaml $affinity | nindent 8 }}
{{- else }}
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
component: worker
topologyKey: kubernetes.io/hostname
weight: 100
{{- end }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
{{- if .Values.workers.hostAliases }}
hostAliases: {{- toYaml .Values.workers.hostAliases | nindent 8 }}
{{- end }}
terminationGracePeriodSeconds: {{ .Values.workers.terminationGracePeriodSeconds }}
restartPolicy: Always
serviceAccountName: {{ include "worker.serviceAccountName" . }}
securityContext: {{ $securityContext | nindent 8 }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
initContainers:
{{- if and $persistence .Values.workers.persistence.fixPermissions }}
- name: volume-permissions
resources: {{- toYaml .Values.workers.resources | nindent 12 }}
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
command:
- chown
- -R
- "{{ include "airflowPodSecurityContextsIds" (list . .Values.workers) }}"
- {{ template "airflow_logs" . }}
securityContext: {{ $containerSecurityContextPersistence | nindent 12 }}
volumeMounts:
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- end }}
{{- if and (semverCompare ">=2.8.0" .Values.airflowVersion) .Values.workers.kerberosInitContainer.enabled }}
- name: kerberos-init
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
args: ["kerberos", "-o"]
resources: {{- toYaml .Values.workers.kerberosInitContainer.resources | nindent 12 }}
volumeMounts:
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- include "airflow_config_mount" . | nindent 12 }}
- name: config
mountPath: {{ .Values.kerberos.configPath | quote }}
subPath: krb5.conf
readOnly: true
- name: kerberos-keytab
subPath: "kerberos.keytab"
mountPath: {{ .Values.kerberos.keytabPath | quote }}
readOnly: true
- name: kerberos-ccache
mountPath: {{ .Values.kerberos.ccacheMountPath | quote }}
readOnly: false
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.workers.extraVolumeMounts }}
{{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
- name: KRB5_CONFIG
value: {{ .Values.kerberos.configPath | quote }}
- name: KRB5CCNAME
value: {{ include "kerberos_ccache_path" . | quote }}
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- end }}
{{- if .Values.workers.waitForMigrations.enabled }}
- name: wait-for-airflow-migrations
resources: {{- toYaml .Values.workers.resources | nindent 12 }}
image: {{ template "airflow_image_for_migrations" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }}
volumeMounts:
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.workers.extraVolumeMounts }}
{{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
args: {{- include "wait-for-migrations-command" . | indent 10 }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- if .Values.workers.waitForMigrations.env }}
{{- tpl (toYaml .Values.workers.waitForMigrations.env) $ | nindent 12 }}
{{- end }}
{{- end }}
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
{{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }}
{{- end }}
{{- if .Values.workers.extraInitContainers }}
{{- tpl (toYaml .Values.workers.extraInitContainers) . | nindent 8 }}
{{- end }}
containers:
- name: worker
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
{{- if .Values.workers.command }}
command: {{ tpl (toYaml .Values.workers.command) . | nindent 12 }}
{{- end }}
{{- if .Values.workers.args }}
args: {{ tpl (toYaml .Values.workers.args) . | nindent 12 }}
{{- end }}
resources: {{- toYaml .Values.workers.resources | nindent 12 }}
{{- if .Values.workers.livenessProbe.enabled }}
livenessProbe:
initialDelaySeconds: {{ .Values.workers.livenessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.workers.livenessProbe.timeoutSeconds }}
failureThreshold: {{ .Values.workers.livenessProbe.failureThreshold }}
periodSeconds: {{ .Values.workers.livenessProbe.periodSeconds }}
exec:
command:
{{- if .Values.workers.livenessProbe.command }}
{{- toYaml .Values.workers.livenessProbe.command | nindent 16 }}
{{- else }}
- sh
- -c
- CONNECTION_CHECK_MAX_COUNT=0 exec /entrypoint python -m celery --app {{ include "celery_executor_namespace" . }} inspect ping -d celery@$(hostname)
{{- end }}
{{- end }}
ports:
{{- if .Values.workers.extraPorts }}
{{- toYaml .Values.workers.extraPorts | nindent 12 }}
{{- end }}
- name: worker-logs
containerPort: {{ .Values.ports.workerLogs }}
volumeMounts:
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.workers.extraVolumeMounts }}
{{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }}
{{- end }}
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if .Values.kerberos.enabled }}
- name: kerberos-keytab
subPath: "kerberos.keytab"
mountPath: {{ .Values.kerberos.keytabPath | quote }}
readOnly: true
- name: config
mountPath: {{ .Values.kerberos.configPath | quote }}
subPath: krb5.conf
readOnly: true
- name: kerberos-ccache
mountPath: {{ .Values.kerberos.ccacheMountPath | quote }}
readOnly: true
{{- end }}
{{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }}
{{- include "airflow_dags_mount" . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
# Only signal the main process, not the process group, to make Warm Shutdown work properly
- name: DUMB_INIT_SETSID
value: "0"
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- include "container_extra_envs" (list . .Values.workers.env) | indent 10 }}
{{- if .Values.workers.kerberosSidecar.enabled }}
- name: KRB5_CONFIG
value: {{ .Values.kerberos.configPath | quote }}
- name: KRB5CCNAME
value: {{ include "kerberos_ccache_path" . | quote }}
{{- end }}
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
{{- include "git_sync_container" . | nindent 8 }}
{{- end }}
{{- if and $persistence .Values.workers.logGroomerSidecar.enabled }}
- name: worker-log-groomer
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContextLogGroomerSidecar | nindent 12 }}
{{- if $containerLifecycleHooksLogGroomerSidecar }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooksLogGroomerSidecar) . | nindent 12 }}
{{- end }}
{{- if .Values.workers.logGroomerSidecar.command }}
command: {{ tpl (toYaml .Values.workers.logGroomerSidecar.command) . | nindent 12 }}
{{- end }}
{{- if .Values.workers.logGroomerSidecar.args }}
args: {{ tpl (toYaml .Values.workers.logGroomerSidecar.args) . | nindent 12 }}
{{- end }}
env:
{{- if .Values.workers.logGroomerSidecar.retentionDays }}
- name: AIRFLOW__LOG_RETENTION_DAYS
value: "{{ .Values.workers.logGroomerSidecar.retentionDays }}"
{{- end }}
{{- if .Values.workers.logGroomerSidecar.frequencyMinutes }}
- name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES
value: "{{ .Values.workers.logGroomerSidecar.frequencyMinutes }}"
{{- end }}
- name: AIRFLOW_HOME
value: "{{ .Values.airflowHome }}"
{{- if .Values.workers.logGroomerSidecar.env }}
{{- tpl (toYaml .Values.workers.logGroomerSidecar.env) $ | nindent 12 }}
{{- end }}
resources: {{- toYaml .Values.workers.logGroomerSidecar.resources | nindent 12 }}
volumeMounts:
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.workers.extraVolumeMounts }}
{{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
{{- end }}
{{- if .Values.workers.kerberosSidecar.enabled }}
- name: worker-kerberos
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContextKerberosSidecar | nindent 12 }}
{{- if $containerLifecycleHooksKerberosSidecar }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooksKerberosSidecar) . | nindent 12 }}
{{- end }}
args: ["kerberos"]
resources: {{- toYaml .Values.workers.kerberosSidecar.resources | nindent 12 }}
volumeMounts:
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- include "airflow_config_mount" . | nindent 12 }}
- name: config
mountPath: {{ .Values.kerberos.configPath | quote }}
subPath: krb5.conf
readOnly: true
- name: kerberos-keytab
subPath: "kerberos.keytab"
mountPath: {{ .Values.kerberos.keytabPath | quote }}
readOnly: true
- name: kerberos-ccache
mountPath: {{ .Values.kerberos.ccacheMountPath | quote }}
readOnly: false
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.workers.extraVolumeMounts }}
{{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
- name: KRB5_CONFIG
value: {{ .Values.kerberos.configPath | quote }}
- name: KRB5CCNAME
value: {{ include "kerberos_ccache_path" . | quote }}
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- end }}
{{- if .Values.workers.extraContainers }}
{{- tpl (toYaml .Values.workers.extraContainers) . | nindent 8 }}
{{- end }}
volumes:
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 8 }}
{{- end }}
{{- if .Values.workers.extraVolumes }}
{{- tpl (toYaml .Values.workers.extraVolumes) . | nindent 8 }}
{{- end }}
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
- name: webserver-config
configMap:
name: {{ template "airflow_webserver_config_configmap_name" . }}
{{- end }}
{{- if .Values.kerberos.enabled }}
- name: kerberos-keytab
secret:
secretName: {{ include "kerberos_keytab_secret" . | quote }}
- name: kerberos-ccache
emptyDir: {}
{{- end }}
{{- if .Values.dags.persistence.enabled }}
- name: dags
persistentVolumeClaim:
claimName: {{ template "airflow_dags_volume_claim" . }}
{{- else if .Values.dags.gitSync.enabled }}
- name: dags
emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 12 }}
{{- if or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey}}
{{- include "git_sync_ssh_key_volume" . | indent 8 }}
{{- end }}
{{- end }}
{{- if .Values.logs.persistence.enabled }}
- name: logs
persistentVolumeClaim:
claimName: {{ template "airflow_logs_volume_claim" . }}
{{- else if not $persistence }}
- name: logs
emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) | nindent 12 }}
{{- else }}
volumeClaimTemplates:
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: logs
{{- if .Values.workers.persistence.annotations }}
annotations: {{- toYaml .Values.workers.persistence.annotations | nindent 10 }}
{{- end }}
spec:
{{- if .Values.workers.persistence.storageClassName }}
storageClassName: {{ tpl .Values.workers.persistence.storageClassName . | quote }}
{{- end }}
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: {{ .Values.workers.persistence.size }}
{{- with .Values.workers.volumeClaimTemplates }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,49 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Worker HPA
#################################
{{- if and (and (not .Values.workers.keda.enabled) .Values.workers.hpa.enabled) (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }}
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "airflow.fullname" . }}-worker
labels:
tier: airflow
component: worker-horizontalpodautoscaler
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
deploymentName: {{ .Release.Name }}-worker
{{- if or (.Values.labels) (.Values.workers.labels) }}
{{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: {{ ternary "StatefulSet" "Deployment" .Values.workers.persistence.enabled }}
name: {{ include "airflow.fullname" . }}-worker
minReplicas: {{ .Values.workers.hpa.minReplicaCount }}
maxReplicas: {{ .Values.workers.hpa.maxReplicaCount }}
metrics: {{- toYaml .Values.workers.hpa.metrics | nindent 4 }}
{{- with .Values.workers.hpa.behavior }}
behavior: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
@@ -0,0 +1,68 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Worker KEDA Scaler
#################################
{{- if and .Values.workers.keda.enabled (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) ) }}
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
name: {{ include "airflow.fullname" . }}-worker
labels:
tier: airflow
component: worker-horizontalpodautoscaler
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
deploymentName: {{ .Release.Name }}-worker
{{- if or (.Values.labels) (.Values.workers.labels) }}
{{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
scaleTargetRef:
kind: {{ ternary "StatefulSet" "Deployment" .Values.workers.persistence.enabled }}
name: {{ include "airflow.fullname" . }}-worker
envSourceContainerName: worker
pollingInterval: {{ .Values.workers.keda.pollingInterval }}
cooldownPeriod: {{ .Values.workers.keda.cooldownPeriod }}
minReplicaCount: {{ .Values.workers.keda.minReplicaCount }}
maxReplicaCount: {{ .Values.workers.keda.maxReplicaCount }}
{{- if .Values.workers.keda.advanced }}
advanced: {{- toYaml .Values.workers.keda.advanced | nindent 4 }}
{{- end }}
triggers:
{{- if eq .Values.data.metadataConnection.protocol "mysql" }}
- type: "mysql"
metadata:
queryValue: "1"
connectionStringFromEnv: KEDA_DB_CONN
query: {{ tpl .Values.workers.keda.query . | quote }}
{{- else }}
- type: "postgresql"
metadata:
targetQueryValue: "1"
{{- if and .Values.pgbouncer.enabled (not .Values.workers.keda.usePgbouncer) }}
connectionFromEnv: KEDA_DB_CONN
{{- else }}
connectionFromEnv: AIRFLOW_CONN_AIRFLOW_DB
{{- end }}
query: {{ tpl .Values.workers.keda.query . | quote }}
{{- end }}
{{- end }}
@@ -0,0 +1,55 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Worker NetworkPolicy
#################################
{{- if and .Values.networkPolicies.enabled (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ include "airflow.fullname" . }}-worker-policy
labels:
tier: airflow
component: airflow-worker-policy
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.workers.labels) }}
{{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
podSelector:
matchLabels:
tier: airflow
component: worker
release: {{ .Release.Name }}
policyTypes:
- Ingress
ingress:
- from:
- podSelector:
matchLabels:
tier: airflow
release: {{ .Release.Name }}
component: webserver
ports:
- protocol: TCP
port: {{ .Values.ports.workerLogs }}
{{- end }}
@@ -0,0 +1,48 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Worker Service
#################################
{{- if or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "airflow.fullname" . }}-worker
labels:
tier: airflow
component: worker
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.workers.labels) }}
{{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
clusterIP: None
selector:
tier: airflow
component: worker
release: {{ .Release.Name }}
ports:
- name: worker-logs
protocol: TCP
port: {{ .Values.ports.workerLogs }}
targetPort: {{ .Values.ports.workerLogs }}
{{- end }}
@@ -0,0 +1,41 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Worker ServiceAccount
#################################
{{- if and .Values.workers.serviceAccount.create (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) (contains "KubernetesExecutor" .Values.executor) (contains "LocalKubernetesExecutor" .Values.executor)) }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.workers.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "worker.serviceAccountName" . }}
labels:
tier: airflow
component: worker
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.workers.labels) }}
{{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.workers.serviceAccount.annotations}}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}