Add chart flink cdc session cluster chart
This commit is contained in:
wbsong111
@@ -0,0 +1,19 @@
|
||||
{{/*
|
||||
Chart label
|
||||
*/}}
|
||||
{{- define "flink-cdc-session.chart" -}}
|
||||
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Common labels
|
||||
*/}}
|
||||
{{- define "flink-cdc-session.labels" -}}
|
||||
helm.sh/chart: {{ include "flink-cdc-session.chart" . }}
|
||||
app.kubernetes.io/name: {{ .Chart.Name }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
{{- if .Chart.AppVersion }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||
{{- end }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,45 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: cdc-pipeline-config
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "flink-cdc-session.labels" . | nindent 4 }}
|
||||
data:
|
||||
flink-cdc.yaml: |
|
||||
parallelism: {{ .Values.flinkCdc.parallelism }}
|
||||
schema.change.behavior: {{ .Values.flinkCdc.schemaChangeBehavior }}
|
||||
|
||||
pipeline.yaml: |
|
||||
source:
|
||||
type: postgres
|
||||
hostname: {{ .Values.postgres.hostname }}
|
||||
port: {{ .Values.postgres.port }}
|
||||
username: {{ .Values.postgres.username }}
|
||||
password: {{ .Values.postgres.password }}
|
||||
decoding.plugin.name: {{ .Values.postgres.decodingPlugin }}
|
||||
slot.name: {{ .Values.postgres.slotName }}
|
||||
tables: {{ .Values.postgres.tables }}
|
||||
|
||||
pipeline:
|
||||
name: {{ .Values.pipeline.name }}
|
||||
parallelism: {{ .Values.pipeline.parallelism }}
|
||||
execution.checkpointing.interval: {{ .Values.pipeline.checkpointInterval }}
|
||||
|
||||
sink:
|
||||
type: iceberg
|
||||
catalog.properties.type: rest
|
||||
catalog.properties.uri: {{ .Values.sink.catalog.uri | quote }}
|
||||
catalog.properties.warehouse: {{ .Values.sink.catalog.warehouse }}
|
||||
catalog.properties.io-impl: org.apache.iceberg.aws.s3.S3FileIO
|
||||
catalog.properties.s3.endpoint: {{ .Values.sink.catalog.s3Endpoint | quote }}
|
||||
catalog.properties.s3.path-style-access: {{ .Values.sink.catalog.s3PathStyleAccess | quote }}
|
||||
catalog.properties.oauth2-server-uri: {{ .Values.sink.catalog.oauth2Uri | quote }}
|
||||
catalog.properties.credential: {{ .Values.sink.catalog.credential }}
|
||||
catalog.properties.scope: {{ .Values.sink.catalog.scope }}
|
||||
|
||||
route:
|
||||
{{- range .Values.route }}
|
||||
- source-table: {{ .sourceTable }}
|
||||
sink-table: {{ .sinkTable }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,103 @@
|
||||
apiVersion: flink.apache.org/v1beta1
|
||||
kind: FlinkDeployment
|
||||
metadata:
|
||||
name: flink-cdc-session
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "flink-cdc-session.labels" . | nindent 4 }}
|
||||
spec:
|
||||
image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
flinkVersion: {{ .Values.flinkVersion }}
|
||||
flinkConfiguration:
|
||||
classloader.resolve-order: parent-first
|
||||
taskmanager.numberOfTaskSlots: {{ .Values.taskManager.taskSlots | quote }}
|
||||
execution.checkpointing.interval: {{ .Values.checkpoint.interval }}
|
||||
execution.checkpointing.mode: {{ .Values.checkpoint.mode }}
|
||||
execution.checkpointing.dir: {{ .Values.checkpointStorage.checkpointDir }}
|
||||
execution.checkpointing.savepoint-dir: {{ .Values.checkpointStorage.savepointDir }}
|
||||
classloader.parent-first-patterns.additional: org.apache.iceberg.
|
||||
{{- if eq .Values.checkpointStorage.storageType "s3" }}
|
||||
{{- if .Values.checkpointStorage.s3.endpoint }}
|
||||
s3.endpoint: {{ .Values.checkpointStorage.s3.endpoint }}
|
||||
s3.path-style-access: {{ .Values.checkpointStorage.s3.pathStyleAccess | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.truststore.enabled }}
|
||||
env.java.opts.jobmanager: >-
|
||||
-Djavax.net.ssl.trustStore=/opt/flink/certs/truststore.jks
|
||||
-Djavax.net.ssl.trustStorePassword={{ .Values.truststore.password }}
|
||||
-Djavax.net.ssl.trustStoreType=JKS
|
||||
env.java.opts.taskmanager: >-
|
||||
-Djavax.net.ssl.trustStore=/opt/flink/certs/truststore.jks
|
||||
-Djavax.net.ssl.trustStorePassword={{ .Values.truststore.password }}
|
||||
-Djavax.net.ssl.trustStoreType=JKS
|
||||
{{- end }}
|
||||
serviceAccount: {{ .Values.serviceAccount }}
|
||||
jobManager:
|
||||
replicas: {{ .Values.jobManager.replicas }}
|
||||
resource:
|
||||
memory: {{ .Values.jobManager.memory }}
|
||||
cpu: {{ .Values.jobManager.cpu }}
|
||||
taskManager:
|
||||
replicas: {{ .Values.taskManager.replicas }}
|
||||
resource:
|
||||
memory: {{ .Values.taskManager.memory }}
|
||||
cpu: {{ .Values.taskManager.cpu }}
|
||||
podTemplate:
|
||||
spec:
|
||||
containers:
|
||||
- name: flink-main-container
|
||||
{{- if eq .Values.checkpointStorage.storageType "gcs" }}
|
||||
env:
|
||||
- name: GOOGLE_APPLICATION_CREDENTIALS
|
||||
value: {{ .Values.checkpointStorage.gcs.credentialPath }}
|
||||
{{- else if eq .Values.checkpointStorage.storageType "s3" }}
|
||||
{{- if or .Values.checkpointStorage.s3.existingSecret .Values.checkpointStorage.s3.accessKeyId }}
|
||||
env:
|
||||
{{- if .Values.checkpointStorage.s3.existingSecret }}
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.checkpointStorage.s3.existingSecret }}
|
||||
key: {{ .Values.checkpointStorage.s3.accessKeyIdKey }}
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.checkpointStorage.s3.existingSecret }}
|
||||
key: {{ .Values.checkpointStorage.s3.secretAccessKeyKey }}
|
||||
{{- else }}
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
value: {{ .Values.checkpointStorage.s3.accessKeyId }}
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
value: {{ .Values.checkpointStorage.s3.secretAccessKey }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
- name: pipeline-config
|
||||
mountPath: /opt/flink/flink-cdc-3.5.0/conf
|
||||
{{- if .Values.truststore.enabled }}
|
||||
- name: truststore
|
||||
mountPath: /opt/flink/certs
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if eq .Values.checkpointStorage.storageType "gcs" }}
|
||||
- name: gcs-key
|
||||
mountPath: /opt/flink/gcs-key
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: pipeline-config
|
||||
configMap:
|
||||
name: cdc-pipeline-config
|
||||
{{- if .Values.truststore.enabled }}
|
||||
- name: truststore
|
||||
secret:
|
||||
secretName: {{ .Values.truststore.secretName }}
|
||||
{{- end }}
|
||||
{{- if eq .Values.checkpointStorage.storageType "gcs" }}
|
||||
- name: gcs-key
|
||||
secret:
|
||||
secretName: {{ .Values.checkpointStorage.gcs.credentialSecret }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,32 @@
|
||||
{{- if .Values.ingress.enabled }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: {{ .Release.Name }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "flink-cdc-session.labels" . | nindent 4 }}
|
||||
{{- if .Values.ingress.annotations }}
|
||||
annotations:
|
||||
{{- toYaml .Values.ingress.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.ingress.className }}
|
||||
ingressClassName: {{ .Values.ingress.className }}
|
||||
{{- end }}
|
||||
rules:
|
||||
- host: {{ .Values.ingress.host }}
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: ImplementationSpecific
|
||||
backend:
|
||||
service:
|
||||
name: {{ .Release.Name }}-rest
|
||||
port:
|
||||
number: 8081
|
||||
tls:
|
||||
- hosts:
|
||||
- {{ .Values.ingress.host }}
|
||||
secretName: {{ .Release.Name }}-tls
|
||||
{{- end }}
|
||||
@@ -0,0 +1,80 @@
|
||||
{{- if .Values.job.enabled }}
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ .Values.job.name }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "flink-cdc-session.labels" . | nindent 4 }}
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
template:
|
||||
spec:
|
||||
serviceAccountName: {{ .Values.serviceAccount }}
|
||||
containers:
|
||||
- name: submit
|
||||
image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
|
||||
{{- if eq .Values.checkpointStorage.storageType "gcs" }}
|
||||
env:
|
||||
- name: GOOGLE_APPLICATION_CREDENTIALS
|
||||
value: {{ .Values.checkpointStorage.gcs.credentialPath }}
|
||||
{{- else if eq .Values.checkpointStorage.storageType "s3" }}
|
||||
{{- if or .Values.checkpointStorage.s3.existingSecret .Values.checkpointStorage.s3.accessKeyId }}
|
||||
env:
|
||||
{{- if .Values.checkpointStorage.s3.existingSecret }}
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.checkpointStorage.s3.existingSecret }}
|
||||
key: {{ .Values.checkpointStorage.s3.accessKeyIdKey }}
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.checkpointStorage.s3.existingSecret }}
|
||||
key: {{ .Values.checkpointStorage.s3.secretAccessKeyKey }}
|
||||
{{- else }}
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
value: {{ .Values.checkpointStorage.s3.accessKeyId }}
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
value: {{ .Values.checkpointStorage.s3.secretAccessKey }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
cd /opt/flink/flink-cdc-3.5.0
|
||||
./bin/flink-cdc.sh \
|
||||
{{ .Values.job.pipelineFile }} \
|
||||
-Dexecution.checkpointing.interval={{ .Values.pipeline.checkpointInterval }} \
|
||||
-Drest.address={{ .Values.job.restAddress }} \
|
||||
-Drest.port={{ .Values.job.restPort }}
|
||||
volumeMounts:
|
||||
- name: pipeline-config
|
||||
mountPath: /opt/flink/flink-cdc-3.5.0/conf
|
||||
{{- if .Values.truststore.enabled }}
|
||||
- name: truststore
|
||||
mountPath: /opt/flink/certs
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if eq .Values.checkpointStorage.storageType "gcs" }}
|
||||
- name: gcs-key
|
||||
mountPath: /opt/flink/gcs-key
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: pipeline-config
|
||||
configMap:
|
||||
name: cdc-pipeline-config
|
||||
{{- if .Values.truststore.enabled }}
|
||||
- name: truststore
|
||||
secret:
|
||||
secretName: {{ .Values.truststore.secretName }}
|
||||
{{- end }}
|
||||
{{- if eq .Values.checkpointStorage.storageType "gcs" }}
|
||||
- name: gcs-key
|
||||
secret:
|
||||
secretName: {{ .Values.checkpointStorage.gcs.credentialSecret }}
|
||||
{{- end }}
|
||||
restartPolicy: Never
|
||||
{{- end }}
|
||||
@@ -0,0 +1,25 @@
|
||||
{{- if .Values.truststore.create }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ .Values.truststore.secretName }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "flink-cdc-session.labels" . | nindent 4 }}
|
||||
type: Opaque
|
||||
data:
|
||||
truststore.jks: {{ required "truststore.base64jks is required when truststore.create is true" .Values.truststore.base64jks }}
|
||||
{{- end }}
|
||||
{{- if and (eq .Values.checkpointStorage.storageType "gcs") .Values.checkpointStorage.gcs.create }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ .Values.checkpointStorage.gcs.credentialSecret }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "flink-cdc-session.labels" . | nindent 4 }}
|
||||
type: Opaque
|
||||
data:
|
||||
key.json: {{ required "checkpointStorage.gcs.base64json is required when checkpointStorage.gcs.create is true" .Values.checkpointStorage.gcs.base64json }}
|
||||
{{- end }}
|
||||
Reference in New Issue
Block a user