From d4fecb4ae507627f4342a004560882ea0be682f0 Mon Sep 17 00:00:00 2001 From: wbsong111 Date: Thu, 29 Jan 2026 16:21:16 +0900 Subject: [PATCH] Update iceberg support --- manifests/helm/flink-sql-gateway/Chart.yaml | 37 +++-- .../helm/flink-sql-gateway/custom-values.yaml | 64 +++++++- .../templates/flink-sql-gateway.yaml | 18 ++- .../templates/hadoop-config.yaml | 96 ++++++++++++ .../templates/sql-client.yaml | 137 ++++++++++-------- manifests/helm/flink-sql-gateway/values.yaml | 25 ++-- 6 files changed, 279 insertions(+), 98 deletions(-) create mode 100644 manifests/helm/flink-sql-gateway/templates/hadoop-config.yaml diff --git a/manifests/helm/flink-sql-gateway/Chart.yaml b/manifests/helm/flink-sql-gateway/Chart.yaml index 0d3ebf4..89ac131 100644 --- a/manifests/helm/flink-sql-gateway/Chart.yaml +++ b/manifests/helm/flink-sql-gateway/Chart.yaml @@ -1,22 +1,21 @@ -apiVersion: v2 -name: flink-sql-gateway -description: A Helm chart for Flink SQL Gateway with Kafka integration -type: application -version: 0.1.0 -appVersion: "2.0.1" -keywords: - - flink - - sql - - gateway - - kafka - - streaming -home: https://github.com/your-org/flink-sql-gateway-helm -sources: - - https://github.com/your-org/flink-sql-gateway-helm -maintainers: - - name: Flink SQL Gateway Team - email: team@example.com -dependencies: [] annotations: category: Analytics licenses: Apache-2.0 +apiVersion: v2 +appVersion: 1.20.0 +description: A Helm chart for Flink SQL Gateway with Kafka integration +home: https://github.com/your-org/flink-sql-gateway-helm +keywords: +- flink +- sql +- gateway +- kafka +- streaming +maintainers: +- email: team@example.com + name: Flink SQL Gateway Team +name: flink-sql-gateway +sources: +- https://github.com/your-org/flink-sql-gateway-helm +type: application +version: 0.1.0 diff --git a/manifests/helm/flink-sql-gateway/custom-values.yaml b/manifests/helm/flink-sql-gateway/custom-values.yaml index 56672df..38acef4 100644 --- a/manifests/helm/flink-sql-gateway/custom-values.yaml +++ b/manifests/helm/flink-sql-gateway/custom-values.yaml @@ -22,14 +22,38 @@ sessionCluster: cpu: 2 # Increased from default 1 flinkConfiguration: taskmanager.numberOfTaskSlots: "4" # Increased from default "2" - volumeMounts: - - name: kafka-certs + # SSL/TLS Configuration for truststore + # security.ssl.internal.enabled: "true" + # security.ssl.internal.truststore: "/opt/flink/certs/ca.p12" + # security.ssl.internal.truststore-type: "PKCS12" + # JVM SSL system properties for JobManager and TaskManager + env.java.opts.jobmanager: "-Djavax.net.ssl.trustStore=/opt/flink/certs/ca.p12 -Djavax.net.ssl.trustStoreType=PKCS12 -Djavax.net.ssl.trustStorePassword=TBnYvX5Hp7y2" + env.java.opts.taskmanager: "-Djavax.net.ssl.trustStore=/opt/flink/certs/ca.p12 -Djavax.net.ssl.trustStoreType=PKCS12 -Djavax.net.ssl.trustStorePassword=TBnYvX5Hp7y2" + # Iceberg REST Catalog SSL Configuration + table.exec.resource.default-parallelism: "2" + + # [추가] s3 스키마를 Hadoop S3AFileSystem으로 처리하도록 매핑 + fs.s3.impl: org.apache.hadoop.fs.s3a.S3AFileSystem + fs.s3a.impl: org.apache.hadoop.fs.s3a.S3AFileSystem + + # [추가] MinIO 사용 시 Path Style Access 필수 + fs.s3a.endpoint: "https://minio.example.org" + fs.s3a.path.style.access: "true" + + env: + - name: TRUSTSTORE_PASSWORD + valueFrom: + secretKeyRef: + name: truststore-secret + key: ca.password + volumeMounts: + - name: truststore-certs mountPath: /opt/flink/certs readOnly: true volumes: - - name: kafka-certs + - name: truststore-certs secret: - secretName: kafka-ca-cert + secretName: truststore-secret # SQL Gateway resource configuration sqlGateway: @@ -40,9 +64,41 @@ sqlGateway: limits: memory: 2Gi # Increased from default 1Gi cpu: 1 # Increased from default 0.5 + flinkConfiguration: + # JVM SSL system properties for SQL Gateway + # env.java.opts.all: "-Djavax.net.ssl.trustStore=/opt/flink/certs/ca.p12 -Djavax.net.ssl.trustStoreType=PKCS12 -Djavax.net.ssl.trustStorePassword=${TRUSTSTORE_PASSWORD}" + env.java.opts: "-Djavax.net.ssl.trustStore=/opt/flink/certs/ca.p12 -Djavax.net.ssl.trustStoreType=PKCS12 -Djavax.net.ssl.trustStorePassword=TBnYvX5Hp7y2" + env: + - name: TRUSTSTORE_PASSWORD + valueFrom: + secretKeyRef: + name: truststore-secret + key: ca.password + volumeMounts: + - name: truststore-certs + mountPath: /opt/flink/certs + readOnly: true + volumes: + - name: truststore-certs + secret: + secretName: truststore-secret sqlClient: enabled: true + env: + - name: TRUSTSTORE_PASSWORD + valueFrom: + secretKeyRef: + name: truststore-secret + key: ca.password + volumeMounts: + - name: truststore-certs + mountPath: /opt/flink/certs + readOnly: true + volumes: + - name: truststore-certs + secret: + secretName: truststore-secret # Optional: Enable Ingress for external access # ingress: # enabled: true diff --git a/manifests/helm/flink-sql-gateway/templates/flink-sql-gateway.yaml b/manifests/helm/flink-sql-gateway/templates/flink-sql-gateway.yaml index f32035b..42bde67 100644 --- a/manifests/helm/flink-sql-gateway/templates/flink-sql-gateway.yaml +++ b/manifests/helm/flink-sql-gateway/templates/flink-sql-gateway.yaml @@ -77,12 +77,19 @@ spec: echo "Flink cluster is ready. Starting SQL Gateway..." - # Create flink-conf.yaml with remote cluster configuration + # Create flink configuration with environment variable substitution cat > /opt/flink/conf/config.yaml << EOF # Remote Flink Cluster Configuration + rest.bind-address: 0.0.0.0 rest.address: {{ .Values.sessionCluster.name }}-rest rest.port: 8081 - execution.target: remote + jobmanager.rpc.address: {{ .Values.sessionCluster.name }} + jobmanager.rpc.port: 6123 + + # Session Cluster at Kubernetes + execution.target: kubernetes-session + kubernetes.namespace: {{ .Release.Namespace }} + kubernetes.cluster-id: {{ .Values.sessionCluster.name }} # SQL Gateway Configuration sql-gateway.endpoint.rest.address: 0.0.0.0 @@ -91,6 +98,13 @@ spec: # Table/SQL Configuration table.exec.source.idle-timeout: 30s table.exec.resource.default-parallelism: 1 + + # Additional Flink Configuration from values + {{- with .Values.sqlGateway.flinkConfiguration }} + {{- range $key, $value := . }} + {{ $key }}: {{ $value }} + {{- end }} + {{- end }} EOF # Start SQL Gateway diff --git a/manifests/helm/flink-sql-gateway/templates/hadoop-config.yaml b/manifests/helm/flink-sql-gateway/templates/hadoop-config.yaml new file mode 100644 index 0000000..48c06ac --- /dev/null +++ b/manifests/helm/flink-sql-gateway/templates/hadoop-config.yaml @@ -0,0 +1,96 @@ +{{- if .Values.sqlGateway.enabled }} +--- +# Hadoop Configuration ConfigMap for SQL Gateway +apiVersion: v1 +kind: ConfigMap +metadata: + name: hadoop-config-{{ .Values.sessionCluster.name }} + namespace: {{ include "flink-sql-gateway.namespace" . }} + labels: + {{- include "flink-sql-gateway.sqlGateway.labels" . | nindent 4 }} +data: + core-site.xml: | + + + + fs.defaultFS + hdfs://localhost:9000 + + + hadoop.tmp.dir + /tmp/hadoop + + + + + fs.s3a.impl + org.apache.hadoop.fs.s3a.S3AFileSystem + + + fs.s3a.aws.credentials.provider + org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider + + + fs.s3a.access.key + ${AWS_ACCESS_KEY_ID} + + + fs.s3a.secret.key + ${AWS_SECRET_ACCESS_KEY} + + + fs.s3a.endpoint + ${S3_ENDPOINT} + + + fs.s3a.path.style.access + true + + + fs.s3a.connection.ssl.enabled + false + + + fs.s3a.fast.upload + true + + + fs.s3a.multipart.size + 67108864 + + + fs.s3a.multipart.threshold + 134217728 + + + + hdfs-site.xml: | + + + + dfs.replication + 1 + + + dfs.namenode.name.dir + /tmp/hadoop/namenode + + + dfs.datanode.data.dir + /tmp/hadoop/datanode + + + + yarn-site.xml: | + + + + yarn.nodemanager.aux-services + mapreduce_shuffle + + + yarn.resourcemanager.hostname + localhost + + +{{- end }} \ No newline at end of file diff --git a/manifests/helm/flink-sql-gateway/templates/sql-client.yaml b/manifests/helm/flink-sql-gateway/templates/sql-client.yaml index 20f30a8..7694fd0 100644 --- a/manifests/helm/flink-sql-gateway/templates/sql-client.yaml +++ b/manifests/helm/flink-sql-gateway/templates/sql-client.yaml @@ -1,8 +1,8 @@ {{- if .Values.sqlClient.enabled }} --- -# SQL Client Pod for interactive SQL queries -apiVersion: v1 -kind: Pod +# SQL Client Deployment for interactive SQL queries +apiVersion: apps/v1 +kind: Deployment metadata: name: {{ .Values.sqlClient.name }} namespace: {{ include "flink-sql-gateway.namespace" . }} @@ -13,71 +13,80 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - serviceAccountName: {{ include "flink-sql-gateway.serviceAccountName" . }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .Values.podSecurityContext }} - securityContext: - {{- toYaml . | nindent 4 }} - {{- end }} - containers: - - name: sql-client - image: {{ include "flink-sql-gateway.image" . }} - imagePullPolicy: {{ .Values.global.image.pullPolicy }} - {{- with .Values.securityContext }} + replicas: 1 + selector: + matchLabels: + app: {{ .Values.sqlClient.name }} + template: + metadata: + labels: + app: {{ .Values.sqlClient.name }} + {{- include "flink-sql-gateway.selectorLabels" . | nindent 8 }} + spec: + serviceAccountName: {{ include "flink-sql-gateway.serviceAccountName" . }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.podSecurityContext }} securityContext: {{- toYaml . | nindent 8 }} {{- end }} - command: ["/bin/bash"] - args: - - -c - - | - echo "Flink SQL Client Ready" - echo "Connect to SQL Gateway: {{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}" - echo "Usage: /opt/flink/bin/sql-client.sh gateway --endpoint http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}" - - # Wait for SQL Gateway to be ready - echo "Waiting for SQL Gateway..." - until curl -f http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}/v1/info; do - echo "Waiting for SQL Gateway..." - sleep 10 - done - - echo "SQL Gateway is ready!" - echo "You can now connect using:" - echo "/opt/flink/bin/sql-client.sh gateway --endpoint http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}" - - # Keep container running - tail -f /dev/null - {{- with .Values.sqlClient.env }} - env: - {{- toYaml . | nindent 8 }} - {{- end }} - volumeMounts: - {{- with .Values.sqlClient.volumeMounts }} + containers: + - name: sql-client + image: {{ include "flink-sql-gateway.image" . }} + imagePullPolicy: {{ .Values.global.image.pullPolicy }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + command: ["/bin/bash"] + args: + - -c + - | + echo "Flink SQL Client Ready" + echo "Connect to SQL Gateway: {{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}" + echo "Usage: /opt/flink/bin/sql-client.sh gateway --endpoint http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}" + + # Wait for SQL Gateway to be ready + echo "Waiting for SQL Gateway..." + until curl -f http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}/v1/info; do + echo "Waiting for SQL Gateway..." + sleep 10 + done + + echo "SQL Gateway is ready!" + echo "You can now connect using:" + echo "/opt/flink/bin/sql-client.sh gateway --endpoint http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}" + + # Keep container running + tail -f /dev/null + {{- with .Values.sqlClient.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + {{- with .Values.sqlClient.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + - name: sql-scripts + mountPath: /opt/sql-scripts + readOnly: true + resources: + {{- toYaml .Values.sqlClient.resources | nindent 12 }} + volumes: + {{- with .Values.sqlClient.volumes }} {{- toYaml . | nindent 8 }} {{- end }} - name: sql-scripts - mountPath: /opt/sql-scripts - readOnly: true - resources: - {{- toYaml .Values.sqlClient.resources | nindent 8 }} - volumes: - {{- with .Values.sqlClient.volumes }} - {{- toYaml . | nindent 4 }} - {{- end }} - - name: sql-scripts - configMap: - name: sql-test-scripts - restartPolicy: Always + configMap: + name: sql-test-scripts {{- end }} \ No newline at end of file diff --git a/manifests/helm/flink-sql-gateway/values.yaml b/manifests/helm/flink-sql-gateway/values.yaml index 059fb5b..61e81de 100644 --- a/manifests/helm/flink-sql-gateway/values.yaml +++ b/manifests/helm/flink-sql-gateway/values.yaml @@ -48,7 +48,7 @@ sessionCluster: volumeMounts: [] # Example: # volumeMounts: - # - name: kafka-certs + # - name: truststore-certs # mountPath: /opt/flink/certs # readOnly: true @@ -56,9 +56,9 @@ sessionCluster: volumes: [] # Example: # volumes: - # - name: kafka-certs + # - name: truststore-certs # secret: - # secretName: kafka-cluster-ca-cert + # secretName: truststore-secret # SQL Gateway configuration sqlGateway: @@ -99,6 +99,13 @@ sqlGateway: enabled: true maxSize: 100 ttl: 3600000 # 1 hour + + # Optional Flink configuration for SQL Gateway + flinkConfiguration: {} + # Example: + # flinkConfiguration: + # env.java.opts: "-Djavax.net.ssl.trustStore=/opt/flink/certs/ca.p12 -Djavax.net.ssl.trustStoreType=PKCS12 -Djavax.net.ssl.trustStorePassword=${TRUSTSTORE_PASSWORD}" + # Optional environment variables for the SQL Gateway env: [] # Example: @@ -110,7 +117,7 @@ sqlGateway: volumeMounts: [] # Example: # volumeMounts: - # - name: kafka-certs + # - name: truststore-certs # mountPath: /opt/flink/certs # readOnly: true @@ -118,9 +125,9 @@ sqlGateway: volumes: [] # Example: # volumes: - # - name: kafka-certs + # - name: truststore-certs # secret: - # secretName: kafka-cluster-ca-cert + # secretName: truststore-secret # SQL Client configuration (optional - for interactive SQL queries) sqlClient: @@ -144,7 +151,7 @@ sqlClient: volumeMounts: [] # Example: # volumeMounts: - # - name: kafka-certs + # - name: truststore-certs # mountPath: /opt/flink/certs # readOnly: true @@ -152,9 +159,9 @@ sqlClient: volumes: [] # Example: # volumes: - # - name: kafka-certs + # - name: truststore-certs # secret: - # secretName: kafka-cluster-ca-cert + # secretName: truststore-secret # Service configuration services: