diff --git a/manifests/helm/flink-sql-gateway/Chart.yaml b/manifests/helm/flink-sql-gateway/Chart.yaml
index 0d3ebf4..89ac131 100644
--- a/manifests/helm/flink-sql-gateway/Chart.yaml
+++ b/manifests/helm/flink-sql-gateway/Chart.yaml
@@ -1,22 +1,21 @@
-apiVersion: v2
-name: flink-sql-gateway
-description: A Helm chart for Flink SQL Gateway with Kafka integration
-type: application
-version: 0.1.0
-appVersion: "2.0.1"
-keywords:
- - flink
- - sql
- - gateway
- - kafka
- - streaming
-home: https://github.com/your-org/flink-sql-gateway-helm
-sources:
- - https://github.com/your-org/flink-sql-gateway-helm
-maintainers:
- - name: Flink SQL Gateway Team
- email: team@example.com
-dependencies: []
annotations:
category: Analytics
licenses: Apache-2.0
+apiVersion: v2
+appVersion: 1.20.0
+description: A Helm chart for Flink SQL Gateway with Kafka integration
+home: https://github.com/your-org/flink-sql-gateway-helm
+keywords:
+- flink
+- sql
+- gateway
+- kafka
+- streaming
+maintainers:
+- email: team@example.com
+ name: Flink SQL Gateway Team
+name: flink-sql-gateway
+sources:
+- https://github.com/your-org/flink-sql-gateway-helm
+type: application
+version: 0.1.0
diff --git a/manifests/helm/flink-sql-gateway/custom-values.yaml b/manifests/helm/flink-sql-gateway/custom-values.yaml
index 56672df..38acef4 100644
--- a/manifests/helm/flink-sql-gateway/custom-values.yaml
+++ b/manifests/helm/flink-sql-gateway/custom-values.yaml
@@ -22,14 +22,38 @@ sessionCluster:
cpu: 2 # Increased from default 1
flinkConfiguration:
taskmanager.numberOfTaskSlots: "4" # Increased from default "2"
- volumeMounts:
- - name: kafka-certs
+ # SSL/TLS Configuration for truststore
+ # security.ssl.internal.enabled: "true"
+ # security.ssl.internal.truststore: "/opt/flink/certs/ca.p12"
+ # security.ssl.internal.truststore-type: "PKCS12"
+ # JVM SSL system properties for JobManager and TaskManager
+ env.java.opts.jobmanager: "-Djavax.net.ssl.trustStore=/opt/flink/certs/ca.p12 -Djavax.net.ssl.trustStoreType=PKCS12 -Djavax.net.ssl.trustStorePassword=TBnYvX5Hp7y2"
+ env.java.opts.taskmanager: "-Djavax.net.ssl.trustStore=/opt/flink/certs/ca.p12 -Djavax.net.ssl.trustStoreType=PKCS12 -Djavax.net.ssl.trustStorePassword=TBnYvX5Hp7y2"
+ # Iceberg REST Catalog SSL Configuration
+ table.exec.resource.default-parallelism: "2"
+
+ # [추가] s3 스키마를 Hadoop S3AFileSystem으로 처리하도록 매핑
+ fs.s3.impl: org.apache.hadoop.fs.s3a.S3AFileSystem
+ fs.s3a.impl: org.apache.hadoop.fs.s3a.S3AFileSystem
+
+ # [추가] MinIO 사용 시 Path Style Access 필수
+ fs.s3a.endpoint: "https://minio.example.org"
+ fs.s3a.path.style.access: "true"
+
+ env:
+ - name: TRUSTSTORE_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: truststore-secret
+ key: ca.password
+ volumeMounts:
+ - name: truststore-certs
mountPath: /opt/flink/certs
readOnly: true
volumes:
- - name: kafka-certs
+ - name: truststore-certs
secret:
- secretName: kafka-ca-cert
+ secretName: truststore-secret
# SQL Gateway resource configuration
sqlGateway:
@@ -40,9 +64,41 @@ sqlGateway:
limits:
memory: 2Gi # Increased from default 1Gi
cpu: 1 # Increased from default 0.5
+ flinkConfiguration:
+ # JVM SSL system properties for SQL Gateway
+ # env.java.opts.all: "-Djavax.net.ssl.trustStore=/opt/flink/certs/ca.p12 -Djavax.net.ssl.trustStoreType=PKCS12 -Djavax.net.ssl.trustStorePassword=${TRUSTSTORE_PASSWORD}"
+ env.java.opts: "-Djavax.net.ssl.trustStore=/opt/flink/certs/ca.p12 -Djavax.net.ssl.trustStoreType=PKCS12 -Djavax.net.ssl.trustStorePassword=TBnYvX5Hp7y2"
+ env:
+ - name: TRUSTSTORE_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: truststore-secret
+ key: ca.password
+ volumeMounts:
+ - name: truststore-certs
+ mountPath: /opt/flink/certs
+ readOnly: true
+ volumes:
+ - name: truststore-certs
+ secret:
+ secretName: truststore-secret
sqlClient:
enabled: true
+ env:
+ - name: TRUSTSTORE_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: truststore-secret
+ key: ca.password
+ volumeMounts:
+ - name: truststore-certs
+ mountPath: /opt/flink/certs
+ readOnly: true
+ volumes:
+ - name: truststore-certs
+ secret:
+ secretName: truststore-secret
# Optional: Enable Ingress for external access
# ingress:
# enabled: true
diff --git a/manifests/helm/flink-sql-gateway/templates/flink-sql-gateway.yaml b/manifests/helm/flink-sql-gateway/templates/flink-sql-gateway.yaml
index f32035b..42bde67 100644
--- a/manifests/helm/flink-sql-gateway/templates/flink-sql-gateway.yaml
+++ b/manifests/helm/flink-sql-gateway/templates/flink-sql-gateway.yaml
@@ -77,12 +77,19 @@ spec:
echo "Flink cluster is ready. Starting SQL Gateway..."
- # Create flink-conf.yaml with remote cluster configuration
+ # Create flink configuration with environment variable substitution
cat > /opt/flink/conf/config.yaml << EOF
# Remote Flink Cluster Configuration
+ rest.bind-address: 0.0.0.0
rest.address: {{ .Values.sessionCluster.name }}-rest
rest.port: 8081
- execution.target: remote
+ jobmanager.rpc.address: {{ .Values.sessionCluster.name }}
+ jobmanager.rpc.port: 6123
+
+ # Session Cluster at Kubernetes
+ execution.target: kubernetes-session
+ kubernetes.namespace: {{ .Release.Namespace }}
+ kubernetes.cluster-id: {{ .Values.sessionCluster.name }}
# SQL Gateway Configuration
sql-gateway.endpoint.rest.address: 0.0.0.0
@@ -91,6 +98,13 @@ spec:
# Table/SQL Configuration
table.exec.source.idle-timeout: 30s
table.exec.resource.default-parallelism: 1
+
+ # Additional Flink Configuration from values
+ {{- with .Values.sqlGateway.flinkConfiguration }}
+ {{- range $key, $value := . }}
+ {{ $key }}: {{ $value }}
+ {{- end }}
+ {{- end }}
EOF
# Start SQL Gateway
diff --git a/manifests/helm/flink-sql-gateway/templates/hadoop-config.yaml b/manifests/helm/flink-sql-gateway/templates/hadoop-config.yaml
new file mode 100644
index 0000000..48c06ac
--- /dev/null
+++ b/manifests/helm/flink-sql-gateway/templates/hadoop-config.yaml
@@ -0,0 +1,96 @@
+{{- if .Values.sqlGateway.enabled }}
+---
+# Hadoop Configuration ConfigMap for SQL Gateway
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: hadoop-config-{{ .Values.sessionCluster.name }}
+ namespace: {{ include "flink-sql-gateway.namespace" . }}
+ labels:
+ {{- include "flink-sql-gateway.sqlGateway.labels" . | nindent 4 }}
+data:
+ core-site.xml: |
+
+
+
+ fs.defaultFS
+ hdfs://localhost:9000
+
+
+ hadoop.tmp.dir
+ /tmp/hadoop
+
+
+
+
+ fs.s3a.impl
+ org.apache.hadoop.fs.s3a.S3AFileSystem
+
+
+ fs.s3a.aws.credentials.provider
+ org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider
+
+
+ fs.s3a.access.key
+ ${AWS_ACCESS_KEY_ID}
+
+
+ fs.s3a.secret.key
+ ${AWS_SECRET_ACCESS_KEY}
+
+
+ fs.s3a.endpoint
+ ${S3_ENDPOINT}
+
+
+ fs.s3a.path.style.access
+ true
+
+
+ fs.s3a.connection.ssl.enabled
+ false
+
+
+ fs.s3a.fast.upload
+ true
+
+
+ fs.s3a.multipart.size
+ 67108864
+
+
+ fs.s3a.multipart.threshold
+ 134217728
+
+
+
+ hdfs-site.xml: |
+
+
+
+ dfs.replication
+ 1
+
+
+ dfs.namenode.name.dir
+ /tmp/hadoop/namenode
+
+
+ dfs.datanode.data.dir
+ /tmp/hadoop/datanode
+
+
+
+ yarn-site.xml: |
+
+
+
+ yarn.nodemanager.aux-services
+ mapreduce_shuffle
+
+
+ yarn.resourcemanager.hostname
+ localhost
+
+
+{{- end }}
\ No newline at end of file
diff --git a/manifests/helm/flink-sql-gateway/templates/sql-client.yaml b/manifests/helm/flink-sql-gateway/templates/sql-client.yaml
index 20f30a8..7694fd0 100644
--- a/manifests/helm/flink-sql-gateway/templates/sql-client.yaml
+++ b/manifests/helm/flink-sql-gateway/templates/sql-client.yaml
@@ -1,8 +1,8 @@
{{- if .Values.sqlClient.enabled }}
---
-# SQL Client Pod for interactive SQL queries
-apiVersion: v1
-kind: Pod
+# SQL Client Deployment for interactive SQL queries
+apiVersion: apps/v1
+kind: Deployment
metadata:
name: {{ .Values.sqlClient.name }}
namespace: {{ include "flink-sql-gateway.namespace" . }}
@@ -13,71 +13,80 @@ metadata:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
- serviceAccountName: {{ include "flink-sql-gateway.serviceAccountName" . }}
- {{- with .Values.nodeSelector }}
- nodeSelector:
- {{- toYaml . | nindent 4 }}
- {{- end }}
- {{- with .Values.affinity }}
- affinity:
- {{- toYaml . | nindent 4 }}
- {{- end }}
- {{- with .Values.tolerations }}
- tolerations:
- {{- toYaml . | nindent 4 }}
- {{- end }}
- {{- with .Values.podSecurityContext }}
- securityContext:
- {{- toYaml . | nindent 4 }}
- {{- end }}
- containers:
- - name: sql-client
- image: {{ include "flink-sql-gateway.image" . }}
- imagePullPolicy: {{ .Values.global.image.pullPolicy }}
- {{- with .Values.securityContext }}
+ replicas: 1
+ selector:
+ matchLabels:
+ app: {{ .Values.sqlClient.name }}
+ template:
+ metadata:
+ labels:
+ app: {{ .Values.sqlClient.name }}
+ {{- include "flink-sql-gateway.selectorLabels" . | nindent 8 }}
+ spec:
+ serviceAccountName: {{ include "flink-sql-gateway.serviceAccountName" . }}
+ {{- with .Values.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.podSecurityContext }}
securityContext:
{{- toYaml . | nindent 8 }}
{{- end }}
- command: ["/bin/bash"]
- args:
- - -c
- - |
- echo "Flink SQL Client Ready"
- echo "Connect to SQL Gateway: {{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}"
- echo "Usage: /opt/flink/bin/sql-client.sh gateway --endpoint http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}"
-
- # Wait for SQL Gateway to be ready
- echo "Waiting for SQL Gateway..."
- until curl -f http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}/v1/info; do
- echo "Waiting for SQL Gateway..."
- sleep 10
- done
-
- echo "SQL Gateway is ready!"
- echo "You can now connect using:"
- echo "/opt/flink/bin/sql-client.sh gateway --endpoint http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}"
-
- # Keep container running
- tail -f /dev/null
- {{- with .Values.sqlClient.env }}
- env:
- {{- toYaml . | nindent 8 }}
- {{- end }}
- volumeMounts:
- {{- with .Values.sqlClient.volumeMounts }}
+ containers:
+ - name: sql-client
+ image: {{ include "flink-sql-gateway.image" . }}
+ imagePullPolicy: {{ .Values.global.image.pullPolicy }}
+ {{- with .Values.securityContext }}
+ securityContext:
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ command: ["/bin/bash"]
+ args:
+ - -c
+ - |
+ echo "Flink SQL Client Ready"
+ echo "Connect to SQL Gateway: {{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}"
+ echo "Usage: /opt/flink/bin/sql-client.sh gateway --endpoint http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}"
+
+ # Wait for SQL Gateway to be ready
+ echo "Waiting for SQL Gateway..."
+ until curl -f http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}/v1/info; do
+ echo "Waiting for SQL Gateway..."
+ sleep 10
+ done
+
+ echo "SQL Gateway is ready!"
+ echo "You can now connect using:"
+ echo "/opt/flink/bin/sql-client.sh gateway --endpoint http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}"
+
+ # Keep container running
+ tail -f /dev/null
+ {{- with .Values.sqlClient.env }}
+ env:
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ volumeMounts:
+ {{- with .Values.sqlClient.volumeMounts }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ - name: sql-scripts
+ mountPath: /opt/sql-scripts
+ readOnly: true
+ resources:
+ {{- toYaml .Values.sqlClient.resources | nindent 12 }}
+ volumes:
+ {{- with .Values.sqlClient.volumes }}
{{- toYaml . | nindent 8 }}
{{- end }}
- name: sql-scripts
- mountPath: /opt/sql-scripts
- readOnly: true
- resources:
- {{- toYaml .Values.sqlClient.resources | nindent 8 }}
- volumes:
- {{- with .Values.sqlClient.volumes }}
- {{- toYaml . | nindent 4 }}
- {{- end }}
- - name: sql-scripts
- configMap:
- name: sql-test-scripts
- restartPolicy: Always
+ configMap:
+ name: sql-test-scripts
{{- end }}
\ No newline at end of file
diff --git a/manifests/helm/flink-sql-gateway/values.yaml b/manifests/helm/flink-sql-gateway/values.yaml
index 059fb5b..61e81de 100644
--- a/manifests/helm/flink-sql-gateway/values.yaml
+++ b/manifests/helm/flink-sql-gateway/values.yaml
@@ -48,7 +48,7 @@ sessionCluster:
volumeMounts: []
# Example:
# volumeMounts:
- # - name: kafka-certs
+ # - name: truststore-certs
# mountPath: /opt/flink/certs
# readOnly: true
@@ -56,9 +56,9 @@ sessionCluster:
volumes: []
# Example:
# volumes:
- # - name: kafka-certs
+ # - name: truststore-certs
# secret:
- # secretName: kafka-cluster-ca-cert
+ # secretName: truststore-secret
# SQL Gateway configuration
sqlGateway:
@@ -99,6 +99,13 @@ sqlGateway:
enabled: true
maxSize: 100
ttl: 3600000 # 1 hour
+
+ # Optional Flink configuration for SQL Gateway
+ flinkConfiguration: {}
+ # Example:
+ # flinkConfiguration:
+ # env.java.opts: "-Djavax.net.ssl.trustStore=/opt/flink/certs/ca.p12 -Djavax.net.ssl.trustStoreType=PKCS12 -Djavax.net.ssl.trustStorePassword=${TRUSTSTORE_PASSWORD}"
+
# Optional environment variables for the SQL Gateway
env: []
# Example:
@@ -110,7 +117,7 @@ sqlGateway:
volumeMounts: []
# Example:
# volumeMounts:
- # - name: kafka-certs
+ # - name: truststore-certs
# mountPath: /opt/flink/certs
# readOnly: true
@@ -118,9 +125,9 @@ sqlGateway:
volumes: []
# Example:
# volumes:
- # - name: kafka-certs
+ # - name: truststore-certs
# secret:
- # secretName: kafka-cluster-ca-cert
+ # secretName: truststore-secret
# SQL Client configuration (optional - for interactive SQL queries)
sqlClient:
@@ -144,7 +151,7 @@ sqlClient:
volumeMounts: []
# Example:
# volumeMounts:
- # - name: kafka-certs
+ # - name: truststore-certs
# mountPath: /opt/flink/certs
# readOnly: true
@@ -152,9 +159,9 @@ sqlClient:
volumes: []
# Example:
# volumes:
- # - name: kafka-certs
+ # - name: truststore-certs
# secret:
- # secretName: kafka-cluster-ca-cert
+ # secretName: truststore-secret
# Service configuration
services: