Update iceberg support

manifests/helm/flink-sql-gateway/Chart.yaml

@@ -1,22 +1,21 @@
+annotations:
+  category: Analytics
+  licenses: Apache-2.0
 apiVersion: v2
-name: flink-sql-gateway
+appVersion: 1.20.0
 description: A Helm chart for Flink SQL Gateway with Kafka integration
-type: application
+home: https://github.com/your-org/flink-sql-gateway-helm
-version: 0.1.0
-appVersion: "2.0.1"
 keywords:
 - flink
 - sql
 - gateway
 - kafka
 - streaming
-home: https://github.com/your-org/flink-sql-gateway-helm
+maintainers:
+- email: team@example.com
+  name: Flink SQL Gateway Team
+name: flink-sql-gateway
 sources:
 - https://github.com/your-org/flink-sql-gateway-helm
-maintainers:
+type: application
-  - name: Flink SQL Gateway Team
+version: 0.1.0
-    email: team@example.com
-dependencies: []
-annotations:
-  category: Analytics
-  licenses: Apache-2.0

manifests/helm/flink-sql-gateway/custom-values.yaml

@@ -22,14 +22,38 @@ sessionCluster:
       cpu: 2         # Increased from default 1
   flinkConfiguration:
     taskmanager.numberOfTaskSlots: "4"  # Increased from default "2"
+    # SSL/TLS Configuration for truststore
+    # security.ssl.internal.enabled: "true"
+    # security.ssl.internal.truststore: "/opt/flink/certs/ca.p12"
+    # security.ssl.internal.truststore-type: "PKCS12"
+    # JVM SSL system properties for JobManager and TaskManager
+    env.java.opts.jobmanager: "-Djavax.net.ssl.trustStore=/opt/flink/certs/ca.p12 -Djavax.net.ssl.trustStoreType=PKCS12 -Djavax.net.ssl.trustStorePassword=TBnYvX5Hp7y2"
+    env.java.opts.taskmanager: "-Djavax.net.ssl.trustStore=/opt/flink/certs/ca.p12 -Djavax.net.ssl.trustStoreType=PKCS12 -Djavax.net.ssl.trustStorePassword=TBnYvX5Hp7y2"
+    # Iceberg REST Catalog SSL Configuration
+    table.exec.resource.default-parallelism: "2"
+    
+    # [추가] s3 스키마를 Hadoop S3AFileSystem으로 처리하도록 매핑
+    fs.s3.impl: org.apache.hadoop.fs.s3a.S3AFileSystem
+    fs.s3a.impl: org.apache.hadoop.fs.s3a.S3AFileSystem
+    
+    # [추가] MinIO 사용 시 Path Style Access 필수
+    fs.s3a.endpoint: "https://minio.example.org"
+    fs.s3a.path.style.access: "true"
+
+  env:
+    - name: TRUSTSTORE_PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: truststore-secret
+          key: ca.password
   volumeMounts:
-    - name: kafka-certs
+    - name: truststore-certs
       mountPath: /opt/flink/certs
       readOnly: true
   volumes:
-    - name: kafka-certs
+    - name: truststore-certs
       secret:
-        secretName: kafka-ca-cert
+        secretName: truststore-secret
 
 # SQL Gateway resource configuration
 sqlGateway:
@@ -40,9 +64,41 @@ sqlGateway:
     limits:
       memory: 2Gi    # Increased from default 1Gi
       cpu: 1         # Increased from default 0.5
+  flinkConfiguration:
+    # JVM SSL system properties for SQL Gateway
+    # env.java.opts.all: "-Djavax.net.ssl.trustStore=/opt/flink/certs/ca.p12 -Djavax.net.ssl.trustStoreType=PKCS12 -Djavax.net.ssl.trustStorePassword=${TRUSTSTORE_PASSWORD}"
+    env.java.opts: "-Djavax.net.ssl.trustStore=/opt/flink/certs/ca.p12 -Djavax.net.ssl.trustStoreType=PKCS12 -Djavax.net.ssl.trustStorePassword=TBnYvX5Hp7y2"
+  env:
+    - name: TRUSTSTORE_PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: truststore-secret
+          key: ca.password
+  volumeMounts:
+    - name: truststore-certs
+      mountPath: /opt/flink/certs
+      readOnly: true
+  volumes:
+    - name: truststore-certs
+      secret:
+        secretName: truststore-secret
 
 sqlClient:
    enabled: true
+   env:
+     - name: TRUSTSTORE_PASSWORD
+       valueFrom:
+         secretKeyRef:
+           name: truststore-secret
+           key: ca.password
+   volumeMounts:
+     - name: truststore-certs
+       mountPath: /opt/flink/certs
+       readOnly: true
+   volumes:
+     - name: truststore-certs
+       secret:
+         secretName: truststore-secret
 # Optional: Enable Ingress for external access
 # ingress:
 #   enabled: true

manifests/helm/flink-sql-gateway/templates/flink-sql-gateway.yaml

@@ -77,12 +77,19 @@ spec:
               
               echo "Flink cluster is ready. Starting SQL Gateway..."
               
-              # Create flink-conf.yaml with remote cluster configuration
+              # Create flink configuration with environment variable substitution
               cat > /opt/flink/conf/config.yaml << EOF
               # Remote Flink Cluster Configuration
+              rest.bind-address: 0.0.0.0
               rest.address: {{ .Values.sessionCluster.name }}-rest
               rest.port: 8081
-              execution.target: remote
+              jobmanager.rpc.address: {{ .Values.sessionCluster.name }}
+              jobmanager.rpc.port: 6123
+              
+              # Session Cluster at Kubernetes
+              execution.target: kubernetes-session
+              kubernetes.namespace: {{ .Release.Namespace }}
+              kubernetes.cluster-id: {{ .Values.sessionCluster.name }}
               
               # SQL Gateway Configuration
               sql-gateway.endpoint.rest.address: 0.0.0.0
@@ -91,6 +98,13 @@ spec:
               # Table/SQL Configuration
               table.exec.source.idle-timeout: 30s
               table.exec.resource.default-parallelism: 1
+              
+              # Additional Flink Configuration from values
+              {{- with .Values.sqlGateway.flinkConfiguration }}
+              {{- range $key, $value := . }}
+              {{ $key }}: {{ $value }}
+              {{- end }}
+              {{- end }}
               EOF
               
               # Start SQL Gateway

manifests/helm/flink-sql-gateway/templates/hadoop-config.yaml

@@ -0,0 +1,96 @@
+{{- if .Values.sqlGateway.enabled }}
+---
+# Hadoop Configuration ConfigMap for SQL Gateway
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: hadoop-config-{{ .Values.sessionCluster.name }}
+  namespace: {{ include "flink-sql-gateway.namespace" . }}
+  labels:
+    {{- include "flink-sql-gateway.sqlGateway.labels" . | nindent 4 }}
+data:
+  core-site.xml: |
+    <?xml version="1.0" encoding="UTF-8"?>
+    <configuration>
+      <property>
+        <name>fs.defaultFS</name>
+        <value>hdfs://localhost:9000</value>
+      </property>
+      <property>
+        <name>hadoop.tmp.dir</name>
+        <value>/tmp/hadoop</value>
+      </property>
+      
+      <!-- S3A Configuration -->
+      <property>
+        <name>fs.s3a.impl</name>
+        <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
+      </property>
+      <property>
+        <name>fs.s3a.aws.credentials.provider</name>
+        <value>org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider</value>
+      </property>
+      <property>
+        <name>fs.s3a.access.key</name>
+        <value>${AWS_ACCESS_KEY_ID}</value>
+      </property>
+      <property>
+        <name>fs.s3a.secret.key</name>
+        <value>${AWS_SECRET_ACCESS_KEY}</value>
+      </property>
+      <property>
+        <name>fs.s3a.endpoint</name>
+        <value>${S3_ENDPOINT}</value>
+      </property>
+      <property>
+        <name>fs.s3a.path.style.access</name>
+        <value>true</value>
+      </property>
+      <property>
+        <name>fs.s3a.connection.ssl.enabled</name>
+        <value>false</value>
+      </property>
+      <property>
+        <name>fs.s3a.fast.upload</name>
+        <value>true</value>
+      </property>
+      <property>
+        <name>fs.s3a.multipart.size</name>
+        <value>67108864</value>
+      </property>
+      <property>
+        <name>fs.s3a.multipart.threshold</name>
+        <value>134217728</value>
+      </property>
+    </configuration>
+  
+  hdfs-site.xml: |
+    <?xml version="1.0" encoding="UTF-8"?>
+    <configuration>
+      <property>
+        <name>dfs.replication</name>
+        <value>1</value>
+      </property>
+      <property>
+        <name>dfs.namenode.name.dir</name>
+        <value>/tmp/hadoop/namenode</value>
+      </property>
+      <property>
+        <name>dfs.datanode.data.dir</name>
+        <value>/tmp/hadoop/datanode</value>
+      </property>
+    </configuration>
+  
+  yarn-site.xml: |
+    <?xml version="1.0" encoding="UTF-8"?>
+    <configuration>
+      <property>
+        <name>yarn.nodemanager.aux-services</name>
+        <value>mapreduce_shuffle</value>
+      </property>
+      <property>
+        <name>yarn.resourcemanager.hostname</name>
+        <value>localhost</value>
+      </property>
+    </configuration>
+{{- end }}

manifests/helm/flink-sql-gateway/templates/sql-client.yaml

@@ -1,8 +1,8 @@
 {{- if .Values.sqlClient.enabled }}
 ---
-# SQL Client Pod for interactive SQL queries
+# SQL Client Deployment for interactive SQL queries
-apiVersion: v1
+apiVersion: apps/v1
-kind: Pod
+kind: Deployment
 metadata:
   name: {{ .Values.sqlClient.name }}
   namespace: {{ include "flink-sql-gateway.namespace" . }}
@@ -12,23 +12,33 @@ metadata:
   annotations:
     {{- toYaml . | nindent 4 }}
   {{- end }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ .Values.sqlClient.name }}
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.sqlClient.name }}
+        {{- include "flink-sql-gateway.selectorLabels" . | nindent 8 }}
     spec:
       serviceAccountName: {{ include "flink-sql-gateway.serviceAccountName" . }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
-    {{- toYaml . | nindent 4 }}
+        {{- toYaml . | nindent 8 }}
       {{- end }}
       {{- with .Values.affinity }}
       affinity:
-    {{- toYaml . | nindent 4 }}
+        {{- toYaml . | nindent 8 }}
       {{- end }}
       {{- with .Values.tolerations }}
       tolerations:
-    {{- toYaml . | nindent 4 }}
+        {{- toYaml . | nindent 8 }}
       {{- end }}
       {{- with .Values.podSecurityContext }}
       securityContext:
-    {{- toYaml . | nindent 4 }}
+        {{- toYaml . | nindent 8 }}
       {{- end }}
       containers:
         - name: sql-client
@@ -36,7 +46,7 @@ spec:
           imagePullPolicy: {{ .Values.global.image.pullPolicy }}
           {{- with .Values.securityContext }}
           securityContext:
-        {{- toYaml . | nindent 8 }}
+            {{- toYaml . | nindent 12 }}
           {{- end }}
           command: ["/bin/bash"]
           args:
@@ -61,23 +71,22 @@ spec:
               tail -f /dev/null
           {{- with .Values.sqlClient.env }}
           env:
-        {{- toYaml . | nindent 8 }}
+            {{- toYaml . | nindent 12 }}
           {{- end }}
           volumeMounts:
             {{- with .Values.sqlClient.volumeMounts }}
-        {{- toYaml . | nindent 8 }}
+            {{- toYaml . | nindent 12 }}
             {{- end }}
             - name: sql-scripts
               mountPath: /opt/sql-scripts
               readOnly: true
           resources:
-        {{- toYaml .Values.sqlClient.resources | nindent 8 }}
+            {{- toYaml .Values.sqlClient.resources | nindent 12 }}
       volumes:
         {{- with .Values.sqlClient.volumes }}
-    {{- toYaml . | nindent 4 }}
+        {{- toYaml . | nindent 8 }}
         {{- end }}
         - name: sql-scripts
           configMap:
             name: sql-test-scripts
-  restartPolicy: Always
 {{- end }}

manifests/helm/flink-sql-gateway/values.yaml

@@ -48,7 +48,7 @@ sessionCluster:
   volumeMounts: []
   # Example:
   # volumeMounts:
-  #   - name: kafka-certs
+  #   - name: truststore-certs
   #     mountPath: /opt/flink/certs
   #     readOnly: true
   
@@ -56,9 +56,9 @@ sessionCluster:
   volumes: []
   # Example:
   # volumes:
-  #   - name: kafka-certs
+  #   - name: truststore-certs
   #     secret:
-  #       secretName: kafka-cluster-ca-cert
+  #       secretName: truststore-secret
 
 # SQL Gateway configuration
 sqlGateway:
@@ -99,6 +99,13 @@ sqlGateway:
         enabled: true
         maxSize: 100
         ttl: 3600000  # 1 hour
+
+  # Optional Flink configuration for SQL Gateway
+  flinkConfiguration: {}
+  # Example:
+  # flinkConfiguration:
+  #   env.java.opts: "-Djavax.net.ssl.trustStore=/opt/flink/certs/ca.p12 -Djavax.net.ssl.trustStoreType=PKCS12 -Djavax.net.ssl.trustStorePassword=${TRUSTSTORE_PASSWORD}"
+        
   # Optional environment variables for the SQL Gateway
   env: []
   # Example:
@@ -110,7 +117,7 @@ sqlGateway:
   volumeMounts: []
   # Example:
   # volumeMounts:
-  #   - name: kafka-certs
+  #   - name: truststore-certs
   #     mountPath: /opt/flink/certs
   #     readOnly: true
   
@@ -118,9 +125,9 @@ sqlGateway:
   volumes: []
   # Example:
   # volumes:
-  #   - name: kafka-certs
+  #   - name: truststore-certs
   #     secret:
-  #       secretName: kafka-cluster-ca-cert
+  #       secretName: truststore-secret
 
 # SQL Client configuration (optional - for interactive SQL queries)
 sqlClient:
@@ -144,7 +151,7 @@ sqlClient:
   volumeMounts: []
   # Example:
   # volumeMounts:
-  #   - name: kafka-certs
+  #   - name: truststore-certs
   #     mountPath: /opt/flink/certs
   #     readOnly: true
   
@@ -152,9 +159,9 @@ sqlClient:
   volumes: []
   # Example:
   # volumes:
-  #   - name: kafka-certs
+  #   - name: truststore-certs
   #     secret:
-  #       secretName: kafka-cluster-ca-cert
+  #       secretName: truststore-secret
 
 # Service configuration
 services: