Update iceberg support

manifests/helm/flink-sql-gateway/Chart.yaml

@@ -1,22 +1,21 @@
-apiVersion: v2
-name: flink-sql-gateway
-description: A Helm chart for Flink SQL Gateway with Kafka integration
-type: application
-version: 0.1.0
-appVersion: "2.0.1"
-keywords:
-  - flink
-  - sql
-  - gateway
-  - kafka
-  - streaming
-home: https://github.com/your-org/flink-sql-gateway-helm
-sources:
-  - https://github.com/your-org/flink-sql-gateway-helm
-maintainers:
-  - name: Flink SQL Gateway Team
-    email: team@example.com
-dependencies: []
 annotations:
   category: Analytics
   licenses: Apache-2.0
+apiVersion: v2
+appVersion: 1.20.0
+description: A Helm chart for Flink SQL Gateway with Kafka integration
+home: https://github.com/your-org/flink-sql-gateway-helm
+keywords:
+- flink
+- sql
+- gateway
+- kafka
+- streaming
+maintainers:
+- email: team@example.com
+  name: Flink SQL Gateway Team
+name: flink-sql-gateway
+sources:
+- https://github.com/your-org/flink-sql-gateway-helm
+type: application
+version: 0.1.0

manifests/helm/flink-sql-gateway/custom-values.yaml

@@ -22,14 +22,38 @@ sessionCluster:
       cpu: 2         # Increased from default 1
   flinkConfiguration:
     taskmanager.numberOfTaskSlots: "4"  # Increased from default "2"
+    # SSL/TLS Configuration for truststore
+    # security.ssl.internal.enabled: "true"
+    # security.ssl.internal.truststore: "/opt/flink/certs/ca.p12"
+    # security.ssl.internal.truststore-type: "PKCS12"
+    # JVM SSL system properties for JobManager and TaskManager
+    env.java.opts.jobmanager: "-Djavax.net.ssl.trustStore=/opt/flink/certs/ca.p12 -Djavax.net.ssl.trustStoreType=PKCS12 -Djavax.net.ssl.trustStorePassword=TBnYvX5Hp7y2"
+    env.java.opts.taskmanager: "-Djavax.net.ssl.trustStore=/opt/flink/certs/ca.p12 -Djavax.net.ssl.trustStoreType=PKCS12 -Djavax.net.ssl.trustStorePassword=TBnYvX5Hp7y2"
+    # Iceberg REST Catalog SSL Configuration
+    table.exec.resource.default-parallelism: "2"
+    
+    # [추가] s3 스키마를 Hadoop S3AFileSystem으로 처리하도록 매핑
+    fs.s3.impl: org.apache.hadoop.fs.s3a.S3AFileSystem
+    fs.s3a.impl: org.apache.hadoop.fs.s3a.S3AFileSystem
+    
+    # [추가] MinIO 사용 시 Path Style Access 필수
+    fs.s3a.endpoint: "https://minio.example.org"
+    fs.s3a.path.style.access: "true"
+
+  env:
+    - name: TRUSTSTORE_PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: truststore-secret
+          key: ca.password
   volumeMounts:
-    - name: kafka-certs
+    - name: truststore-certs
       mountPath: /opt/flink/certs
       readOnly: true
   volumes:
-    - name: kafka-certs
+    - name: truststore-certs
       secret:
-        secretName: kafka-ca-cert
+        secretName: truststore-secret
 
 # SQL Gateway resource configuration
 sqlGateway:
@@ -40,9 +64,41 @@ sqlGateway:
     limits:
       memory: 2Gi    # Increased from default 1Gi
       cpu: 1         # Increased from default 0.5
+  flinkConfiguration:
+    # JVM SSL system properties for SQL Gateway
+    # env.java.opts.all: "-Djavax.net.ssl.trustStore=/opt/flink/certs/ca.p12 -Djavax.net.ssl.trustStoreType=PKCS12 -Djavax.net.ssl.trustStorePassword=${TRUSTSTORE_PASSWORD}"
+    env.java.opts: "-Djavax.net.ssl.trustStore=/opt/flink/certs/ca.p12 -Djavax.net.ssl.trustStoreType=PKCS12 -Djavax.net.ssl.trustStorePassword=TBnYvX5Hp7y2"
+  env:
+    - name: TRUSTSTORE_PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: truststore-secret
+          key: ca.password
+  volumeMounts:
+    - name: truststore-certs
+      mountPath: /opt/flink/certs
+      readOnly: true
+  volumes:
+    - name: truststore-certs
+      secret:
+        secretName: truststore-secret
 
 sqlClient:
    enabled: true
+   env:
+     - name: TRUSTSTORE_PASSWORD
+       valueFrom:
+         secretKeyRef:
+           name: truststore-secret
+           key: ca.password
+   volumeMounts:
+     - name: truststore-certs
+       mountPath: /opt/flink/certs
+       readOnly: true
+   volumes:
+     - name: truststore-certs
+       secret:
+         secretName: truststore-secret
 # Optional: Enable Ingress for external access
 # ingress:
 #   enabled: true

manifests/helm/flink-sql-gateway/templates/flink-sql-gateway.yaml

@@ -77,12 +77,19 @@ spec:
               
               echo "Flink cluster is ready. Starting SQL Gateway..."
               
-              # Create flink-conf.yaml with remote cluster configuration
+              # Create flink configuration with environment variable substitution
               cat > /opt/flink/conf/config.yaml << EOF
               # Remote Flink Cluster Configuration
+              rest.bind-address: 0.0.0.0
               rest.address: {{ .Values.sessionCluster.name }}-rest
               rest.port: 8081
-              execution.target: remote
+              jobmanager.rpc.address: {{ .Values.sessionCluster.name }}
+              jobmanager.rpc.port: 6123
+              
+              # Session Cluster at Kubernetes
+              execution.target: kubernetes-session
+              kubernetes.namespace: {{ .Release.Namespace }}
+              kubernetes.cluster-id: {{ .Values.sessionCluster.name }}
               
               # SQL Gateway Configuration
               sql-gateway.endpoint.rest.address: 0.0.0.0
@@ -91,6 +98,13 @@ spec:
               # Table/SQL Configuration
               table.exec.source.idle-timeout: 30s
               table.exec.resource.default-parallelism: 1
+              
+              # Additional Flink Configuration from values
+              {{- with .Values.sqlGateway.flinkConfiguration }}
+              {{- range $key, $value := . }}
+              {{ $key }}: {{ $value }}
+              {{- end }}
+              {{- end }}
               EOF
               
               # Start SQL Gateway

manifests/helm/flink-sql-gateway/templates/hadoop-config.yaml

@@ -0,0 +1,96 @@
+{{- if .Values.sqlGateway.enabled }}
+---
+# Hadoop Configuration ConfigMap for SQL Gateway
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: hadoop-config-{{ .Values.sessionCluster.name }}
+  namespace: {{ include "flink-sql-gateway.namespace" . }}
+  labels:
+    {{- include "flink-sql-gateway.sqlGateway.labels" . | nindent 4 }}
+data:
+  core-site.xml: |
+    <?xml version="1.0" encoding="UTF-8"?>
+    <configuration>
+      <property>
+        <name>fs.defaultFS</name>
+        <value>hdfs://localhost:9000</value>
+      </property>
+      <property>
+        <name>hadoop.tmp.dir</name>
+        <value>/tmp/hadoop</value>
+      </property>
+      
+      <!-- S3A Configuration -->
+      <property>
+        <name>fs.s3a.impl</name>
+        <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
+      </property>
+      <property>
+        <name>fs.s3a.aws.credentials.provider</name>
+        <value>org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider</value>
+      </property>
+      <property>
+        <name>fs.s3a.access.key</name>
+        <value>${AWS_ACCESS_KEY_ID}</value>
+      </property>
+      <property>
+        <name>fs.s3a.secret.key</name>
+        <value>${AWS_SECRET_ACCESS_KEY}</value>
+      </property>
+      <property>
+        <name>fs.s3a.endpoint</name>
+        <value>${S3_ENDPOINT}</value>
+      </property>
+      <property>
+        <name>fs.s3a.path.style.access</name>
+        <value>true</value>
+      </property>
+      <property>
+        <name>fs.s3a.connection.ssl.enabled</name>
+        <value>false</value>
+      </property>
+      <property>
+        <name>fs.s3a.fast.upload</name>
+        <value>true</value>
+      </property>
+      <property>
+        <name>fs.s3a.multipart.size</name>
+        <value>67108864</value>
+      </property>
+      <property>
+        <name>fs.s3a.multipart.threshold</name>
+        <value>134217728</value>
+      </property>
+    </configuration>
+  
+  hdfs-site.xml: |
+    <?xml version="1.0" encoding="UTF-8"?>
+    <configuration>
+      <property>
+        <name>dfs.replication</name>
+        <value>1</value>
+      </property>
+      <property>
+        <name>dfs.namenode.name.dir</name>
+        <value>/tmp/hadoop/namenode</value>
+      </property>
+      <property>
+        <name>dfs.datanode.data.dir</name>
+        <value>/tmp/hadoop/datanode</value>
+      </property>
+    </configuration>
+  
+  yarn-site.xml: |
+    <?xml version="1.0" encoding="UTF-8"?>
+    <configuration>
+      <property>
+        <name>yarn.nodemanager.aux-services</name>
+        <value>mapreduce_shuffle</value>
+      </property>
+      <property>
+        <name>yarn.resourcemanager.hostname</name>
+        <value>localhost</value>
+      </property>
+    </configuration>
+{{- end }}

manifests/helm/flink-sql-gateway/templates/sql-client.yaml

@@ -1,8 +1,8 @@
 {{- if .Values.sqlClient.enabled }}
 ---
-# SQL Client Pod for interactive SQL queries
+# SQL Client Deployment for interactive SQL queries
-apiVersion: v1
+apiVersion: apps/v1
-kind: Pod
+kind: Deployment
 metadata:
   name: {{ .Values.sqlClient.name }}
   namespace: {{ include "flink-sql-gateway.namespace" . }}
@@ -13,71 +13,80 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
 spec:
-  serviceAccountName: {{ include "flink-sql-gateway.serviceAccountName" . }}
+  replicas: 1
-  {{- with .Values.nodeSelector }}
+  selector:
-  nodeSelector:
+    matchLabels:
-    {{- toYaml . | nindent 4 }}
+      app: {{ .Values.sqlClient.name }}
-  {{- end }}
+  template:
-  {{- with .Values.affinity }}
+    metadata:
-  affinity:
+      labels:
-    {{- toYaml . | nindent 4 }}
+        app: {{ .Values.sqlClient.name }}
-  {{- end }}
+        {{- include "flink-sql-gateway.selectorLabels" . | nindent 8 }}
-  {{- with .Values.tolerations }}
+    spec:
-  tolerations:
+      serviceAccountName: {{ include "flink-sql-gateway.serviceAccountName" . }}
-    {{- toYaml . | nindent 4 }}
+      {{- with .Values.nodeSelector }}
-  {{- end }}
+      nodeSelector:
-  {{- with .Values.podSecurityContext }}
+        {{- toYaml . | nindent 8 }}
-  securityContext:
+      {{- end }}
-    {{- toYaml . | nindent 4 }}
+      {{- with .Values.affinity }}
-  {{- end }}
+      affinity:
-  containers:
+        {{- toYaml . | nindent 8 }}
-    - name: sql-client
+      {{- end }}
-      image: {{ include "flink-sql-gateway.image" . }}
+      {{- with .Values.tolerations }}
-      imagePullPolicy: {{ .Values.global.image.pullPolicy }}
+      tolerations:
-      {{- with .Values.securityContext }}
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.podSecurityContext }}
       securityContext:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      command: ["/bin/bash"]
+      containers:
-      args:
+        - name: sql-client
-        - -c
+          image: {{ include "flink-sql-gateway.image" . }}
-        - |
+          imagePullPolicy: {{ .Values.global.image.pullPolicy }}
-          echo "Flink SQL Client Ready"
+          {{- with .Values.securityContext }}
-          echo "Connect to SQL Gateway: {{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}"
+          securityContext:
-          echo "Usage: /opt/flink/bin/sql-client.sh gateway --endpoint http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}"
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          command: ["/bin/bash"]
+          args:
+            - -c
+            - |
+              echo "Flink SQL Client Ready"
+              echo "Connect to SQL Gateway: {{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}"
+              echo "Usage: /opt/flink/bin/sql-client.sh gateway --endpoint http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}"
               
-          # Wait for SQL Gateway to be ready
+              # Wait for SQL Gateway to be ready
-          echo "Waiting for SQL Gateway..."
+              echo "Waiting for SQL Gateway..."
-          until curl -f http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}/v1/info; do
+              until curl -f http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}/v1/info; do
-            echo "Waiting for SQL Gateway..."
+                echo "Waiting for SQL Gateway..."
-            sleep 10
+                sleep 10
-          done
+              done
               
-          echo "SQL Gateway is ready!"
+              echo "SQL Gateway is ready!"
-          echo "You can now connect using:"
+              echo "You can now connect using:"
-          echo "/opt/flink/bin/sql-client.sh gateway --endpoint http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}"
+              echo "/opt/flink/bin/sql-client.sh gateway --endpoint http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}"
               
-          # Keep container running
+              # Keep container running
-          tail -f /dev/null
+              tail -f /dev/null
-      {{- with .Values.sqlClient.env }}
+          {{- with .Values.sqlClient.env }}
-      env:
+          env:
-        {{- toYaml . | nindent 8 }}
+            {{- toYaml . | nindent 12 }}
-      {{- end }}
+          {{- end }}
-      volumeMounts:
+          volumeMounts:
-        {{- with .Values.sqlClient.volumeMounts }}
+            {{- with .Values.sqlClient.volumeMounts }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+            - name: sql-scripts
+              mountPath: /opt/sql-scripts
+              readOnly: true
+          resources:
+            {{- toYaml .Values.sqlClient.resources | nindent 12 }}
+      volumes:
+        {{- with .Values.sqlClient.volumes }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
         - name: sql-scripts
-          mountPath: /opt/sql-scripts
+          configMap:
-          readOnly: true
+            name: sql-test-scripts
-      resources:
-        {{- toYaml .Values.sqlClient.resources | nindent 8 }}
-  volumes:
-    {{- with .Values.sqlClient.volumes }}
-    {{- toYaml . | nindent 4 }}
-    {{- end }}
-    - name: sql-scripts
-      configMap:
-        name: sql-test-scripts
-  restartPolicy: Always
 {{- end }}

manifests/helm/flink-sql-gateway/values.yaml

@@ -48,7 +48,7 @@ sessionCluster:
   volumeMounts: []
   # Example:
   # volumeMounts:
-  #   - name: kafka-certs
+  #   - name: truststore-certs
   #     mountPath: /opt/flink/certs
   #     readOnly: true
   
@@ -56,9 +56,9 @@ sessionCluster:
   volumes: []
   # Example:
   # volumes:
-  #   - name: kafka-certs
+  #   - name: truststore-certs
   #     secret:
-  #       secretName: kafka-cluster-ca-cert
+  #       secretName: truststore-secret
 
 # SQL Gateway configuration
 sqlGateway:
@@ -99,6 +99,13 @@ sqlGateway:
         enabled: true
         maxSize: 100
         ttl: 3600000  # 1 hour
+
+  # Optional Flink configuration for SQL Gateway
+  flinkConfiguration: {}
+  # Example:
+  # flinkConfiguration:
+  #   env.java.opts: "-Djavax.net.ssl.trustStore=/opt/flink/certs/ca.p12 -Djavax.net.ssl.trustStoreType=PKCS12 -Djavax.net.ssl.trustStorePassword=${TRUSTSTORE_PASSWORD}"
+        
   # Optional environment variables for the SQL Gateway
   env: []
   # Example:
@@ -110,7 +117,7 @@ sqlGateway:
   volumeMounts: []
   # Example:
   # volumeMounts:
-  #   - name: kafka-certs
+  #   - name: truststore-certs
   #     mountPath: /opt/flink/certs
   #     readOnly: true
   
@@ -118,9 +125,9 @@ sqlGateway:
   volumes: []
   # Example:
   # volumes:
-  #   - name: kafka-certs
+  #   - name: truststore-certs
   #     secret:
-  #       secretName: kafka-cluster-ca-cert
+  #       secretName: truststore-secret
 
 # SQL Client configuration (optional - for interactive SQL queries)
 sqlClient:
@@ -144,7 +151,7 @@ sqlClient:
   volumeMounts: []
   # Example:
   # volumeMounts:
-  #   - name: kafka-certs
+  #   - name: truststore-certs
   #     mountPath: /opt/flink/certs
   #     readOnly: true
   
@@ -152,9 +159,9 @@ sqlClient:
   volumes: []
   # Example:
   # volumes:
-  #   - name: kafka-certs
+  #   - name: truststore-certs
   #     secret:
-  #       secretName: kafka-cluster-ca-cert
+  #       secretName: truststore-secret
 
 # Service configuration
 services: