Update iceberg support

2026-01-29 16:21:16 +09:00
parent bb69f903f4
commit d4fecb4ae5
6 changed files with 279 additions and 98 deletions
@@ -77,12 +77,19 @@ spec:
              
              echo "Flink cluster is ready. Starting SQL Gateway..."
              
-              # Create flink-conf.yaml with remote cluster configuration
+              # Create flink configuration with environment variable substitution
              cat > /opt/flink/conf/config.yaml << EOF
              # Remote Flink Cluster Configuration
+              rest.bind-address: 0.0.0.0
              rest.address: {{ .Values.sessionCluster.name }}-rest
              rest.port: 8081
-              execution.target: remote
+              jobmanager.rpc.address: {{ .Values.sessionCluster.name }}
+              jobmanager.rpc.port: 6123
+              
+              # Session Cluster at Kubernetes
+              execution.target: kubernetes-session
+              kubernetes.namespace: {{ .Release.Namespace }}
+              kubernetes.cluster-id: {{ .Values.sessionCluster.name }}
              
              # SQL Gateway Configuration
              sql-gateway.endpoint.rest.address: 0.0.0.0
@@ -91,6 +98,13 @@ spec:
              # Table/SQL Configuration
              table.exec.source.idle-timeout: 30s
              table.exec.resource.default-parallelism: 1
+              
+              # Additional Flink Configuration from values
+              {{- with .Values.sqlGateway.flinkConfiguration }}
+              {{- range $key, $value := . }}
+              {{ $key }}: {{ $value }}
+              {{- end }}
+              {{- end }}
              EOF
              
              # Start SQL Gateway
@@ -0,0 +1,96 @@
+{{- if .Values.sqlGateway.enabled }}
+---
+# Hadoop Configuration ConfigMap for SQL Gateway
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: hadoop-config-{{ .Values.sessionCluster.name }}
+  namespace: {{ include "flink-sql-gateway.namespace" . }}
+  labels:
+    {{- include "flink-sql-gateway.sqlGateway.labels" . | nindent 4 }}
+data:
+  core-site.xml: |
+    <?xml version="1.0" encoding="UTF-8"?>
+    <configuration>
+      <property>
+        <name>fs.defaultFS</name>
+        <value>hdfs://localhost:9000</value>
+      </property>
+      <property>
+        <name>hadoop.tmp.dir</name>
+        <value>/tmp/hadoop</value>
+      </property>
+      
+      <!-- S3A Configuration -->
+      <property>
+        <name>fs.s3a.impl</name>
+        <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
+      </property>
+      <property>
+        <name>fs.s3a.aws.credentials.provider</name>
+        <value>org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider</value>
+      </property>
+      <property>
+        <name>fs.s3a.access.key</name>
+        <value>${AWS_ACCESS_KEY_ID}</value>
+      </property>
+      <property>
+        <name>fs.s3a.secret.key</name>
+        <value>${AWS_SECRET_ACCESS_KEY}</value>
+      </property>
+      <property>
+        <name>fs.s3a.endpoint</name>
+        <value>${S3_ENDPOINT}</value>
+      </property>
+      <property>
+        <name>fs.s3a.path.style.access</name>
+        <value>true</value>
+      </property>
+      <property>
+        <name>fs.s3a.connection.ssl.enabled</name>
+        <value>false</value>
+      </property>
+      <property>
+        <name>fs.s3a.fast.upload</name>
+        <value>true</value>
+      </property>
+      <property>
+        <name>fs.s3a.multipart.size</name>
+        <value>67108864</value>
+      </property>
+      <property>
+        <name>fs.s3a.multipart.threshold</name>
+        <value>134217728</value>
+      </property>
+    </configuration>
+  
+  hdfs-site.xml: |
+    <?xml version="1.0" encoding="UTF-8"?>
+    <configuration>
+      <property>
+        <name>dfs.replication</name>
+        <value>1</value>
+      </property>
+      <property>
+        <name>dfs.namenode.name.dir</name>
+        <value>/tmp/hadoop/namenode</value>
+      </property>
+      <property>
+        <name>dfs.datanode.data.dir</name>
+        <value>/tmp/hadoop/datanode</value>
+      </property>
+    </configuration>
+  
+  yarn-site.xml: |
+    <?xml version="1.0" encoding="UTF-8"?>
+    <configuration>
+      <property>
+        <name>yarn.nodemanager.aux-services</name>
+        <value>mapreduce_shuffle</value>
+      </property>
+      <property>
+        <name>yarn.resourcemanager.hostname</name>
+        <value>localhost</value>
+      </property>
+    </configuration>
+{{- end }}
@@ -1,8 +1,8 @@
 {{- if .Values.sqlClient.enabled }}
 ---
-# SQL Client Pod for interactive SQL queries
-apiVersion: v1
-kind: Pod
+# SQL Client Deployment for interactive SQL queries
+apiVersion: apps/v1
+kind: Deployment
 metadata:
  name: {{ .Values.sqlClient.name }}
  namespace: {{ include "flink-sql-gateway.namespace" . }}
@@ -13,71 +13,80 @@ metadata:
    {{- toYaml . | nindent 4 }}
  {{- end }}
 spec:
-  serviceAccountName: {{ include "flink-sql-gateway.serviceAccountName" . }}
-  {{- with .Values.nodeSelector }}
-  nodeSelector:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  {{- with .Values.affinity }}
-  affinity:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  {{- with .Values.tolerations }}
-  tolerations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  {{- with .Values.podSecurityContext }}
-  securityContext:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  containers:
-    - name: sql-client
-      image: {{ include "flink-sql-gateway.image" . }}
-      imagePullPolicy: {{ .Values.global.image.pullPolicy }}
-      {{- with .Values.securityContext }}
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ .Values.sqlClient.name }}
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.sqlClient.name }}
+        {{- include "flink-sql-gateway.selectorLabels" . | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "flink-sql-gateway.serviceAccountName" . }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.podSecurityContext }}
      securityContext:
        {{- toYaml . | nindent 8 }}
      {{- end }}
-      command: ["/bin/bash"]
-      args:
-        - -c
-        - |
-          echo "Flink SQL Client Ready"
-          echo "Connect to SQL Gateway: {{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}"
-          echo "Usage: /opt/flink/bin/sql-client.sh gateway --endpoint http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}"
-          
-          # Wait for SQL Gateway to be ready
-          echo "Waiting for SQL Gateway..."
-          until curl -f http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}/v1/info; do
-            echo "Waiting for SQL Gateway..."
-            sleep 10
-          done
-          
-          echo "SQL Gateway is ready!"
-          echo "You can now connect using:"
-          echo "/opt/flink/bin/sql-client.sh gateway --endpoint http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}"
-          
-          # Keep container running
-          tail -f /dev/null
-      {{- with .Values.sqlClient.env }}
-      env:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      volumeMounts:
-        {{- with .Values.sqlClient.volumeMounts }}
+      containers:
+        - name: sql-client
+          image: {{ include "flink-sql-gateway.image" . }}
+          imagePullPolicy: {{ .Values.global.image.pullPolicy }}
+          {{- with .Values.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          command: ["/bin/bash"]
+          args:
+            - -c
+            - |
+              echo "Flink SQL Client Ready"
+              echo "Connect to SQL Gateway: {{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}"
+              echo "Usage: /opt/flink/bin/sql-client.sh gateway --endpoint http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}"
+              
+              # Wait for SQL Gateway to be ready
+              echo "Waiting for SQL Gateway..."
+              until curl -f http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}/v1/info; do
+                echo "Waiting for SQL Gateway..."
+                sleep 10
+              done
+              
+              echo "SQL Gateway is ready!"
+              echo "You can now connect using:"
+              echo "/opt/flink/bin/sql-client.sh gateway --endpoint http://{{ .Values.sqlGateway.name }}:{{ .Values.sqlGateway.port }}"
+              
+              # Keep container running
+              tail -f /dev/null
+          {{- with .Values.sqlClient.env }}
+          env:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            {{- with .Values.sqlClient.volumeMounts }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+            - name: sql-scripts
+              mountPath: /opt/sql-scripts
+              readOnly: true
+          resources:
+            {{- toYaml .Values.sqlClient.resources | nindent 12 }}
+      volumes:
+        {{- with .Values.sqlClient.volumes }}
        {{- toYaml . | nindent 8 }}
        {{- end }}
        - name: sql-scripts
-          mountPath: /opt/sql-scripts
-          readOnly: true
-      resources:
-        {{- toYaml .Values.sqlClient.resources | nindent 8 }}
-  volumes:
-    {{- with .Values.sqlClient.volumes }}
-    {{- toYaml . | nindent 4 }}
-    {{- end }}
-    - name: sql-scripts
-      configMap:
-        name: sql-test-scripts
-  restartPolicy: Always
+          configMap:
+            name: sql-test-scripts
 {{- end }}