kafka-cluster

2025-12-02 13:33:13 +09:00
parent 688d6c7c34
commit 257c6915f1
13 changed files with 1184 additions and 0 deletions
@@ -0,0 +1,260 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaNodePool
+metadata:
+  name: controller
+  namespace: "{{ .Namespace }}"
+  labels:
+    strimzi.io/cluster: "{{ .Name }}"
+spec:
+  replicas: 3
+  roles:
+    - controller
+  storage:
+    type: jbod
+    volumes:
+      - id: 0
+        type: persistent-claim
+        size: 50Gi
+        kraftMetadata: shared
+        class: longhorn
+  resources:
+    requests:
+      cpu: 500m
+      memory: 1Gi
+    limits:
+      cpu: 1
+      memory: 2Gi        
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaNodePool
+metadata:
+  name: broker
+  namespace: "{{ .Namespace }}"
+  labels:
+    strimzi.io/cluster: "{{ .Name }}"
+spec:
+  replicas: 3
+  roles:
+    - broker
+  storage:
+    type: jbod
+    volumes:
+      - id: 0
+        type: persistent-claim
+        size: 50Gi
+        class: longhorn
+        kraftMetadata: shared
+  resources:
+    requests:
+      cpu: 500m
+      memory: 1Gi
+    limits:
+      cpu: 1
+      memory: 2Gi        
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: Kafka
+metadata:
+  name: "{{ .Name }}"
+  namespace: "{{ .Namespace }}"
+  annotations:
+    strimzi.io/node-pools: enabled
+    strimzi.io/kraft: enabled
+spec:
+  kafka:
+    version: 4.0.0
+    metadataVersion: "4.0"
+    template:
+      kafkaContainer:
+        env:
+          - name: KAFKA_OPTS
+            value: "-Duser.timezone=Asia/Seoul"
+    listeners:
+      - name: tls
+        type: cluster-ip
+        port: 9093
+        tls: false
+        authentication:
+          type: oauth
+          clientId: "{{.KEYCLOAK_CLIENT_ID}}"
+          clientSecret:
+            key: client-secret
+            secretName: kafka-cluster-oauth-secret
+          checkIssuer: true
+          checkAccessTokenType: true
+          accessTokenIsJwt: true
+          checkAudience: false
+          enableOauthBearer: true
+          validIssuerUri: "{{.KEYCLOAK_URL}}/realms/$KEYCLOAK_REALM"
+          jwksEndpointUri: "{{.KEYCLOAK_URL}}/realms/$KEYCLOAK_REALM/protocol/openid-connect/certs"
+          userNameClaim: preferred_username
+          customClaimCheck: "'{{.KEYCLOAK_CLIENT_ID}}' in @.realm_access.roles"
+          tlsTrustedCertificates:
+            - secretName: keycloak-tls
+              certificate: ca.crt
+    config:
+      offsets.topic.replication.factor: 3
+      transaction.state.log.replication.factor: 3
+      transaction.state.log.min.isr: 2
+      default.replication.factor: 3
+      min.insync.replicas: 2
+      auto.create.topics.enable: false
+      num.partitions: 3
+      delete.topic.enable: true
+    authorization:
+      type: simple
+    logging:
+      type: inline
+      loggers:
+        kafka.root.logger.level: INFO
+  entityOperator:
+  entityOperator:
+    topicOperator:
+      resources:
+        requests:
+          memory: 128Mi
+          cpu: 100m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+    userOperator:
+      resources:
+        requests:
+          memory: 128Mi
+          cpu: 100m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaConnect
+metadata:
+  name: "{{ .Name }}"
+  namespace: "{{ .Namespace }}"
+  labels:
+    strimzi.io/cluster: "{{ .Name }}"
+  annotations:
+    strimzi.io/use-connector-resources: "true"
+spec:
+  image: paasup/kafka-connect:0.2
+  replicas: 1
+  bootstrapServers: "kafka-cluster-kafka-tls-bootstrap.{{ .Namespace }}.svc.cluster.local:9093"
+  config:
+    group.id: "{{ .Name }}-default-connect"
+    offset.storage.topic: "{{ .Name }}.connect-offsets"
+    config.storage.topic: "{{ .Name }}.connect-configs"
+    status.storage.topic: "{{ .Name }}.connect-status"
+    key.converter: org.apache.kafka.connect.json.JsonConverter
+    value.converter: org.apache.kafka.connect.json.JsonConverter
+    plugin.path: /opt/kafka/plugins
+    topic.creation.enable: "true"
+  authentication:
+    type: oauth
+    tokenEndpointUri: "{{.KEYCLOAK_URL}}/realms/$KEYCLOAK_REALM/protocol/openid-connect/token"
+    clientId: "{{.KEYCLOAK_CLIENT_ID}}-kafka-connect"
+    clientSecret:
+      secretName: kafka-connect-oauth-secret
+      key: client-secret
+    tlsTrustedCertificates:
+      - secretName: keycloak-tls
+        pattern: "ca.crt"
+
+  template:
+    connectContainer:
+      volumeMounts:
+        - name: truststore-volume
+          mountPath: /mnt/truststore/truststore.jks
+          subPath: truststore.jks
+          readOnly: true
+    pod:
+      volumes:
+        - name: truststore-volume
+          secret:
+            secretName: truststore
+
+  logging:
+    type: inline
+    loggers:
+      rootLogger.level: INFO
+
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaUser
+metadata:
+  name: "service-account-{{.KEYCLOAK_CLIENT_ID}}-kafka-connect"
+  namespace: "{{ .Namespace }}"
+  labels:
+    strimzi.io/cluster: "{{ .Name }}"
+spec:
+  authorization:
+    type: simple
+    acls:
+      - resource:
+          type: topic
+          name: "*"
+          patternType: literal
+        operations:
+          - Read
+          - Describe
+          - DescribeConfigs
+          - Write
+      - resource:
+          type: group
+          name: "*"
+          patternType: literal
+        operations:
+          - Read
+          - Write
+          - Describe
+
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  labels:
+    strimzi.io/cluster: kafka-cluster
+  name: "{{ .Name }}-connect-offsets"
+  namespace: "{{ .Namespace }}"
+spec:
+  partitions: 1
+  replicas: 3
+  topicName: "{{ .Name }}.connect-offsets"
+  config:
+    cleanup.policy: compact
+    retention.ms: 604800000
+    segment.bytes: 1073741824
+
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  labels:
+    strimzi.io/cluster: kafka-cluster
+  name: "{{ .Name }}-connect-configs"
+  namespace: "{{ .Namespace }}"
+spec:
+  partitions: 1
+  replicas: 3
+  topicName: "{{ .Name }}.connect-configs"
+  config:
+    cleanup.policy: compact
+    retention.ms: 604800000
+    segment.bytes: 1073741824
+
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  labels:
+    strimzi.io/cluster: kafka-cluster
+  name: "{{ .Name }}-connect-status"
+  namespace: "{{ .Namespace }}"
+spec:
+  partitions: 1
+  replicas: 3
+  topicName: "{{ .Name }}.connect-status"
+  config:
+    cleanup.policy: compact
+    retention.ms: 604800000
+    segment.bytes: 1073741824