Add VictoriaMetrics observability stack + sync catalog for monitoring test

- VM stack 10 charts: victoria-metrics-cluster/auth, victoria-logs-cluster, victoria-metrics-agent/alert, opentelemetry-collector, kube-state-metrics, prometheus-node-exporter, alertmanager, perses (JWT/OIDC, Infisical-ready) - ArgoCD ApplicationSet (syncWave) + per-chart dip-values overlays - doc/victoria-metrics-architecture.md, define-chart-resources updates - includes pending working-tree changes (mlflow, kubeflow, apisix, CLAUDE.md) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-25 11:10:51 +09:00
parent a55427730e
commit 6290322f1b
514 changed files with 68103 additions and 40 deletions
@@ -0,0 +1,110 @@
+# =============================================================================
+# vmalert — PaaSup 커스텀 오버라이드 (알림 규칙 평가 엔진)
+# 차트: victoria-metrics/victoria-metrics-alert
+#
+# 연동:
+#   - datasource : vmselect multitenant 엔드포인트 직결 (클러스터 전체 집계 조회)
+#       vmalert는 내부 서비스이므로 vmauth(JWT)를 경유하지 않는다.
+#       사용자/외부 접근만 vmauth JWT를 거친다. vmauth 경유가 필요하면
+#       Keycloak service-account 토큰(bearer)을 datasource에 설정한다(README 참조).
+#   - remoteWrite: ALERTS 메트릭 → vminsert accountID 0
+#   - notifier   : Alertmanager:9093
+# =============================================================================
+
+server:
+  datasource:
+    url: "http://vmcluster-victoria-metrics-cluster-vmselect.monitoring.svc.cluster.local:8481/select/multitenant/prometheus"
+
+  # vmalert 차트가 /api/v1/write를 자동으로 붙이므로 URL에 포함하지 않는다.
+  remoteWrite:
+    url: "http://vmcluster-victoria-metrics-cluster-vminsert.monitoring.svc.cluster.local:8480/insert/0/prometheus"
+
+  notifier:
+    url: "http://alertmanager.monitoring.svc.cluster.local:9093"
+
+  extraArgs:
+    envflag.enable: "true"
+    envflag.prefix: VM_
+    loggerFormat: json
+    evaluationInterval: 30s
+
+  # 클러스터 전체 플랫폼 레벨 알림 규칙
+  config:
+    alerts:
+      groups:
+        - name: cluster-health
+          rules:
+            - alert: NodeDown
+              expr: kube_node_status_condition{condition="Ready",status="true"} == 0
+              for: 2m
+              labels:
+                severity: critical
+              annotations:
+                summary: "Node {{ $labels.node }} is down"
+                description: "Node {{ $labels.node }} has been unready for more than 2 minutes."
+
+            - alert: NodeHighCPU
+              expr: avg by(node)(1 - rate(node_cpu_seconds_total{mode="idle"}[5m])) > 0.85
+              for: 5m
+              labels:
+                severity: warning
+              annotations:
+                summary: "Node {{ $labels.node }} CPU > 85%"
+                description: "CPU usage on node {{ $labels.node }} has exceeded 85% for 5 minutes."
+
+            - alert: NodeHighMemory
+              expr: (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes > 0.85
+              for: 5m
+              labels:
+                severity: warning
+              annotations:
+                summary: "Node {{ $labels.instance }} memory > 85%"
+                description: "Memory usage on {{ $labels.instance }} has exceeded 85% for 5 minutes."
+
+            - alert: NodeDiskPressure
+              expr: kube_node_status_condition{condition="DiskPressure",status="true"} == 1
+              for: 2m
+              labels:
+                severity: warning
+              annotations:
+                summary: "Node {{ $labels.node }} disk pressure"
+                description: "Node {{ $labels.node }} has been under disk pressure for more than 2 minutes."
+
+            - alert: PodCrashLooping
+              expr: increase(kube_pod_container_status_restarts_total[1h]) > 5
+              for: 0m
+              labels:
+                severity: critical
+              annotations:
+                summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is crash looping"
+                description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} restarted more than 5 times in the last hour."
+
+            - alert: PodNotReady
+              expr: |
+                kube_pod_status_ready{condition="false"} == 1
+                  and on(namespace, pod)
+                  kube_pod_status_phase{phase=~"Running|Pending"} == 1
+              for: 5m
+              labels:
+                severity: warning
+              annotations:
+                summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} not ready"
+                description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for more than 5 minutes."
+
+            - alert: DeploymentReplicasMismatch
+              expr: kube_deployment_spec_replicas != kube_deployment_status_available_replicas
+              for: 5m
+              labels:
+                severity: warning
+              annotations:
+                summary: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} replica mismatch"
+                description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has fewer available replicas than desired for 5 minutes."
+
+            - alert: PVCFillingUp
+              expr: kubelet_volume_stats_used_bytes / kubelet_volume_stats_capacity_bytes > 0.85
+              for: 5m
+              labels:
+                severity: warning
+              annotations:
+                summary: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} > 85%"
+                description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is using more than 85% of its capacity."