Add VictoriaMetrics observability stack + sync catalog for monitoring test
- VM stack 10 charts: victoria-metrics-cluster/auth, victoria-logs-cluster, victoria-metrics-agent/alert, opentelemetry-collector, kube-state-metrics, prometheus-node-exporter, alertmanager, perses (JWT/OIDC, Infisical-ready) - ArgoCD ApplicationSet (syncWave) + per-chart dip-values overlays - doc/victoria-metrics-architecture.md, define-chart-resources updates - includes pending working-tree changes (mlflow, kubeflow, apisix, CLAUDE.md) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
wbsong111
@@ -0,0 +1,186 @@
|
||||
# =============================================================================
|
||||
# OpenTelemetry Collector (DaemonSet) — PaaSup 커스텀 오버라이드
|
||||
# 차트: open-telemetry/opentelemetry-collector
|
||||
# 배포: DaemonSet (모든 노드에서 파드 로그 수집)
|
||||
# 릴리스명: otelcol
|
||||
#
|
||||
# 파이프라인 (로그 전용):
|
||||
# logs/demo01 : filelog → filter(namespace=demo01-*) → vlinsert AccountID 1
|
||||
# logs/demo02 : filelog → filter(namespace=demo02-*) → vlinsert AccountID 2
|
||||
# logs/platform: filelog → filter(나머지) → vlinsert AccountID 0
|
||||
#
|
||||
# 참고: K8s 이벤트 수집은 Deployment 모드의 custom-values-events.yaml로 별도 배포.
|
||||
# =============================================================================
|
||||
|
||||
mode: daemonset
|
||||
|
||||
image:
|
||||
repository: otel/opentelemetry-collector-contrib
|
||||
|
||||
# 로그 전용 — 차트 기본 trace/OTLP 인바운드 포트 비활성화 (filelog는 호스트 파일을 읽음)
|
||||
ports:
|
||||
otlp:
|
||||
enabled: false
|
||||
otlp-http:
|
||||
enabled: false
|
||||
jaeger-compact:
|
||||
enabled: false
|
||||
jaeger-thrift:
|
||||
enabled: false
|
||||
jaeger-grpc:
|
||||
enabled: false
|
||||
zipkin:
|
||||
enabled: false
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
||||
# 호스트 로그 접근
|
||||
extraVolumes:
|
||||
- name: varlogpods
|
||||
hostPath:
|
||||
path: /var/log/pods
|
||||
- name: varlibdockercontainers
|
||||
hostPath:
|
||||
path: /var/lib/docker/containers
|
||||
extraVolumeMounts:
|
||||
- name: varlogpods
|
||||
mountPath: /var/log/pods
|
||||
readOnly: true
|
||||
- name: varlibdockercontainers
|
||||
mountPath: /var/lib/docker/containers
|
||||
readOnly: true
|
||||
|
||||
config:
|
||||
# 차트 기본 트레이스/메트릭 컴포넌트 제거 (로그 전용 collector)
|
||||
receivers:
|
||||
jaeger: null
|
||||
zipkin: null
|
||||
otlp: null
|
||||
prometheus: null
|
||||
filelog:
|
||||
include:
|
||||
- /var/log/pods/*/*/*.log
|
||||
exclude:
|
||||
- /var/log/pods/monitoring_otelcol*/*/*.log
|
||||
start_at: end
|
||||
include_file_path: true
|
||||
include_file_name: false
|
||||
operators:
|
||||
- type: router
|
||||
id: get-format
|
||||
routes:
|
||||
- output: parser-docker
|
||||
expr: 'body matches "^\\{"'
|
||||
- output: parser-crio
|
||||
expr: 'body matches "^[^ ]+ "'
|
||||
- type: json_parser
|
||||
id: parser-docker
|
||||
output: extract-metadata-from-filepath
|
||||
- type: regex_parser
|
||||
id: parser-crio
|
||||
regex: '^(?P<time>[^ ]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]+) ?(?P<log>.*)$'
|
||||
output: extract-metadata-from-filepath
|
||||
timestamp:
|
||||
parse_from: attributes.time
|
||||
layout: '%Y-%m-%dT%H:%M:%S.%LZ'
|
||||
- type: regex_parser
|
||||
id: extract-metadata-from-filepath
|
||||
regex: '^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]+)\/(?P<container_name>[^\._]+)\/(?P<restart_count>\d+)\.log$'
|
||||
parse_from: attributes["log.file.path"]
|
||||
- type: move
|
||||
from: attributes.log
|
||||
to: body
|
||||
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 5s
|
||||
limit_percentage: 80
|
||||
spike_limit_percentage: 25
|
||||
batch:
|
||||
send_batch_size: 10000
|
||||
timeout: 10s
|
||||
k8sattributes:
|
||||
auth_type: serviceAccount
|
||||
passthrough: false
|
||||
extract:
|
||||
metadata:
|
||||
- k8s.namespace.name
|
||||
- k8s.pod.name
|
||||
- k8s.container.name
|
||||
- k8s.node.name
|
||||
pod_association:
|
||||
- sources:
|
||||
- from: resource_attribute
|
||||
name: k8s.pod.ip
|
||||
- sources:
|
||||
- from: connection
|
||||
|
||||
# namespace는 filelog regex가 log record attributes에 저장 → OTTL attributes["namespace"]
|
||||
filter/demo01:
|
||||
logs:
|
||||
log_record:
|
||||
- 'not IsMatch(attributes["namespace"], "^demo01-.*")'
|
||||
filter/demo02:
|
||||
logs:
|
||||
log_record:
|
||||
- 'not IsMatch(attributes["namespace"], "^demo02-.*")'
|
||||
filter/platform:
|
||||
logs:
|
||||
log_record:
|
||||
- 'IsMatch(attributes["namespace"], "^(demo01|demo02)-.*")'
|
||||
|
||||
exporters:
|
||||
otlphttp/vlogs-0:
|
||||
endpoint: "http://vlogs-victoria-logs-cluster-vlinsert.monitoring.svc.cluster.local:9481/insert/opentelemetry"
|
||||
tls:
|
||||
insecure: true
|
||||
headers:
|
||||
VictoriaLogs-AccountID: "0"
|
||||
otlphttp/vlogs-1:
|
||||
endpoint: "http://vlogs-victoria-logs-cluster-vlinsert.monitoring.svc.cluster.local:9481/insert/opentelemetry"
|
||||
tls:
|
||||
insecure: true
|
||||
headers:
|
||||
VictoriaLogs-AccountID: "1"
|
||||
otlphttp/vlogs-2:
|
||||
endpoint: "http://vlogs-victoria-logs-cluster-vlinsert.monitoring.svc.cluster.local:9481/insert/opentelemetry"
|
||||
tls:
|
||||
insecure: true
|
||||
headers:
|
||||
VictoriaLogs-AccountID: "2"
|
||||
|
||||
service:
|
||||
pipelines:
|
||||
# 차트 기본 파이프라인 제거 (로그 전용)
|
||||
traces: null
|
||||
metrics: null
|
||||
logs: null
|
||||
logs/demo01:
|
||||
receivers: [filelog]
|
||||
processors: [memory_limiter, k8sattributes, filter/demo01, batch]
|
||||
exporters: [otlphttp/vlogs-1]
|
||||
logs/demo02:
|
||||
receivers: [filelog]
|
||||
processors: [memory_limiter, k8sattributes, filter/demo02, batch]
|
||||
exporters: [otlphttp/vlogs-2]
|
||||
logs/platform:
|
||||
receivers: [filelog]
|
||||
processors: [memory_limiter, k8sattributes, filter/platform, batch]
|
||||
exporters: [otlphttp/vlogs-0]
|
||||
|
||||
# k8sattributes 프로세서용 RBAC
|
||||
clusterRole:
|
||||
create: true
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: [pods, namespaces, nodes]
|
||||
verbs: [get, list, watch]
|
||||
- apiGroups: ["apps"]
|
||||
resources: [replicasets]
|
||||
verbs: [get, list, watch]
|
||||
Reference in New Issue
Block a user