init vllm/0.0.11

charts/vllm/templates/_helpers.tpl

@@ -0,0 +1,142 @@
+{{/*
+Define ports for the pods
+*/}}
+{{- define "chart.container-port" -}}
+{{-  default "8000" .Values.servingEngineSpec.containerPort }}
+{{- end }}
+
+{{/*
+Define service port
+*/}}
+{{- define "chart.service-port" -}}
+{{-  if .Values.servingEngineSpec.servicePort }}
+{{-    .Values.servingEngineSpec.servicePort }}
+{{-  else }}
+{{-    include "chart.container-port" . }}
+{{-  end }}
+{{- end }}
+
+{{/*
+Define service port name
+*/}}
+{{- define "chart.service-port-name" -}}
+"service-port"
+{{- end }}
+
+{{/*
+Define container port name
+*/}}
+{{- define "chart.container-port-name" -}}
+"container-port"
+{{- end }}
+
+{{/*
+Define engine deployment strategy.
+If .Values.engineStrategy is defined, use it.
+Otherwise, fall back to the default rolling update strategy.
+*/}}
+{{- define "chart.engineStrategy" -}}
+strategy:
+{{- if .Values.servingEngineSpec.strategy }}
+{{- toYaml .Values.servingEngineSpec.strategy | nindent 2 }}
+{{- else }}
+  rollingUpdate:
+    maxSurge: 100%
+    maxUnavailable: 0
+{{- end }}
+{{- end }}
+
+{{/*
+Define router deployment strategy.
+If .Values.routerStrategy is defined, use it.
+Otherwise, fall back to the default rolling update strategy.
+*/}}
+{{- define "chart.routerStrategy" -}}
+strategy:
+{{- if .Values.routerSpec.strategy }}
+{{- toYaml .Values.routerSpec.strategy | nindent 2 }}
+{{- else }}
+  rollingUpdate:
+    maxSurge: 100%
+    maxUnavailable: 0
+{{- end }}
+{{- end }}
+
+{{/*
+Define additional ports
+*/}}
+{{- define "chart.extraPorts" }}
+{{-   with .Values.servingEngineSpec.extraPorts }}
+{{      toYaml . }}
+{{-   end }}
+{{- end }}
+
+
+{{/*
+Define liveness et readiness probes
+*/}}
+{{- define "chart.probes" -}}
+{{-   if .Values.servingEngineSpec.startupProbe  }}
+startupProbe:
+{{-     with .Values.servingEngineSpec.startupProbe }}
+{{-       toYaml . | nindent 2 }}
+{{-     end }}
+{{-   end }}
+{{-   if .Values.servingEngineSpec.livenessProbe  }}
+livenessProbe:
+{{-     with .Values.servingEngineSpec.livenessProbe }}
+{{-       toYaml . | nindent 2 }}
+{{-     end }}
+{{-   end }}
+{{- end }}
+
+{{/*
+Define resources with a variable model spec
+*/}}
+{{- define "chart.resources" -}}
+{{- $modelSpec := . -}}
+requests:
+  memory: {{ required "Value 'modelSpec.requestMemory' must be defined !" ($modelSpec.requestMemory | quote) }}
+  cpu: {{ required "Value 'modelSpec.requestCPU' must be defined !" ($modelSpec.requestCPU | quote) }}
+  {{- if (gt (int $modelSpec.requestGPU) 0) }}
+  nvidia.com/gpu: {{ required "Value 'modelSpec.requestGPU' must be defined !" (index $modelSpec.requestGPU | quote) }}
+  {{- end }}
+limits:
+  memory: {{ required "Value 'modelSpec.requestMemory' must be defined !" ($modelSpec.requestMemory | quote) }}
+  cpu: {{ required "Value 'modelSpec.requestCPU' must be defined !" ($modelSpec.requestCPU | quote) }}
+  {{- if (gt (int $modelSpec.requestGPU) 0) }}
+  nvidia.com/gpu: {{ required "Value 'modelSpec.requestGPU' must be defined !" (index $modelSpec.requestGPU | quote) }}
+  {{- end }}
+{{- end }}
+
+
+{{/*
+  Define labels for serving engine and its service
+*/}}
+{{- define "chart.engineLabels" -}}
+{{-   with .Values.servingEngineSpec.labels -}}
+{{      toYaml . }}
+{{-   end }}
+{{- end }}
+
+{{/*
+  Define labels for router and its service
+*/}}
+{{- define "chart.routerLabels" -}}
+{{-   with .Values.routerSpec.labels -}}
+{{      toYaml . }}
+{{-   end }}
+{{- end }}
+
+{{/*
+  Define helper function to convert labels to a comma separated list
+*/}}
+{{- define "labels.toCommaSeparatedList" -}}
+{{- $labels := . -}}
+{{- $result := "" -}}
+{{- range $key, $value := $labels -}}
+  {{- if $result }},{{ end -}}
+  {{ $key }}={{ $value }}
+  {{- $result = "," -}}
+{{- end -}}
+{{- end -}}

charts/vllm/templates/configmap.yaml

@@ -0,0 +1,11 @@
+{{- if .Values.servingEngineSpec.configs -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "{{ .Release.Name }}-configs"
+  namespace: {{ .Release.Namespace }}
+data:
+  {{- with .Values.servingEngineSpec.configs }}
+  {{- toYaml . | nindent 2 }}
+  {{- end }}
+{{- end -}}

charts/vllm/templates/deployment-router.yaml

@@ -0,0 +1,85 @@
+{{- if .Values.routerSpec.enableRouter -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: "{{ .Release.Name }}-deployment-router"
+  namespace: {{ .Release.Namespace }}
+  labels:
+  {{- include "chart.routerLabels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.routerSpec.replicaCount }}
+  {{- include "chart.routerStrategy" . | nindent 2 }}
+  selector:
+    matchLabels:
+    {{- include "chart.routerLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+      {{- include "chart.routerLabels" . | nindent 8 }}
+    spec:
+      serviceAccountName: {{ .Release.Name }}-router-service-account
+      containers:
+      - name: router-container
+        image: "{{ .Values.routerSpec.repository | default "lmcache/lmstack-router" }}:{{ .Values.routerSpec.tag | default "latest" }}"
+        imagePullPolicy: "{{ .Values.routerSpec.imagePullPolicy | default "Always" }}"
+        args:
+          - "--host"
+          - "0.0.0.0"
+          - "--port"
+          - "{{ .Values.routerSpec.containerPort }}"
+          - "--service-discovery"
+          - "{{ default "k8s" .Values.routerSpec.serviceDiscovery }}"
+          {{- if eq .Values.routerSpec.serviceDiscovery "k8s" }}
+          - "--k8s-namespace"
+          - "{{ .Release.Namespace }}"
+          - "--k8s-label-selector"
+          - {{ include "labels.toCommaSeparatedList" .Values.servingEngineSpec.labels }}
+          {{- end }}
+          {{- if eq .Values.routerSpec.serviceDiscovery "static" }}
+          - "--static-backends"
+          - "{{ required "When using static service discovery, .Values.routerSpec.staticBackends is a required value" .Values.routerSpec.staticBackends }}"
+          - "--static-models"
+          - "{{ required "When using static service discovery, .Values.routerSpec.staticModels is a required value" .Values.routerSpec.staticModels }}"
+          {{- end }}
+          - "--routing-logic"
+          - "{{ .Values.routerSpec.routingLogic }}"
+          {{- if .Values.routerSpec.sessionKey }}
+          - "--session-key"
+          - "{{ .Values.routerSpec.sessionKey }}"
+          {{- end }}
+          {{- if .Values.routerSpec.engineScrapeInterval }}
+          - "--engine-stats-interval"
+          - "{{ .Values.routerSpec.engineScrapeInterval }}"
+          {{- end }}
+          {{- if .Values.routerSpec.requestStatsWindow }}
+          - "--request-stats-window"
+          - "{{ .Values.routerSpec.requestStatsWindow }}"
+          {{- end }}
+          {{- if .Values.routerSpec.extraArgs }}
+          {{- toYaml .Values.routerSpec.extraArgs | nindent 10 }}
+          {{- end }}
+        {{- if .Values.routerSpec.resources }}
+        resources:
+          {{- if .Values.routerSpec.resources.requests }}
+          requests:
+            cpu: "{{ .Values.routerSpec.resources.requests.cpu }}"
+            memory: "{{ .Values.routerSpec.resources.requests.memory }}"
+          {{- end }}
+          {{- if .Values.routerSpec.resources.limits }}
+          limits:
+            cpu: "{{ .Values.routerSpec.resources.limits.cpu }}"
+            memory: "{{ .Values.routerSpec.resources.limits.memory }}"
+          {{- end }}
+        {{- end }}
+        ports:
+          - name: "router-cport"
+            containerPort: {{ .Values.routerSpec.containerPort }}
+
+        livenessProbe:
+          initialDelaySeconds: 30
+          periodSeconds: 5
+          failureThreshold: 3
+          httpGet:
+            path: /health
+            port: {{ .Values.routerSpec.containerPort }}
+{{- end }}

charts/vllm/templates/deployment-vllm-multi.yaml

@@ -0,0 +1,168 @@
+{{- range $modelSpec := .Values.servingEngineSpec.modelSpec }}
+{{- with $ -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: "{{ .Release.Name }}-{{$modelSpec.name}}-deployment-vllm"
+  namespace: {{ .Release.Namespace }}
+  labels:
+  {{- include "chart.engineLabels" . | nindent 4 }}
+spec:
+  replicas: {{ $modelSpec.replicaCount }}
+  {{- include "chart.engineStrategy" . | nindent 2 }}
+  selector:
+    matchLabels:
+    {{- include "chart.engineLabels" . | nindent 6 }}
+  progressDeadlineSeconds: 1200
+  template:
+    metadata:
+      labels:
+      {{- include "chart.engineLabels" . | nindent 8 }}
+    spec:
+      containers:
+        - name: "vllm"
+          image: "{{ required "Required value 'modelSpec.repository' must be defined !" $modelSpec.repository }}:{{ required "Required value 'modelSpec.tag' must be defined !" $modelSpec.tag }}"
+
+          command:
+          - "vllm"
+          - "serve"
+          - {{ $modelSpec.modelURL | quote }}
+          - "--host"
+          - "0.0.0.0"
+          - "--port"
+          - {{ include "chart.container-port" . | quote }}
+          {{- with $modelSpec.vllmConfig }}
+          {{-   if hasKey . "enableChunkedPrefill" }}
+          - "--enable-chunked-prefill"
+          - {{ .enableChunkedPrefill | quote }}
+          {{-   end }}
+          {{-   if .enablePrefixCaching }}
+          - "--enable-prefix-caching"
+          {{-   end }}
+          {{-   if hasKey . "maxModelLen" }}
+          - "--max-model-len"
+          - {{ .maxModelLen | quote }}
+          {{-   end }}
+          {{-   if hasKey . "dtype" }}
+          - "--dtype"
+          - {{ .dtype | quote }}
+          {{-   end }}
+          {{-   if hasKey . "tensorParallelSize" }}
+          - "--tensor-parallel-size"
+          - {{ .tensorParallelSize | quote }}
+          {{-   end }}
+          {{-   if .extraArgs }}
+          {{-     range .extraArgs }}
+          - {{ . | quote }}
+          {{-     end }}
+          {{-   end }}
+          {{- end }}
+          {{- if $modelSpec.lmcacheConfig }}
+          {{-   if $modelSpec.lmcacheConfig.enabled }}
+          - "--kv-transfer-config"
+          - '{"kv_connector":"LMCacheConnector","kv_role":"kv_both"}'
+          {{-   end }}
+          {{- end }}
+          securityContext:
+            runAsNonRoot: false
+          imagePullPolicy: IfNotPresent
+          env:
+          - name: HF_HOME
+            {{- if hasKey $modelSpec "pvcStorage" }}
+            value: /data
+            {{- else }}
+            value: /tmp
+            {{- end }}
+          {{- with $modelSpec.vllmConfig}}
+          {{- if hasKey . "v1" }}
+          - name: VLLM_USE_V1
+            value: {{ default 0 $modelSpec.vllmConfig.v1 | quote }}
+          {{- end}}
+          {{- end}}
+          {{- if $modelSpec.hf_token }}
+          - name: HF_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: {{ .Release.Name }}-secrets
+                key: hf_token_{{ $modelSpec.name }}
+          {{- end }}
+          {{- with $modelSpec.env }}
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          {{- if $modelSpec.lmcacheConfig }}
+          {{-   if $modelSpec.lmcacheConfig.enabled }}
+          - name: LMCACHE_USE_EXPERIMENTAL
+            value: "True"
+          - name: VLLM_RPC_TIMEOUT
+            value: "1000000"
+          {{-   end }}
+          {{-   if $modelSpec.lmcacheConfig.cpuOffloadingBufferSize }}
+          - name: LMCACHE_LOCAL_CPU
+            value: "True"
+          - name: LMCACHE_MAX_LOCAL_CPU_SIZE
+            value: "{{ $modelSpec.lmcacheConfig.cpuOffloadingBufferSize }}"
+          {{-   end }}
+          {{-   if $modelSpec.lmcacheConfig.diskOffloadingBufferSize }}
+          - name: LMCACHE_LOCAL_DISK
+            value: "True"
+          - name: LMCACHE_MAX_LOCAL_DISK_SIZE
+            value: "{{ $modelSpec.lmcacheConfig.diskOffloadingBufferSize }}"
+          {{-   end }}
+          {{- end }}
+          {{- if .Values.servingEngineSpec.configs }}
+          envFrom:
+            - configMapRef:
+                name: "{{ .Release.Name }}-configs"
+          {{- end }}
+          ports:
+            - name: {{ include "chart.container-port-name" . }}
+              containerPort: {{ include "chart.container-port" . }}
+          {{- include "chart.probes" . | indent 10 }}
+          resources: {{- include "chart.resources" $modelSpec | nindent 12 }}
+          {{- if hasKey $modelSpec "pvcStorage" }}
+          volumeMounts:
+          - name: {{ .Release.Name }}-storage
+            mountPath: /data
+          {{- end }}
+          {{- with $modelSpec.vllmConfig }}
+          {{- if hasKey $modelSpec.vllmConfig "tensorParallelSize"}}
+          - name: shm
+            mountPath: /dev/shm
+          {{- end}}
+          {{- end}}
+      volumes:
+        {{- if hasKey $modelSpec "pvcStorage" }}
+        - name: {{ .Release.Name }}-storage
+          persistentVolumeClaim:
+            claimName: "{{ .Release.Name }}-{{$modelSpec.name}}-storage-claim"
+        {{- end }}
+        {{- with $modelSpec.vllmConfig }}
+        {{- if hasKey $modelSpec.vllmConfig "tensorParallelSize"}}
+        - name: shm
+          emptyDir:
+            medium: Memory
+            sizeLimit: {{ default "20Gi" $modelSpec.shmSize }}
+        {{- end}}
+        {{- end}}
+      {{- if .Values.servingEngineSpec.tolerations }}
+      {{-   with .Values.servingEngineSpec.tolerations }}
+      tolerations:
+        {{-   toYaml . | nindent 8 }}
+      {{-   end }}
+      {{- end }}
+
+      {{- if .Values.servingEngineSpec.runtimeClassName }}
+      runtimeClassName: {{ .Values.servingEngineSpec.runtimeClassName }}
+      {{- end }}
+      {{- if $modelSpec.nodeSelectorTerms}}
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            {{- with $modelSpec.nodeSelectorTerms }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+      {{- end }}
+{{- end }}
+---
+{{- end }}

charts/vllm/templates/ingress.yaml

@@ -0,0 +1,42 @@
+{{- if and (.Values.routerSpec.enableRouter) (.Values.routerSpec.ingress.enabled) -}}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: "{{ .Release.Name }}-ingress-router"
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "chart.routerLabels" . | nindent 4 }}
+  {{- with .Values.routerSpec.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if .Values.routerSpec.ingress.className }}
+  ingressClassName: {{ .Values.routerSpec.ingress.className }}
+  {{- end }}
+  {{- if .Values.routerSpec.ingress.tls }}
+  tls:
+    {{- range .Values.routerSpec.ingress.tls }}
+    - hosts:
+        {{- range .hosts }}
+        - {{ . | quote }}
+        {{- end }}
+      secretName: {{ .secretName }}
+    {{- end }}
+  {{- end }}
+  rules:
+    {{- range .Values.routerSpec.ingress.hosts }}
+    - host: {{ .host | quote }}
+      http:
+        paths:
+          {{- range .paths }}
+          - path: {{ .path }}
+            pathType: {{ .pathType }}
+            backend:
+              service:
+                name: "{{ $.Release.Name }}-router-service"
+                port:
+                  number: {{ $.Values.routerSpec.servicePort }}
+          {{- end }}
+    {{- end }}
+{{- end }}

charts/vllm/templates/poddisruptionbudget.yaml

@@ -0,0 +1,7 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: "{{ .Release.Name }}-pdb"
+  namespace: {{ .Release.Namespace }}
+spec:
+  maxUnavailable: {{ default 1 .Values.servingEngineSpec.maxUnavailablePodDisruptionBudget }}

charts/vllm/templates/pvc.yaml

@@ -0,0 +1,30 @@
+{{- range $modelSpec := .Values.servingEngineSpec.modelSpec }}
+{{- with $ -}}
+{{- if and (hasKey $modelSpec "pvcStorage") (not (empty $modelSpec.pvcStorage)) }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: "{{ .Release.Name }}-{{$modelSpec.name}}-storage-claim"
+  namespace: {{ .Release.Namespace }}
+spec:
+  accessModes:
+    {{- if $modelSpec.pvcAccessMode }}
+    {{- toYaml $modelSpec.pvcAccessMode | nindent 4 }}
+    {{- else }}
+    - ReadWriteOnce
+    {{- end }}
+  resources:
+    requests:
+      storage: {{ $modelSpec.pvcStorage | default "20Gi" }}  # Default to 40Gi if not set
+  {{- if hasKey $modelSpec "storageClass" }}
+  storageClassName: "{{ $modelSpec.storageClass }}"
+  {{- end }}
+  {{- if not (empty $modelSpec.pvcMatchLabels) }}
+  selector:
+    matchLabels:
+      {{- toYaml $modelSpec.pvcMatchLabels | nindent 8 }}
+  {{- end }}
+{{- end }}
+{{- end }}
+---
+{{- end }}

charts/vllm/templates/role.yaml

@@ -0,0 +1,11 @@
+{{- if .Values.routerSpec.enableRouter -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: "{{ .Release.Name }}-pod-reader"
+  namespace: {{ .Release.Namespace }}
+rules:
+- apiGroups: [""] # "" indicates the core API group
+  resources: ["pods"]
+  verbs: ["get", "watch", "list"]
+{{- end }}

charts/vllm/templates/rolebinding.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.routerSpec.enableRouter -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ .Release.Name }}-deployment-access-binding
+  namespace: {{ .Release.Namespace }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Release.Name }}-router-service-account
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: Role
+  name: {{ .Release.Name }}-pod-reader
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}

charts/vllm/templates/secrets.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "{{ .Release.Name }}-secrets"
+  namespace: {{ .Release.Namespace }}
+type: Opaque
+data:
+  {{- range $modelSpec := .Values.servingEngineSpec.modelSpec }}
+  {{- with $ -}}
+  {{-   if $modelSpec.hf_token }}
+  hf_token_{{ $modelSpec.name }}: {{ $modelSpec.hf_token | b64enc | quote }}
+  {{-   end }}
+  {{- end }}
+  {{- end }}

charts/vllm/templates/service-router.yaml

@@ -0,0 +1,18 @@
+{{- if .Values.routerSpec.enableRouter -}}
+apiVersion: v1
+kind: Service
+metadata:
+  name: "{{ .Release.Name }}-router-service"
+  namespace: {{ .Release.Namespace }}
+  labels:
+  {{- include "chart.routerLabels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+    - name: "router-sport"
+      port: {{ .Values.routerSpec.servicePort }}
+      targetPort: {{ .Values.routerSpec.containerPort }}
+      protocol: TCP
+  selector:
+  {{- include "chart.routerLabels" . | nindent 4 }}
+{{- end }}

charts/vllm/templates/service-vllm.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: "{{ .Release.Name }}-engine-service"
+  namespace: {{ .Release.Namespace }}
+  labels:
+  {{- include "chart.engineLabels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+    - name: {{ include "chart.service-port-name" . }}
+      port: {{ include "chart.service-port" . }}
+      targetPort: {{ include "chart.container-port-name" . }}
+      protocol: TCP
+  selector:
+  {{- include "chart.engineLabels" . | nindent 4 }}

charts/vllm/templates/serviceaccount.yaml

@@ -0,0 +1,7 @@
+{{- if .Values.routerSpec.enableRouter -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: "{{ .Release.Name }}-router-service-account"
+  namespace: {{ .Release.Namespace }}
+{{- end }}