init vllm/0.0.11

2025-02-27 04:44:13 +00:00
parent 4a90238056
commit 1e33c0cb92
17 changed files with 874 additions and 0 deletions
@@ -0,0 +1,168 @@
+{{- range $modelSpec := .Values.servingEngineSpec.modelSpec }}
+{{- with $ -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: "{{ .Release.Name }}-{{$modelSpec.name}}-deployment-vllm"
+  namespace: {{ .Release.Namespace }}
+  labels:
+  {{- include "chart.engineLabels" . | nindent 4 }}
+spec:
+  replicas: {{ $modelSpec.replicaCount }}
+  {{- include "chart.engineStrategy" . | nindent 2 }}
+  selector:
+    matchLabels:
+    {{- include "chart.engineLabels" . | nindent 6 }}
+  progressDeadlineSeconds: 1200
+  template:
+    metadata:
+      labels:
+      {{- include "chart.engineLabels" . | nindent 8 }}
+    spec:
+      containers:
+        - name: "vllm"
+          image: "{{ required "Required value 'modelSpec.repository' must be defined !" $modelSpec.repository }}:{{ required "Required value 'modelSpec.tag' must be defined !" $modelSpec.tag }}"
+
+          command:
+          - "vllm"
+          - "serve"
+          - {{ $modelSpec.modelURL | quote }}
+          - "--host"
+          - "0.0.0.0"
+          - "--port"
+          - {{ include "chart.container-port" . | quote }}
+          {{- with $modelSpec.vllmConfig }}
+          {{-   if hasKey . "enableChunkedPrefill" }}
+          - "--enable-chunked-prefill"
+          - {{ .enableChunkedPrefill | quote }}
+          {{-   end }}
+          {{-   if .enablePrefixCaching }}
+          - "--enable-prefix-caching"
+          {{-   end }}
+          {{-   if hasKey . "maxModelLen" }}
+          - "--max-model-len"
+          - {{ .maxModelLen | quote }}
+          {{-   end }}
+          {{-   if hasKey . "dtype" }}
+          - "--dtype"
+          - {{ .dtype | quote }}
+          {{-   end }}
+          {{-   if hasKey . "tensorParallelSize" }}
+          - "--tensor-parallel-size"
+          - {{ .tensorParallelSize | quote }}
+          {{-   end }}
+          {{-   if .extraArgs }}
+          {{-     range .extraArgs }}
+          - {{ . | quote }}
+          {{-     end }}
+          {{-   end }}
+          {{- end }}
+          {{- if $modelSpec.lmcacheConfig }}
+          {{-   if $modelSpec.lmcacheConfig.enabled }}
+          - "--kv-transfer-config"
+          - '{"kv_connector":"LMCacheConnector","kv_role":"kv_both"}'
+          {{-   end }}
+          {{- end }}
+          securityContext:
+            runAsNonRoot: false
+          imagePullPolicy: IfNotPresent
+          env:
+          - name: HF_HOME
+            {{- if hasKey $modelSpec "pvcStorage" }}
+            value: /data
+            {{- else }}
+            value: /tmp
+            {{- end }}
+          {{- with $modelSpec.vllmConfig}}
+          {{- if hasKey . "v1" }}
+          - name: VLLM_USE_V1
+            value: {{ default 0 $modelSpec.vllmConfig.v1 | quote }}
+          {{- end}}
+          {{- end}}
+          {{- if $modelSpec.hf_token }}
+          - name: HF_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: {{ .Release.Name }}-secrets
+                key: hf_token_{{ $modelSpec.name }}
+          {{- end }}
+          {{- with $modelSpec.env }}
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          {{- if $modelSpec.lmcacheConfig }}
+          {{-   if $modelSpec.lmcacheConfig.enabled }}
+          - name: LMCACHE_USE_EXPERIMENTAL
+            value: "True"
+          - name: VLLM_RPC_TIMEOUT
+            value: "1000000"
+          {{-   end }}
+          {{-   if $modelSpec.lmcacheConfig.cpuOffloadingBufferSize }}
+          - name: LMCACHE_LOCAL_CPU
+            value: "True"
+          - name: LMCACHE_MAX_LOCAL_CPU_SIZE
+            value: "{{ $modelSpec.lmcacheConfig.cpuOffloadingBufferSize }}"
+          {{-   end }}
+          {{-   if $modelSpec.lmcacheConfig.diskOffloadingBufferSize }}
+          - name: LMCACHE_LOCAL_DISK
+            value: "True"
+          - name: LMCACHE_MAX_LOCAL_DISK_SIZE
+            value: "{{ $modelSpec.lmcacheConfig.diskOffloadingBufferSize }}"
+          {{-   end }}
+          {{- end }}
+          {{- if .Values.servingEngineSpec.configs }}
+          envFrom:
+            - configMapRef:
+                name: "{{ .Release.Name }}-configs"
+          {{- end }}
+          ports:
+            - name: {{ include "chart.container-port-name" . }}
+              containerPort: {{ include "chart.container-port" . }}
+          {{- include "chart.probes" . | indent 10 }}
+          resources: {{- include "chart.resources" $modelSpec | nindent 12 }}
+          {{- if hasKey $modelSpec "pvcStorage" }}
+          volumeMounts:
+          - name: {{ .Release.Name }}-storage
+            mountPath: /data
+          {{- end }}
+          {{- with $modelSpec.vllmConfig }}
+          {{- if hasKey $modelSpec.vllmConfig "tensorParallelSize"}}
+          - name: shm
+            mountPath: /dev/shm
+          {{- end}}
+          {{- end}}
+      volumes:
+        {{- if hasKey $modelSpec "pvcStorage" }}
+        - name: {{ .Release.Name }}-storage
+          persistentVolumeClaim:
+            claimName: "{{ .Release.Name }}-{{$modelSpec.name}}-storage-claim"
+        {{- end }}
+        {{- with $modelSpec.vllmConfig }}
+        {{- if hasKey $modelSpec.vllmConfig "tensorParallelSize"}}
+        - name: shm
+          emptyDir:
+            medium: Memory
+            sizeLimit: {{ default "20Gi" $modelSpec.shmSize }}
+        {{- end}}
+        {{- end}}
+      {{- if .Values.servingEngineSpec.tolerations }}
+      {{-   with .Values.servingEngineSpec.tolerations }}
+      tolerations:
+        {{-   toYaml . | nindent 8 }}
+      {{-   end }}
+      {{- end }}
+
+      {{- if .Values.servingEngineSpec.runtimeClassName }}
+      runtimeClassName: {{ .Values.servingEngineSpec.runtimeClassName }}
+      {{- end }}
+      {{- if $modelSpec.nodeSelectorTerms}}
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            {{- with $modelSpec.nodeSelectorTerms }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+      {{- end }}
+{{- end }}
+---
+{{- end }}