service-catalog/manifests/helm/vllm/0.0.11/templates/deployment-vllm-multi.yaml

{{- range $modelSpec := .Values.servingEngineSpec.modelSpec }}
{{- with $ -}}
apiVersion: apps/v1
kind: Deployment
metadata:
  name: "{{ .Release.Name }}-{{$modelSpec.name}}-deployment-vllm"
  namespace: {{ .Release.Namespace }}
  labels:
  {{- include "chart.engineLabels" . | nindent 4 }}
spec:
  replicas: {{ $modelSpec.replicaCount }}
  {{- include "chart.engineStrategy" . | nindent 2 }}
  selector:
    matchLabels:
    {{- include "chart.engineLabels" . | nindent 6 }}
  progressDeadlineSeconds: 1200
  template:
    metadata:
      labels:
      {{- include "chart.engineLabels" . | nindent 8 }}
    spec:
      containers:
        - name: "vllm"
          image: "{{ required "Required value 'modelSpec.repository' must be defined !" $modelSpec.repository }}:{{ required "Required value 'modelSpec.tag' must be defined !" $modelSpec.tag }}"

          command:
          - "vllm"
          - "serve"
          - {{ $modelSpec.modelURL | quote }}
          - "--host"
          - "0.0.0.0"
          - "--port"
          - {{ include "chart.container-port" . | quote }}
          {{- with $modelSpec.vllmConfig }}
          {{-   if hasKey . "enableChunkedPrefill" }}
          - "--enable-chunked-prefill"
          - {{ .enableChunkedPrefill | quote }}
          {{-   end }}
          {{-   if .enablePrefixCaching }}
          - "--enable-prefix-caching"
          {{-   end }}
          {{-   if hasKey . "maxModelLen" }}
          - "--max-model-len"
          - {{ .maxModelLen | quote }}
          {{-   end }}
          {{-   if hasKey . "dtype" }}
          - "--dtype"
          - {{ .dtype | quote }}
          {{-   end }}
          {{-   if hasKey . "tensorParallelSize" }}
          - "--tensor-parallel-size"
          - {{ .tensorParallelSize | quote }}
          {{-   end }}
          {{-   if .extraArgs }}
          {{-     range .extraArgs }}
          - {{ . | quote }}
          {{-     end }}
          {{-   end }}
          {{- end }}
          {{- if $modelSpec.lmcacheConfig }}
          {{-   if $modelSpec.lmcacheConfig.enabled }}
          - "--kv-transfer-config"
          - '{"kv_connector":"LMCacheConnector","kv_role":"kv_both"}'
          {{-   end }}
          {{- end }}
          securityContext:
            runAsNonRoot: false
          imagePullPolicy: IfNotPresent
          env:
          - name: HF_HOME
            {{- if hasKey $modelSpec "pvcStorage" }}
            value: /data
            {{- else }}
            value: /tmp
            {{- end }}
          {{- with $modelSpec.vllmConfig}}
          {{- if hasKey . "v1" }}
          - name: VLLM_USE_V1
            value: {{ default 0 $modelSpec.vllmConfig.v1 | quote }}
          {{- end}}
          {{- end}}
          {{- if $modelSpec.hf_token }}
          - name: HF_TOKEN
            valueFrom:
              secretKeyRef:
                name: {{ .Release.Name }}-secrets
                key: hf_token_{{ $modelSpec.name }}
          {{- end }}
          {{- with $modelSpec.env }}
          {{- toYaml . | nindent 10 }}
          {{- end }}
          {{- if $modelSpec.lmcacheConfig }}
          {{-   if $modelSpec.lmcacheConfig.enabled }}
          - name: LMCACHE_USE_EXPERIMENTAL
            value: "True"
          - name: VLLM_RPC_TIMEOUT
            value: "1000000"
          {{-   end }}
          {{-   if $modelSpec.lmcacheConfig.cpuOffloadingBufferSize }}
          - name: LMCACHE_LOCAL_CPU
            value: "True"
          - name: LMCACHE_MAX_LOCAL_CPU_SIZE
            value: "{{ $modelSpec.lmcacheConfig.cpuOffloadingBufferSize }}"
          {{-   end }}
          {{-   if $modelSpec.lmcacheConfig.diskOffloadingBufferSize }}
          - name: LMCACHE_LOCAL_DISK
            value: "True"
          - name: LMCACHE_MAX_LOCAL_DISK_SIZE
            value: "{{ $modelSpec.lmcacheConfig.diskOffloadingBufferSize }}"
          {{-   end }}
          {{- end }}
          {{- if .Values.servingEngineSpec.configs }}
          envFrom:
            - configMapRef:
                name: "{{ .Release.Name }}-configs"
          {{- end }}
          ports:
            - name: {{ include "chart.container-port-name" . }}
              containerPort: {{ include "chart.container-port" . }}
          {{- include "chart.probes" . | indent 10 }}
          resources: {{- include "chart.resources" $modelSpec | nindent 12 }}
          {{- if hasKey $modelSpec "pvcStorage" }}
          volumeMounts:
          - name: {{ .Release.Name }}-storage
            mountPath: /data
          {{- end }}
          {{- with $modelSpec.vllmConfig }}
          {{- if hasKey $modelSpec.vllmConfig "tensorParallelSize"}}
          - name: shm
            mountPath: /dev/shm
          {{- end}}
          {{- end}}
      volumes:
        {{- if hasKey $modelSpec "pvcStorage" }}
        - name: {{ .Release.Name }}-storage
          persistentVolumeClaim:
            claimName: "{{ .Release.Name }}-{{$modelSpec.name}}-storage-claim"
        {{- end }}
        {{- with $modelSpec.vllmConfig }}
        {{- if hasKey $modelSpec.vllmConfig "tensorParallelSize"}}
        - name: shm
          emptyDir:
            medium: Memory
            sizeLimit: {{ default "20Gi" $modelSpec.shmSize }}
        {{- end}}
        {{- end}}
      {{- if .Values.servingEngineSpec.tolerations }}
      {{-   with .Values.servingEngineSpec.tolerations }}
      tolerations:
        {{-   toYaml . | nindent 8 }}
      {{-   end }}
      {{- end }}

      {{- if .Values.servingEngineSpec.runtimeClassName }}
      runtimeClassName: {{ .Values.servingEngineSpec.runtimeClassName }}
      {{- end }}
      {{- if $modelSpec.nodeSelectorTerms}}
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            {{- with $modelSpec.nodeSelectorTerms }}
            {{- toYaml . | nindent 12 }}
            {{- end }}
      {{- end }}
{{- end }}
---
{{- end }}