init vllm/0.0.11
This commit is contained in:
ChanghoWoo
@@ -0,0 +1,7 @@
|
||||
apiVersion: v2
|
||||
description: The stack deployment of vLLM
|
||||
maintainers:
|
||||
- name: apostac
|
||||
name: vllm-stack
|
||||
type: application
|
||||
version: 0.0.11
|
||||
@@ -0,0 +1,27 @@
|
||||
# vLLM Production Stack helm chart
|
||||
|
||||
This helm chart lets users deploy multiple serving engines and a router into the Kubernetes cluster.
|
||||
|
||||
## Key features
|
||||
|
||||
- Support running multiple serving engines with multiple different models
|
||||
- Load the model weights directly from the existing PersistentVolumes
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. A running Kubernetes cluster with GPU. (You can set it up through `minikube`: <https://minikube.sigs.k8s.io/docs/tutorials/nvidia/>)
|
||||
2. [Helm](https://helm.sh/docs/intro/install/)
|
||||
|
||||
## Install the helm chart
|
||||
|
||||
```bash
|
||||
helm install llmstack . -f values-example.yaml
|
||||
```
|
||||
|
||||
## Uninstall the deployment
|
||||
|
||||
run `helm uninstall llmstack`
|
||||
|
||||
## Configure the deployments
|
||||
|
||||
See `helm/values.yaml` for mode details.
|
||||
@@ -0,0 +1,37 @@
|
||||
servingEngineSpec:
|
||||
modelSpec: []
|
||||
resources:
|
||||
requests:
|
||||
cpu: "4"
|
||||
memory: "16G"
|
||||
limits:
|
||||
cpu: "8"
|
||||
memory: "32G"
|
||||
|
||||
routerSpec:
|
||||
# -- The docker image of the router. The following values are defaults:
|
||||
repository: "lmcache/lmstack-router"
|
||||
tag: "latest"
|
||||
resources:
|
||||
requests:
|
||||
cpu: "2"
|
||||
memory: "8G"
|
||||
limits:
|
||||
cpu: "4"
|
||||
memory: "16G"
|
||||
ingress:
|
||||
enabled: true
|
||||
className: ""
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: "selfsigned-issuer"
|
||||
cert-manager.io/duration: 8760h
|
||||
cert-manager.io/renew-before: 720h
|
||||
hosts:
|
||||
- host: vllm.example.com
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
tls:
|
||||
- secretName: vllm-tls-secret
|
||||
hosts:
|
||||
- vllm.example.com
|
||||
@@ -0,0 +1,142 @@
|
||||
{{/*
|
||||
Define ports for the pods
|
||||
*/}}
|
||||
{{- define "chart.container-port" -}}
|
||||
{{- default "8000" .Values.servingEngineSpec.containerPort }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Define service port
|
||||
*/}}
|
||||
{{- define "chart.service-port" -}}
|
||||
{{- if .Values.servingEngineSpec.servicePort }}
|
||||
{{- .Values.servingEngineSpec.servicePort }}
|
||||
{{- else }}
|
||||
{{- include "chart.container-port" . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Define service port name
|
||||
*/}}
|
||||
{{- define "chart.service-port-name" -}}
|
||||
"service-port"
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Define container port name
|
||||
*/}}
|
||||
{{- define "chart.container-port-name" -}}
|
||||
"container-port"
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Define engine deployment strategy.
|
||||
If .Values.engineStrategy is defined, use it.
|
||||
Otherwise, fall back to the default rolling update strategy.
|
||||
*/}}
|
||||
{{- define "chart.engineStrategy" -}}
|
||||
strategy:
|
||||
{{- if .Values.servingEngineSpec.strategy }}
|
||||
{{- toYaml .Values.servingEngineSpec.strategy | nindent 2 }}
|
||||
{{- else }}
|
||||
rollingUpdate:
|
||||
maxSurge: 100%
|
||||
maxUnavailable: 0
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Define router deployment strategy.
|
||||
If .Values.routerStrategy is defined, use it.
|
||||
Otherwise, fall back to the default rolling update strategy.
|
||||
*/}}
|
||||
{{- define "chart.routerStrategy" -}}
|
||||
strategy:
|
||||
{{- if .Values.routerSpec.strategy }}
|
||||
{{- toYaml .Values.routerSpec.strategy | nindent 2 }}
|
||||
{{- else }}
|
||||
rollingUpdate:
|
||||
maxSurge: 100%
|
||||
maxUnavailable: 0
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Define additional ports
|
||||
*/}}
|
||||
{{- define "chart.extraPorts" }}
|
||||
{{- with .Values.servingEngineSpec.extraPorts }}
|
||||
{{ toYaml . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
|
||||
{{/*
|
||||
Define liveness et readiness probes
|
||||
*/}}
|
||||
{{- define "chart.probes" -}}
|
||||
{{- if .Values.servingEngineSpec.startupProbe }}
|
||||
startupProbe:
|
||||
{{- with .Values.servingEngineSpec.startupProbe }}
|
||||
{{- toYaml . | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.servingEngineSpec.livenessProbe }}
|
||||
livenessProbe:
|
||||
{{- with .Values.servingEngineSpec.livenessProbe }}
|
||||
{{- toYaml . | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Define resources with a variable model spec
|
||||
*/}}
|
||||
{{- define "chart.resources" -}}
|
||||
{{- $modelSpec := . -}}
|
||||
requests:
|
||||
memory: {{ required "Value 'modelSpec.requestMemory' must be defined !" ($modelSpec.requestMemory | quote) }}
|
||||
cpu: {{ required "Value 'modelSpec.requestCPU' must be defined !" ($modelSpec.requestCPU | quote) }}
|
||||
{{- if (gt (int $modelSpec.requestGPU) 0) }}
|
||||
nvidia.com/gpu: {{ required "Value 'modelSpec.requestGPU' must be defined !" (index $modelSpec.requestGPU | quote) }}
|
||||
{{- end }}
|
||||
limits:
|
||||
memory: {{ required "Value 'modelSpec.requestMemory' must be defined !" ($modelSpec.requestMemory | quote) }}
|
||||
cpu: {{ required "Value 'modelSpec.requestCPU' must be defined !" ($modelSpec.requestCPU | quote) }}
|
||||
{{- if (gt (int $modelSpec.requestGPU) 0) }}
|
||||
nvidia.com/gpu: {{ required "Value 'modelSpec.requestGPU' must be defined !" (index $modelSpec.requestGPU | quote) }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
|
||||
{{/*
|
||||
Define labels for serving engine and its service
|
||||
*/}}
|
||||
{{- define "chart.engineLabels" -}}
|
||||
{{- with .Values.servingEngineSpec.labels -}}
|
||||
{{ toYaml . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Define labels for router and its service
|
||||
*/}}
|
||||
{{- define "chart.routerLabels" -}}
|
||||
{{- with .Values.routerSpec.labels -}}
|
||||
{{ toYaml . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Define helper function to convert labels to a comma separated list
|
||||
*/}}
|
||||
{{- define "labels.toCommaSeparatedList" -}}
|
||||
{{- $labels := . -}}
|
||||
{{- $result := "" -}}
|
||||
{{- range $key, $value := $labels -}}
|
||||
{{- if $result }},{{ end -}}
|
||||
{{ $key }}={{ $value }}
|
||||
{{- $result = "," -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
@@ -0,0 +1,11 @@
|
||||
{{- if .Values.servingEngineSpec.configs -}}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: "{{ .Release.Name }}-configs"
|
||||
namespace: {{ .Release.Namespace }}
|
||||
data:
|
||||
{{- with .Values.servingEngineSpec.configs }}
|
||||
{{- toYaml . | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
@@ -0,0 +1,85 @@
|
||||
{{- if .Values.routerSpec.enableRouter -}}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: "{{ .Release.Name }}-deployment-router"
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "chart.routerLabels" . | nindent 4 }}
|
||||
spec:
|
||||
replicas: {{ .Values.routerSpec.replicaCount }}
|
||||
{{- include "chart.routerStrategy" . | nindent 2 }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "chart.routerLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "chart.routerLabels" . | nindent 8 }}
|
||||
spec:
|
||||
serviceAccountName: {{ .Release.Name }}-router-service-account
|
||||
containers:
|
||||
- name: router-container
|
||||
image: "{{ .Values.routerSpec.repository | default "lmcache/lmstack-router" }}:{{ .Values.routerSpec.tag | default "latest" }}"
|
||||
imagePullPolicy: "{{ .Values.routerSpec.imagePullPolicy | default "Always" }}"
|
||||
args:
|
||||
- "--host"
|
||||
- "0.0.0.0"
|
||||
- "--port"
|
||||
- "{{ .Values.routerSpec.containerPort }}"
|
||||
- "--service-discovery"
|
||||
- "{{ default "k8s" .Values.routerSpec.serviceDiscovery }}"
|
||||
{{- if eq .Values.routerSpec.serviceDiscovery "k8s" }}
|
||||
- "--k8s-namespace"
|
||||
- "{{ .Release.Namespace }}"
|
||||
- "--k8s-label-selector"
|
||||
- {{ include "labels.toCommaSeparatedList" .Values.servingEngineSpec.labels }}
|
||||
{{- end }}
|
||||
{{- if eq .Values.routerSpec.serviceDiscovery "static" }}
|
||||
- "--static-backends"
|
||||
- "{{ required "When using static service discovery, .Values.routerSpec.staticBackends is a required value" .Values.routerSpec.staticBackends }}"
|
||||
- "--static-models"
|
||||
- "{{ required "When using static service discovery, .Values.routerSpec.staticModels is a required value" .Values.routerSpec.staticModels }}"
|
||||
{{- end }}
|
||||
- "--routing-logic"
|
||||
- "{{ .Values.routerSpec.routingLogic }}"
|
||||
{{- if .Values.routerSpec.sessionKey }}
|
||||
- "--session-key"
|
||||
- "{{ .Values.routerSpec.sessionKey }}"
|
||||
{{- end }}
|
||||
{{- if .Values.routerSpec.engineScrapeInterval }}
|
||||
- "--engine-stats-interval"
|
||||
- "{{ .Values.routerSpec.engineScrapeInterval }}"
|
||||
{{- end }}
|
||||
{{- if .Values.routerSpec.requestStatsWindow }}
|
||||
- "--request-stats-window"
|
||||
- "{{ .Values.routerSpec.requestStatsWindow }}"
|
||||
{{- end }}
|
||||
{{- if .Values.routerSpec.extraArgs }}
|
||||
{{- toYaml .Values.routerSpec.extraArgs | nindent 10 }}
|
||||
{{- end }}
|
||||
{{- if .Values.routerSpec.resources }}
|
||||
resources:
|
||||
{{- if .Values.routerSpec.resources.requests }}
|
||||
requests:
|
||||
cpu: "{{ .Values.routerSpec.resources.requests.cpu }}"
|
||||
memory: "{{ .Values.routerSpec.resources.requests.memory }}"
|
||||
{{- end }}
|
||||
{{- if .Values.routerSpec.resources.limits }}
|
||||
limits:
|
||||
cpu: "{{ .Values.routerSpec.resources.limits.cpu }}"
|
||||
memory: "{{ .Values.routerSpec.resources.limits.memory }}"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- name: "router-cport"
|
||||
containerPort: {{ .Values.routerSpec.containerPort }}
|
||||
|
||||
livenessProbe:
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 5
|
||||
failureThreshold: 3
|
||||
httpGet:
|
||||
path: /health
|
||||
port: {{ .Values.routerSpec.containerPort }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,168 @@
|
||||
{{- range $modelSpec := .Values.servingEngineSpec.modelSpec }}
|
||||
{{- with $ -}}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: "{{ .Release.Name }}-{{$modelSpec.name}}-deployment-vllm"
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "chart.engineLabels" . | nindent 4 }}
|
||||
spec:
|
||||
replicas: {{ $modelSpec.replicaCount }}
|
||||
{{- include "chart.engineStrategy" . | nindent 2 }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "chart.engineLabels" . | nindent 6 }}
|
||||
progressDeadlineSeconds: 1200
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "chart.engineLabels" . | nindent 8 }}
|
||||
spec:
|
||||
containers:
|
||||
- name: "vllm"
|
||||
image: "{{ required "Required value 'modelSpec.repository' must be defined !" $modelSpec.repository }}:{{ required "Required value 'modelSpec.tag' must be defined !" $modelSpec.tag }}"
|
||||
|
||||
command:
|
||||
- "vllm"
|
||||
- "serve"
|
||||
- {{ $modelSpec.modelURL | quote }}
|
||||
- "--host"
|
||||
- "0.0.0.0"
|
||||
- "--port"
|
||||
- {{ include "chart.container-port" . | quote }}
|
||||
{{- with $modelSpec.vllmConfig }}
|
||||
{{- if hasKey . "enableChunkedPrefill" }}
|
||||
- "--enable-chunked-prefill"
|
||||
- {{ .enableChunkedPrefill | quote }}
|
||||
{{- end }}
|
||||
{{- if .enablePrefixCaching }}
|
||||
- "--enable-prefix-caching"
|
||||
{{- end }}
|
||||
{{- if hasKey . "maxModelLen" }}
|
||||
- "--max-model-len"
|
||||
- {{ .maxModelLen | quote }}
|
||||
{{- end }}
|
||||
{{- if hasKey . "dtype" }}
|
||||
- "--dtype"
|
||||
- {{ .dtype | quote }}
|
||||
{{- end }}
|
||||
{{- if hasKey . "tensorParallelSize" }}
|
||||
- "--tensor-parallel-size"
|
||||
- {{ .tensorParallelSize | quote }}
|
||||
{{- end }}
|
||||
{{- if .extraArgs }}
|
||||
{{- range .extraArgs }}
|
||||
- {{ . | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if $modelSpec.lmcacheConfig }}
|
||||
{{- if $modelSpec.lmcacheConfig.enabled }}
|
||||
- "--kv-transfer-config"
|
||||
- '{"kv_connector":"LMCacheConnector","kv_role":"kv_both"}'
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
runAsNonRoot: false
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: HF_HOME
|
||||
{{- if hasKey $modelSpec "pvcStorage" }}
|
||||
value: /data
|
||||
{{- else }}
|
||||
value: /tmp
|
||||
{{- end }}
|
||||
{{- with $modelSpec.vllmConfig}}
|
||||
{{- if hasKey . "v1" }}
|
||||
- name: VLLM_USE_V1
|
||||
value: {{ default 0 $modelSpec.vllmConfig.v1 | quote }}
|
||||
{{- end}}
|
||||
{{- end}}
|
||||
{{- if $modelSpec.hf_token }}
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Release.Name }}-secrets
|
||||
key: hf_token_{{ $modelSpec.name }}
|
||||
{{- end }}
|
||||
{{- with $modelSpec.env }}
|
||||
{{- toYaml . | nindent 10 }}
|
||||
{{- end }}
|
||||
{{- if $modelSpec.lmcacheConfig }}
|
||||
{{- if $modelSpec.lmcacheConfig.enabled }}
|
||||
- name: LMCACHE_USE_EXPERIMENTAL
|
||||
value: "True"
|
||||
- name: VLLM_RPC_TIMEOUT
|
||||
value: "1000000"
|
||||
{{- end }}
|
||||
{{- if $modelSpec.lmcacheConfig.cpuOffloadingBufferSize }}
|
||||
- name: LMCACHE_LOCAL_CPU
|
||||
value: "True"
|
||||
- name: LMCACHE_MAX_LOCAL_CPU_SIZE
|
||||
value: "{{ $modelSpec.lmcacheConfig.cpuOffloadingBufferSize }}"
|
||||
{{- end }}
|
||||
{{- if $modelSpec.lmcacheConfig.diskOffloadingBufferSize }}
|
||||
- name: LMCACHE_LOCAL_DISK
|
||||
value: "True"
|
||||
- name: LMCACHE_MAX_LOCAL_DISK_SIZE
|
||||
value: "{{ $modelSpec.lmcacheConfig.diskOffloadingBufferSize }}"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.servingEngineSpec.configs }}
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: "{{ .Release.Name }}-configs"
|
||||
{{- end }}
|
||||
ports:
|
||||
- name: {{ include "chart.container-port-name" . }}
|
||||
containerPort: {{ include "chart.container-port" . }}
|
||||
{{- include "chart.probes" . | indent 10 }}
|
||||
resources: {{- include "chart.resources" $modelSpec | nindent 12 }}
|
||||
{{- if hasKey $modelSpec "pvcStorage" }}
|
||||
volumeMounts:
|
||||
- name: {{ .Release.Name }}-storage
|
||||
mountPath: /data
|
||||
{{- end }}
|
||||
{{- with $modelSpec.vllmConfig }}
|
||||
{{- if hasKey $modelSpec.vllmConfig "tensorParallelSize"}}
|
||||
- name: shm
|
||||
mountPath: /dev/shm
|
||||
{{- end}}
|
||||
{{- end}}
|
||||
volumes:
|
||||
{{- if hasKey $modelSpec "pvcStorage" }}
|
||||
- name: {{ .Release.Name }}-storage
|
||||
persistentVolumeClaim:
|
||||
claimName: "{{ .Release.Name }}-{{$modelSpec.name}}-storage-claim"
|
||||
{{- end }}
|
||||
{{- with $modelSpec.vllmConfig }}
|
||||
{{- if hasKey $modelSpec.vllmConfig "tensorParallelSize"}}
|
||||
- name: shm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: {{ default "20Gi" $modelSpec.shmSize }}
|
||||
{{- end}}
|
||||
{{- end}}
|
||||
{{- if .Values.servingEngineSpec.tolerations }}
|
||||
{{- with .Values.servingEngineSpec.tolerations }}
|
||||
tolerations:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.servingEngineSpec.runtimeClassName }}
|
||||
runtimeClassName: {{ .Values.servingEngineSpec.runtimeClassName }}
|
||||
{{- end }}
|
||||
{{- if $modelSpec.nodeSelectorTerms}}
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
{{- with $modelSpec.nodeSelectorTerms }}
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
---
|
||||
{{- end }}
|
||||
@@ -0,0 +1,42 @@
|
||||
{{- if and (.Values.routerSpec.enableRouter) (.Values.routerSpec.ingress.enabled) -}}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: "{{ .Release.Name }}-ingress-router"
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "chart.routerLabels" . | nindent 4 }}
|
||||
{{- with .Values.routerSpec.ingress.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.routerSpec.ingress.className }}
|
||||
ingressClassName: {{ .Values.routerSpec.ingress.className }}
|
||||
{{- end }}
|
||||
{{- if .Values.routerSpec.ingress.tls }}
|
||||
tls:
|
||||
{{- range .Values.routerSpec.ingress.tls }}
|
||||
- hosts:
|
||||
{{- range .hosts }}
|
||||
- {{ . | quote }}
|
||||
{{- end }}
|
||||
secretName: {{ .secretName }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
rules:
|
||||
{{- range .Values.routerSpec.ingress.hosts }}
|
||||
- host: {{ .host | quote }}
|
||||
http:
|
||||
paths:
|
||||
{{- range .paths }}
|
||||
- path: {{ .path }}
|
||||
pathType: {{ .pathType }}
|
||||
backend:
|
||||
service:
|
||||
name: "{{ $.Release.Name }}-router-service"
|
||||
port:
|
||||
number: {{ $.Values.routerSpec.servicePort }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,7 @@
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: "{{ .Release.Name }}-pdb"
|
||||
namespace: {{ .Release.Namespace }}
|
||||
spec:
|
||||
maxUnavailable: {{ default 1 .Values.servingEngineSpec.maxUnavailablePodDisruptionBudget }}
|
||||
@@ -0,0 +1,30 @@
|
||||
{{- range $modelSpec := .Values.servingEngineSpec.modelSpec }}
|
||||
{{- with $ -}}
|
||||
{{- if and (hasKey $modelSpec "pvcStorage") (not (empty $modelSpec.pvcStorage)) }}
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: "{{ .Release.Name }}-{{$modelSpec.name}}-storage-claim"
|
||||
namespace: {{ .Release.Namespace }}
|
||||
spec:
|
||||
accessModes:
|
||||
{{- if $modelSpec.pvcAccessMode }}
|
||||
{{- toYaml $modelSpec.pvcAccessMode | nindent 4 }}
|
||||
{{- else }}
|
||||
- ReadWriteOnce
|
||||
{{- end }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ $modelSpec.pvcStorage | default "20Gi" }} # Default to 40Gi if not set
|
||||
{{- if hasKey $modelSpec "storageClass" }}
|
||||
storageClassName: "{{ $modelSpec.storageClass }}"
|
||||
{{- end }}
|
||||
{{- if not (empty $modelSpec.pvcMatchLabels) }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- toYaml $modelSpec.pvcMatchLabels | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
---
|
||||
{{- end }}
|
||||
@@ -0,0 +1,11 @@
|
||||
{{- if .Values.routerSpec.enableRouter -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: "{{ .Release.Name }}-pod-reader"
|
||||
namespace: {{ .Release.Namespace }}
|
||||
rules:
|
||||
- apiGroups: [""] # "" indicates the core API group
|
||||
resources: ["pods"]
|
||||
verbs: ["get", "watch", "list"]
|
||||
{{- end }}
|
||||
@@ -0,0 +1,15 @@
|
||||
{{- if .Values.routerSpec.enableRouter -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-deployment-access-binding
|
||||
namespace: {{ .Release.Namespace }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ .Release.Name }}-router-service-account
|
||||
namespace: {{ .Release.Namespace }}
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: {{ .Release.Name }}-pod-reader
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
{{- end }}
|
||||
@@ -0,0 +1,14 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: "{{ .Release.Name }}-secrets"
|
||||
namespace: {{ .Release.Namespace }}
|
||||
type: Opaque
|
||||
data:
|
||||
{{- range $modelSpec := .Values.servingEngineSpec.modelSpec }}
|
||||
{{- with $ -}}
|
||||
{{- if $modelSpec.hf_token }}
|
||||
hf_token_{{ $modelSpec.name }}: {{ $modelSpec.hf_token | b64enc | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,18 @@
|
||||
{{- if .Values.routerSpec.enableRouter -}}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: "{{ .Release.Name }}-router-service"
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "chart.routerLabels" . | nindent 4 }}
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- name: "router-sport"
|
||||
port: {{ .Values.routerSpec.servicePort }}
|
||||
targetPort: {{ .Values.routerSpec.containerPort }}
|
||||
protocol: TCP
|
||||
selector:
|
||||
{{- include "chart.routerLabels" . | nindent 4 }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,16 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: "{{ .Release.Name }}-engine-service"
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "chart.engineLabels" . | nindent 4 }}
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- name: {{ include "chart.service-port-name" . }}
|
||||
port: {{ include "chart.service-port" . }}
|
||||
targetPort: {{ include "chart.container-port-name" . }}
|
||||
protocol: TCP
|
||||
selector:
|
||||
{{- include "chart.engineLabels" . | nindent 4 }}
|
||||
@@ -0,0 +1,7 @@
|
||||
{{- if .Values.routerSpec.enableRouter -}}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: "{{ .Release.Name }}-router-service-account"
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,237 @@
|
||||
# -- Default values for llmstack helm chart
|
||||
# -- Declare variables to be passed into your templates.
|
||||
|
||||
|
||||
# -- Serving engine configuratoon
|
||||
servingEngineSpec:
|
||||
# -- Customized labels for the serving engine deployment
|
||||
labels:
|
||||
environment: "test"
|
||||
release: "test"
|
||||
|
||||
# modelSpec - configuring multiple serving engines deployments that runs different models
|
||||
# Each entry in the modelSpec array should contain the following fields:
|
||||
# - name: (string) The name of the model, e.g., "example-model"
|
||||
# - repository: (string) The repository of the model, e.g., "vllm/vllm-openai"
|
||||
# - tag: (string) The tag of the model, e.g., "latest"
|
||||
# - modelURL: (string) The URL of the model, e.g., "facebook/opt-125m"
|
||||
#
|
||||
# - replicaCount: (int) The number of replicas for the model, e.g. 1
|
||||
# - requestCPU: (int) The number of CPUs requested for the model, e.g. 6
|
||||
# - requestMemory: (string) The amount of memory requested for the model, e.g., "16Gi"
|
||||
# - requestGPU: (int) The number of GPUs requested for the model, e.g., 1
|
||||
#
|
||||
# - pvcStorage: (Optional, string) The amount of storage requested for the model, e.g., "50Gi".
|
||||
# - pvcAccessMode: (Optional, list) The access mode policy for the mounted volume, e.g., ["ReadWriteOnce"]
|
||||
# - storageClass: (Optional, String) The storage class of the PVC e.g., "", default is ""
|
||||
# - pvcMatchLabels: (Optional, map) The labels to match the PVC, e.g., {model: "opt125m"}
|
||||
#
|
||||
# - vllmConfig: (optional, map) The configuration for the VLLM model, supported options are:
|
||||
# - enablePrefixCaching: (optional, bool) Enable prefix caching, e.g., false
|
||||
# - enableChunkedPrefill: (optional, bool) Enable chunked prefill, e.g., false
|
||||
# - maxModelLen: (optional, int) The maximum model length, e.g., 16384
|
||||
# - dtype: (optional, string) The data type, e.g., "bfloat16"
|
||||
# - tensorParallelSize: (optional, int) The degree of tensor parallelism, e.g., 2
|
||||
# - extraArgs: (optional, list) Extra command line arguments to pass to vLLM, e.g., ["--disable-log-requests"]
|
||||
#
|
||||
# - lmcacheConfig: (optional, map) The configuration of the LMCache for KV offloading, supported options are:
|
||||
# - enabled: (optional, bool) Enable LMCache, e.g., true
|
||||
# - cpuOffloadingBufferSize: (optional, string) The CPU offloading buffer size, e.g., "30"
|
||||
#
|
||||
# - hf_token: (optional, string) the Huggingface tokens for this model
|
||||
#
|
||||
# - env: (optional, list) The environment variables to set in the container, e.g., your HF_TOKEN
|
||||
#
|
||||
# - nodeSelectorTerms: (optional, list) The node selector terms to match the nodes
|
||||
#
|
||||
# - shmSize: (optional, string) The size of the shared memory, e.g., "20Gi"
|
||||
#
|
||||
# Example:
|
||||
# modelSpec:
|
||||
# - name: "mistral"
|
||||
# repository: "lmcache/vllm-openai"
|
||||
# tag: "latest"
|
||||
# modelURL: "mistralai/Mistral-7B-Instruct-v0.2"
|
||||
# replicaCount: 1
|
||||
#
|
||||
# requestCPU: 10
|
||||
# requestMemory: "64Gi"
|
||||
# requestGPU: 1
|
||||
#
|
||||
# pvcStorage: "50Gi"
|
||||
# pvcAccessMode:
|
||||
# - ReadWriteOnce
|
||||
# pvcMatchLabels:
|
||||
# model: "mistral"
|
||||
#
|
||||
# vllmConfig:
|
||||
# enableChunkedPrefill: false
|
||||
# enablePrefixCaching: false
|
||||
# maxModelLen: 16384
|
||||
# dtype: "bfloat16"
|
||||
# extraArgs: ["--disable-log-requests", "--gpu-memory-utilization", "0.8"]
|
||||
#
|
||||
# lmcacheConfig:
|
||||
# enabled: true
|
||||
# cpuOffloadingBufferSize: "30"
|
||||
#
|
||||
# hf_token: <HUGGING_FACE_TOKEN>
|
||||
#
|
||||
# nodeSelectorTerms:
|
||||
# - matchExpressions:
|
||||
# - key: nvidia.com/gpu.product
|
||||
# operator: "In"
|
||||
# values:
|
||||
# - "NVIDIA-RTX-A6000"
|
||||
modelSpec: []
|
||||
|
||||
# -- Container port
|
||||
containerPort: 8000
|
||||
# -- Service port
|
||||
servicePort: 80
|
||||
|
||||
# -- Set other environment variables from config map
|
||||
configs: {}
|
||||
|
||||
# -- deployment strategy
|
||||
strategy: {}
|
||||
|
||||
# -- Readiness probe configuration
|
||||
startupProbe:
|
||||
# -- Number of seconds after the container has started before startup probe is initiated
|
||||
initialDelaySeconds: 15
|
||||
# -- How often (in seconds) to perform the startup probe
|
||||
periodSeconds: 10
|
||||
# -- Number of times after which if a probe fails in a row, Kubernetes considers that the overall check has failed: the container is not ready
|
||||
failureThreshold: 60
|
||||
# -- Configuration of the Kubelet http request on the server
|
||||
httpGet:
|
||||
# -- Path to access on the HTTP server
|
||||
path: /health
|
||||
# -- Name or number of the port to access on the container, on which the server is listening
|
||||
port: 8000
|
||||
|
||||
# -- Liveness probe configuration
|
||||
livenessProbe:
|
||||
# -- Number of seconds after the container has started before liveness probe is initiated
|
||||
initialDelaySeconds: 15
|
||||
# -- Number of times after which if a probe fails in a row, Kubernetes considers that the overall check has failed: the container is not alive
|
||||
failureThreshold: 3
|
||||
# -- How often (in seconds) to perform the liveness probe
|
||||
periodSeconds: 10
|
||||
# -- Configuration of the Kubelet http request on the server
|
||||
httpGet:
|
||||
# -- Path to access on the HTTP server
|
||||
path: /health
|
||||
# -- Name or number of the port to access on the container, on which the server is listening
|
||||
port: 8000
|
||||
|
||||
# -- Disruption Budget Configuration
|
||||
maxUnavailablePodDisruptionBudget: ""
|
||||
|
||||
# -- Tolerations configuration (when there are taints on nodes)
|
||||
# Example:
|
||||
# tolerations:
|
||||
# - key: "node-role.kubernetes.io/control-plane"
|
||||
# operator: "Exists"
|
||||
# effect: "NoSchedule"
|
||||
tolerations: []
|
||||
|
||||
# -- RuntimeClassName configuration, set to "nvidia" if the model requires GPU
|
||||
runtimeClassName: "nvidia"
|
||||
|
||||
routerSpec:
|
||||
# -- The docker image of the router. The following values are defaults:
|
||||
repository: "lmcache/lmstack-router"
|
||||
tag: "latest"
|
||||
imagePullPolicy: "Always"
|
||||
|
||||
# -- Whether to enable the router service
|
||||
enableRouter: true
|
||||
|
||||
# -- Number of replicas
|
||||
replicaCount: 1
|
||||
|
||||
# -- Container port
|
||||
containerPort: 8000
|
||||
|
||||
# -- Service port
|
||||
servicePort: 80
|
||||
|
||||
# -- Service discovery mode, supports "k8s" or "static". Defaults to "k8s" if not set.
|
||||
serviceDiscovery: "k8s"
|
||||
|
||||
# -- If serviceDiscovery is set to "static", the comma-separated values below are required. There needs to be the same number of backends and models
|
||||
staticBackends: ""
|
||||
staticModels: ""
|
||||
|
||||
# -- routing logic, could be "roundrobin" or "session"
|
||||
routingLogic: "roundrobin"
|
||||
|
||||
# -- session key if using "session" routing logic
|
||||
sessionKey: ""
|
||||
|
||||
# -- extra router commandline arguments
|
||||
extraArgs: []
|
||||
|
||||
# -- Interval in seconds to scrape the serving engine metrics
|
||||
engineScrapeInterval: 15
|
||||
|
||||
# -- Window size in seconds to calculate the request statistics
|
||||
requestStatsWindow: 60
|
||||
|
||||
# -- deployment strategy
|
||||
strategy: {}
|
||||
|
||||
# -- router resource requests and limits
|
||||
resources:
|
||||
requests:
|
||||
cpu: "4"
|
||||
memory: "16G"
|
||||
limits:
|
||||
cpu: "8"
|
||||
memory: "32G"
|
||||
|
||||
# -- Customized labels for the router deployment
|
||||
labels:
|
||||
environment: "router"
|
||||
release: "router"
|
||||
|
||||
ingress:
|
||||
# -- Enable ingress controller resource
|
||||
enabled: false
|
||||
|
||||
# -- IngressClass that will be used to implement the Ingress
|
||||
className: ""
|
||||
|
||||
# -- Additional annotations for the Ingress resource
|
||||
annotations: {}
|
||||
# kubernetes.io/ingress.class: alb
|
||||
# kubernetes.io/ingress.class: nginx
|
||||
# kubernetes.io/tls-acme: "true"
|
||||
|
||||
# The list of hostnames to be covered with this ingress record.
|
||||
hosts:
|
||||
- host: vllm-router.local
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
|
||||
# -- The tls configuration for hostnames to be covered with this ingress record.
|
||||
tls: []
|
||||
# - secretName: vllm-router-tls
|
||||
# hosts:
|
||||
# - vllm-router.local
|
||||
|
||||
# -- TODO: Readiness probe configuration
|
||||
#startupProbe:
|
||||
# # -- Number of seconds after the container has started before startup probe is initiated
|
||||
# initialDelaySeconds: 5
|
||||
# # -- How often (in seconds) to perform the startup probe
|
||||
# periodSeconds: 5
|
||||
# # -- Number of times after which if a probe fails in a row, Kubernetes considers that the overall check has failed: the container is not ready
|
||||
# failureThreshold: 100
|
||||
# # -- Configuration of the Kubelet http request on the server
|
||||
# httpGet:
|
||||
# # -- Path to access on the HTTP server
|
||||
#
|
||||
Reference in New Issue
Block a user