diff --git a/charts/vllm/custom-values.yaml b/charts/vllm/custom-values.yaml index 54cef51..88aec8d 100644 --- a/charts/vllm/custom-values.yaml +++ b/charts/vllm/custom-values.yaml @@ -1,27 +1,34 @@ servingEngineSpec: modelSpec: - - name: "llama3" - repository: "vllm/vllm-openai" - tag: "latest" - modelURL: "meta-llama/Llama-3.1-8B-Instruct" - replicaCount: 1 + - name: "qwen" + repository: "vllm/vllm-openai" + tag: "latest" + modelURL: "Qwen/Qwen2.5-Coder-14B-Instruct" + replicaCount: 1 - requestCPU: 10 - requestMemory: "16Gi" - requestGPU: 1 + requestCPU: 16 + requestMemory: "32Gi" + requestGPU: 1 - pvcStorage: "50Gi" - - vllmConfig: - enableChunkedPrefill: false - enablePrefixCaching: false - maxModelLen: 24576 - dtype: "float16" - extraArgs: ["--disable-log-requests", "--gpu-memory-utilization", "0.8"] - hf_token: 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 + pvcStorage: "10Gi" + pvcAccessMode: + - ReadWriteMany + storageClass: "nfs" + vllmConfig: + enableChunkedPrefill: false + enablePrefixCaching: false + maxModelLen: 4096 + dtype: "float16" + extraArgs: + [ + "--disable-log-requests", + "--gpu-memory-utilization", + "0.95", + "--download-dir", + "/data/qwen", + ] routerSpec: - # -- The docker image of the router. The following values are defaults: repository: "lmcache/lmstack-router" tag: "latest" enableRouter: true @@ -36,8 +43,8 @@ routerSpec: enabled: true className: "" annotations: - cert-manager.io/cluster-issuer: "selfsigned-issuer" - cert-manager.io/duration: 8760h + cert-manager.io/cluster-issuer: "selfsigned-issuer" + cert-manager.io/duration: 8760h cert-manager.io/renew-before: 720h hosts: - host: vllm.example.com @@ -45,6 +52,6 @@ routerSpec: - path: / pathType: Prefix tls: - - secretName: vllm-tls-secret - hosts: - - vllm.example.com + - secretName: vllm-tls-secret + hosts: + - vllm.example.com