diff --git a/charts/vllm/custom-values.yaml b/charts/vllm/custom-values.yaml index cefca80..7c795d4 100644 --- a/charts/vllm/custom-values.yaml +++ b/charts/vllm/custom-values.yaml @@ -1,5 +1,24 @@ servingEngineSpec: - modelSpec: [] + modelSpec: + - name: "llama3" + repository: "vllm/vllm-openai" + tag: "latest" + modelURL: "meta-llama/Llama-3.1-8B-Instruct" + replicaCount: 1 + + requestCPU: 10 + requestMemory: "16Gi" + requestGPU: 1 + + pvcStorage: "50Gi" + + vllmConfig: + enableChunkedPrefill: false + enablePrefixCaching: false + maxModelLen: 24576 + dtype: "float16" + extraArgs: ["--disable-log-requests", "--gpu-memory-utilization", "0.8"] + hf_token: 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 resources: requests: cpu: "4" @@ -12,6 +31,7 @@ routerSpec: # -- The docker image of the router. The following values are defaults: repository: "lmcache/lmstack-router" tag: "latest" + enableRouter: true resources: requests: cpu: "2"