udpate vllm

charts/vllm/custom-values.yaml

@@ -1,27 +1,34 @@
 servingEngineSpec:
   modelSpec:
-  - name: "llama3"
+    - name: "qwen"
-    repository: "vllm/vllm-openai"
+      repository: "vllm/vllm-openai"
-    tag: "latest"
+      tag: "latest"
-    modelURL: "meta-llama/Llama-3.1-8B-Instruct"
+      modelURL: "Qwen/Qwen2.5-Coder-14B-Instruct"
-    replicaCount: 1
+      replicaCount: 1
 
-    requestCPU: 10
+      requestCPU: 16
-    requestMemory: "16Gi"
+      requestMemory: "32Gi"
-    requestGPU: 1
+      requestGPU: 1
 
-    pvcStorage: "50Gi"
+      pvcStorage: "10Gi"
-
+      pvcAccessMode:
-    vllmConfig:
+        - ReadWriteMany
-      enableChunkedPrefill: false
+      storageClass: "nfs"
-      enablePrefixCaching: false
-      maxModelLen: 24576
-      dtype: "float16"
-      extraArgs: ["--disable-log-requests", "--gpu-memory-utilization", "0.8"]
-    hf_token: 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
 
+      vllmConfig:
+        enableChunkedPrefill: false
+        enablePrefixCaching: false
+        maxModelLen: 4096
+        dtype: "float16"
+        extraArgs:
+          [
+            "--disable-log-requests",
+            "--gpu-memory-utilization",
+            "0.95",
+            "--download-dir",
+            "/data/qwen",
+          ]
 routerSpec:
-  # -- The docker image of the router. The following values are defaults:
   repository: "lmcache/lmstack-router"
   tag: "latest"
   enableRouter: true
@@ -45,6 +52,6 @@ routerSpec:
           - path: /
             pathType: Prefix
     tls:
-     - secretName: vllm-tls-secret
+      - secretName: vllm-tls-secret
-       hosts:
+        hosts:
-         - vllm.example.com
+          - vllm.example.com