You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
60 lines
1.3 KiB
60 lines
1.3 KiB
servingEngineSpec:
|
|
modelSpec:
|
|
- name: "qwen"
|
|
repository: "vllm/vllm-openai"
|
|
tag: "latest"
|
|
modelURL: "Qwen/Qwen2.5-Coder-14B-Instruct"
|
|
replicaCount: 1
|
|
|
|
requestCPU: 16
|
|
requestMemory: "32Gi"
|
|
requestGPU: 1
|
|
|
|
pvcStorage: "10Gi"
|
|
pvcAccessMode:
|
|
- ReadWriteMany
|
|
storageClass: "nfs"
|
|
|
|
vllmConfig:
|
|
enableChunkedPrefill: false
|
|
enablePrefixCaching: false
|
|
maxModelLen: 4096
|
|
dtype: "float16"
|
|
extraArgs:
|
|
[
|
|
"--disable-log-requests",
|
|
"--gpu-memory-utilization",
|
|
"0.95",
|
|
"--download-dir",
|
|
"/data/qwen",
|
|
]
|
|
env:
|
|
- name: HF_HUB_OFFLINE
|
|
value: "1"
|
|
routerSpec:
|
|
repository: "lmcache/lmstack-router"
|
|
tag: "latest"
|
|
enableRouter: true
|
|
resources:
|
|
requests:
|
|
cpu: "2"
|
|
memory: "8G"
|
|
limits:
|
|
cpu: "4"
|
|
memory: "16G"
|
|
ingress:
|
|
enabled: true
|
|
className: ""
|
|
annotations:
|
|
cert-manager.io/cluster-issuer: "selfsigned-issuer"
|
|
cert-manager.io/duration: 8760h
|
|
cert-manager.io/renew-before: 720h
|
|
hosts:
|
|
- host: vllm.example.com
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
tls:
|
|
- secretName: vllm-tls-secret
|
|
hosts:
|
|
- vllm.example.com
|
|
|