ChanghoWoo
58 lines
1.3 KiB
YAML
58 lines
1.3 KiB
YAML
servingEngineSpec:
|
|
modelSpec:
|
|
- name: "llama3"
|
|
repository: "vllm/vllm-openai"
|
|
tag: "latest"
|
|
modelURL: "meta-llama/Llama-3.1-8B-Instruct"
|
|
replicaCount: 1
|
|
|
|
requestCPU: 10
|
|
requestMemory: "16Gi"
|
|
requestGPU: 1
|
|
|
|
pvcStorage: "50Gi"
|
|
|
|
vllmConfig:
|
|
enableChunkedPrefill: false
|
|
enablePrefixCaching: false
|
|
maxModelLen: 24576
|
|
dtype: "float16"
|
|
extraArgs: ["--disable-log-requests", "--gpu-memory-utilization", "0.8"]
|
|
hf_token: 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
|
|
resources:
|
|
requests:
|
|
cpu: "4"
|
|
memory: "16G"
|
|
limits:
|
|
cpu: "8"
|
|
memory: "32G"
|
|
|
|
routerSpec:
|
|
# -- The docker image of the router. The following values are defaults:
|
|
repository: "lmcache/lmstack-router"
|
|
tag: "latest"
|
|
enableRouter: true
|
|
resources:
|
|
requests:
|
|
cpu: "2"
|
|
memory: "8G"
|
|
limits:
|
|
cpu: "4"
|
|
memory: "16G"
|
|
ingress:
|
|
enabled: true
|
|
className: ""
|
|
annotations:
|
|
cert-manager.io/cluster-issuer: "selfsigned-issuer"
|
|
cert-manager.io/duration: 8760h
|
|
cert-manager.io/renew-before: 720h
|
|
hosts:
|
|
- host: vllm.example.com
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
tls:
|
|
- secretName: vllm-tls-secret
|
|
hosts:
|
|
- vllm.example.com
|