You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
56 lines
1.3 KiB
56 lines
1.3 KiB
servingEngineSpec:
|
|
modelSpec:
|
|
- name: "llama3"
|
|
repository: "vllm/vllm-openai"
|
|
tag: "latest"
|
|
modelURL: "$servingEngineSpec.modelSpec.modelURL"
|
|
replicaCount: 1
|
|
|
|
requestCPU: 10
|
|
requestMemory: "16Gi"
|
|
requestGPU: 1
|
|
|
|
pvcStorage: "50Gi"
|
|
|
|
vllmConfig:
|
|
enableChunkedPrefill: false
|
|
enablePrefixCaching: false
|
|
maxModelLen: 24576
|
|
dtype: "float16"
|
|
extraArgs: ["--disable-log-requests", "--gpu-memory-utilization", "0.8"]
|
|
hf_token: "$servingEngineSpec.modelSpec.hf_token"
|
|
resources:
|
|
requests:
|
|
cpu: "4"
|
|
memory: "16G"
|
|
limits:
|
|
cpu: "8"
|
|
memory: "32G"
|
|
|
|
routerSpec:
|
|
repository: "lmcache/lmstack-router"
|
|
tag: "latest"
|
|
resources:
|
|
requests:
|
|
cpu: "2"
|
|
memory: "8G"
|
|
limits:
|
|
cpu: "4"
|
|
memory: "16G"
|
|
ingress:
|
|
enabled: true
|
|
className: "apisix"
|
|
annotations:
|
|
cert-manager.io/cluster-issuer: "root-ca-issuer"
|
|
cert-manager.io/duration: 8760h
|
|
cert-manager.io/renew-before: 720h
|
|
k8s.apisix.apache.org/plugin-config-name: oidc-keycloak-authz
|
|
hosts:
|
|
- host: "{{ .Name }}.{{ .Domain }}"
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
tls:
|
|
- secretName: "{{ .Name }}-tls-secret"
|
|
hosts:
|
|
- "{{ .Name }}.{{ .Domain }}"
|
|
|