apiVersion: serving.kserve.io/v1alpha1 kind: ClusterServingRuntime metadata: name: kserve-huggingfaceserver spec: annotations: prometheus.kserve.io/path: /metrics prometheus.kserve.io/port: "8080" containers: - args: - --model_name={{.Name}} image: kserve/huggingfaceserver:v0.13.0 name: kserve-container resources: limits: cpu: "1" memory: 2Gi requests: cpu: "1" memory: 2Gi protocolVersions: - v2 - v1 supportedModelFormats: - autoSelect: true name: huggingface priority: 1 version: "1" --- apiVersion: serving.kserve.io/v1alpha1 kind: ClusterServingRuntime metadata: name: kserve-lgbserver spec: annotations: prometheus.kserve.io/path: /metrics prometheus.kserve.io/port: "8080" containers: - args: - --model_name={{.Name}} - --model_dir=/mnt/models - --http_port=8080 - --nthread=1 image: kserve/lgbserver:v0.13.0 name: kserve-container resources: limits: cpu: "1" memory: 2Gi requests: cpu: "1" memory: 2Gi protocolVersions: - v1 - v2 supportedModelFormats: - autoSelect: true name: lightgbm priority: 1 version: "3" --- apiVersion: serving.kserve.io/v1alpha1 kind: ClusterServingRuntime metadata: name: kserve-mlserver spec: annotations: prometheus.kserve.io/path: /metrics prometheus.kserve.io/port: "8080" containers: - env: - name: MLSERVER_MODEL_IMPLEMENTATION value: '{{.Labels.modelClass}}' - name: MLSERVER_HTTP_PORT value: "8080" - name: MLSERVER_GRPC_PORT value: "9000" - name: MODELS_DIR value: /mnt/models image: docker.io/seldonio/mlserver:1.3.2 name: kserve-container resources: limits: cpu: "1" memory: 2Gi requests: cpu: "1" memory: 2Gi protocolVersions: - v2 supportedModelFormats: - autoSelect: true name: sklearn priority: 2 version: "0" - autoSelect: true name: sklearn priority: 2 version: "1" - autoSelect: true name: xgboost priority: 2 version: "1" - autoSelect: true name: lightgbm priority: 2 version: "3" - autoSelect: true name: mlflow priority: 1 version: "1" --- apiVersion: serving.kserve.io/v1alpha1 kind: ClusterServingRuntime metadata: name: kserve-paddleserver spec: annotations: prometheus.kserve.io/path: /metrics prometheus.kserve.io/port: "8080" containers: - args: - --model_name={{.Name}} - --model_dir=/mnt/models - --http_port=8080 image: kserve/paddleserver:v0.13.0 name: kserve-container resources: limits: cpu: "1" memory: 2Gi requests: cpu: "1" memory: 2Gi protocolVersions: - v1 - v2 supportedModelFormats: - autoSelect: true name: paddle priority: 1 version: "2" --- apiVersion: serving.kserve.io/v1alpha1 kind: ClusterServingRuntime metadata: name: kserve-pmmlserver spec: annotations: prometheus.kserve.io/path: /metrics prometheus.kserve.io/port: "8080" containers: - args: - --model_name={{.Name}} - --model_dir=/mnt/models - --http_port=8080 image: kserve/pmmlserver:v0.13.0 name: kserve-container resources: limits: cpu: "1" memory: 2Gi requests: cpu: "1" memory: 2Gi protocolVersions: - v1 - v2 supportedModelFormats: - autoSelect: true name: pmml priority: 1 version: "3" - autoSelect: true name: pmml priority: 1 version: "4" --- apiVersion: serving.kserve.io/v1alpha1 kind: ClusterServingRuntime metadata: name: kserve-sklearnserver spec: annotations: prometheus.kserve.io/path: /metrics prometheus.kserve.io/port: "8080" containers: - args: - --model_name={{.Name}} - --model_dir=/mnt/models - --http_port=8080 image: kserve/sklearnserver:v0.13.0 name: kserve-container resources: limits: cpu: "1" memory: 2Gi requests: cpu: "1" memory: 2Gi protocolVersions: - v1 - v2 supportedModelFormats: - autoSelect: true name: sklearn priority: 1 version: "1" --- apiVersion: serving.kserve.io/v1alpha1 kind: ClusterServingRuntime metadata: name: kserve-tensorflow-serving spec: annotations: prometheus.kserve.io/path: /metrics prometheus.kserve.io/port: "8080" containers: - args: - --model_name={{.Name}} - --port=9000 - --rest_api_port=8080 - --model_base_path=/mnt/models - --rest_api_timeout_in_ms=60000 command: - /usr/bin/tensorflow_model_server image: tensorflow/serving:2.6.2 name: kserve-container resources: limits: cpu: "1" memory: 2Gi requests: cpu: "1" memory: 2Gi protocolVersions: - v1 - grpc-v1 supportedModelFormats: - autoSelect: true name: tensorflow priority: 2 version: "1" - autoSelect: true name: tensorflow priority: 2 version: "2" --- apiVersion: serving.kserve.io/v1alpha1 kind: ClusterServingRuntime metadata: name: kserve-torchserve spec: annotations: prometheus.kserve.io/path: /metrics prometheus.kserve.io/port: "8082" containers: - args: - torchserve - --start - --model-store=/mnt/models/model-store - --ts-config=/mnt/models/config/config.properties env: - name: TS_SERVICE_ENVELOPE value: '{{.Labels.serviceEnvelope}}' image: pytorch/torchserve-kfs:0.9.0 name: kserve-container resources: limits: cpu: "1" memory: 2Gi requests: cpu: "1" memory: 2Gi protocolVersions: - v1 - v2 - grpc-v2 supportedModelFormats: - autoSelect: true name: pytorch priority: 2 version: "1" --- apiVersion: serving.kserve.io/v1alpha1 kind: ClusterServingRuntime metadata: name: kserve-tritonserver spec: annotations: prometheus.kserve.io/path: /metrics prometheus.kserve.io/port: "8002" containers: - args: - tritonserver - --model-store=/mnt/models - --grpc-port=9000 - --http-port=8080 - --allow-grpc=true - --allow-http=true image: nvcr.io/nvidia/tritonserver:23.05-py3 name: kserve-container resources: limits: cpu: "1" memory: 2Gi requests: cpu: "1" memory: 2Gi protocolVersions: - v2 - grpc-v2 supportedModelFormats: - autoSelect: true name: tensorrt priority: 1 version: "8" - autoSelect: true name: tensorflow priority: 1 version: "1" - autoSelect: true name: tensorflow priority: 1 version: "2" - autoSelect: true name: onnx priority: 1 version: "1" - name: pytorch version: "1" - autoSelect: true name: triton priority: 1 version: "2" --- apiVersion: serving.kserve.io/v1alpha1 kind: ClusterServingRuntime metadata: name: kserve-xgbserver spec: annotations: prometheus.kserve.io/path: /metrics prometheus.kserve.io/port: "8080" containers: - args: - --model_name={{.Name}} - --model_dir=/mnt/models - --http_port=8080 - --nthread=1 image: kserve/xgbserver:v0.13.0 name: kserve-container resources: limits: cpu: "1" memory: 2Gi requests: cpu: "1" memory: 2Gi protocolVersions: - v1 - v2 supportedModelFormats: - autoSelect: true name: xgboost priority: 1 version: "1" --- apiVersion: serving.kserve.io/v1alpha1 kind: ClusterStorageContainer metadata: name: default spec: container: image: kserve/storage-initializer:v0.13.0 name: storage-initializer resources: limits: cpu: "1" memory: 1Gi requests: cpu: 100m memory: 100Mi supportedUriFormats: - prefix: gs:// - prefix: s3:// - prefix: hdfs:// - prefix: webhdfs:// - regex: https://(.+?).blob.core.windows.net/(.+) - regex: https://(.+?).file.core.windows.net/(.+) - regex: https?://(.+)/(.+)