init nemo chart

This commit is contained in:
Your Name
2025-06-05 08:36:12 +00:00
parent dbfbebb13c
commit f2203cd282
1235 changed files with 151843 additions and 0 deletions
+39
View File
@@ -0,0 +1,39 @@
dependencies:
- name: nemo-guardrails
repository: file://components/guardrails
version: 0.2.0
- name: nemo-entity-store
repository: file://components/entity-store
version: 0.2.0
- name: nemo-datastore
repository: file://components/datastore
version: 0.5.1
- name: nemo-customizer
repository: file://components/customizer
version: 0.11.0-alpha.13
- name: nemo-evaluator
repository: file://components/evaluator
version: 0.2.4
- name: nemo-operator
repository: file://components/nemo-operator
version: 0.2.0
- name: nemo-deployment-management
repository: file://components/deployment-management
version: 0.2.1
- name: nemo-nim-proxy
repository: file://components/nim-proxy
version: 0.2.0
- name: nim-llm
repository: https://helm.ngc.nvidia.com/nvidian/nemo-llm
version: 1.7.0-rc1
- name: k8s-nim-operator
repository: https://helm.ngc.nvidia.com/nvidia/
version: 1.0.1
- name: dgxc-admission-controller
repository: https://helm.ngc.nvidia.com/nvidian/nemo-llm
version: 1.642.0
- name: volcano
repository: https://volcano-sh.github.io/helm-charts
version: 1.9.0
digest: sha256:a182a437e6ff1efa1a0a6b56cf7533e612fe9a6ee087ad7f813193f63bcf0bd4
generated: "2025-04-16T22:58:40.429920984Z"
+68
View File
@@ -0,0 +1,68 @@
apiVersion: v2
dependencies:
- alias: guardrails
condition: guardrails.enabled
name: nemo-guardrails
repository: file://components/guardrails
version: '>=0.1.0-0'
- alias: entity-store
condition: entity-store.enabled
name: nemo-entity-store
repository: file://components/entity-store
version: '>=0.1.0-0'
- alias: data-store
condition: data-store.enabled
name: nemo-datastore
repository: file://components/datastore
version: '>=0.1.0-0'
- alias: customizer
condition: customizer.enabled
name: nemo-customizer
repository: file://components/customizer
version: '>=0.1.0-0'
- alias: evaluator
condition: evaluator.enabled
name: nemo-evaluator
repository: file://components/evaluator
version: '>=0.1.0-0'
- alias: nemo-operator
condition: nemo-operator.enabled
name: nemo-operator
repository: file://components/nemo-operator
version: '>=0.1.0-0'
- alias: deployment-management
condition: deployment-management.enabled
name: nemo-deployment-management
repository: file://components/deployment-management
version: '>=0.1.0-0'
- alias: nim-proxy
condition: nim-proxy.enabled
name: nemo-nim-proxy
repository: file://components/nim-proxy
version: '>=0.1.0-0'
- alias: nim
condition: nim.enabled
name: nim-llm
repository: https://helm.ngc.nvidia.com/nvidian/nemo-llm
version: 1.7.0-rc1
- alias: nim-operator
condition: nim-operator.enabled
name: k8s-nim-operator
repository: https://helm.ngc.nvidia.com/nvidia/
version: 1.0.1
- condition: dgxc-admission-controller.enabled
name: dgxc-admission-controller
repository: https://helm.ngc.nvidia.com/nvidian/nemo-llm
version: 1.642.0
- condition: volcano.enabled
name: volcano
repository: https://volcano-sh.github.io/helm-charts
version: 1.9.0
description: NeMo Microservices Helm Chart
home: https://nvidia.com
maintainers:
- name: NVIDIA, Inc.
url: https://www.nvidia.com
name: nemo-microservices-helm-chart
type: application
version: 25.4.0
+152
View File
@@ -0,0 +1,152 @@
# NeMo Microservices Helm Chart
![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
For deployment guide, see [Admin Setup](https://docs.nvidia.com/nemo/microservices/latest/set-up/index.html) in the NeMo Microservices documentation.
## Values
| Key | Type | Default | Description |
|-----|------|---------|-------------|
| customizer.customizerConfig.entityStoreURL | string | `"http://nemo-entity-store:8000"` | Specifies the internal K8s DNS record for the NeMo Entity Store service. |
| customizer.customizerConfig.mlflowURL | string | `""` | Specifies the internal K8s DNS record for the MLflow service. |
| customizer.customizerConfig.models | object | `{"meta/llama-3.1-8b-instruct":{"enabled":true}}` | Specifies the model information to configure with the NeMo Customizer microservice. To find a complete list of models that NeMo Customizer supports, see [Model Catalog](https://docs.nvidia.com/nemo/microservices/latest/fine-tune/models/index.html). |
| customizer.customizerConfig.models."meta/llama-3.1-8b-instruct".enabled | bool | `true` | Specifies whether to use the `meta/llama-3.1-8b-instruct` model for a customization job. |
| customizer.customizerConfig.nemoDataStoreURL | string | `"http://nemo-data-store:3000"` | Specifies the internal K8s DNS record for the NeMo Data Store service. |
| customizer.customizerConfig.training.pvc | object | `{"storageClass":""}` | Specifies where to store the customization job information during training. |
| customizer.enabled | bool | `true` | Specifies whether to install the NeMo Customizer microservice. |
| customizer.modelsStorage | object | `{"accessModes":["ReadWriteOnce"],"enabled":true,"size":"20Gi","storageClassName":""}` | Specifies whether to enable the PVC for storing base models. |
| customizer.modelsStorage.accessModes | list | `["ReadWriteOnce"]` | Specifies the access modes for the PVC. |
| customizer.modelsStorage.size | string | `"20Gi"` | Specifies the size of the PVC. |
| customizer.modelsStorage.storageClassName | string | `""` | Specifies the storage class to use for the PVC. |
| customizer.nemoDataStoreTools | object | `{"imagePullSecret":"nvcrimagepullsecret"}` | Specifies the image pull secret for the NeMo Data Store tools. |
| customizer.postgresql.nameOverride | string | `"customizerdb"` | The name override for the PostgreSQL database. |
| customizer.postgresql.serviceAccount.create | bool | `true` | Specifies whether to create a new service account for PostgreSQL. |
| customizer.postgresql.serviceAccount.name | string | `"customizer-postgresql"` | |
| customizer.serviceName | string | `"nemo-customizer"` | |
| customizer.workspaceStorage.enabled | bool | `false` | Specifies whether to enable the PVC for storing a training job's workspace. |
| data-store.enabled | bool | `true` | Specifies whether to install the NeMo Data Store microservice. |
| data-store.objectStore.enabled | bool | `false` | Specifies whether to enable object storage for the NeMo Data Store microservice. |
| data-store.persistence.accessModes | list | `["ReadWriteOnce"]` | The access modes for the persistent volume. |
| data-store.persistence.claimName | string | `"datastore-shared-storage"` | The name of the persistent volume claim. |
| data-store.persistence.enabled | bool | `true` | Specifies whether to enable persistent volume for the NeMo Data Store microservice. |
| data-store.persistence.size | string | `"1Gi"` | The size of the persistent volume. |
| data-store.persistence.storageClass | string | `""` | The storage class to use for the persistent volume. |
| data-store.replicaCount | int | `1` | The number of replicas for the NeMo Data Store microservice deployment. |
| data-store.serviceAccount.create | bool | `true` | Specifies whether to create a service account for the NeMo Data Store microservice. |
| data-store.serviceAccount.name | string | `"gitea"` | The service account name for the NeMo Data Store microservice. |
| data-store.serviceName | string | `"nemo-data-store"` | The service name for the NeMo Data Store microservice. |
| data-store.strategy.type | string | `"Recreate"` | The type of deployment strategy to use for the NeMo Data Store microservice. |
| deployment-management.deployments.entityStoreUrl | string | `"http://nemo-entity-store:8000"` | The URL for the NeMo Entity Store service. |
| deployment-management.deployments.modelSyncPeriod | string | `"30"` | The period in seconds for model synchronization. |
| deployment-management.deployments.nimImagePullSecrets | list | `["nvcrimagepullsecret"]` | The image pull secrets for the NeMo Deployment Management microservice. |
| deployment-management.deployments.nimPeftSource | string | `"http://nemo-entity-store:8000"` | The URL for the NIM PEFT source. |
| deployment-management.enabled | bool | `true` | Specifies whether to install the NeMo Deployment Management microservice. |
| deployment-management.image.pullPolicy | string | `"IfNotPresent"` | The pull policy for the NeMo Deployment Management container image. |
| deployment-management.serviceName | string | `"nemo-deployment-management"` | The service name for the NeMo Deployment Management microservice. |
| dgxc-admission-controller.cloudProviderSpec.dummy | object | `{"type":"dummy"}` | The type of the dummy cloud provider. |
| dgxc-admission-controller.dgxcController.image | string | `"nvcr.io/nvidia/nemo-microservices/dgxc-admission-controller:1.642.0"` | The image for the DGX Cloud Admission Controller. |
| dgxc-admission-controller.enableDGXCRbac | bool | `false` | |
| dgxc-admission-controller.enableIntegrationsSupport | bool | `false` | Specifies whether to enable integrations support. |
| dgxc-admission-controller.enableLabeler | bool | `false` | Specifies whether to enable the labeler. |
| dgxc-admission-controller.enableNetworkPolicies | bool | `false` | Specifies whether to enable network policies. |
| dgxc-admission-controller.enabled | bool | `false` | Specifies whether to install the DGX Cloud Admission Controller. |
| dgxc-admission-controller.imagePullSecret | string | `"nvcrimagepullsecret"` | The image pull secret for the DGX Cloud Admission Controller. |
| dgxc-admission-controller.kubernetesAffinity.enabled | bool | `false` | Specifies whether to enable Kubernetes affinity. |
| entity-store.appConfig.BASE_URL_DATASTORE | string | `"http://nemo-data-store:3000/v1/hf"` | The base URL for the NeMo Data Store microservice. |
| entity-store.appConfig.BASE_URL_NIM | string | `"http://nemo-nim-proxy:8000"` | The base URL for the NIM service used by the NeMo Entity Store microservice. |
| entity-store.enabled | bool | `true` | Specifies whether to install the NeMo Entity Store microservice. |
| entity-store.postgresql.nameOverride | string | `"entity-storedb"` | The name override for the PostgreSQL database. |
| entity-store.postgresql.serviceAccount.create | bool | `true` | Specifies whether to create a new service account for PostgreSQL. |
| entity-store.postgresql.serviceAccount.name | string | `"entity-store-postgresql"` | The service account name for PostgreSQL. |
| entity-store.serviceName | string | `"nemo-entity-store"` | The service name for the NeMo Entity Store microservice. |
| evaluator.argoWorkflows.controller.serviceAccount.name | string | `"argo-workflows-controller"` | The name of the service account for the Argo Workflows controller. |
| evaluator.argoWorkflows.crds.install | bool | `true` | Specifies whether to install the CRDs for Argo Workflows. |
| evaluator.argoWorkflows.enabled | bool | `true` | Specifies whether to enable the default Argo Workflows integration for the NeMo Evaluator microservice. |
| evaluator.argoWorkflows.server.authModes[0] | string | `"server"` | |
| evaluator.argoWorkflows.server.secure | bool | `true` | Specifies whether to secure the connection to the Argo Workflows server. |
| evaluator.argoWorkflows.server.serviceAccount.name | string | `"argo-workflows-server"` | The name of the service account for the Argo Workflows server. |
| evaluator.argoWorkflows.server.servicePort | int | `2746` | The service port for the Argo Workflows server. |
| evaluator.argoWorkflows.serviceName | string | `"argo-workflows-server"` | The Argo Workflows service name for the NeMo Evaluator microservice. |
| evaluator.enabled | bool | `true` | Specifies whether to install the NeMo Evaluator microservice. |
| evaluator.external.dataStore | object | `{"endpoint":"http://nemo-data-store:3000/v1/hf"}` | The external URL of the NeMo Data Store microservice. |
| evaluator.milvus.standalone.persistence.persistentVolumeClaim.storageClass | string | `""` | Specifies the storage class to use for the Milvus PVC. |
| evaluator.postgresql.nameOverride | string | `"evaluatordb"` | The name override for the default PostgreSQL database. |
| evaluator.postgresql.primary.networkPolicy.enabled | bool | `false` | Specifies whether to enable the network policy for the PostgreSQL service. |
| evaluator.postgresql.serviceAccount.name | string | `"evaluator-postgresql"` | The name of the service account for PostgreSQL. |
| evaluator.serviceName | string | `"nemo-evaluator"` | |
| existingImagePullSecret | string | `""` | You can specify an existing Kubernetes image pull secret for pulling images from the container registry. The chart doesn't reference any image pull secret if you leave this empty. |
| existingSecret | string | `""` | You can use an existing Kubernetes secret for pulling images. The chart uses the `ngcAPIKey` value to generate the secret if you leave this empty. |
| global.imagePullSecrets | list | `[{"name":"nvcrimagepullsecret"}]` | The name of the image pull secret to use globally across all services. |
| guardrails.enabled | bool | `true` | Specifies whether to install the NeMo Guardrails microservice. |
| guardrails.env | object | `{"NIM_ENDPOINT_URL":"http://nemo-nim-proxy:8000/v1"}` | The environment variables for the NeMo Guardrails microservice. |
| guardrails.env.NIM_ENDPOINT_URL | string | `"http://nemo-nim-proxy:8000/v1"` | The NIM endpoint URL for the NeMo Guardrails microservice. |
| guardrails.serviceName | string | `"nemo-guardrails"` | The service name for the NeMo Guardrails microservice. |
| imagePullSecrets | list | `[{"name":"nvcrimagepullsecret","password":"YOUR-NGC-API-KEY","registry":"nvcr.io","username":"$$oauthtoken"}]` | List of image pull secrets. Existing secrets override these values if you specify them. Use this only for experimentation when you want to hardcode a secret in your values file. |
| ingress.annotations | object | `{}` | Annotations for the ingress resource. |
| ingress.className | string | `""` | The ingress class to use if your cluster has more than one class. |
| ingress.enabled | bool | `false` | Specifies whether to enable the ingress. |
| ingress.hosts | object | `{"dataStore":{"name":"data-store.test","paths":[{"path":"/","pathType":"Prefix","port":3000,"service":"nemo-data-store"}]},"default":{"name":"","paths":[{"path":"/v1/namespaces","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/projects","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/datasets","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/repos","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/models","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/customization","pathType":"Prefix","port":8000,"service":"nemo-customizer"},{"path":"/v1/evaluation","pathType":"Prefix","port":7331,"service":"nemo-evaluator"},{"path":"/v1/guardrail","pathType":"Prefix","port":7331,"service":"nemo-guardrails"},{"path":"/v1/deployment","pathType":"Prefix","port":8000,"service":"nemo-deployment-management"}]},"nimProxy":{"name":"nim.test","paths":[{"path":"/v1/completions","pathType":"Prefix","port":8000,"service":"nemo-nim-proxy"},{"path":"/v1/chat","pathType":"Prefix","port":8000,"service":"nemo-nim-proxy"},{"path":"/v1/models","pathType":"Prefix","port":8000,"service":"nemo-nim-proxy"}]}}` | A map of hosts and their corresponding paths for the ingress. |
| ingress.hosts.dataStore.name | string | `"data-store.test"` | The host name for the third ingress host for the NeMo Data Store microservice. |
| ingress.hosts.dataStore.paths | list | `[{"path":"/","pathType":"Prefix","port":3000,"service":"nemo-data-store"}]` | The path rules for the third ingress host. |
| ingress.hosts.default.name | string | `""` | The host name for the default ingress host. |
| ingress.hosts.default.paths | list | `[{"path":"/v1/namespaces","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/projects","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/datasets","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/repos","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/models","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/customization","pathType":"Prefix","port":8000,"service":"nemo-customizer"},{"path":"/v1/evaluation","pathType":"Prefix","port":7331,"service":"nemo-evaluator"},{"path":"/v1/guardrail","pathType":"Prefix","port":7331,"service":"nemo-guardrails"},{"path":"/v1/deployment","pathType":"Prefix","port":8000,"service":"nemo-deployment-management"}]` | The path rules for the default ingress host. |
| ingress.hosts.nimProxy.name | string | `"nim.test"` | The host name for the second ingress host for the NIM Proxy microservice. |
| ingress.hosts.nimProxy.paths | list | `[{"path":"/v1/completions","pathType":"Prefix","port":8000,"service":"nemo-nim-proxy"},{"path":"/v1/chat","pathType":"Prefix","port":8000,"service":"nemo-nim-proxy"},{"path":"/v1/models","pathType":"Prefix","port":8000,"service":"nemo-nim-proxy"}]` | The path rules for the second ingress host. |
| ingress.tls | list | `[]` | TLS configurations. |
| nemo-operator.controllerManager.manager.scheduler | string | `"volcano"` | The scheduler to use for the NeMo Operator. Available options: `volcano` and `runai`. |
| nemo-operator.enabled | bool | `true` | Specifies whether to install the NeMo Operator microservice. |
| ngcAPIKey | string | `"YOUR-NGC-API-KEY"` | Your NVIDIA GPU Cloud (NGC) API key authenticates and enables pulling images from the NGC container registry. The existing secret overrides this key if you provide one to the `existingSecret` key. |
| nim-operator.enabled | bool | `true` | Specifies whether to install the NIM Operator. |
| nim-operator.nfd.nodeFeatureRules.deviceID | bool | `false` | Specifies whether to enable device ID feature rules. |
| nim-proxy.enabled | bool | `true` | Specifies whether to install the NIM Proxy microservice. |
| nim-proxy.monitoring.enabled | bool | `false` | Specifies whether to enable monitoring for the NIM Proxy microservice. |
| nim-proxy.serviceName | string | `"nemo-nim-proxy"` | The service name for the NIM Proxy microservice. |
| nim.enabled | bool | `false` | Specifies whether to deploy a NIM for LLM during the Helm installation of the chart. You can deploy a single static NIM by enabling this object and its values. When enabled, the chart deploys `meta/llama-3.1-8b-instruct` as the default NIM. |
| nim.env | list | `[{"name":"NIM_PEFT_SOURCE","value":"http://nemo-entity-store:8000"},{"name":"NIM_PEFT_REFRESH_INTERVAL","value":"30"},{"name":"NIM_MAX_CPU_LORAS","value":"16"},{"name":"NIM_MAX_GPU_LORAS","value":"8"}]` | Environment variables for the NIM service. |
| nim.image.repository | string | `"nvcr.io/nim/meta/llama-3.1-8b-instruct"` | |
| nim.image.tag | string | `"1.8.3"` | |
| nim.imagePullSecrets[0].name | string | `"nvcrimagepullsecret"` | |
| nim.model.name | string | `"meta/llama-3.1-8b-instruct"` | The name of the model to deploy as NIM. |
| nim.model.ngcAPISecret | string | `"ngc-api"` | The NGC API secret for model access. |
| nim.persistence.annotations | object | `{"helm.sh/resource-policy":"keep"}` | Annotations for the PVC. |
| nim.persistence.enabled | bool | `true` | Specifies whether to enable persistence volume claim (PVC) for the NIM service. |
| nim.persistence.storageClass | string | `""` | Specifies the storage class to use for the PVC. |
| nim.resources | object | `{"limits":{"nvidia.com/gpu":1},"requests":{"nvidia.com/gpu":1}}` | Specifies resources for the NIM service. |
| nim.resources.requests | object | `{"nvidia.com/gpu":1}` | Specifies requests for the NIM service. |
| nim.service.labels | object | `{"app.nvidia.com/nim-type":"inference"}` | Labels for the NIM service. |
| nim.statefulSet.enabled | bool | `false` | Specifies whether to enable a stateful set for the NIM service. |
| virtualService | object | A virtual service configuration template. | Specifies whether to enable the virtual service. If you are not using istio and virtualservices, it can be useful to create some virtual services for the NeMo Microservices system. Don't enable unless you use istio. |
| virtualService.additional.data-store | object | `{"entries":{"data-store":{"corsPolicy":{},"match":[{"uri":{"prefix":"/"}}],"route":[{"destination":{"host":"nemo-data-store","port":{"number":3000}}}]}},"gateways":[],"hosts":[]}` | Additional virtual service configurations. |
| virtualService.additional.data-store.entries | object | `{"data-store":{"corsPolicy":{},"match":[{"uri":{"prefix":"/"}}],"route":[{"destination":{"host":"nemo-data-store","port":{"number":3000}}}]}}` | The entries for the virtual NeMo Data Store service. |
| virtualService.additional.data-store.entries.data-store.corsPolicy | object | `{}` | The CORS policy for the virtual NeMo Data Store service. |
| virtualService.additional.data-store.entries.data-store.match | list | `[{"uri":{"prefix":"/"}}]` | The match for the virtual NeMo Data Store service. |
| virtualService.additional.data-store.entries.data-store.route | list | `[{"destination":{"host":"nemo-data-store","port":{"number":3000}}}]` | The route for the virtual NeMo Data Store service. |
| virtualService.additional.data-store.gateways | list | `[]` | The gateways for the virtual NeMo Data Store service. |
| virtualService.additional.data-store.hosts | list | `[]` | The hosts for the virtual NeMo Data Store service. |
| virtualService.additional.nim-proxy.entries.nim-proxy.corsPolicy | object | `{}` | The CORS policy for the virtual NIM Proxy service. |
| virtualService.additional.nim-proxy.entries.nim-proxy.match | list | `[{"uri":{"prefix":"/"}}]` | The match for the virtual NIM Proxy service. |
| virtualService.additional.nim-proxy.entries.nim-proxy.route | list | `[{"destination":{"host":"nemo-nim-proxy","port":{"number":8000}}}]` | The route for the virtual NIM Proxy service. |
| virtualService.additional.nim-proxy.gateways | list | `[]` | The gateways for the virtual NIM Proxy service. |
| virtualService.additional.nim-proxy.hosts | list | `[]` | The hosts for the virtual NIM Proxy service. |
| virtualService.annotations | object | `{}` | Annotations for the virtual service. |
| virtualService.enabled | bool | `false` | Specifies whether to enable the virtual service. |
| virtualService.labels | object | `{}` | Labels for the virtual service. |
| virtualService.main.entries.customizer.corsPolicy | object | `{}` | The CORS policy for the virtual NeMo Customizer service. |
| virtualService.main.entries.customizer.match | list | `[{"uri":{"prefix":"/v1/customization"}}]` | The match for the virtual NeMo Customizer service. |
| virtualService.main.entries.customizer.route | list | `[{"destination":{"host":"nemo-customizer","port":{"number":8000}}}]` | The route for the virtual NeMo Customizer service. |
| virtualService.main.entries.deployment-management.corsPolicy | object | `{}` | The CORS policy for the virtual NeMo Deployment Management service. |
| virtualService.main.entries.deployment-management.match | list | `[{"uri":{"prefix":"/v1/deployment"}}]` | The match for the virtual NeMo Deployment Management service. |
| virtualService.main.entries.deployment-management.route | list | `[{"destination":{"host":"nemo-deployment-management","port":{"number":8000}}}]` | The route for the virtual NeMo Deployment Management service. |
| virtualService.main.entries.entity-store.corsPolicy | object | `{}` | The CORS policy for the virtual NeMo Entity Store service. |
| virtualService.main.entries.entity-store.match | list | `[{"uri":{"prefix":"/v1/namespaces"}},{"uri":{"prefix":"/v1/projects"}},{"uri":{"prefix":"/v1/datasets"}},{"uri":{"prefix":"/v1/repos"}},{"uri":{"prefix":"/v1/models"}}]` | The match for the virtual NeMo Entity Store service. |
| virtualService.main.entries.entity-store.route | list | `[{"destination":{"host":"nemo-entity-store","port":{"number":8000}}}]` | The route for the virtual NeMo Entity Store service. |
| virtualService.main.entries.evaluator.corsPolicy | object | `{}` | The CORS policy for the virtual NeMo Evaluator service. |
| virtualService.main.entries.evaluator.match | list | `[{"uri":{"prefix":"/v1/evaluation"}}]` | The match for the virtual NeMo Evaluator service. |
| virtualService.main.entries.evaluator.route | list | `[{"destination":{"host":"nemo-evaluator","port":{"number":7331}}}]` | The route for the virtual NeMo Evaluator service. |
| virtualService.main.entries.guardrails.corsPolicy | object | `{}` | The CORS policy for the virtual NeMo Guardrails service. |
| virtualService.main.entries.guardrails.match | list | `[{"uri":{"prefix":"/v1/guardrail"}}]` | The match for the virtual NeMo Guardrails service. |
| virtualService.main.entries.guardrails.route | list | `[{"destination":{"host":"nemo-guardrails","port":{"number":7331}}}]` | The route for the virtual NeMo Guardrails service. |
| virtualService.main.gateways | list | `[]` | A list of gateways for the virtual service. |
| virtualService.main.hosts | list | `[]` | A list of hosts for the virtual service. |
| volcano.enabled | bool | `false` | Specifies whether to enable the default Volcano scheduler installation. To learn more, see [Volcano](https://docs.nvidia.com/nemo/microservices/latest/set-up/deploy-as-microservices/customizer.html#volcano). |
@@ -0,0 +1,5 @@
apiVersion: v1
appVersion: 0.0.1
description: A Helm chart for Kubernetes
name: dgxc-admission-controller
version: 1.642.0
@@ -0,0 +1,220 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.16.3
name: nvstoragelocations.storage.dgxc.nvidia.com
spec:
group: storage.dgxc.nvidia.com
names:
kind: NvStorageLocation
listKind: NvStorageLocationList
plural: nvstoragelocations
singular: nvstoragelocation
scope: Namespaced
versions:
- additionalPrinterColumns:
- description: Bucket
jsonPath: .spec.bucket
name: Bucket
type: string
name: v1beta1
schema:
openAPIV3Schema:
description: NvStorageLocation defines storage for DGXC Run:ai accelerated
compute environments.
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
spec:
properties:
description:
type: string
mountPath:
type: string
pvc:
description: '`pvc` represents a PVC to manage.'
properties:
capacity:
description: Requested size of the volume (required for a managed
volume)
format: ByteSize
type: string
mountOptions:
items:
type: string
type: array
objectSpec:
description: '`object` represents a volume that uses object storage.'
properties:
bucket:
type: string
endpointUrl:
type: string
required:
- bucket
- endpointUrl
type: object
parameters:
additionalProperties:
type: string
description: |-
Parameters contains an arbitrary set of key-value pairs
to supply to whatever storage backend we're using
according to our rules for that object storage backend.
Currently the only thing supported is the GCS OIDC ConfigMap
this parameter is "gcs-oidc-configmap" with the value being
the name of the configmap
type: object
required:
- capacity
- objectSpec
type: object
volumeName:
description: volumeName will be used to name the PVC and if needed
the PVs
type: string
required:
- description
- mountPath
type: object
status:
properties:
conditions:
items:
description: Condition contains details for one aspect of the current
state of this API Resource.
properties:
lastTransitionTime:
description: |-
lastTransitionTime is the last time the condition transitioned from one status to another.
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
format: date-time
type: string
message:
description: |-
message is a human readable message indicating details about the transition.
This may be an empty string.
maxLength: 32768
type: string
observedGeneration:
description: |-
observedGeneration represents the .metadata.generation that the condition was set based upon.
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
with respect to the current state of the instance.
format: int64
minimum: 0
type: integer
reason:
description: |-
reason contains a programmatic identifier indicating the reason for the condition's last transition.
Producers of specific condition types may define expected values and meanings for this field,
and whether the values are considered a guaranteed API.
The value should be a CamelCase string.
This field may not be empty.
maxLength: 1024
minLength: 1
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
type: string
status:
description: status of the condition, one of True, False, Unknown.
enum:
- "True"
- "False"
- Unknown
type: string
type:
description: type of condition in CamelCase or in foo.example.com/CamelCase.
maxLength: 316
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
type: string
required:
- lastTransitionTime
- message
- reason
- status
- type
type: object
type: array
pendingCount:
description: The number of PVCs monitored by this NVStorageLocation
that are not in a ready state (e.g. missing, error state).
type: integer
pvcs:
description: |-
A list of PVC statuses. This is a list because we expect to support multiple namespaces,
but it may very well be a list of one object.
items:
properties:
externalReferences:
items:
properties:
clusterName:
type: string
externalID:
type: string
platform:
type: string
required:
- clusterName
- externalID
- platform
type: object
type: array
namespace:
description: The namespace in which the PersistentVolumeClaim
exists.
type: string
pv:
type: string
state:
description: The current state of this volume.
type: string
volumeName:
description: The full name of the PersistentVolumeClaim.
type: string
required:
- namespace
- pv
- state
- volumeName
type: object
type: array
readyCount:
description: The number of PVCs monitored by this NVStorageLocation
that are in a ready state.
type: integer
summary:
description: |-
A short human-readable summary of the status of all managed and monitored PVCs.
Machine-readable data should be sourced from the `Ready` and `NotReady` counts, or
from pvcs[].State for detailed information.
type: string
required:
- pendingCount
- pvcs
- readyCount
type: object
required:
- spec
type: object
served: true
storage: true
subresources:
status: {}
@@ -0,0 +1 @@
No notes
@@ -0,0 +1,64 @@
{{/* vim: set filetype=mustache: */}}
{{/*
Expand the name of the chart.
*/}}
{{- define "dgxcController.name" -}}
{{- default "controller" .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "dgxcController.fullname" -}}
{{- if .Values.dgxcController.deploymentFullNameOverride -}}
{{- .Values.dgxcController.deploymentFullNameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "dgxcController.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "dgx-application-deploy" -}}
{{ .Release.Name }}-{{ .Release.Revision }}
{{- end -}}
{{- define "kubernetesAffinityConfig" }}
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: {{ .Values.kubernetesAffinity.antiAffinityKey | default "reason-to-avoid" }}
operator: DoesNotExist
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: "nvidia.com/dgx-application-deploy"
operator: In
values:
- {{ include "dgx-application-deploy" . }}
topologyKey: {{ .Values.kubernetesAffinity.topologyKey | default "kubernetes.io/hostname" }}
{{- if .Values.kubernetesAffinity.runOnMasters }}
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
operator: Exists
{{- end }}
{{- end }}
@@ -0,0 +1,250 @@
{{- if .Values.enabledComponents.dgxcAdmissionController }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ .Values.dgxcController.name}}-config
data:
config.yaml: |-
apiVersion: runai.dgxc.nvidia.com/v1
kind: DGXCRunaiMutatingWebhook
certs:
namespace: dgxc-admission-controller
service: {{ .Values.serviceName }}
port: 9443
organization: {{ .Values.webhooks.organization }}
mountPath: {{ .Values.dgxcController.certs.mountPath }}
certName: {{ .Values.dgxcController.certs.certName }}
keyName: {{ .Values.dgxcController.certs.keyName }}
podMutatingConfig:
name: dgxc-create-pod-mwh
kind: {{ .Values.webhooks.podMutatingWebhook.kind }}
fqdn: {{ .Values.webhooks.podMutatingWebhook.fqdn }}
path: {{ .Values.webhooks.podMutatingWebhook.path }}
failurePolicy: {{ .Values.webhooks.podMutatingWebhook.failurePolicy }}
timeoutSeconds: {{ .Values.webhooks.podMutatingWebhook.timeoutSeconds }}
fqdn: {{ .Values.webhooks.podMutatingWebhook.fqdn }}
sideEffects: {{ .Values.webhooks.podMutatingWebhook.sideEffects | default "NoneOnDryRun" }}
admissionReviewVersions:
{{- range $i, $value := .Values.webhooks.podMutatingWebhook.admissionReviewVersions }}
- {{ $value }}
{{- end }}
rules:
{{- range $i, $value := .Values.webhooks.podMutatingWebhook.rules }}
- apiGroups:
{{- range $j, $value2 := $value.apiGroups }}
- {{ $value2 | quote }}
{{- end }}
apiVersions:
{{- range $j, $value2 := $value.apiVersions }}
- {{ $value2 | quote }}
{{- end }}
operations:
{{- range $j, $value2 := $value.operations }}
- {{ $value2 | quote }}
{{- end }}
resources:
{{- range $j, $value2 := $value.resources }}
- {{ $value2 | quote }}
{{- end }}
scope: {{ $value.scope | quote }}
{{- end }}
nsExcludeLabels:
- key: kubernetes.io/metadata.name
values:
- dgxc-admission-controller
- argocd
- nvdiag
objectMatchLabels:
{{- range $i, $value := .Values.webhooks.podMutatingWebhook.objectLabelSelector }}
- key: {{ $value.key}}
values:
{{- range $j, $value2 := $value.values }}
- {{ $value2 | quote }}
{{- end }}
{{- end }}
podMutatingSpec:
{{- with (first .Values.webhooks.podMutatingWebhook.customSettings) }}
addRdmaEnabled: {{ .addRdmaEnabled }}
rdmaResourcePerGpu: {{ .rdmaResourcePerGpu }}
networkAttachmentDefinition: {{ .networkAttachmentDefinition }}
rdmaResourceName: {{ .rdmaResourceName }}
addScratchEnabled: {{ .scratchEnabled }}
scratchVolName: {{ .scratchVolName | quote }}
scratchMountPath: {{ .scratchMountpath | quote }}
scratchVolSize: {{ .scratchVolSize | quote }}
scratchVolSizePerGpu: {{ .scratchVolSizePerGpu }}
scratchVolEnforceLimit: {{ .scratchVolEnforceLimit }}
{{- if .scratchVolSource }}
scratchVolSource:
{{ .scratchVolSource | toYaml | nindent 8 }}
{{- end}}
scratchOnCpu: {{ .scratchOnCpu | default false }}
addTcpxSidecarEnabled: {{ .addTcpxSidecarEnabled }}
addTcpxoSidecarEnabled: {{ .addTcpxoSidecarEnabled }}
addTcpxNcclVars: {{ .addTcpxNCCLVars }}
addTcpxoNcclVars: {{ .addTcpxoNCCLVars }}
addNcclVars: {{ .addNcclVars }}
csp: {{ .csp}}
addGcpFuseAnnotation: {{ .addGcpFuseAnnotation}}
gcpFuseAnnotationKey: {{ .gcpFuseAnnotationKey | quote }}
gcpFuseWICredentialConfigMapAnnotationKey: {{ .gcpFuseWICredentialConfigMapAnnotationKey | quote }}
gcpFuseStorageClasses:
{{- range .gcpFuseStorageClasses}}
- {{ . | quote }}
{{- end }}
clusterName: {{ .clusterName }}
injectRunaiJob: true
{{- end}}
systemNamespaceNames:
{{- range .Values.clusterScopedKinds }}
{{- if eq .kind "Namespace" }}
{{- range .systemResources}}
- {{ . | quote }}
{{- end }}
{{- end }}
{{- end }}
pvcMutatingConfig:
name: dgxc-create-pvc-mwh
kind: {{ .Values.webhooks.pvcMutatingWebhook.kind }}
fqdn: {{ .Values.webhooks.pvcMutatingWebhook.fqdn }}
path: {{ .Values.webhooks.pvcMutatingWebhook.path }}
failurePolicy: {{ .Values.webhooks.pvcMutatingWebhook.failurePolicy }}
timeoutSeconds: {{ .Values.webhooks.pvcMutatingWebhook.timeoutSeconds }}
fqdn: {{ .Values.webhooks.pvcMutatingWebhook.fqdn }}
sideEffects: {{ .Values.webhooks.pvcMutatingWebhook.sideEffects | default "NoneOnDryRun" }}
admissionReviewVersions:
{{- range $i, $value := .Values.webhooks.pvcMutatingWebhook.admissionReviewVersions }}
- {{ $value }}
{{- end }}
rules:
{{- range $i, $value := .Values.webhooks.pvcMutatingWebhook.rules }}
- apiGroups:
{{- range $j, $value2 := $value.apiGroups }}
- {{ $value2 | quote }}
{{- end }}
apiVersions:
{{- range $j, $value2 := $value.apiVersions }}
- {{ $value2 | quote }}
{{- end }}
operations:
{{- range $j, $value2 := $value.operations }}
- {{ $value2 | quote }}
{{- end }}
resources:
{{- range $j, $value2 := $value.resources }}
- {{ $value2 | quote }}
{{- end }}
scope: {{ $value.scope | quote }}
{{- end }}
nsMatchLabels:
{{- range $i, $value := .Values.webhooks.pvcMutatingWebhook.namespaceLabelSelector }}
- key: {{ $value.key}}
values:
{{- range $j, $value2 := $value.values }}
- {{ $value2 | quote }}
{{- end }}
{{- end }}
objectMatchLabels:
{{- range $i, $value := .Values.webhooks.pvcMutatingWebhook.objectLabelSelector }}
- key: {{ $value.key}}
values:
{{- range $j, $value2 := $value.values }}
- {{ $value2 | quote }}
{{- end }}
{{- end }}
pvcMutatingSpec:
rules:
{{- range $i, $value := .Values.webhooks.pvcMutatingWebhook.customSettings }}
- storageClasses:
{{- range $j, $value2 := $value.storageClasses }}
- {{ $value2 | quote }}
{{- end }}
incrementAdjustmentEnabled: {{ $value.incrementCheckEnabled }}
incrementAdjustmentValue: {{ $value.incrementValue | quote }}
unitConversionEnabled: {{ $value.unitConversionEnabled }}
unitConversionFormat: {{ $value.unitConversionFormat }}
{{- if $value.minSizeValue }}
adjustedMinimumSize: {{ $value.minSizeValue | quote }}
{{- end }}
{{- end }}
pvcValidatingConfig:
name: dgxc-create-pvc-vwh
kind: {{ .Values.webhooks.pvcValidatingWebhook.kind }}
fqdn: {{ .Values.webhooks.pvcValidatingWebhook.fqdn }}
path: {{ .Values.webhooks.pvcValidatingWebhook.path }}
failurePolicy: {{ .Values.webhooks.pvcValidatingWebhook.failurePolicy }}
timeoutSeconds: {{ .Values.webhooks.pvcValidatingWebhook.timeoutSeconds }}
fqdn: {{ .Values.webhooks.pvcValidatingWebhook.fqdn }}
sideEffects: {{ .Values.webhooks.pvcValidatingWebhook.sideEffects | default "NoneOnDryRun" }}
admissionReviewVersions:
{{- range $i, $value := .Values.webhooks.pvcValidatingWebhook.admissionReviewVersions }}
- {{ $value }}
{{- end }}
rules:
{{- range $i, $value := .Values.webhooks.pvcValidatingWebhook.rules }}
- apiGroups:
{{- range $j, $value2 := $value.apiGroups }}
- {{ $value2 | quote }}
{{- end }}
apiVersions:
{{- range $j, $value2 := $value.apiVersions }}
- {{ $value2 | quote }}
{{- end }}
operations:
{{- range $j, $value2 := $value.operations }}
- {{ $value2 | quote }}
{{- end }}
resources:
{{- range $j, $value2 := $value.resources }}
- {{ $value2 | quote }}
{{- end }}
scope: {{ $value.scope | quote }}
{{- end }}
nsMatchLabels:
{{- range $i, $value := .Values.webhooks.pvcValidatingWebhook.namespaceLabelSelector }}
- key: {{ $value.key}}
values:
{{- range $j, $value2 := $value.values }}
- {{ $value2 | quote }}
{{- end }}
{{- end }}
objectMatchLabels:
{{- range $i, $value := .Values.webhooks.pvcValidatingWebhook.objectLabelSelector }}
- key: {{ $value.key}}
values:
{{- range $j, $value2 := $value.values }}
- {{ $value2 | quote }}
{{- end }}
{{- end }}
pvcValidatingSpec:
rules:
{{- range $i, $value := .Values.webhooks.pvcValidatingWebhook.customSettings }}
- name: {{ $value.name }}
storageClasses:
{{- range $j, $value2 := $value.storageClasses }}
- {{ $value2 | quote }}
{{- end }}
minsizeCheckEnabled: {{ $value.minSizeCheckEnabled }}
minsizeValue: {{ $value.minSizeValue | quote }}
maxsizeCheckEnabled: {{ $value.maxSizeCheckEnabled }}
maxsizeValue: {{ $value.maxSizeValue | quote }}
{{- end }}
systemNamespaceNames:
{{- range .Values.clusterScopedKinds }}
{{- if eq .kind "Namespace" }}
{{- range .systemResources}}
- {{ . | quote }}
{{- end }}
{{- end }}
{{- end }}
{{ end }}
@@ -0,0 +1,161 @@
{{ if .Values.enabledComponents.dgxcRunaiController }}
apiVersion: v1
kind: ConfigMap
metadata:
name: dgxc-runai-controller
namespace: {{ .Release.Namespace }}
data:
config.yaml: |-
apiVersion: runai.dgxc.nvidia.com/v1
kind: Controller
{{- if .Values.prober }}
prober:
intervalInSeconds: {{ .Values.prober.intervalInSeconds }}
timeoutInSeconds: {{ .Values.prober.timeoutInSeconds }}
endpoints:
{{- range .Values.prober.endpoints }}
- name: {{ .name }}
url: {{ .url }}
{{- end }}
{{- end }}
{{- if .Values.sso }}
cluster: {{ .Values.sso.cluster }}
sso:
controlPlaneDomain: {{ .Values.sso.controlPlaneDomain }}
realm: {{ .Values.sso.realm }}
pollIntervalSeconds: {{ .Values.sso.pollIntervalSeconds }}
pollAttempts: {{ .Values.sso.pollAttempts }}
{{- end }}
tenantMetadata:
cluster: {{ .Values.tenantMetadata.cluster }}
platform: {{ .Values.tenantMetadata.platform }}
orgName: {{ .Values.tenantMetadata.orgName }}
tenantName: {{ .Values.tenantMetadata.tenantName }}
tenantType: {{ .Values.tenantMetadata.tenantType }}
startDate: {{ .Values.tenantMetadata.startDate }}
endDate: {{ .Values.tenantMetadata.endDate }}
CSP: {{ .Values.tenantMetadata.CSP }}
allocatedCapacityGPU: {{ .Values.tenantMetadata.allocatedCapacityGPU }}
allocatedCapacityCPU: {{ .Values.tenantMetadata.allocatedCapacityCPU }}
committedCapacityGPU: {{ .Values.tenantMetadata.committedCapacityGPU }}
committedCapacityCPU: {{ .Values.tenantMetadata.committedCapacityCPU }}
CSPProject: {{ .Values.tenantMetadata.CSPProject }}
dataCenterRegion: {{ .Values.tenantMetadata.dataCenterRegion }}
storageType: {{ .Values.tenantMetadata.storageType }}
storageQuota: {{ .Values.tenantMetadata.storageQuota }}
{{- if .Values.initSettings }}
appID: {{ .Values.initSettings.appID }}
tenant: {{ .Values.initSettings.tenant }}
controlPlane: {{ .Values.initSettings.controlPlane }}
initSettings:
supportEmailAddress: {{ .Values.initSettings.supportEmailAddress }}
customerEmailAddress: {{ .Values.initSettings.customerEmailAddress }}
bootstrapEmailAddress: {{ .Values.initSettings.bootstrapEmailAddress }}
{{- end }}
dgxcAdmins:
{{- range .Values.dgxcAdmins }}
- {{ . }}
{{- end }}
tenantSubjects:
{{- range .Values.tenantSubjects.User }}
- {{ . }}
{{- end }}
reservedCPU: {{ .Values.reserved.CPU }}
reservedMemory: {{ .Values.reserved.Memory }}
maxCPUJobs: {{ .Values.maxCPUJobs }}
cloudProviderSpec:
type: {{ .Values.cloudProviderSpec.type }}
{{- if eq .Values.cloudProviderSpec.type "gcp" }}
gcp:
projectId: {{ .Values.cloudProviderSpec.gcp.projectId }}
projectNumber: {{ .Values.cloudProviderSpec.gcp.projectNumber | int64 }}
securityPolicyRuleName: {{ .Values.cloudProviderSpec.gcp.securityPolicyRuleName }}
region: {{ .Values.cloudProviderSpec.gcp.region }}
{{- else if eq .Values.cloudProviderSpec.type "aws" }}
aws:
accountName: {{ .Values.cloudProviderSpec.aws.accountName }}
clusterName: {{ .Values.cloudProviderSpec.aws.clusterName }}
region: {{ .Values.cloudProviderSpec.aws.region }}
securityGroupARN: {{ .Values.cloudProviderSpec.aws.securityGroupARN }}
statefulRuleGroupName: {{ .Values.cloudProviderSpec.aws.statefulRuleGroupName }}
{{- else if eq .Values.cloudProviderSpec.type "oci" }}
oci:
compartmentID: {{ .Values.cloudProviderSpec.oci.compartmentID }}
clusterID: {{ .Values.cloudProviderSpec.oci.clusterID }}
fssAvailabilityDomain: "{{ .Values.cloudProviderSpec.oci.fssAvailabilityDomain }}"
publicSubnetFirewallID: "{{ .Values.cloudProviderSpec.oci.publicSubnetFirewallID }}"
privateSubnetFirewallID: "{{ .Values.cloudProviderSpec.oci.privateSubnetFirewallID }}"
baseFirewallPolicyID: "{{ .Values.cloudProviderSpec.oci.baseFirewallPolicyID }}"
{{- else if eq .Values.cloudProviderSpec.type "azure" }}
azure:
subscriptionId: {{ .Values.cloudProviderSpec.azure.subscriptionId }}
resourceGroupName: {{ .Values.cloudProviderSpec.azure.resourceGroupName }}
ingressNsgName: {{ .Values.cloudProviderSpec.azure.ingressNsgName }}
egressNsgName: {{ .Values.cloudProviderSpec.azure.egressNsgName }}
standardStorageAccountName: {{ .Values.cloudProviderSpec.azure.standardStorageAccountName }}
premiumStorageAccountName: {{ .Values.cloudProviderSpec.azure.premiumStorageAccountName }}
logExporterAnalyticsWorkspaceId: {{ .Values.cloudProviderSpec.azure.logExporterAnalyticsWorkspaceId }}
{{- end }}
logExporterControllerSpec:
enabled: {{ .Values.logExporterControllerSpec.enabled }}
exportFrequency: {{ .Values.logExporterControllerSpec.exportFrequency }}
exportBuffer: {{ .Values.logExporterControllerSpec.exportBuffer }}
fetchTimeout: {{ .Values.logExporterControllerSpec.fetchTimeout }}
systemResources:
{{- range $i, $csk := .Values.clusterScopedKinds }}
{{- $group := $csk.apiGroup | default "core" -}}
{{- range $j, $kind := $csk.resources }}
{{- range $csk.systemResources}}
{{- if and (eq $kind "namespaces") (eq . "dgxc-tenant-cluster-policies") | not }}
- {{ print $group "/" $kind "/" . | quote }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
networkControllerSpec:
enabled: {{ .Values.networkControllerSpec.enabled }}
defaultIngressCIDRs:
{{- .Values.bootstrapIngressCIDR | toYaml | nindent 6 }}
{{- if eq .Values.cloudProviderSpec.type "gcp" }}
storageControllerSpec:
enabled: {{ .Values.storageControllerSpecGCP.enabled }}
defaultCustomerPVReclaimPolicy: {{ .Values.storageControllerSpecGCP.defaultCustomerPVReclaimPolicy | default "Retain" }}
defaultStorageTypes:
{{- .Values.storageControllerSpecGCP.defaultStorageTypes | toYaml | nindent 6 }}
{{- else if eq .Values.cloudProviderSpec.type "aws" }}
storageControllerSpec:
enabled: {{ .Values.storageControllerSpecAWS.enabled }}
defaultCustomerPVReclaimPolicy: {{ .Values.storageControllerSpecAWS.defaultCustomerPVReclaimPolicy | default "Retain" }}
defaultStorageTypes:
{{- .Values.storageControllerSpecAWS.defaultStorageTypes | toYaml | nindent 6 }}
{{- else if eq .Values.cloudProviderSpec.type "oci" }}
storageControllerSpec:
enabled: {{ .Values.storageControllerSpecOCI.enabled }}
defaultCustomerPVReclaimPolicy: {{ .Values.storageControllerSpecOCI.defaultCustomerPVReclaimPolicy | default "Retain" }}
defaultStorageTypes:
{{- .Values.storageControllerSpecOCI.defaultStorageTypes | toYaml | nindent 6 }}
{{- else if eq .Values.cloudProviderSpec.type "azure" }}
storageControllerSpec:
enabled: {{ .Values.storageControllerSpecAzure.enabled }}
defaultCustomerPVReclaimPolicy: {{ .Values.storageControllerSpecAzure.defaultCustomerPVReclaimPolicy | default "Retain" }}
defaultStorageTypes:
{{- .Values.storageControllerSpecAzure.defaultStorageTypes | toYaml | nindent 6 }}
{{- end }}
nvStorageControllerSpec:
enabled: {{ .Values.nvStorage.enabled }}
storageClass: {{ .Values.nvStorage.storageClass }}
{{- if .Values.nvStorage.platform }}
platform:
provider: {{ .Values.nvStorage.platform.provider }}
clusterName: {{ .Values.nvStorage.platform.clusterName }}
config:
{{- .Values.nvStorage.platform.config | toYaml | nindent 10 }}
{{- end }}
{{- if .Values.nvStorage.csi }}
csi:
driver: {{ .Values.nvStorage.csi.driver }}
volumeAttributes:
{{- .Values.nvStorage.csi.volumeAttributes | toYaml | nindent 10 }}
{{- end}}
{{ end}}
@@ -0,0 +1,28 @@
{{ if .Values.enabledComponents.dgxcRunaiController }}
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: dgxc-runai-controller
rules:
- apiGroups: [""]
resources: ["services", "configmaps", "secrets"]
verbs: ["*"]
- apiGroups: ["authentication.gke.io"]
resources: ["clientconfigs"]
verbs: ["*"]
- apiGroups: ["run.ai"]
resources: ["runaiconfigs"]
verbs: ["*"]
- apiGroups: ["runai.dgxc.nvidia.com", "storage.dgxc.nvidia.com", "kyverno.io"]
resources: ["*"]
verbs: ["*"]
- apiGroups: [""]
resources: ["persistentvolumes", "persistentvolumeclaims"]
verbs: ["*"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["clusterrolebindings", "clusterroles", "roles", "rolebindings"]
verbs: ["*"]
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["get", "list", "watch"]
{{ end }}
@@ -0,0 +1,14 @@
{{ if .Values.enabledComponents.dgxcRunaiController }}
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: dgxc-runai-controller
subjects:
- kind: ServiceAccount
name: dgxc-runai-controller
namespace: {{ .Release.Namespace }}
roleRef:
kind: ClusterRole
name: dgxc-runai-controller
apiGroup: rbac.authorization.k8s.io
{{ end }}
@@ -0,0 +1,68 @@
{{ if .Values.enabledComponents.dgxcRunaiController }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: "dgxc-runai-controller"
namespace: {{ .Release.Namespace }}
labels:
app.kubernetes.io/name: "dgxc-runai-controller"
helm.sh/chart: {{ include "dgxcController.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: "dgxc-runai-controller"
app.kubernetes.io/instance: {{ .Release.Name }}
template:
metadata:
labels:
app: "dgxc-runai-controller"
app.kubernetes.io/name: "dgxc-runai-controller"
app.kubernetes.io/instance: {{ .Release.Name }}
{{- with .Values.extraControllerPodLabels }}
{{ toYaml . }}
{{- end }}
spec:
serviceAccountName: dgxc-runai-controller
imagePullSecrets:
- name: {{ .Values.imagePullSecret | required ".Values.imagePullSecret" }}
containers:
- name: controller
image: {{ .Values.dgxcController.image | required "dgxcController.image is required." | quote }}
args:
- controller
- --zap-log-level=debug
imagePullPolicy: Always
{{- if eq .Values.cloudProviderSpec.type "oci" }}
env:
- name: OCI_RESOURCE_PRINCIPAL_VERSION
value: {{ .Values.cloudProviderSpec.oci.resourcePrincipalVersion | quote }}
- name: OCI_RESOURCE_PRINCIPAL_REGION
value: {{ .Values.cloudProviderSpec.oci.region }}
{{ end }}
ports:
- containerPort: 8080
{{- if and (hasKey .Values.nvStorage "platform") (eq .Values.nvStorage.platform.provider "runai" )}}
volumeMounts:
- name: run-ai-secret
mountPath: {{ .Values.nvStorage.platform.config.secret_path | required "config.secret_path is required for runai setup" | quote }}
readOnly: true
{{- end }}
tolerations:
- effect: NoSchedule
key: "{{ .Values.systemTolerationKeyName }}"
operator: Equal
value: "{{ .Values.systemTolerationValueName }}"
- effect: NoExecute
key: "{{ .Values.systemTolerationKeyName }}"
operator: Equal
value: "{{ .Values.systemTolerationValueName }}"
{{- if and (hasKey .Values.nvStorage "platform") (eq .Values.nvStorage.platform.provider "runai" )}}
volumes:
- name: run-ai-secret
secret:
secretName: run-ai-secret
{{- end}}
{{ end }}
@@ -0,0 +1,13 @@
{{ if .Values.enabledComponents.dgxcRunaiController }}
{{- if .Values.controllerServiceAccount.create }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ .Values.controllerServiceAccount.name }}
namespace: {{ .Release.Namespace }}
{{- if .Values.controllerServiceAccount.annotations }}
annotations:
{{ toYaml .Values.controllerServiceAccount.annotations }}
{{- end }}
{{ end }}
{{ end }}
@@ -0,0 +1,13 @@
{{ if .Values.enabledComponents.dgxcAdmissionController }}
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ .Values.dgxcController.name }}
rules:
- apiGroups: [""]
resources: ["pods", "secrets", "configmaps", "persistentvolumeclaims", "storageclass"]
verbs: ["*"]
- apiGroups: ["admissionregistration.k8s.io"]
resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"]
verbs: ["*"]
{{ end }}
@@ -0,0 +1,14 @@
{{ if .Values.enabledComponents.dgxcAdmissionController }}
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ .Values.dgxcController.name }}
subjects:
- kind: ServiceAccount
name: {{ .Values.serviceAccount.name }}
namespace: {{ .Release.Namespace }}
roleRef:
kind: ClusterRole
name: {{ .Values.dgxcController.name }}
apiGroup: rbac.authorization.k8s.io
{{ end }}
@@ -0,0 +1,71 @@
{{- if eq .Values.cloudProviderSpec.type "gcp" }}
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: fix-hostname
namespace: kube-system
labels:
nodeGroup: customer-gpu
spec:
selector:
matchLabels:
nodeGroup: customer-gpu
template:
metadata:
labels:
nodeGroup: customer-gpu
spec:
nodeSelector:
nodeGroup: customer-gpu
hostNetwork: true
containers:
- image: gcr.io/google-containers/pause:3.2
name: pause
initContainers:
- name: fix-hostname
image: ubuntu
env: [ {name: NODE_NAME, valueFrom: { fieldRef: {fieldPath: spec.nodeName } } } ]
command:
- /bin/bash
- -c
- |
set -xeuo pipefail
hostname $(NODE_NAME)
cat << EOF > /host/etc/systemd/network/97-temp.network
[Match]
Name=eth* !eth0
[Network]
DHCP=yes
IPv6AcceptRA=yes
# non-eth0 modification
[DHCPv4]
RoutesToDNS=false
RoutesToNTP=false
[DHCP]
UseMTU=true
UseDomains=true
# non-eth0 modification
UseHostname=false
EOF
chroot /host networkctl reload
echo "Success"
securityContext: { privileged: true, capabilities: { add: [ NET_ADMIN ] } }
volumeMounts:
- name: host
mountPath: /host
tolerations:
- effect: NoExecute
operator: Exists
- effect: NoSchedule
operator: Exists
volumes:
- name: host
hostPath:
path: /
{{- end }}
@@ -0,0 +1,97 @@
{{ if .Values.enabledComponents.dgxcAdmissionController }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "dgxcController.fullname" . }}
labels:
app.kubernetes.io/name: {{ include "dgxcController.name" . }}
helm.sh/chart: {{ include "dgxcController.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
replicas: {{ .Values.dgxcController.replicas | required ".Values.dgxcController.replicas required" }}
selector:
matchLabels:
app.kubernetes.io/name: {{ include "dgxcController.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
strategy:
rollingUpdate:
maxSurge: 1
maxUnavailable: 50%
type: RollingUpdate
template:
metadata:
labels:
app: {{ .Values.dgxcController.name }}
app.kubernetes.io/name: {{ include "dgxcController.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
nvidia.com/dgx-application-deploy: {{ include "dgx-application-deploy" . }}
{{- with .Values.extraPodLabels }}
{{ toYaml . }}
{{- end }}
{{- with .Values.podAnnotations }}
annotations:
{{ toYaml . }}
{{- end }}
spec:
serviceAccountName: {{ .Values.serviceAccount.name }}
imagePullSecrets:
- name: {{ .Values.imagePullSecret | required ".Values.imagePullSecret" }}
initContainers:
- name: init
image: {{ .Values.dgxcController.image | required ".Values.dgxcController.image is required." | quote }}
args: [ "admission-init", "--config", "/etc/{{ .Values.dgxcController.name}}/config.yaml", "--zap-log-level=debug" ]
imagePullPolicy: Always
volumeMounts:
- name: {{ .Values.dgxcController.name}}-config
mountPath: /etc/{{ .Values.dgxcController.name}}/
readOnly: true
- name: {{ .Values.dgxcController.name}}-certs
mountPath: {{ .Values.dgxcController.certs.mountPath }}
containers:
- name: controller
image: {{ .Values.dgxcController.image | required "dgxcController.image is required." | quote }}
args: ["admission", "--config", "/etc/{{ .Values.dgxcController.name}}/config.yaml", "--zap-log-level=debug"]
imagePullPolicy: Always
ports: []
volumeMounts:
- name: {{ .Values.dgxcController.name}}-certs
mountPath: {{ .Values.dgxcController.certs.mountPath }}
readOnly: true
- name: {{ .Values.dgxcController.name}}-config
mountPath: /etc/{{ .Values.dgxcController.name}}/
readOnly: true
{{/* - name: app-share-ns*/}}
{{/* command:*/}}
{{/* - sleep*/}}
{{/* - infinity*/}}
{{/* image: nvcr.io/nvidian/nsvis-corestorage/ubuntu:latest*/}}
{{/* imagePullPolicy: IfNotPresent*/}}
{{/* terminationMessagePath: /dev/termination-log*/}}
{{/* terminationMessagePolicy: File*/}}
{{/* volumeMounts:*/}}
{{/* - name: {{ .Values.dgxcController.name}}-certs*/}}
{{/* mountPath: {{ .Values.dgxcController.certs.mountPath }}*/}}
{{/* readOnly: true*/}}
{{/* - name: {{ .Values.dgxcController.name}}-config*/}}
{{/* mountPath: /etc/{{ .Values.dgxcController.name}}-/*/}}
{{/* readOnly: true*/}}
{{ if .Values.kubernetesAffinity.enabled }}
{{- include "kubernetesAffinityConfig" . | nindent 6 }}
{{ end }}
tolerations:
- effect: NoSchedule
key: "{{ .Values.systemTolerationKeyName }}"
operator: Equal
value: "{{ .Values.systemTolerationValueName }}"
- effect: NoExecute
key: "{{ .Values.systemTolerationKeyName }}"
operator: Equal
value: "{{ .Values.systemTolerationValueName }}"
volumes:
- name: {{ .Values.dgxcController.name}}-certs
emptyDir: {}
- name: {{ .Values.dgxcController.name}}-config
configMap:
name: {{ .Values.dgxcController.name}}-config
{{ end }}
@@ -0,0 +1,55 @@
{{ if .Values.enabledComponents.dgxcExporter }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: dgxc-exporter
namespace: {{ .Release.Namespace }}
labels:
app.kubernetes.io/name: dgxc-exporter
helm.sh/chart: {{ include "dgxcController.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: dgxc-exporter
app.kubernetes.io/instance: {{ .Release.Name }}
template:
metadata:
labels:
app: dgxc-exporter
app.kubernetes.io/name: dgxc-exporter
app.kubernetes.io/instance: {{ .Release.Name }}
spec:
serviceAccount: dgxc-exporter
imagePullSecrets:
- name: {{ .Values.imagePullSecret | required ".Values.imagePullSecret" }}
containers:
- name: exporter
image: {{ .Values.dgxcController.image | required "dgxcController.image is required." | quote }}
args:
- exporter
- --port={{ .Values.exporter.metricsPort }}
- --zap-log-level=debug
imagePullPolicy: Always
{{- if eq .Values.cloudProviderSpec.type "oci" }}
env:
- name: OCI_RESOURCE_PRINCIPAL_VERSION
value: {{ .Values.cloudProviderSpec.oci.resourcePrincipalVersion | quote }}
- name: OCI_RESOURCE_PRINCIPAL_REGION
value: {{ .Values.cloudProviderSpec.oci.region }}
{{ end }}
ports:
- containerPort: {{ .Values.exporter.metricsPort }}
protocol: TCP
tolerations:
- effect: NoSchedule
key: "{{ .Values.systemTolerationKeyName }}"
operator: Equal
value: "{{ .Values.systemTolerationValueName }}"
- effect: NoExecute
key: "{{ .Values.systemTolerationKeyName }}"
operator: Equal
value: "{{ .Values.systemTolerationValueName }}"
{{ end }}
@@ -0,0 +1,32 @@
{{ if .Values.enabledComponents.dgxcExporter }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: dgxc-exporter
namespace: {{ .Release.Namespace }}
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: dgxc-exporter
rules:
- apiGroups: [kyverno.io]
resources: [updaterequests]
verbs: [get, list, watch]
- apiGroups: [""]
resources: [configmaps]
verbs: [get, list, watch]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: dgxc-exporter
subjects:
- kind: ServiceAccount
name: dgxc-exporter
namespace: {{ .Release.Namespace }}
roleRef:
kind: ClusterRole
name: dgxc-exporter
apiGroup: rbac.authorization.k8s.io
{{ end }}
@@ -0,0 +1,416 @@
{{- if .Values.enableIntegrationsSupport }}
# cluster roles required by kyverno to label cluster scoped resources
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: kyverno:dgxc-integrations
labels:
app.kubernetes.io/component: background-controller
app.kubernetes.io/instance: kyverno
app.kubernetes.io/part-of: kyverno
rules:
{{- range .Values.clusterScopedKinds }}
{{- if gt (len .systemResources) 0 }}
- apiGroups: [ {{ .apiGroup | quote }} ]
resources:
{{- range .resources }}
- {{ . | quote }}
{{- end }}
verbs: ['create', 'delete', 'deletecollection', 'get', 'list', 'patch', 'update', 'watch']
{{- end }}
{{- end }}
- apiGroups: ['rbac.authorization.k8s.io']
resources: ['clusterroles']
verbs: ['escalate', 'bind']
---
# label system namespaces with dgxc/system-managed: "true"
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: dgxc-integrations-set-tolerations
labels:
dgxc/system-managed: "true"
spec:
background: false # must be done as part of request
validationFailureAction: {{ .Values.validationFailureAction }}
rules:
- name: inject-system-pod-tolerations # inject system node tolerations on system pods
preconditions: # pod doesn't already have system toleration
any:
- key:
- NoExecute
- NoSchedule
operator: AnyNotIn
value: |
{{`{{`}} request.object.spec.tolerations[?(key == '{{ .Values.tolerations.system.key }}' && value == '{{ .Values.tolerations.system.value }}')].effect {{`}}`}}
match:
all:
- resources: # system pods
kinds: [ Pod ]
namespaceSelector:
matchExpressions: [ { key: kubernetes.io/metadata.name , operator: In, values: [ "runai" ] } ]
mutate:
patchesJson6902: |-
- op: add
path: "/spec/tolerations/-"
value:
key: {{ .Values.tolerations.system.key | quote }}
operator: Equal
value: {{ .Values.tolerations.system.value | quote }}
effect: NoSchedule
- op: add
path: "/spec/tolerations/-"
value:
key: {{ .Values.tolerations.system.key | quote }}
operator: Equal
value: {{ .Values.tolerations.system.value | quote }}
effect: NoExecute
- name: inject-customer-pod-tolerations
preconditions: # pod doesn't already have customer toleration
any:
- key:
- NoExecute
- NoSchedule
operator: AnyNotIn
value: |
{{`{{`}} request.object.spec.tolerations[?(key == '{{ .Values.tolerations.customer.key }}' && value == '{{ .Values.tolerations.customer.value }}')].effect {{`}}`}}
match:
all:
- resources:
kinds: [ Pod ]
exclude:
all:
- resources:
namespaceSelector:
matchExpressions: [ { key: dgxc/system-managed , operator: In, values: [ "true" ] } ]
mutate:
patchesJson6902: |-
- op: add
path: "/spec/tolerations/-"
value:
key: {{ .Values.tolerations.customer.key | quote }}
operator: Equal
value: {{ .Values.tolerations.customer.value | quote }}
effect: NoSchedule
- op: add
path: "/spec/tolerations/-"
value:
key: {{ .Values.tolerations.customer.key | quote }}
operator: Equal
value: {{ .Values.tolerations.customer.value | quote }}
effect: NoExecute
- name: inject-customer-ds-tolerations
match:
all:
- resources:
kinds: [ DaemonSet ]
exclude:
all:
- resources:
namespaceSelector:
matchExpressions: [ { key: dgxc/system-managed , operator: In, values: [ "true" ] } ]
preconditions:
any:
- key:
- NoExecute
- NoSchedule
operator: AnyNotIn
value: |
{{`{{`}} request.object.spec.template.spec.tolerations[?(key=='{{ .Values.tolerations.customer.key }}' && value=='{{ .Values.tolerations.customer.value }}')].effect {{`}}`}}
mutate:
patchesJson6902: |-
- op: add
path: "/spec/template/spec/tolerations/-"
value:
key: {{ .Values.tolerations.customer.key | quote }}
operator: Equal
value: {{ .Values.tolerations.customer.value | quote }}
effect: NoSchedule
- op: add
path: "/spec/template/spec/tolerations/-"
value:
key: {{ .Values.tolerations.customer.key | quote }}
operator: Equal
value: {{ .Values.tolerations.customer.value | quote }}
effect: NoExecute
- name: deny-customer-pod-tolerations
match:
all:
- resources:
kinds: [ Pod ]
exclude:
all:
- resources:
namespaceSelector: # resources in system namespaces
matchExpressions: [ { key: dgxc/system-managed , operator: In, values: [ "true" ] } ]
preconditions:
any:
- key:
- NoExecute
- NoSchedule
operator: AnyIn
value: |
{{`{{ request.object.spec.tolerations[?(key==`}}'{{ .Values.tolerations.system.key }}'{{` && value==`}}'{{ .Values.tolerations.system.value }}'{{` || key=='' || key == null)].effect }}`}}
- key: "true" # explicitly bound to system node
operator: Equals
value: |
{{`{{`}} spec.nodename && contains(spec.nodename, 'system') {{`}}`}}
validate:
message: "Customer pods may not have tolerations with an empty key or the key \"{{ .Values.tolerations.system.key }}\" or a nodeName containing 'system' as these allow them to schedule on system pods."
deny: {}
- name: deny-customer-ds-blanket-tolerations
match:
all:
- resources:
kinds: [ DaemonSet ]
exclude:
all:
- resources:
namespaceSelector: # resources in system namespaces
matchExpressions: [ { key: dgxc/system-managed , operator: In, values: [ "true" ] } ]
validate:
message: "Customer daemonsets may not have tolerations with an empty key or empty value."
pattern:
spec:
template:
spec:
=(tolerations):
- key: "?*"
value: "?*"
---
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: dgxc-integrations-deny-changes-to-system-resources
labels:
dgxc/system-managed: "true"
spec:
background: false
validationFailureAction: {{ .Values.validationFailureAction }}
rules:
# Deny all writes to resources to cluster scoped system resources not made by system accounts
- name: deny-cluster-scoped
match:
any:
# Only below resources are considered dgxc/system-managed
{{- range .Values.clusterScopedKinds }}
{{- if gt (len .systemResources) 0 }}
- resources:
kinds: [ {{ .kind | quote }} ]
names:
{{- range .systemResources }}
- {{ . | quote }}
{{- end }}
{{- end }}
{{- end }}
exclude: # allow writes by system accounts
any:
- subjects:
# allow writes from service accounts in system namespaces
{{- range .Values.clusterScopedKinds }}
{{- if eq .kind "Namespace" }}
{{- range .systemResources}}
- kind: ServiceAccount
namespace: "{{ . }}"
name: "*"
{{- end }}
{{- end }}
{{- end }}
# allow writes from system users and groups
{{- range $k, $v := .Values.systemSubjects }}
{{- range $v }}
- kind: {{ $k | quote }}
name: {{ . | quote }}
{{- end }}
{{- end }}
# allow writes from system CSP service accounts
{{- range $k, $v := .Values.cspSubjects }}
{{- range $v }}
- kind: {{ $k | quote }}
name: {{ . | quote }}
{{- end }}
{{- end }}
validate:
message: "Changes to system resources are not allowed."
deny: {}
# Deny all writes to resources in system namespaces not made by system accounts
- name: deny-namespace-scoped
match:
any:
- resources:
namespaceSelector: # resources in system namespaces
matchExpressions: [ { key: dgxc/system-managed , operator: In, values: [ "true" ] } ]
kinds:
{{- $kinds := dict }}
{{- range .Values.namespacedKinds }}
{{- if .kind }}
{{- if not (hasKey $kinds .kind) }}
- {{ .kind | quote }}
{{ $_ := set $kinds .kind .kind }}
{{- end }}
{{- end }}
{{- end }}
- 'ServiceAccount/token'
exclude: # allow writes by system accounts
any:
- subjects:
# allow writes from service accounts in system namespaces
{{- range .Values.clusterScopedKinds }}
{{- if eq .kind "Namespace" }}
{{- range .systemResources}}
- kind: ServiceAccount
namespace: "{{ . }}"
name: "*"
{{- end }}
{{- end }}
{{- end }}
# allow writes from system users and groups
{{- range $k, $v := .Values.systemSubjects }}
{{- range $v }}
- kind: {{ $k | quote }}
name: {{ . | quote }}
{{- end }}
{{- end }}
# allow writes from system CSP service accounts
{{- range $k, $v := .Values.cspSubjects }}
{{- range $v }}
- kind: {{ $k | quote }}
name: {{ . | quote }}
{{- end }}
{{- end }}
validate:
message: "Changes to system resources are not allowed."
deny: {}
# Deny customer from applying the dgxc/system-managed label
- name: deny-system-label-on-user-namespaces
match:
any:
{{- range .Values.clusterScopedKinds }}
{{- if gt (len .systemResources) 0 }}
{{- if eq .kind "Namespace" }}
- resources:
kinds: [ {{ .kind | quote }} ]
names:
{{- range .systemResources }}
- {{ . | quote }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
exclude: # allow writes by system accounts
any:
- subjects:
# allow writes from service accounts in system namespaces
{{- range .Values.clusterScopedKinds }}
{{- if eq .kind "Namespace" }}
{{- range .systemResources}}
- kind: ServiceAccount
namespace: "{{ . }}"
name: "*"
{{- end }}
{{- end }}
{{- end }}
# allow writes from system users and groups
{{- range $k, $v := .Values.systemSubjects }}
{{- range $v }}
- kind: {{ $k | quote }}
name: {{ . | quote }}
{{- end }}
{{- end }}
# allow writes from system CSP service accounts
{{- range $k, $v := .Values.cspSubjects }}
{{- range $v }}
- kind: {{ $k | quote }}
name: {{ . | quote }}
{{- end }}
{{- end }}
validate:
message: "Setting label dgxc/system-managed not allowed"
deny:
conditions:
- key: "{{`{{ request.object.metadata.labels.\"dgxc/system-managed\" || '' }}`}}"
operator: Equals
value: "true"
- name: deny-system-connect
match:
any:
- resources:
kinds: [ 'Pod/exec', 'Pod/portforward', 'Pod/attach', 'Pod/ephemeralcontainers', 'Pod/proxy' ]
namespaceSelector: # resources in system namespaces
matchExpressions: [ { key: dgxc/system-managed , operator: In, values: [ "true" ] } ]
exclude: # allow writes by system accounts
any:
- subjects:
# allow writes from service accounts in system namespaces
{{- range .Values.clusterScopedKinds }}
{{- if eq .kind "Namespace" }}
{{- range .systemResources}}
- kind: ServiceAccount
namespace: "{{ . }}"
name: "*"
{{- end }}
{{- end }}
{{- end }}
# allow writes from system users and groups
{{- range $k, $v := .Values.systemSubjects }}
{{- range $v }}
- kind: {{ $k | quote }}
name: {{ . | quote }}
{{- end }}
{{- end }}
# allow writes from system CSP service accounts
{{- range $k, $v := .Values.cspSubjects }}
{{- range $v }}
- kind: {{ $k | quote }}
name: {{ . | quote }}
{{- end }}
{{- end }}
preconditions:
all:
- key: "{{`{{ request.operation || 'BACKGROUND' }}`}}"
operator: Equals
value: CONNECT
validate:
message: "Changes to system resources are not allowed."
deny: {}
---
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: dgxc-integrations-generate-rbac
annotations:
argocd.argoproj.io/sync-options: Force=true,Replace=true
labels:
dgxc/system-managed: "true"
spec:
rules:
- name: generate-rbac-for-crd
match: { any: [ resources: { kinds: [ "CustomResourceDefinition" ] } ] }
exclude:
any:
- resources:
names:
{{- range .Values.clusterScopedKinds }}
{{- if eq .kind "CustomResourceDefinition" }}
{{- range .systemResources }}
- {{ . | quote }}
{{- end }}
{{- end }}
{{- end }}
generate:
synchronize: true
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
name: "dgxc-customer-app-admin-{{ `{{ request.object.spec.names.plural }}` }}"
data:
metadata:
labels:
dgxc/customer-app-admin: "true"
dgxc/system-managed: "true"
dgxc/generated: "true"
rules:
- apiGroups: [ "{{ `{{request.object.spec.group}}` }}" ]
resources: [ "{{ `{{request.object.spec.names.plural}}` }}", "{{ `{{request.object.spec.names.plural}}` }}/finalizers", "{{ `{{request.object.spec.names.plural}}` }}/status", "{{ `{{request.object.spec.names.plural}}` }}/scale" ]
verbs: ['create', 'delete', 'deletecollection', 'get', 'list', 'patch', 'update', 'watch']
---
{{- end }}
@@ -0,0 +1,37 @@
{{- if .Values.enableIntegrationsSupport }}
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: dgxc-customer-app-admin-base
labels:
dgxc/customer-app-admin: "true"
dgxc/system-managed: "true"
rules:
{{- range .Values.namespacedKinds }}
{{- if gt (len .verbs) 0 }}
- apiGroups: [ {{ .apiGroup | quote }} ]
resources:
{{- range .resources }}
- {{ . | quote }}
{{- end }}
verbs:
{{- range .verbs }}
- {{ . | quote }}
{{- end }}
{{- end }}
{{- end }}
{{- range .Values.clusterScopedKinds }}
{{- if gt (len .verbs) 0 }}
- apiGroups: [ {{ .apiGroup | quote }} ]
resources:
{{- range .resources }}
- {{ . | quote }}
{{- end }}
verbs:
{{- range .verbs }}
- {{ . | quote }}
{{- end }}
{{- end }}
{{- end }}
---
{{- end }}
@@ -0,0 +1,102 @@
{{- if .Values.enableLabeler }}
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: kyverno:watch-update-namespaces
labels:
app.kubernetes.io/component: background-controller
app.kubernetes.io/instance: kyverno
app.kubernetes.io/part-of: kyverno
rules:
- apiGroups:
- ""
resources:
- namespaces
verbs: ["get", "watch", "list", "patch", "update"]
---
# label system namespaces with dgxc/system-managed: "true"
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: dgxc-label-system-resources
labels:
dgxc/system-managed: "true"
spec:
mutateExistingOnPolicyUpdate: true
rules:
- name: label-cluster-scoped-system-resources
mutate:
targets:
- apiVersion: "{{`{{request.object.apiVersion}}`}}"
kind: "{{`{{request.object.kind}}`}}"
name: "{{`{{request.object.metadata.name}}`}}"
patchStrategicMerge:
metadata:
labels:
"dgxc/system-managed": "true"
match:
any:
{{- range .Values.clusterScopedKinds }}
{{- if gt (len .systemResources) 0 }}
{{- if eq .kind "Namespace" }}
- resources:
kinds: [ {{ .kind | quote }} ]
names:
{{- range .systemResources }}
- {{ . | quote }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
---
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: dgxc-copy-node-group-to-node-role
spec:
rules:
- name: cleanup-node-roles
match:
any:
- resources:
kinds:
- Node
context:
- name: nodeGroup
variable:
jmesPath: request.object.metadata.labels.nodeGroup
default: ""
mutate:
foreach:
- list: keys(request.object.metadata.labels)[?starts_with(@, 'node-role.kubernetes.io')]
patchesJson6902: |-
- op: remove
path: "/metadata/labels/{{`{{ replace_all('{{ element }}', '/', '~1') }}`}}"
preconditions:
all:
- key: "{{`{{ element }}`}}"
operator: NotEquals
value: "{{`{{ nodeGroup }}`}}"
- name: copy-node-role
match:
any:
- resources:
kinds:
- Node
context:
- name: nodeGroup
variable:
jmesPath: request.object.metadata.labels.nodeGroup
default: ""
mutate:
patchStrategicMerge:
metadata:
labels:
node-role.kubernetes.io/{{`{{ nodeGroup }}`}}: ""
preconditions:
all:
- key: "{{`{{ nodeGroup }}`}}"
operator: NotEquals
value: ""
---
{{- end }}
@@ -0,0 +1,19 @@
{{- if .Values.enableServiceMonitor }}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: dgxc-admission-controller
labels:
app: {{ .Values.dgxcController.name }}-metrics
spec:
endpoints:
- interval: 30s
path: /metrics
port: metrics
namespaceSelector:
matchNames:
- {{ .Release.Namespace }}
selector:
matchLabels:
app: {{ .Values.dgxcController.name }}-metrics
{{- end }}
@@ -0,0 +1,400 @@
{{- if and (.Values.cloudProviderSpec) (eq .Values.cloudProviderSpec.type "aws") ((.Values.cloudProviderSpec.aws).efa)}}
# EFA config changes:
# add volume /opt/amazon-efa-ofi to pod, env vars and volumeMount to applicable containers in pod
# Applies to:
# 1) Worker pods resulting from MPIJob or PyTorchJob, and containers only when requested GPUs == gpuAllocatable
# 2) Batch Jobs - these act as launchers for MPIJob and PyTorchJob. Generally no GPU requested.
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
labels:
dgxc/system-managed: "true"
name: dgxc-eks-efa-configs
spec:
background: false
rules:
# attach hostpath volume to Pytorch and MPI workers that are full GPU
- match:
any:
- resources:
kinds:
- Pod
operations:
- CREATE
exclude:
any:
- resources:
annotations:
disable-auto-efa: "true"
mutate:
patchStrategicMerge:
spec:
volumes:
- hostPath:
path: /opt/amazon-efa-ofi
type: Directory
name: amazon-efa
preconditions:
all:
- key: "{{`{{ request.object.spec.containers[].resources.requests.\"nvidia.com/gpu\" || '' }}`}}"
operator: AnyIn
value: [ "{{ .Values.cloudProviderSpec.aws.gpuNode.gpuAllocatable }}" ]
- key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}"
operator: AnyIn
value: [ "PyTorchJob", "MPIJob", "Job", "RunaiJob" ]
- key: "{{`{{ request.object.metadata.ownerReferences[].apiVersion.split(@, '/')[0] }}`}}"
operator: AnyIn
value: [ "batch.volcano.sh", "kubeflow.org", "run.ai" ]
name: efa-pod-volumes-workers
# PyTorch and MPI launchers can be CPU-only Job, still need the hostpath volume
- match:
any:
- resources:
kinds:
- Pod
operations:
- CREATE
exclude:
any:
- resources:
annotations:
disable-auto-efa: "true"
mutate:
patchStrategicMerge:
spec:
volumes:
- hostPath:
path: /opt/amazon-efa-ofi
type: Directory
name: amazon-efa
preconditions:
all:
- key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}"
operator: AnyIn
value: [ "Job" ]
name: efa-pod-volumes-launchers
# Make container changes for EFA - mount and resources
- match:
any:
- resources:
kinds:
- Pod
operations:
- CREATE
exclude:
any:
- resources:
annotations:
disable-auto-efa: "true"
preconditions:
all:
- key: "{{`{{ request.object.spec.containers[].resources.requests.\"nvidia.com/gpu\" || '' }}`}}"
operator: AnyIn
value: [ "{{ .Values.cloudProviderSpec.aws.gpuNode.gpuAllocatable }}" ]
- key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}"
operator: AnyIn
value: [ "PyTorchJob", "MPIJob", "Job", "RunaiJob" ]
- key: "{{`{{ request.object.metadata.ownerReferences[].apiVersion.split(@, '/')[0] }}`}}"
operator: AnyIn
value: [ "batch.volcano.sh", "kubeflow.org", "run.ai" ]
mutate:
foreach:
- list: "request.object.spec.containers"
patchStrategicMerge:
spec:
containers:
- (name): "{{`{{ element.name }}`}}"
volumeMounts:
- mountPath: /opt/amazon-efa-ofi
name: amazon-efa
readOnly: true
resources:
requests:
"vpc.amazonaws.com/efa": "{{ .Values.cloudProviderSpec.aws.gpuNode.efaAllocatable }}"
hugepages-2Mi: "{{`{{ element.resources.requests.\"hugepages-2Mi\" || '`}}{{ .Values.cloudProviderSpec.aws.gpuNode.hugePagesAllocatable }}{{`' }}`}}"
cpu: "{{`{{ element.resources.requests.cpu || '100m' }}`}}"
limits:
"vpc.amazonaws.com/efa": "{{ .Values.cloudProviderSpec.aws.gpuNode.efaAllocatable }}"
hugepages-2Mi: "{{`{{ element.resources.limits.\"hugepages-2Mi\" || '`}}{{ .Values.cloudProviderSpec.aws.gpuNode.hugePagesAllocatable }}{{`' }}`}}"
preconditions:
any:
- key: "{{`{{ element.resources.requests.\"nvidia.com/gpu\" || '' }}`}}"
operator: Equals
value: "{{ .Values.cloudProviderSpec.aws.gpuNode.gpuAllocatable }}"
name: container-efa-volume-mounts-worker
# We treat the launcher differently, doesn't have GPU
- match:
any:
- resources:
kinds:
- Pod
operations:
- CREATE
exclude:
any:
- resources:
annotations:
disable-auto-efa: "true"
preconditions:
all:
- key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}"
operator: AnyIn
value: [ "Job" ]
mutate:
foreach:
- list: "request.object.spec.containers"
patchStrategicMerge:
spec:
containers:
- (name): "{{`{{ element.name }}`}}"
volumeMounts:
- mountPath: /opt/amazon-efa-ofi
name: amazon-efa
readOnly: true
name: container-efa-volume-mounts-launcher
# worker pods need LD_LIBRARY_PATH updated, but we limit this to containers that have gpuAllocatable GPU
- match:
any:
- resources:
kinds:
- Pod
operations:
- CREATE
exclude:
any:
- resources:
annotations:
disable-auto-efa: "true"
preconditions:
all:
- key: "{{`{{ request.object.spec.containers[].resources.requests.\"nvidia.com/gpu\" || '' }}`}}"
operator: AnyIn
value: [ "{{ .Values.cloudProviderSpec.aws.gpuNode.gpuAllocatable }}" ]
- key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}"
operator: AnyIn
value: [ "PyTorchJob", "MPIJob", "Job", "RunaiJob" ]
- key: "{{`{{ request.object.metadata.ownerReferences[].apiVersion.split(@, '/')[0] }}`}}"
operator: AnyIn
value: [ "batch.volcano.sh", "kubeflow.org", "run.ai" ]
mutate:
foreach:
- list: "request.object.spec.containers"
context:
- name: baselibpath
variable:
jmesPath: element.env[?(name=='LD_LIBRARY_PATH')].value | [0]
default: "/usr/lib:/usr/lib64"
- name: extralibpathprefix
variable:
value: "/opt/amazon-efa-ofi/ofi/lib:/opt/amazon-efa-ofi/efa/lib:/opt/amazon-efa-ofi/openmpi/lib:/usr/local/nvidia/lib64"
- name: extralibpathsuffix
variable:
value: "/usr/lib/x86_64-linux-gnu"
patchStrategicMerge:
spec:
containers:
- (name): "{{`{{ element.name }}`}}"
env:
- name: LD_LIBRARY_PATH
value: "{{`{{ extralibpathprefix }}:{{ baselibpath }}:{{ extralibpathsuffix }}`}}"
preconditions:
all:
- key: "{{`{{ element.resources.requests.\"nvidia.com/gpu\" || '' }}`}}"
operator: Equals
value: "{{ .Values.cloudProviderSpec.aws.gpuNode.gpuAllocatable }}"
- key: "{{`{{ element.[env[?(name=='LD_LIBRARY_PATH' && contains(value, 'amazon'))] || `}}`[]`{{` ][] | length(@) }}`}}"
operator: Equals
value: 0
name: container-worker-library-path
# launcher pods need LD_LIBRARY_PATH updated
- match:
any:
- resources:
kinds:
- Pod
operations:
- CREATE
exclude:
any:
- resources:
annotations:
disable-auto-efa: "true"
preconditions:
all:
- key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}"
operator: AnyIn
value: [ "Job" ]
- key: "{{`{{ request.object.metadata.ownerReferences[].apiVersion.split(@, '/')[0] }}`}}"
operator: AnyIn
value: [ "batch" ]
mutate:
foreach:
- list: "request.object.spec.containers"
context:
- name: baselibpath
variable:
jmesPath: element.env[?(name=='LD_LIBRARY_PATH')].value | [0]
default: "/usr/lib:/usr/lib64"
- name: extralibpathprefix
variable:
value: "/opt/amazon-efa-ofi/ofi/lib:/opt/amazon-efa-ofi/efa/lib:/opt/amazon-efa-ofi/openmpi/lib:/usr/local/nvidia/lib64"
- name: extralibpathsuffix
variable:
value: "/usr/lib/x86_64-linux-gnu"
patchStrategicMerge:
spec:
containers:
- (name): "{{`{{ element.name }}`}}"
env:
- name: LD_LIBRARY_PATH
value: "{{`{{ extralibpathprefix }}:{{ baselibpath }}:{{ extralibpathsuffix }}`}}"
preconditions:
all:
- key: "{{`{{ element.[env[?(name=='LD_LIBRARY_PATH' && contains(value, 'amazon'))] || `}}`[]`{{` ][] | length(@) }}`}}"
operator: Equals
value: 0
name: container-launcher-library-path
{{- if and ((.Values.cloudProviderSpec.aws).efa) (((.Values.envVars).user).efa)}}
- match:
any:
- resources:
kinds:
- Pod
operations:
- CREATE
exclude:
any:
- resources:
annotations:
disable-auto-efa: "true"
preconditions:
all:
- key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}"
operator: AnyIn
value: [ "PyTorchJob", "MPIJob", "Job", "RunaiJob" ]
- key: "{{`{{ request.object.metadata.ownerReferences[].apiVersion.split(@, '/')[0] }}`}}"
operator: AnyIn
value: [ "batch.volcano.sh", "kubeflow.org", "run.ai" ]
mutate:
foreach:
{{- range $e := .Values.envVars.user.efa }}
- list: "request.object.spec.containers"
patchStrategicMerge:
spec:
containers:
- (name): "{{`{{ element.name }}`}}"
env:
- name: {{ $e.name }}
value: "{{ $e.value }}"
preconditions:
all:
- key: {{ $e.name }}
operator: AnyNotIn
value: "{{`{{ element.env[].name || '' }}`}}"
- key: "{{`{{ element.resources.requests.\"nvidia.com/gpu\" || '' }}`}}"
operator: Equals
value: "{{ $.Values.cloudProviderSpec.aws.gpuNode.gpuAllocatable }}"
{{- end }}
name: extra-gpu-env-vars-workers
- match:
any:
- resources:
kinds:
- Pod
operations:
- CREATE
exclude:
any:
- resources:
annotations:
disable-auto-efa: "true"
preconditions:
all:
- key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}"
operator: AnyIn
value: [ "Job" ]
- key: "{{`{{ request.object.metadata.ownerReferences[].apiVersion.split(@, '/')[0] }}`}}"
operator: AnyIn
value: [ "batch" ]
mutate:
foreach:
{{- range $e := .Values.envVars.user.efa }}
- list: "request.object.spec.containers"
patchStrategicMerge:
spec:
containers:
- (name): "{{`{{ element.name }}`}}"
env:
- name: {{ $e.name }}
value: "{{ $e.value }}"
preconditions:
all:
- key: {{ $e.name }}
operator: AnyNotIn
value: "{{`{{ element.env[].name || '' }}`}}"
{{- end }}
name: extra-gpu-env-vars-launchers
{{- end}}
---
{{- end}}
{{- if and (.Values.cloudProviderSpec) (eq .Values.cloudProviderSpec.type "azure") (eq .Values.cloudProviderSpec.azure.instanceType "Standard_ND96amsr_A100_v4")}}
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: nccl-topo-configmap-policy
spec:
generateExisting: true
rules:
- name: generate-nccl-topo-configmap
match:
any:
- resources:
kinds:
- Namespace
name: "{{ .Values.ncclTopoConfigMapNamespaces }}"
generate:
apiVersion: v1
kind: ConfigMap
name: nccl-topo
synchronize: true
namespace: "{{`{{request.object.metadata.name}}`}}"
data:
data:
nccl-topo.xml: |
<system version="1">
<cpu numaid="0" affinity="0000ffff,0000ffff" arch="x86_64" vendor="AuthenticAMD" familyid="23" modelid="49">
<pci busid="ffff:ff:01.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0003:00:00.0" class="0x030200" link_speed="16 GT/s" link_width="16"/>
<pci busid="0103:00:00.0" class="0x020700" link_speed="16 GT/s" link_width="16"/>
<pci busid="0004:00:00.0" class="0x030200" link_speed="16 GT/s" link_width="16"/>
<pci busid="0104:00:00.0" class="0x020700" link_speed="16 GT/s" link_width="16"/>
</pci>
</cpu>
<cpu numaid="1" affinity="0000ffff,0000ffff" arch="x86_64" vendor="AuthenticAMD" familyid="23" modelid="49">
<pci busid="ffff:ff:02.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0001:00:00.0" class="0x030200" link_speed="16 GT/s" link_width="16"/>
<pci busid="0101:00:00.0" class="0x020700" link_speed="16 GT/s" link_width="16"/>
<pci busid="0002:00:00.0" class="0x030200" link_speed="16 GT/s" link_width="16"/>
<pci busid="0102:00:00.0" class="0x020700" link_speed="16 GT/s" link_width="16"/>
</pci>
</cpu>
<cpu numaid="2" affinity="0000ffff,0000ffff" arch="x86_64" vendor="AuthenticAMD" familyid="23" modelid="49">
<pci busid="ffff:ff:03.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="000d:00:00.0" class="0x030200" link_speed="16 GT/s" link_width="16"/>
<pci busid="0107:00:00.0" class="0x020700" link_speed="16 GT/s" link_width="16"/>
<pci busid="000e:00:00.0" class="0x030200" link_speed="16 GT/s" link_width="16"/>
<pci busid="0108:00:00.0" class="0x020700" link_speed="16 GT/s" link_width="16"/>
</pci>
</cpu>
<cpu numaid="3" affinity="0000ffff,0000ffff" arch="x86_64" vendor="AuthenticAMD" familyid="23" modelid="49">
<pci busid="ffff:ff:04.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="000b:00:00.0" class="0x030200" link_speed="16 GT/s" link_width="16"/>
<pci busid="0105:00:00.0" class="0x020700" link_speed="16 GT/s" link_width="16"/>
<pci busid="000c:00:00.0" class="0x030200" link_speed="16 GT/s" link_width="16"/>
<pci busid="0106:00:00.0" class="0x020700" link_speed="16 GT/s" link_width="16"/>
</pci>
</cpu>
</system>
---
{{- end}}
@@ -0,0 +1,9 @@
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: {{ include "dgxcController.fullname" . }}
spec:
minAvailable: 1
selector:
matchLabels:
app.kubernetes.io/name: {{ include "dgxcController.name" . }}
@@ -0,0 +1,29 @@
{{- if .Values.enableRunAIRbac }}
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: dgxc-customer-app-admin
labels:
dgxc/system-managed: "true"
aggregationRule:
clusterRoleSelectors:
- matchLabels:
dgxc/customer-app-admin: "true"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: dgxc-customer-app-admin
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: dgxc-customer-app-admin
subjects:
{{- range $k, $v := .Values.tenantSubjects }}
{{- range $v }}
- apiGroup: rbac.authorization.k8s.io
kind: {{ $k | quote }}
name: {{ . | quote }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,152 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.16.3
name: runaidgxclogexporters.runai.dgxc.nvidia.com
spec:
group: runai.dgxc.nvidia.com
names:
kind: RunaiDGXCLogExporter
listKind: RunaiDGXCLogExporterList
plural: runaidgxclogexporters
singular: runaidgxclogexporter
scope: Namespaced
versions:
- additionalPrinterColumns:
- jsonPath: .spec.destination
name: Destination
type: string
- jsonPath: .status.watermark
name: Watermark
type: date
name: v1beta1
schema:
openAPIV3Schema:
description: RunaiDGXCLogExporter exports audit logs to an object storage
bucket.
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
spec:
properties:
destination:
description: |-
Destination is the cloud storage bucket to write logs to.
If set, logs of each type specified in LogTypes will be exported to the
bucket periodically. The destination is created and managed by the user.
The Status.WriterPrincipal field will specify the principal that must be
granted write access to the destination.
type: string
logTypes:
default:
- apiserver-audit
description: LogTypes specifies the types of logs to export.
items:
enum:
- apiserver-audit
type: string
minItems: 1
type: array
required:
- destination
- logTypes
type: object
status:
properties:
conditions:
items:
description: Condition contains details for one aspect of the current
state of this API Resource.
properties:
lastTransitionTime:
description: |-
lastTransitionTime is the last time the condition transitioned from one status to another.
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
format: date-time
type: string
message:
description: |-
message is a human readable message indicating details about the transition.
This may be an empty string.
maxLength: 32768
type: string
observedGeneration:
description: |-
observedGeneration represents the .metadata.generation that the condition was set based upon.
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
with respect to the current state of the instance.
format: int64
minimum: 0
type: integer
reason:
description: |-
reason contains a programmatic identifier indicating the reason for the condition's last transition.
Producers of specific condition types may define expected values and meanings for this field,
and whether the values are considered a guaranteed API.
The value should be a CamelCase string.
This field may not be empty.
maxLength: 1024
minLength: 1
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
type: string
status:
description: status of the condition, one of True, False, Unknown.
enum:
- "True"
- "False"
- Unknown
type: string
type:
description: type of condition in CamelCase or in foo.example.com/CamelCase.
maxLength: 316
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
type: string
required:
- lastTransitionTime
- message
- reason
- status
- type
type: object
type: array
x-kubernetes-list-map-keys:
- type
x-kubernetes-list-type: map
watermark:
description: Watermark is the latest timestamp up until which logs
are exported.
format: date-time
type: string
writerPrincipal:
description: |-
WriterPrincipal is the cloud provider-specific identifier of the log writer.
The principal must be granted write access to the destination in order for log
writes to succeed.
type: string
type: object
required:
- spec
type: object
served: true
storage: true
subresources:
status: {}
@@ -0,0 +1,151 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.16.3
name: runaidgxcnetworkpolicies.runai.dgxc.nvidia.com
spec:
group: runai.dgxc.nvidia.com
names:
kind: RunaiDGXCNetworkPolicy
listKind: RunaiDGXCNetworkPolicyList
plural: runaidgxcnetworkpolicies
singular: runaidgxcnetworkpolicy
scope: Namespaced
versions:
- name: v1beta1
schema:
openAPIV3Schema:
description: RunaiDGXCNetworkPolicy defines tenant cluster scoped network
policies.
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
spec:
properties:
clusterAccessPolicy:
description: ClusterAccessPolicy defines how the cluster may be accessed
through the apiserver.
properties:
permittedCIDR:
description: PermittedCIDR defines IP blocks that may access the
cluster through the apiserver.
items:
type: string
type: array
required:
- permittedCIDR
type: object
egressPolicy:
description: EgressPolicy if defined limits the egress NetworkPolicies
that may be defined per-namespace. Applies to all namespaces.
properties:
permittedCIDR:
description: PermittedCIDR defines permitted egress CIDR ranges
items:
properties:
cidr:
description: CIDR defines the CIDR range
type: string
port:
description: Port defines the allowed port
type: string
required:
- cidr
- port
type: object
type: array
required:
- permittedCIDR
type: object
ingressPolicy:
description: IngressPolicy if defined limits the ingress NetworkPolicies
that may be defined per-namespace. Applies to all namespaces.
properties:
permittedCIDR:
description: PermittedCIDR defines permitted ingress CIDR ranges
items:
type: string
type: array
required:
- permittedCIDR
type: object
type: object
status:
properties:
clusterAccessPolicy:
description: ClusterAccessPolicy defines how the cluster may be accessed
through the apiserver.
properties:
permittedCIDR:
description: PermittedCIDR defines IP blocks that may access the
cluster through the apiserver.
items:
type: string
type: array
required:
- permittedCIDR
type: object
egressPolicy:
description: EgressPolicy is the egressPolicy for the last fully reconciled
spec
properties:
permittedCIDR:
description: PermittedCIDR defines permitted egress CIDR ranges
items:
properties:
cidr:
description: CIDR defines the CIDR range
type: string
port:
description: Port defines the allowed port
type: string
required:
- cidr
- port
type: object
type: array
required:
- permittedCIDR
type: object
ingressPolicy:
description: IngressPolicy is the ingressPolicy for the last fully
reconciled spec
properties:
permittedCIDR:
description: PermittedCIDR defines permitted ingress CIDR ranges
items:
type: string
type: array
required:
- permittedCIDR
type: object
reconcileErrors:
description: ReconcileErrors are errors seen during the last reconciliation.
items:
type: string
type: array
type: object
required:
- spec
type: object
served: true
storage: true
subresources:
status: {}
@@ -0,0 +1,236 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.16.3
name: runaidgxcstorages.runai.dgxc.nvidia.com
spec:
group: runai.dgxc.nvidia.com
names:
kind: RunaiDGXCStorage
listKind: RunaiDGXCStorageList
plural: runaidgxcstorages
singular: runaidgxcstorage
scope: Namespaced
versions:
- additionalPrinterColumns:
- description: Storage classes
jsonPath: .status.storageClassesDisplayString
name: StorageClasses
type: string
- description: Percent of storage quota used for volumes
jsonPath: .status.storageUsedPct
name: QuotaUsedPct
type: string
- description: Storage quota
jsonPath: .status.storageQuota
name: Quota
type: string
- description: Storage quota used by PVs
jsonPath: .status.storageRequested
name: QuotaUsed
type: string
- description: Storage utilized by PVs
jsonPath: .status.storageConsumed
name: StorageConsumed
type: string
name: v1beta1
schema:
openAPIV3Schema:
description: RunaiDGXCStorage defines storage for DGXC Run:ai accelerated
compute environments.
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
spec:
description: RunaiDGXCStorageSpec defines the desired storage state
properties:
instances:
additionalProperties:
properties:
name:
description: Name of the PV
type: string
persistentVolumeReclaimPolicy:
description: PersistentVolumeReclaimPolicy describes a policy
for end-of-life maintenance of persistent volumes.
type: string
type: object
type: object
quota:
type: string
quotaID:
type: string
x-kubernetes-validations:
- message: quotaID is immutable
rule: self == oldSelf
tiers:
items:
type: string
type: array
required:
- quotaID
type: object
status:
properties:
altNames:
description: Alternative names of the storage type
items:
type: string
type: array
conditions:
description: Conditions contains storage conditions
items:
description: Condition contains details for one aspect of the current
state of this API Resource.
properties:
lastTransitionTime:
description: |-
lastTransitionTime is the last time the condition transitioned from one status to another.
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
format: date-time
type: string
message:
description: |-
message is a human readable message indicating details about the transition.
This may be an empty string.
maxLength: 32768
type: string
observedGeneration:
description: |-
observedGeneration represents the .metadata.generation that the condition was set based upon.
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
with respect to the current state of the instance.
format: int64
minimum: 0
type: integer
reason:
description: |-
reason contains a programmatic identifier indicating the reason for the condition's last transition.
Producers of specific condition types may define expected values and meanings for this field,
and whether the values are considered a guaranteed API.
The value should be a CamelCase string.
This field may not be empty.
maxLength: 1024
minLength: 1
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
type: string
status:
description: status of the condition, one of True, False, Unknown.
enum:
- "True"
- "False"
- Unknown
type: string
type:
description: type of condition in CamelCase or in foo.example.com/CamelCase.
maxLength: 316
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
type: string
required:
- lastTransitionTime
- message
- reason
- status
- type
type: object
type: array
description:
description: Short description for the quota
type: string
docs:
description: Documentation for the storage class
type: string
instances:
additionalProperties:
properties:
capacity:
description: Capacity of the PV
type: string
creationTime:
description: When the volume was created
format: date-time
type: string
fileShares:
additionalProperties:
properties:
name:
description: Name of the fileshare
type: string
orphaned:
description: Orphaned is true if the volume is not mounted
in any cluster
type: boolean
values:
additionalProperties:
type: string
description: Usage metrics
type: object
required:
- name
type: object
description: Fileshare used for the volume
type: object
location:
description: Region where the resource is provisioned
type: string
name:
description: Name of the PV
type: string
persistentVolumeReclaimPolicy:
description: PersistentVolumeReclaimPolicy describes a policy
for end-of-life maintenance of persistent volumes.
type: string
state:
description: Current state of the resource
type: string
tier:
description: Storage tier
type: string
type: object
description: PVs consuming the quota
type: object
storageClasses:
description: Storage classes that are bound to this quota
items:
type: string
type: array
storageClassesDisplayString:
description: StorageClassesDisplayString has a display string for
printing the storage classes from kubectl
type: string
storageConsumed:
description: Amount of storage consumed from the requested
type: string
storageQuota:
description: Amount of quota allocated
type: string
storageRequested:
description: Amount of storage requested from the quota
type: string
storageUsedPct:
type: string
type: object
required:
- spec
type: object
served: true
storage: true
subresources:
status: {}
@@ -0,0 +1,12 @@
{{ if .Values.enabledComponents.dgxcAdmissionController }}
{{- if .Values.serviceAccount.create }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ .Values.serviceAccount.name }}
{{- if .Values.serviceAccount.annotations }}
annotations:
{{ toYaml .Values.serviceAccount.annotations }}
{{- end }}
{{ end }}
{{ end }}
@@ -0,0 +1,308 @@
{{- if .Values.enableNetworkPolicies }}
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: kyverno:watch-update-services
labels:
app.kubernetes.io/component: background-controller
app.kubernetes.io/instance: kyverno
app.kubernetes.io/part-of: kyverno
rules:
- apiGroups:
- ""
resources:
- services
verbs: ["get", "watch", "list", "patch", "update"]
---
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: add-runai-cluster-redis-networkpolicy
spec:
generateExisting: true
rules:
- generate:
apiVersion: networking.k8s.io/v1
data:
metadata:
labels:
dgxc/generated: "true"
spec:
ingress:
- from:
- namespaceSelector:
matchExpressions:
- key: runai/queue
operator: Exists
podSelector:
matchLabels:
run.ai/control-plane-managed: "true"
ports:
- port: 6379
protocol: TCP
podSelector:
matchLabels:
app: cluster-redis
policyTypes:
- Ingress
kind: NetworkPolicy
name: allow-runai-cluster-redis-ingress
namespace: "{{`{{request.object.metadata.name}}`}}"
synchronize: true
match:
any:
- resources:
kinds:
- Namespace
selector:
matchLabels:
kubernetes.io/metadata.name: "runai"
name: allow-runai-cluster-redis-ingress
---
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: add-networkpolicy-allow-system-ingress
spec:
generateExisting: true
rules:
{{- if and (.Values.cloudProviderSpec) (eq .Values.cloudProviderSpec.type "oci") }}
- name: allow-system-ingress-via-ciliumnetworkpolicy
match:
any:
- resources:
kinds:
- Namespace
selector:
matchLabels:
dgxc/system-managed: "true"
generate:
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
name: allow-system-ingress
synchronize: true
namespace: "{{`{{request.object.metadata.name}}`}}"
data:
metadata:
labels:
dgxc/generated: "true"
spec:
endpointSelector: {}
ingress:
- fromEntities:
- remote-node
{{- end }}
- name: allow-system-ingress-via-clusterpolicy
match:
any:
- resources:
kinds:
- Namespace
selector:
matchLabels:
dgxc/system-managed: "true"
generate:
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
name: allow-system-ingress
synchronize: true
namespace: "{{`{{request.object.metadata.name}}`}}"
data:
metadata:
labels:
dgxc/generated: "true"
spec:
podSelector: {}
policyTypes:
- Ingress
ingress:
- from:
{{- range .Values.network.systemCIDRs }}
- ipBlock:
cidr: {{ . | quote }}
{{- end }}
- namespaceSelector:
matchExpressions:
- key: dgxc/system-managed
operator: In
values: ["true"]
- name: default-deny-ingress-via-clusterpolicy
match:
any:
- resources:
kinds:
- Namespace
selector:
matchLabels:
dgxc/system-managed: "true"
generate:
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
name: default-deny
namespace: "{{`{{request.object.metadata.name}}`}}"
synchronize: true
data:
metadata:
labels:
dgxc/generated: "true"
spec:
podSelector: {}
policyTypes:
- Ingress
---
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: manage-loadbalancer-networkpolicy
spec:
generateExisting: true
rules:
- name: allow-internet-access-for-loadbalancer
match:
resources:
kinds:
- Service
preconditions:
any:
- key: "{{`{{request.object.spec.type}}`}}"
operator: Equals
value: "LoadBalancer"
- key: "{{`{{request.object.spec.type}}`}}"
operator: Equals
value: "NodePort"
generate:
kind: NetworkPolicy
name: allow-internet-access
apiVersion: networking.k8s.io/v1
namespace: "{{`{{request.object.metadata.namespace}}`}}"
synchronize: true
data:
metadata:
labels:
dgxc/generated: "true"
spec:
podSelector:
matchLabels: "{{`{{request.object.spec.selector}}`}}"
policyTypes:
- Ingress
ingress:
- {}
---
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: deny-metadata-access-policy
annotations:
argocd.argoproj.io/sync-options: Force=true,Replace=true
spec:
generateExisting: true
rules:
- name: deny-metadata-access-rule
match:
any:
- resources:
kinds:
- Namespace
exclude:
any:
{{- range .Values.clusterScopedKinds }}
{{- if gt (len .systemResources) 0 }}
{{- if eq .kind "Namespace" }}
- resources:
kinds: [ {{ .kind | quote }} ]
names:
{{- range .systemResources }}
- {{ . | quote }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
generate:
kind: NetworkPolicy
name: deny-metadata-access
apiVersion: networking.k8s.io/v1
namespace: "{{`{{request.object.metadata.name}}`}}"
synchronize: true
data:
metadata:
labels:
dgxc/generated: "true"
spec:
podSelector: {}
policyTypes:
- Egress
egress:
- to:
- ipBlock:
cidr: 0.0.0.0/0
{{- if not (and
(hasKey .Values "cloudProviderSpec")
(hasKey .Values.cloudProviderSpec "type")
(eq .Values.cloudProviderSpec.type "gcp")
(hasKey .Values.cloudProviderSpec "gcp")
(hasKey .Values.cloudProviderSpec.gcp "workloadIdentityEnabled")
(.Values.cloudProviderSpec.gcp.workloadIdentityEnabled)
) }}
except:
- 169.254.0.0/16
{{- end }}
{{- if and (.Values.cloudProviderSpec) (or (eq .Values.cloudProviderSpec.type "gcp") (eq .Values.cloudProviderSpec.type "oci") (eq .Values.cloudProviderSpec.type "azure")) }}
- namespaceSelector: {}
- podSelector: {}
{{- end }}
{{- if and (.Values.cloudProviderSpec) (eq .Values.cloudProviderSpec.type "aws")}}
---
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: deny-node-traffic
spec:
generateExisting: true
rules:
- name: deny-node-traffic
match:
any:
- resources:
kinds:
- Namespace
exclude:
any:
{{- range .Values.clusterScopedKinds }}
{{- if gt (len .systemResources) 0 }}
{{- if eq .kind "Namespace" }}
- resources:
kinds: [ {{ .kind | quote }} ]
names:
{{- range .systemResources }}
- {{ . | quote }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
context:
- name: nodesIPs
apiCall:
urlPath: "/api/v1/nodes"
jmesPath: |
map(&join('', [@, '/32']), items[].status[].addresses[?type=='InternalIP'].address | [])
generate:
synchronize: true
kind: NetworkPolicy
name: deny-node-access
apiVersion: networking.k8s.io/v1
namespace: "{{`{{request.object.metadata.name}}`}}"
data:
metadata:
labels:
dgxc/generated: "true"
spec:
podSelector: {}
policyTypes:
- Egress
egress:
- to:
- ipBlock:
cidr: 0.0.0.0/0
except: "{{`{{ nodesIPs }}`}}"
{{- end }}
{{- end }}
@@ -0,0 +1,28 @@
{{- if .Values.enableNetworkPolicies }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-dns
namespace: kube-system
spec:
podSelector:
matchLabels:
k8s-app: "kube-dns"
policyTypes:
- Ingress
ingress:
- from: []
{{- if and (.Values.cloudProviderSpec) (eq .Values.cloudProviderSpec.type "oci") }}
---
apiVersion: cilium.io/v2
kind: CiliumClusterwideNetworkPolicy
metadata:
name: allow-egress-to-all
spec:
endpointSelector: {}
egress:
- toEntities:
- all
{{- end }}
---
{{- end }}
@@ -0,0 +1,27 @@
apiVersion: v1
kind: Service
metadata:
name: {{ .Values.serviceName }}
labels:
app: {{ .Values.dgxcController.name }}
spec:
ports:
- port: 443
targetPort: 9443
selector:
app: {{ .Values.dgxcController.name }}
---
apiVersion: v1
kind: Service
metadata:
name: dgx-controller-metrics
labels:
app: {{ .Values.dgxcController.name }}-metrics
spec:
ports:
- name: metrics
port: 80
protocol: TCP
targetPort: 8080
selector:
app.kubernetes.io/instance: {{ .Release.Name }}
@@ -0,0 +1,222 @@
{{- if .Values.enableDGXCRbac }}
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: sre-cluster-scoped-access
rules:
- apiGroups: [ "" ]
resources:
- nodes
- nodes/proxy
- nodes/status
- componentstatuses
verbs: [ "*" ]
- apiGroups: [ "apiregistration.k8s.io" ]
resources:
- apiservices
- apiservices/status
verbs: [ "*" ]
- apiGroups: [ "authentication.k8s.io" ]
resources:
- selfsubjectreviews
- tokenreviews
verbs: [ "*" ]
- apiGroups: [ "authorization.k8s.io" ]
resources:
- selfsubjectaccessreviews
- selfsubjectrulesreviews
- subjectaccessreviews
verbs: [ "*" ]
- apiGroups: [ "certificates.k8s.io" ]
resources:
- certificatesigningrequests
- certificatesigningrequests/approval
- certificatesigningrequests/status
verbs: [ "*" ]
- apiGroups: [ "networking.k8s.io" ]
resources:
- ingressclasses
verbs: [ "*" ]
- apiGroups: [ "rbac.authorization.k8s.io" ]
resources:
- clusterrolebindings
- clusterroles
verbs: [ "*" ]
- apiGroups: [ "storage.k8s.io" ]
resources:
- csidrivers
- csinodes
- storageclasses
- volumeattachments
- volumeattachments/status
verbs: [ "*" ]
- apiGroups: [ "admissionregistration.k8s.io" ]
resources:
- mutatingwebhookconfigurations
- validatingwebhookconfigurations
verbs: [ "*" ]
- apiGroups: [ "apiextensions.k8s.io" ]
resources:
- customresourcedefinitions
- customresourcedefinitions/status
verbs: [ "*" ]
- apiGroups: [ "scheduling.k8s.io" ]
resources:
- priorityclasses
verbs: [ "*" ]
- apiGroups: [ "node.k8s.io" ]
resources:
- runtimeclasses
verbs: [ "*" ]
- apiGroups: [ "flowcontrol.apiserver.k8s.io" ]
resources:
- flowschemas
- flowschemas/status
- prioritylevelconfigurations
- prioritylevelconfigurations/status
verbs: [ "*" ]
- apiGroups: [ "auto.gke.io" ]
resources:
- allowlistedworkloads
- allowlistedv2workloads
verbs: [ "*" ]
- apiGroups: [ "cilium.io" ]
resources:
- ciliumexternalworkloads
- ciliumexternalworkloads/status
- ciliumidentities
- ciliumidentities/status
- ciliumnodes
- ciliumnodes/status
verbs: [ "*" ]
- apiGroups: [ "engine.run.ai" ]
resources:
- configs
- configs/status
verbs: [ "*" ]
- apiGroups: [ "external-secrets.io" ]
resources:
- clusterexternalsecrets
- clusterexternalsecrets/status
- clustersecretstores
- clustersecretstores/status
verbs: [ "*" ]
- apiGroups: [ "hub.gke.io" ]
resources:
- memberships
verbs: [ "*" ]
- apiGroups: [ "kyverno.io" ]
resources:
- clusterpolicies
- clusterpolicies/status
verbs: [ "*" ]
- apiGroups: [ "monitoring.googleapis.com" ]
resources:
- clusterpodmonitorings
- clusterpodmonitorings/status
- clusterrules
- clusterrules/status
- globalrules
- globalrules/status
- clusternodemonitorings
- clusternodemonitorings/status
verbs: [ "*" ]
- apiGroups: [ "networking.gke.io" ]
resources:
- gkenetworkparamsets
- gkenetworkparamsets/status
- networks
- networks/status
verbs: [ "*" ]
- apiGroups: [ "nfd.k8s-sigs.io" ]
resources:
- nodefeaturerules
verbs: [ "*" ]
- apiGroups: [ "reports.kyverno.io" ]
resources:
- clusterephemeralreports
verbs: [ "*" ]
- apiGroups: [ "skyhook.nvidia.com" ]
resources:
- skyhooks
- skyhooks/status
verbs: [ "*" ]
- apiGroups: [ "snapshot.storage.k8s.io" ]
resources:
- volumesnapshotcontents
- volumesnapshotcontents/status
- volumesnapshotclasses
verbs: [ "*" ]
- apiGroups: [ "warden.gke.io" ]
resources:
- audits
- audits/status
verbs: [ "*" ]
- apiGroups: [ "wgpolicyk8s.io" ]
resources:
- clusterpolicyreports
verbs: [ "*" ]
- apiGroups: [ "metrics.k8s.io" ]
resources:
- nodes
verbs: [ "*" ]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: sre-cluster-scoped-access
subjects:
- kind: Group
name: {{ .Values.sre.group.default }}
apiGroup: rbac.authorization.k8s.io
roleRef:
kind: ClusterRole
name: sre-cluster-scoped-access
apiGroup: rbac.authorization.k8s.io
---
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: dgxc-sre-generate-namespace-rolebindings
annotations:
argocd.argoproj.io/sync-options: Force=true,Replace=true
labels:
dgxc/system-managed: "true"
spec:
generateExisting: true
rules:
- name: generate-sre-namespace-rolebinding
match:
any:
- resources:
kinds:
- Namespace
names:
{{- range .Values.clusterScopedKinds }}
{{- if eq .kind "Namespace" }}
{{- range .systemResources }}
- {{ . | quote }}
{{- end }}
{{- end }}
{{- end }}
generate:
synchronize: true
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
name: "sre-ns-{{ `{{ request.object.metadata.name }}` }}-binding"
namespace: "{{ `{{ request.object.metadata.name }}` }}"
data:
metadata:
labels:
dgxc/customer-app-admin: "true"
dgxc/system-managed: "true"
dgxc/generated: "true"
subjects:
- kind: Group
name: {{ .Values.sre.group.default }}
apiGroup: rbac.authorization.k8s.io
roleRef:
kind: ClusterRole
name: cluster-admin
apiGroup: rbac.authorization.k8s.io
{{- end }}
@@ -0,0 +1,14 @@
{{- if .Values.enableDGXCRbac }}
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: sre-full-access-binding
subjects:
- kind: Group
name: {{ .Values.sre.group.privileged }}
apiGroup: rbac.authorization.k8s.io
roleRef:
kind: ClusterRole
name: cluster-admin
apiGroup: rbac.authorization.k8s.io
{{- end }}
@@ -0,0 +1,3 @@
{{- if .Values.nvStorage.enabled }}
{{ $.Files.Get "files/storage.dgxc.nvidia.com_nvstoragelocations.yaml" }}
{{- end }}
@@ -0,0 +1,26 @@
{{- if .Values.enableDGXCAPIBootstrap }}
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: dgxc-customer-network-policy
labels:
dgxc/customer-app-admin: "true"
dgxc/system-managed: "true"
rules:
- apiGroups: [ "runai.dgxc.nvidia.com" ]
resources:
- runaidgxcnetworkpolicies
verbs: ["get", "list", "watch", "update", "patch"]
- apiGroups: [ "runai.dgxc.nvidia.com" ]
resources:
- runaidgxcstorages
verbs: ["get", "list", "watch", "update", "patch"]
- apiGroups: [ "runai.dgxc.nvidia.com" ]
resources:
- runaidgxclogexporters
verbs: ["get", "list", "watch", "update", "patch"]
- apiGroups: [ "storage.dgxc.nvidia.com" ]
resources:
- nvstoragelocations
verbs: ["*"]
{{- end }}
@@ -0,0 +1,6 @@
{{- if .Values.enableDGXCAPIBootstrap }}
apiVersion: v1
kind: Namespace
metadata:
name: dgxc-tenant-cluster-policies
{{- end }}
@@ -0,0 +1,66 @@
---
kubernetesAffinity:
enabled: false
systemTolerationKeyName: dedicated
systemTolerationValueName: "system-workload"
tolerations:
customer:
key: "dedicated"
value: "user-workload"
system:
key: "dedicated"
value: "system-workload"
cloudProviderSpec:
type: aws
aws:
accountName: nv-dgxc-non-prod-runai-dev
clusterName: t11111111-dgxc-k8s-runai-aws-us-east-1-dev
efa: true
region: us-east-1
securityGroupARN: arn:aws:ec2:us-east-1:111111111111:security-group/sg-11111111111111111
statefulRuleGroupName: dgxc-runai-dev-fw-nacls
gpuNode:
gpuAllocatable: "8"
hugePagesAllocatable: 42242Mi
efaAllocatable: "32"
envVars:
user:
efa:
- name: FI_EFA_USE_DEVICE_RDMA
value: "1"
- name: OPAL_PREFIX
value: "/opt/amazon-efa-ofi/openmpi"
- name: NVIDIA_GDRCOPY
value: "enabled"
storageControllerSpecAWS:
enabled: false
defaultStorageTypes:
- apiVersion: runai.dgxc.nvidia.com/v1beta1
kind: RunaiDGXCStorage
metadata:
name: amazon-fsx-for-lustre
namespace: dgxc-tenant-cluster-policies
spec:
quotaID: L-C8640C82
status:
altNames: [ "fsx" ]
storageClasses:
- lustre-sc
- dgxc-enterprise-file
- dgxc-standard-file
storageClassesDisplayString: "lustre-sc"
docs: https://aws.amazon.com/fsx/when-to-choose-fsx/
nvStorage:
# NVStorageLocation CRD management
enabled: false
storageClass: "dgxc-standard-object"
csi:
driver: "s3.csi.aws.com"
volumeAttributes:
authenticationSource: pod
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,78 @@
kubernetesAffinity:
enabled: false
systemTolerationKeyName: dedicated
systemTolerationValueName: "system-workload"
tolerations:
customer:
key: "dedicated"
value: "user-workload"
system:
key: "dedicated"
value: "system-workload"
cloudProviderSpec:
type: azure
azure:
subscriptionId: 99e004ca-5389-45c5-9e7b-402c0a46550c
subscriptionName: sub-dgxc-runai-non-prod-test
resourceGroupName: rg-runai-scus-dev-test
logExporterAnalyticsWorkspaceId: workspace-id
ingressNsgName: ingress-name
egressNsgName: egress-name
standardStorageAccountName: standardtz990000
premiumStorageAccountName: premiumtz990000
instanceType: Standard_ND96amsr_A100_v4
storageControllerSpecAzure:
enabled: false
defaultStorageTypes:
- apiVersion: runai.dgxc.nvidia.com/v1beta1
kind: RunaiDGXCStorage
metadata:
name: dgxc-azurefile
namespace: dgxc-tenant-cluster-policies
spec:
quotaID: AzureFileStandard
status:
altNames: [ "dgxc-azurefile" ]
storageClasses:
- dgxc-azurefile
storageClassesDisplayString: "dgxc-azurefile"
docs: https://learn.microsoft.com/en-us/azure/storage/files/storage-files-planning
- apiVersion: runai.dgxc.nvidia.com/v1beta1
kind: RunaiDGXCStorage
metadata:
name: dgxc-azurefile-premium-pool-1
namespace: dgxc-tenant-cluster-policies
spec:
quotaID: AzureFilePremiumPool1
status:
altNames: [ "dgxc-azurefile-premium-pool-1" ]
storageClasses:
- dgxc-azurefile-premium-pool-1
storageClassesDisplayString: "dgxc-azurefile-premium-pool-1"
- apiVersion: runai.dgxc.nvidia.com/v1beta1
kind: RunaiDGXCStorage
metadata:
name: dgxc-azurefile-premium-pool-2
namespace: dgxc-tenant-cluster-policies
spec:
quotaID: AzureFilePremiumPool2
status:
altNames: [ "dgxc-azurefile-premium-pool-2" ]
storageClasses:
- dgxc-azurefile-premium-pool-2
storageClassesDisplayString: "dgxc-azurefile-premium-pool-2"
podMutatingWebhook:
customSettings:
- name: modify-pod-rule
addRdmaEnabled: true
rdmaResourcePerGpu: 1
rdmaResourceName: nvidia.com/mlnxnics
addNcclVars: true
csp: "azure"
ncclTopoConfigMapNamespaces: runai-*
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,107 @@
---
kubernetesAffinity:
enabled: true
# we will make an effort to not take up valuable GPU resources
# so try and run on a CPU node that is not reserved for the Kubernetes Master.
# matchExpressions:
#- key: "node-role.kubernetes.io/fss-workload"
# operator: Exists
# matchExpressions:
#- key: "kubernetes.io/role"
# operator: In
# values: ["fss-workload"]
#
# The below configurationa replaces the match expressions.
# The default setting is the handle legacy clusters without the above
# matching updated label.
antiAffinityKey: cloud.google.com/gke-gpu
antiAffinityValue: true
# The below is used for pod anti-affinity to ensure pods are running on
# different nodes
topologyKey: kubernetes.io/hostname
# Run on Masters should remain false unless there is a very specific need
# (ie. if you have been given a minimal dev cluster, with _only_ masters)
runOnMasters: false
systemTolerationKeyName: components.gke.io/gke-managed-components
systemTolerationValueName: "true"
tolerations:
customer:
key: "dedicated"
value: "user-workload"
system:
key: "components.gke.io/gke-managed-components"
value: "true"
cloudProviderSpec:
type: gcp
gcp:
projectId: proj-dgxc-runai-np-dev-mega
projectNumber: 574718269555
securityPolicyRuleName: dgxc-runai-us-east5-ngc-00546849-system-lb-backend-policy
region: us-east5
# GCP Workload Identity requires access to the metadata server
# Setting this flag, disables the kyverno security policy preventing
# Pods for accessing the metadata server.
workloadIdentityEnabled: false
storageControllerSpecGCP:
enabled: false
defaultStorageTypes:
- apiVersion: runai.dgxc.nvidia.com/v1beta1
kind: RunaiDGXCStorage
metadata:
name: "filestore-enterprise"
namespace: dgxc-tenant-cluster-policies
spec:
quotaID: "EnterpriseStorageGibPerRegion"
status:
altNames: [ "enterprise" ]
storageClasses:
- enterprise-multishare-rwx
- enterprise-rwx
storageClassesDisplayString: "enterprise-multishare-rwx,enterprise-rwx"
docs: https://cloud.google.com/filestore/docs/service-tiers
- apiVersion: runai.dgxc.nvidia.com/v1beta1
kind: RunaiDGXCStorage
metadata:
name: "filestore-premium"
namespace: dgxc-tenant-cluster-policies
spec:
quotaID: "PremiumStorageGbPerRegion"
status:
altNames: [ "premium" ]
storageClasses:
- premium-rwo
- premium-rwx
storageClassesDisplayString: "premium-rwo,premium-rwx"
docs: https://cloud.google.com/filestore/docs/service-tiers
- apiVersion: runai.dgxc.nvidia.com/v1beta1
kind: RunaiDGXCStorage
metadata:
name: "filestore-standard"
namespace: dgxc-tenant-cluster-policies
spec:
quotaID: "StandardStorageGbPerRegion"
status:
altNames: [ "filestore-standard" ]
storageClasses:
- standard
- standard-rwo
- standard-rwx
storageClassesDisplayString: "standard,standard-rwo,standard-rwx"
docs: https://cloud.google.com/filestore/docs/service-tiers
- apiVersion: runai.dgxc.nvidia.com/v1beta1
kind: RunaiDGXCStorage
metadata:
name: "filestore-high-scale"
namespace: dgxc-tenant-cluster-policies
spec:
quotaID: "HighScaleSSDStorageGibPerRegion"
status:
altNames: [ "zonal" ]
storageClasses:
- zonal-rwx
storageClassesDisplayString: "zonal-rwx"
docs: https://cloud.google.com/filestore/docs/service-tiers
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,61 @@
---
kubernetesAffinity:
enabled: false
systemTolerationKeyName: CriticalAddonsOnly
systemTolerationValueName: "true"
tolerations:
customer:
key: "dedicated"
value: "user-workload"
system:
key: "CriticalAddonsOnly"
value: "true"
cloudProviderSpec:
type: oci
oci:
compartmentID: ocid1.compartment.oc1..aaaaaaaadt4vviaydftj2dknihhot2dhpmb4cvjn6lr57idmp4q2tbnhj6kq
clusterID: ocid1.cluster.oc1.iad.aaaaaaaalrxcu6og5prnj4hm2kcp7qvkyu26spioeic34i5ytc5bnhgiz67q
fssAvailabilityDomain: VSwX:US-ASHBURN-AD-1
publicSubnetFirewallID: ocid1.networkfirewall.oc1.iad.amaaaaaaaqwjupqaedtz64tuorazbidw36a647cyluuqs2gf4kwev4ayf3yq
privateSubnetFirewallID: ocid1.networkfirewall.oc1.iad.amaaaaaaaqwjupqadoaciovfm5lpledoh4vbnx3iuytj2gwhzu74hlpfcfha
baseFirewallPolicyID: ocid1.fpid.oc1.iad.aaaaaaaalrxcu6og5prnj4hm2kcp7qvkyu26spioeic34i5ytc5bnhgiz67q
resourcePrincipalVersion: "2.2"
region: us-ashburn-1
storageControllerSpecOCI:
enabled: true
defaultStorageTypes:
- apiVersion: runai.dgxc.nvidia.com/v1beta1
kind: RunaiDGXCStorage
metadata:
name: fss
namespace: dgxc-tenant-cluster-policies
spec:
quotaID: unlimited
status:
altNames: [ "fss" ]
storageClasses:
- fss
storageClassesDisplayString: "fss"
webhooks:
registeredWebhooks:
- name: dgxc-create-pvc-vwh
enabled: false
- name: dgxc-create-pvc-mwh
enabled: false
- name: dgxc-create-pod-mwh
enabled: true
podMutatingWebhook:
customSettings:
- name: modify-pod-rule
addRdmaEnabled: true
rdmaResourcePerGpu: 2
networkAttachmentDefinition: network-operator/sriov-net
rdmaResourceName: nvidia.com/mlnxnics
addNcclVars: true
csp: "oci"
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,282 @@
---
imagePullSecret: "nvidia-ngcuser-pull-secret"
enabledComponents:
dgxcAdmissionController: true
dgxcRunaiController: true
dgxcExporter: true
# The storage controller is the thing we're running
dgxcController:
name: dgxc-admission-controller
replicas: 3
image: "nvcr.io/nv-ngc-devops/dgxc-admission-controller:latest"
# shared between the init container and the controller container
certs:
mountPath: "/tmp/k8s-webhook-server/serving-certs"
certName: tls.crt
keyName: tls.key
nvStorage:
# NVStorageLocation CRD management
enabled: false
storageClass: ""
csi: {}
serviceName: dgx-controller-svc
serviceAccount:
create: true
name: dgxc-admission-controller-sa
annotations: {}
controllerServiceAccount:
create: true
name: dgxc-runai-controller
annotations: {}
webhooks:
organization: dgxc.nvidia.com
registeredWebhooks:
- name: dgxc-create-pvc-vwh
type: PvcValidatingWebhook
cmName: dgxc-create-pvc-vwh-config
enabled: true
- name: dgxc-create-pvc-mwh
type: PvcMutatingWebhook
cmName: dgxc-create-pvc-mwh-config
enabled: true
- name: dgxc-create-pod-mwh
type: PodMutatingWebhook
cmName: dgxc-create-pod-mwh-config
enabled: true
pvcValidatingWebhook:
cmName: dgxc-create-pvc-vwh-config
type: PvcValidatingWebhook
kind: ValidatingWebhookConfiguration
fqdn: dgxc-create-pvc-vwh.dgxc.nvidia.com
path: "/pvccreate/v1/validation"
failurePolicy: Fail
timeoutSeconds: 10
sideEffects: NoneOnDryRun
admissionReviewVersions:
- v1
rules:
- apiGroups:
- ""
apiVersions:
- "v1"
operations:
- CREATE
resources:
- persistentvolumeclaims
scope: "*"
namespaceLabelSelector:
- key: runai/namespace-version
values:
- "v1"
- "v2"
objectLabelSelector:
customSettings:
- name: filestore-pvc-premium-rule
storageClasses:
- zonal-rwx
minSizeCheckEnabled: false
minSizeValue: 10Ti
maxSizeCheckEnabled: false
maxSizeValue: 100Ti
- name: filestore-pvc-standard-rule
storageClasses:
- standard-rwx
minSizeCheckEnabled: false
minSizeValue: 1Ti
maxSizeCheckEnabled: false
maxSizeValue: 10Ti
pvcMutatingWebhook:
cmName: dgxc-create-pvc-mwh-config
type: PvcMutatingWebhook
kind: MutatingWebhookConfiguration
fqdn: dgxc-create-pvc-mwh.dgxc.nvidia.com
path: "/pvccreate/v1/mutation"
failurePolicy: Fail
timeoutSeconds: 10
sideEffects: NoneOnDryRun
admissionReviewVersions:
- v1
rules:
- apiGroups:
- ""
apiVersions:
- "v1"
operations:
- CREATE
resources:
- persistentvolumeclaims
scope: "*"
namespaceLabelSelector:
- key: runai/namespace-version
values:
- "v1"
- "v2"
objectLabelSelector:
podMutatingWebhook:
cmName: dgxc-create-pod-mwh-config
kind: MutatingWebhookConfiguration
type: PodMutatingWebhook
fqdn: dgxc-create-pod-mwh.dgxc.mvp.nvidia.com
path: "/podcreate/v1/mutation"
failurePolicy: Fail
timeoutSeconds: 10
sideEffects: NoneOnDryRun
admissionReviewVersions:
- v1
rules:
- apiGroups:
- ""
apiVersions:
- "v1"
operations:
- CREATE
resources:
- pods
scope: "*"
namespaceLabelSelector:
objectLabelSelector:
customSettings:
- name: modify-pod-rule
scratchEnabled: true
scratchVolName: "scratch"
scratchMountpath: "/scratch"
scratchVolSize: "200Gi"
scratchVolSizePerGpu: true
scratchVolEnforceLimit: true
addTcpxSidecarEnabled: true
addTcpxoSidecarEnabled: false
clusterName: "test"
addTcpxNCCLVars: true
addTcpxoNCCLVars: false
addRdmaEnabled: false
addRoceOciNcclVars: false
addGcpFuseAnnotation: true
gcpFuseAnnotationKey: "gke-gcsfuse/volumes"
gcpFuseWICredentialConfigMapAnnotationKey: "gke-gcsfuse/workload-identity-credential-configmap" # This is the annotation key for the ConfigMap that contains the GCP Workload Identity credentials.
gcpFuseStorageClasses:
- dgxc-standard-object
prober:
intervalInSeconds: 60
timeoutInSeconds: 5
endpoints:
- name: dev-cluster
url: https://dgxc-runai-us-east5-ngc-11918940.dgxc.nonprod-nvkong.com
sso:
controlPlaneDomain: run.ai
cluster: "TODO"
realm: dgxc
pollIntervalSeconds: 10
pollAttempts: 180 # 30 minutes
initSettings:
enabled: false
appID: dgxc
tenant: dgxc
controlPlane: run.ai
supportEmailAddress: support@nvidia.com
customerEmailAddress: customer@nvidia.com
bootstrapEmailAddress: bootstrap@nvidia.com
crdRBACName: "{{request.object.spec.names.plural}}"
validationFailureAction: Enforce
tenantSubjects:
User:
- ebalduf@test.nvidia.com
- kelhussein@test.nvidia.com
- ladithyav@test.nvidia.com
- pwittrock@test.nvidia.com
tolerations:
customer:
key: "dedicated"
value: "user-workload"
system:
key: "components.gke.io/gke-managed-components"
value: "true"
bootstrapIngressCIDR: [ "216.228.125.128/30", "12.97.180.32/27", "216.228.127.128/30", "202.164.25.0/27", "120.31.162.32/28", "203.18.50.0/27", "103.74.125.160/28", "216.228.112.0/26" ]
dgxcAdmins:
- Run:ai
- svctdcsqa1@nvidia.com
reserved:
CPU: 2
Memory: 5Gi
maxCPUJobs: 10
logExporterControllerSpec:
enabled: false
exportFrequency: 5m
exportBuffer: 15m
fetchTimeout: 5m
networkControllerSpec:
enabled: false
storageControllerSpecGCP:
enabled: false
storageControllerSpecAWS:
enabled: false
storageControllerSpecOCI:
enabled: false
storageControllerSpecAzure:
enabled: false
exporter:
metricsPort: 8080
sre:
group:
default: "non-existing-engineer@nvidia.com"
privileged: "non-existing-admin@nvidia.com"
tenantMetadata:
cluster: TODO
platform: runai
orgName: nvidian
tenantName: test-mega
tenantType: Internal
startDate: '8/02/24'
endDate: '01/01/99'
CSP: gcp
allocatedCapacityGPU: 2
allocatedCapacityCPU: 2
committedCapacityGPU: 2
committedCapacityCPU: 2
CSPProject: proj-dgxc-runai-np-test-mega
dataCenterRegion: us-central1
storageType: GCPFilestore
storageQuota: 1
cspSubjects:
User:
- "107878228028626601324"
network:
systemCIDRs:
- "10.0.3.0/24"
- "10.0.4.0/24"
enableIntegrationsSupport: true
enableLabeler: true
enableRunAIRbac: true
enableNetworkPolicies: true
enableDGXCRbac: true
enableDGXCAPIBootstrap: true
enableServiceMonitor: true
@@ -0,0 +1,23 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
@@ -0,0 +1,17 @@
apiVersion: v2
appVersion: 1.0.1
description: NVIDIA NIM Operator creates/configures/manages NVIDIA NIM microservices
in Kubernetes
home: https://docs.nvidia.com/nim-operator/latest/index.html
icon: https://assets.nvidiagrid.net/ngc/logos/NIMOperator.png
keywords:
- gpu
- nim
- nemo
- operator
- ai
name: k8s-nim-operator
sources:
- https://github.com/NVIDIA/k8s-nim-operator
type: application
version: 1.0.1
@@ -0,0 +1,500 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.16.2
name: nimcaches.apps.nvidia.com
spec:
group: apps.nvidia.com
names:
kind: NIMCache
listKind: NIMCacheList
plural: nimcaches
singular: nimcache
scope: Namespaced
versions:
- additionalPrinterColumns:
- jsonPath: .status.state
name: Status
type: string
- jsonPath: .status.pvc
name: PVC
type: string
- format: date-time
jsonPath: .metadata.creationTimestamp
name: Age
type: date
name: v1alpha1
schema:
openAPIV3Schema:
description: NIMCache is the Schema for the nimcaches API
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
spec:
description: NIMCacheSpec defines the desired state of NIMCache
properties:
certConfig:
description: |-
CertConfig is the name of the ConfigMap containing the custom certificates.
for secure communication.
properties:
mountPath:
description: MountPath is the path where the certificates should
be mounted in the container.
type: string
name:
description: Name of the ConfigMap containing the certificate
data.
type: string
required:
- mountPath
- name
type: object
env:
description: Env are the additional custom environment variabes for
the caching job
items:
description: EnvVar represents an environment variable present in
a Container.
properties:
name:
description: Name of the environment variable. Must be a C_IDENTIFIER.
type: string
value:
description: |-
Variable references $(VAR_NAME) are expanded
using the previously defined environment variables in the container and
any service environment variables. If a variable cannot be resolved,
the reference in the input string will be unchanged. Double $$ are reduced
to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e.
"$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)".
Escaped references will never be expanded, regardless of whether the variable
exists or not.
Defaults to "".
type: string
valueFrom:
description: Source for the environment variable's value. Cannot
be used if value is not empty.
properties:
configMapKeyRef:
description: Selects a key of a ConfigMap.
properties:
key:
description: The key to select.
type: string
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
type: string
optional:
description: Specify whether the ConfigMap or its key
must be defined
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
fieldRef:
description: |-
Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['<KEY>']`, `metadata.annotations['<KEY>']`,
spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.
properties:
apiVersion:
description: Version of the schema the FieldPath is
written in terms of, defaults to "v1".
type: string
fieldPath:
description: Path of the field to select in the specified
API version.
type: string
required:
- fieldPath
type: object
x-kubernetes-map-type: atomic
resourceFieldRef:
description: |-
Selects a resource of the container: only resources limits and requests
(limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.
properties:
containerName:
description: 'Container name: required for volumes,
optional for env vars'
type: string
divisor:
anyOf:
- type: integer
- type: string
description: Specifies the output format of the exposed
resources, defaults to "1"
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
resource:
description: 'Required: resource to select'
type: string
required:
- resource
type: object
x-kubernetes-map-type: atomic
secretKeyRef:
description: Selects a key of a secret in the pod's namespace
properties:
key:
description: The key of the secret to select from. Must
be a valid secret key.
type: string
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
type: string
optional:
description: Specify whether the Secret or its key must
be defined
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
type: object
required:
- name
type: object
type: array
groupID:
description: GroupID is the group ID for the caching job
format: int64
type: integer
nodeSelector:
additionalProperties:
type: string
description: NodeSelector is the node selector labels to schedule
the caching job.
type: object
resources:
description: Resources defines the minimum resources required for
the caching job to run(cpu, memory, gpu).
properties:
cpu:
anyOf:
- type: integer
- type: string
description: CPU indicates the minimum number of CPUs to use while
caching NIM
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
memory:
anyOf:
- type: integer
- type: string
description: |-
Memory indicates the minimum amount of memory to use while caching NIM
Valid values are numbers followed by one of the suffixes Ki, Mi, Gi, or Ti (e.g. "4Gi", "4096Mi").
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
type: object
runtimeClassName:
description: RuntimeClassName is the runtimeclass for the caching
job
type: string
source:
description: Source is the NIM model source to cache
properties:
dataStore:
description: NGCSource represents models stored in NVIDIA DataStore
service
properties:
authSecret:
description: The name of an existing auth secret containing
the AUTH_TOKEN"
type: string
checkpointName:
type: string
datasetName:
type: string
endpoint:
description: The endpoint for datastore
type: string
modelName:
description: Name of either model/checkpoint or dataset to
download
type: string
modelPuller:
description: ModelPuller is the container image that can pull
the model
type: string
pullSecret:
description: PullSecret for the model puller image
type: string
required:
- authSecret
- endpoint
- modelPuller
type: object
ngc:
description: NGCSource represents models stored in NGC
properties:
authSecret:
description: The name of an existing pull secret containing
the NGC_API_KEY
type: string
model:
description: Model spec for caching
properties:
buildable:
description: Buildable indicates generic model profiles
that can be optimized with an NVIDIA engine for any
GPUs
type: boolean
engine:
description: Engine is the backend engine (tensort_llm,
vllm)
type: string
gpus:
description: GPU is the spec for matching GPUs for caching
optimized models
items:
description: GPUSpec is the spec required to cache models
for selected gpu type
properties:
ids:
description: IDs are the device-ids for a specific
GPU SKU
items:
type: string
type: array
product:
description: Product is the GPU product string (h100,
a100, l40s)
type: string
type: object
type: array
lora:
description: Lora indicates a finetuned model with LoRa
adapters
type: boolean
precision:
description: Precision is the precision for model quantization
type: string
profiles:
description: Profiles are the specific model profiles
to cache. When these are provided, rest of the model
parameters for profile selection are ignored
items:
type: string
type: array
qosProfile:
description: QoSProfile is the supported QoS profile types
for the models (throughput, latency)
type: string
tensorParallelism:
description: TensorParallelism is the minimum GPUs required
for the model computations
type: string
type: object
modelPuller:
description: ModelPuller is the container image that can pull
the model
type: string
x-kubernetes-validations:
- message: modelPuller is an immutable field. Please create
a new NIMCache resource instead when you want to change
this container.
rule: self == oldSelf
pullSecret:
description: PullSecret to pull the model puller image
type: string
required:
- authSecret
- modelPuller
type: object
type: object
storage:
description: Storage is the target storage for caching NIM model
properties:
hostPath:
description: HostPath is the host path volume for caching NIM
type: string
pvc:
description: PersistentVolumeClaim is the pvc volume used for
caching NIM
properties:
create:
description: Create indicates to create a new PVC
type: boolean
name:
description: Name is the name of the PVC
type: string
size:
description: Size of the NIM cache in Gi, used during PVC
creation
type: string
storageClass:
description: StorageClass to be used for PVC creation. Leave
it as empty if the PVC is already created.
type: string
subPath:
type: string
volumeAccessMode:
description: VolumeAccessMode is the volume access mode of
the PVC
type: string
type: object
type: object
tolerations:
description: Tolerations for running the job to cache the NIM model
items:
description: |-
The pod this Toleration is attached to tolerates any taint that matches
the triple <key,value,effect> using the matching operator <operator>.
properties:
effect:
description: |-
Effect indicates the taint effect to match. Empty means match all taint effects.
When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
type: string
key:
description: |-
Key is the taint key that the toleration applies to. Empty means match all taint keys.
If the key is empty, operator must be Exists; this combination means to match all values and all keys.
type: string
operator:
description: |-
Operator represents a key's relationship to the value.
Valid operators are Exists and Equal. Defaults to Equal.
Exists is equivalent to wildcard for value, so that a pod can
tolerate all taints of a particular category.
type: string
tolerationSeconds:
description: |-
TolerationSeconds represents the period of time the toleration (which must be
of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,
it is not set, which means tolerate the taint forever (do not evict). Zero and
negative values will be treated as 0 (evict immediately) by the system.
format: int64
type: integer
value:
description: |-
Value is the taint value the toleration matches to.
If the operator is Exists, the value should be empty, otherwise just a regular string.
type: string
type: object
type: array
userID:
description: UserID is the user ID for the caching job
format: int64
type: integer
required:
- source
- storage
type: object
status:
description: NIMCacheStatus defines the observed state of NIMCache
properties:
conditions:
items:
description: Condition contains details for one aspect of the current
state of this API Resource.
properties:
lastTransitionTime:
description: |-
lastTransitionTime is the last time the condition transitioned from one status to another.
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
format: date-time
type: string
message:
description: |-
message is a human readable message indicating details about the transition.
This may be an empty string.
maxLength: 32768
type: string
observedGeneration:
description: |-
observedGeneration represents the .metadata.generation that the condition was set based upon.
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
with respect to the current state of the instance.
format: int64
minimum: 0
type: integer
reason:
description: |-
reason contains a programmatic identifier indicating the reason for the condition's last transition.
Producers of specific condition types may define expected values and meanings for this field,
and whether the values are considered a guaranteed API.
The value should be a CamelCase string.
This field may not be empty.
maxLength: 1024
minLength: 1
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
type: string
status:
description: status of the condition, one of True, False, Unknown.
enum:
- "True"
- "False"
- Unknown
type: string
type:
description: type of condition in CamelCase or in foo.example.com/CamelCase.
maxLength: 316
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
type: string
required:
- lastTransitionTime
- message
- reason
- status
- type
type: object
type: array
profiles:
items:
description: NIMProfile defines the profiles that were cached
properties:
config:
additionalProperties:
type: string
type: object
model:
type: string
name:
type: string
release:
type: string
type: object
type: array
pvc:
type: string
state:
type: string
type: object
type: object
served: true
storage: true
subresources:
status: {}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,58 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "k8s-nim-operator.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "k8s-nim-operator.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "k8s-nim-operator.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "k8s-nim-operator.labels" -}}
helm.sh/chart: {{ include "k8s-nim-operator.chart" . }}
{{ include "k8s-nim-operator.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "k8s-nim-operator.selectorLabels" -}}
app.kubernetes.io/name: {{ include "k8s-nim-operator.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Full image name with tag
*/}}
{{- define "k8s-nim-operator.fullimage" -}}
{{- .Values.operator.image.repository -}}:{{- .Values.operator.image.tag | default (printf "v%s" .Chart.AppVersion) -}}
{{- end }}
@@ -0,0 +1,78 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "k8s-nim-operator.fullname" . }}
labels:
app.kubernetes.io/component: manager
app.kubernetes.io/created-by: k8s-nim-operator
app.kubernetes.io/part-of: k8s-nim-operator
control-plane: controller-manager
{{- include "k8s-nim-operator.labels" . | nindent 4 }}
spec:
replicas: {{ .Values.operator.replicas }}
selector:
matchLabels:
control-plane: controller-manager
{{- include "k8s-nim-operator.selectorLabels" . | nindent 6 }}
template:
metadata:
labels:
control-plane: controller-manager
{{- include "k8s-nim-operator.selectorLabels" . | nindent 8 }}
annotations:
kubectl.kubernetes.io/default-container: manager
spec:
containers:
- args: {{- toYaml .Values.operator.args | nindent 8 }}
command:
- /manager
image: {{ include "k8s-nim-operator.fullimage" . }}
imagePullPolicy: {{ .Values.operator.image.pullPolicy }}
env:
- name: WATCH_NAMESPACE
value: ""
- name: OPERATOR_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
livenessProbe:
httpGet:
path: /healthz
port: 8081
initialDelaySeconds: 15
periodSeconds: 20
name: manager
readinessProbe:
httpGet:
path: /readyz
port: 8081
initialDelaySeconds: 5
periodSeconds: 10
resources: {{- toYaml .Values.operator.resources | nindent 10 }}
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
imagePullSecrets:
{{- range .Values.operator.image.pullSecrets }}
- name: {{ . }}
{{- end }}
securityContext:
seccompProfile:
type: RuntimeDefault
runAsNonRoot: true
serviceAccountName: k8s-nim-operator
terminationGracePeriodSeconds: 10
{{- with .Values.operator.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.operator.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.operator.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
@@ -0,0 +1,59 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: k8s-nim-operator-leader-election-role
labels:
app.kubernetes.io/component: rbac
app.kubernetes.io/created-by: k8s-nim-operator
app.kubernetes.io/part-of: k8s-nim-operator
{{- include "k8s-nim-operator.labels" . | nindent 4 }}
rules:
- apiGroups:
- ""
resources:
- configmaps
verbs:
- get
- list
- watch
- create
- update
- patch
- delete
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs:
- get
- list
- watch
- create
- update
- patch
- delete
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: k8s-nim-operator-leader-election-rolebinding
labels:
app.kubernetes.io/component: rbac
app.kubernetes.io/created-by: k8s-nim-operator
app.kubernetes.io/part-of: k8s-nim-operator
{{- include "k8s-nim-operator.labels" . | nindent 4 }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: k8s-nim-operator-leader-election-role
subjects:
- kind: ServiceAccount
name: k8s-nim-operator
namespace: '{{ .Release.Namespace }}'
@@ -0,0 +1,315 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: k8s-nim-operator-role
labels:
{{- include "k8s-nim-operator.labels" . | nindent 4 }}
rules:
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- configmaps
- persistentvolumeclaims
- secrets
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- update
- patch
- apiGroups:
- ""
resources:
- endpoints
- pods
- pods/log
- pods/eviction
- serviceaccounts
- services
- services/finalizers
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
- pods
verbs:
- create
- delete
- get
- list
- watch
- apiGroups:
- apps
resources:
- deployments
- statefulsets
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- apps.nvidia.com
resources:
- nimcaches
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- apps.nvidia.com
resources:
- nimcaches/finalizers
verbs:
- update
- apiGroups:
- apps.nvidia.com
resources:
- nimcaches/status
verbs:
- get
- patch
- update
- apiGroups:
- apps.nvidia.com
resources:
- nimpipelines
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- apps.nvidia.com
resources:
- nimpipelines/finalizers
verbs:
- update
- apiGroups:
- apps.nvidia.com
resources:
- nimpipelines/status
verbs:
- get
- patch
- update
- apiGroups:
- apps.nvidia.com
resources:
- nimservices
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- apps.nvidia.com
resources:
- nimservices/finalizers
verbs:
- update
- apiGroups:
- apps.nvidia.com
resources:
- nimservices/status
verbs:
- get
- patch
- update
- apiGroups:
- batch
resources:
- jobs
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- config.openshift.io
resources:
- clusterversions
- proxies
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- persistentvolumeclaims
verbs:
- create
- delete
- get
- list
- apiGroups:
- monitoring.coreos.com
resources:
- prometheusrules
- servicemonitors
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- rbac.authorization.k8s.io
resources:
- rolebindings
- roles
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- route.openshift.io
resources:
- routes
verbs:
- create
- get
- list
- patch
- update
- watch
- apiGroups:
- networking.k8s.io
resources:
- ingresses
verbs:
- create
- get
- list
- patch
- update
- watch
- delete
- apiGroups:
- autoscaling
resources:
- horizontalpodautoscalers
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- scheduling.k8s.io
resources:
- priorityclasses
verbs:
- create
- get
- list
- watch
- apiGroups:
- security.openshift.io
resourceNames:
- nonroot
resources:
- securitycontextconstraints
verbs:
- use
- apiGroups:
- security.openshift.io
resources:
- securitycontextconstraints
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- storage.k8s.io
resources:
- storageclasses
verbs:
- get
- list
- watch
- apiGroups:
- authentication.k8s.io
resources:
- tokenreviews
verbs:
- create
- apiGroups:
- authorization.k8s.io
resources:
- subjectaccessreviews
verbs:
- create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: k8s-nim-operator-rolebinding
labels:
app.kubernetes.io/component: rbac
app.kubernetes.io/created-by: k8s-nim-operator
app.kubernetes.io/part-of: k8s-nim-operator
{{- include "k8s-nim-operator.labels" . | nindent 4 }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: k8s-nim-operator-role
subjects:
- kind: ServiceAccount
name: k8s-nim-operator
namespace: {{ .Release.Namespace }}
@@ -0,0 +1,14 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: k8s-nim-operator-metrics-reader
labels:
app.kubernetes.io/component: kube-rbac-proxy
app.kubernetes.io/created-by: k8s-nim-operator
app.kubernetes.io/part-of: k8s-nim-operator
{{- include "k8s-nim-operator.labels" . | nindent 4 }}
rules:
- nonResourceURLs:
- /metrics
verbs:
- get
@@ -0,0 +1,17 @@
apiVersion: v1
kind: Service
metadata:
name: k8s-nim-operator-metrics-service
labels:
app.kubernetes.io/component: kube-rbac-proxy
app.kubernetes.io/created-by: k8s-nim-operator
app.kubernetes.io/part-of: k8s-nim-operator
control-plane: controller-manager
{{- include "k8s-nim-operator.labels" . | nindent 4 }}
spec:
type: {{ .Values.metricsService.type }}
selector:
control-plane: controller-manager
{{- include "k8s-nim-operator.selectorLabels" . | nindent 4 }}
ports:
{{- .Values.metricsService.ports | toYaml | nindent 2 -}}
@@ -0,0 +1,17 @@
{{- if .Values.nfd.nodeFeatureRules.deviceID }}
kind: NodeFeatureRule
apiVersion: nfd.k8s-sigs.io/v1alpha1
metadata:
name: nfd-pci-device-rule
spec:
rules:
- name: "NVIDIA PCIe Device Label"
labelsTemplate: |
{{`{{ range .pci.device }}pci-{{ .vendor }}-{{ .device }}.present=true`}}
{{`{{ end }}`}}
matchFeatures:
- feature: pci.device
matchExpressions:
vendor: {op: In, value: ["10de"]}
class: {op: In, value: ["0300", "0302"]}
{{- end }}
@@ -0,0 +1,11 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: k8s-nim-operator
labels:
app.kubernetes.io/component: rbac
app.kubernetes.io/created-by: k8s-nim-operator
app.kubernetes.io/part-of: k8s-nim-operator
{{- include "k8s-nim-operator.labels" . | nindent 4 }}
annotations:
{{- toYaml .Values.operator.annotations | nindent 4 }}
@@ -0,0 +1,100 @@
{{- if .Values.operator.upgradeCRD }}
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: nim-operator-upgrade-crd-hook-sa
annotations:
helm.sh/hook: pre-upgrade
helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation
helm.sh/hook-weight: "0"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: nim-operator-upgrade-crd-hook-role
annotations:
helm.sh/hook: pre-upgrade
helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation
helm.sh/hook-weight: "0"
rules:
- apiGroups:
- apiextensions.k8s.io
resources:
- customresourcedefinitions
verbs:
- create
- get
- list
- watch
- patch
- update
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nim-operator-upgrade-crd-hook-binding
annotations:
helm.sh/hook: pre-upgrade
helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation
helm.sh/hook-weight: "0"
subjects:
- kind: ServiceAccount
name: nim-operator-upgrade-crd-hook-sa
namespace: {{ .Release.Namespace }}
roleRef:
kind: ClusterRole
name: nim-operator-upgrade-crd-hook-role
apiGroup: rbac.authorization.k8s.io
---
apiVersion: batch/v1
kind: Job
metadata:
name: nim-operator-upgrade-crd
namespace: {{ .Release.Namespace }}
annotations:
"helm.sh/hook": pre-upgrade
"helm.sh/hook-weight": "1"
"helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation
labels:
{{- include "k8s-nim-operator.labels" . | nindent 4 }}
app.kubernetes.io/component: "nim-operator"
spec:
template:
metadata:
name: nim-operator-upgrade-crd
labels:
{{- include "k8s-nim-operator.labels" . | nindent 8 }}
app.kubernetes.io/component: "nim-operator"
spec:
serviceAccountName: nim-operator-upgrade-crd-hook-sa
{{- if .Values.operator.image.pullSecrets }}
imagePullSecrets:
{{- range .Values.operator.image.pullSecrets }}
- name: {{ . }}
{{- end }}
{{- end }}
{{- with .Values.operator.tolerations }}
securityContext:
seccompProfile:
type: RuntimeDefault
runAsNonRoot: true
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: upgrade-crd
image: {{ include "k8s-nim-operator.fullimage" . }}
imagePullPolicy: {{ .Values.operator.image.pullPolicy }}
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
command: ["kubectl"]
args:
- "apply"
- "-f"
- "/opt/nim-operator/crds/*.yaml"
restartPolicy: OnFailure
{{- end }}
@@ -0,0 +1,46 @@
operator:
replicas: 1
upgradeCRD: true
image:
repository: nvcr.io/nvidia/cloud-native/k8s-nim-operator
tag: v1.0.1
pullSecrets: []
pullPolicy: Always
args:
- --health-probe-bind-address=:8081
- --metrics-bind-address=:8080
- --leader-elect
resources:
limits:
cpu: "1"
memory: 256Mi
requests:
cpu: 500m
memory: 128Mi
nodeSelector: {}
annotations: {}
tolerations:
- key: "node-role.kubernetes.io/control-plane"
operator: "Equal"
value: ""
effect: "NoSchedule"
affinity:
nodeAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 1
preference:
matchExpressions:
- key: "node-role.kubernetes.io/control-plane"
operator: In
values: [""]
metricsService:
ports:
- name: metrics
port: 8080
protocol: TCP
type: ClusterIP
nfd:
nodeFeatureRules:
deviceID: true
@@ -0,0 +1,27 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
# Our example-config.yaml is just an example, users should provide their own values.
example-config.yaml
ephemeral.values.yaml
@@ -0,0 +1,12 @@
dependencies:
- name: postgresql
repository: oci://dockerhub.nvidia.com/bitnamicharts
version: 13.3.1
- name: opentelemetry-collector
repository: https://open-telemetry.github.io/opentelemetry-helm-charts
version: 0.93.3
- name: nemo-common
repository: file://../../../library/
version: 0.1.0
digest: sha256:a30bc670003a830025a3cc57257082923e852c35a108629d30768e9aa2e69c5e
generated: "2025-04-16T22:06:20.120607586Z"
@@ -0,0 +1,18 @@
apiVersion: v2
appVersion: "25.04"
dependencies:
- condition: postgresql.enabled
name: postgresql
repository: oci://dockerhub.nvidia.com/bitnamicharts
version: 13.3.1
- condition: opentelemetry-collector.enabled
name: opentelemetry-collector
repository: https://open-telemetry.github.io/opentelemetry-helm-charts
version: 0.93.3
- name: nemo-common
repository: file://../../../library/
version: '>=0.1.0-0'
description: NeMo Customizer Microservice Helm Chart
name: nemo-customizer
type: application
version: 25.4.0
@@ -0,0 +1,188 @@
# NeMo Customizer Microservice Helm Chart
![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
For deployment guide, see [Admin Setup](https://docs.nvidia.com/nemo/microservices/latest/set-up/index.html) in the NeMo Microservices documentation.
## Values
| Key | Type | Default | Description |
|-----|------|---------|-------------|
| apiImage | object | This object has the following default values for the NeMo Customizer API only image. | Customizer API only image configuration. |
| apiImage.imagePullPolicy | string | `"IfNotPresent"` | Image pull policy for the NeMo Customizer API image. |
| apiImage.registry | string | `"nvcr.io"` | Registry for the NeMo Customizer API image. |
| apiImage.repository | string | `"nvidia/nemo-microservices/customizer-api"` | Repository for the NeMo Customizer API image. |
| customizerConfig | object | This object has default values for the following fields. | Configuration for the NeMo Customizer microservice. |
| customizerConfig.entityStoreURL | string | `"http://nemo-gateway.nemo-gateway.svc.cluster.local:8000"` | URL for the NeMo Entity Store microservice. |
| customizerConfig.mlflowURL | string | `"http://mlflow-tracking.mlflow-system.svc.cluster.local:80"` | URL for the MLflow tracking server. |
| customizerConfig.models | object | This object has default values for the supported models. | Models configuration. |
| customizerConfig.models."meta/llama-3.1-70b-instruct" | object | This object has the following default values for the Llama 3.1 70B Instruct model. | Llama 3.1 70B Instruct model configuration. |
| customizerConfig.models."meta/llama-3.1-70b-instruct".enabled | bool | `false` | Whether to enable the model. |
| customizerConfig.models."meta/llama-3.1-70b-instruct".max_seq_length | int | `4096` | Maximum sequence length for input tokens. |
| customizerConfig.models."meta/llama-3.1-70b-instruct".micro_batch_size | int | `1` | Micro batch size for training. |
| customizerConfig.models."meta/llama-3.1-70b-instruct".model_path | string | `"llama-3_1-70b-instruct_0_0_1"` | Path where model files are stored. |
| customizerConfig.models."meta/llama-3.1-70b-instruct".model_uri | string | `"ngc://nvidia/nemo/llama-3_1-70b-instruct-nemo:2.0"` | NGC model URI for Llama 3.1 70B Instruct model. |
| customizerConfig.models."meta/llama-3.1-70b-instruct".num_parameters | int | `70000000000` | Number of model parameters. |
| customizerConfig.models."meta/llama-3.1-70b-instruct".precision | string | `"bf16-mixed"` | Model precision format. |
| customizerConfig.models."meta/llama-3.1-70b-instruct".prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. |
| customizerConfig.models."meta/llama-3.1-70b-instruct".training_options | list | `[{"finetuning_type":"lora","num_gpus":4,"num_nodes":1,"tensor_parallel_size":4,"training_type":"sft"}]` | Training options for different fine-tuning methods. |
| customizerConfig.models."meta/llama-3.1-8b-instruct" | object | This object has the following default values for the Llama 3.1 8B Instruct model. | Llama 3.1 8B Instruct model configuration. |
| customizerConfig.models."meta/llama-3.1-8b-instruct".enabled | bool | `false` | Whether to enable the model. |
| customizerConfig.models."meta/llama-3.1-8b-instruct".max_seq_length | int | `4096` | Maximum sequence length for input tokens. |
| customizerConfig.models."meta/llama-3.1-8b-instruct".micro_batch_size | int | `1` | Micro batch size for training. |
| customizerConfig.models."meta/llama-3.1-8b-instruct".model_path | string | `"llama-3_1-8b-instruct_0_0_1"` | Path where model files are stored. |
| customizerConfig.models."meta/llama-3.1-8b-instruct".model_uri | string | `"ngc://nvidia/nemo/llama-3_1-8b-instruct-nemo:2.0"` | NGC model URI for Llama 3.1 8B Instruct model. |
| customizerConfig.models."meta/llama-3.1-8b-instruct".num_parameters | int | `8000000000` | Number of model parameters. |
| customizerConfig.models."meta/llama-3.1-8b-instruct".precision | string | `"bf16-mixed"` | Model precision format. |
| customizerConfig.models."meta/llama-3.1-8b-instruct".prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. |
| customizerConfig.models."meta/llama-3.1-8b-instruct".training_options | list | `[{"finetuning_type":"lora","num_gpus":1,"training_type":"sft"},{"finetuning_type":"all_weights","num_gpus":8,"num_nodes":1,"tensor_parallel_size":4,"training_type":"sft"}]` | Training options for different fine-tuning methods. |
| customizerConfig.models."meta/llama-3.2-1b" | object | This object has the following default values for the Llama 3.2 1B model. | Llama 3.2 1B model configuration. |
| customizerConfig.models."meta/llama-3.2-1b".enabled | bool | `false` | Whether to enable the model. |
| customizerConfig.models."meta/llama-3.2-1b".max_seq_length | int | `4096` | Maximum sequence length for input tokens. |
| customizerConfig.models."meta/llama-3.2-1b".micro_batch_size | int | `1` | Micro batch size for training. |
| customizerConfig.models."meta/llama-3.2-1b".model_path | string | `"llama32_1b"` | Path where model files are stored. |
| customizerConfig.models."meta/llama-3.2-1b".model_uri | string | `"ngc://nvidia/nemo/llama-3_2-1b:2.0"` | NGC model URI for Llama 3.2 1B model. |
| customizerConfig.models."meta/llama-3.2-1b".num_parameters | int | `1000000000` | Number of model parameters. |
| customizerConfig.models."meta/llama-3.2-1b".precision | string | `"bf16-mixed"` | Model precision format. |
| customizerConfig.models."meta/llama-3.2-1b".prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. |
| customizerConfig.models."meta/llama-3.2-1b".training_options | list | `[{"finetuning_type":"lora","num_gpus":1,"num_nodes":1,"tensor_parallel_size":1,"training_type":"sft"},{"finetuning_type":"all_weights","num_gpus":1,"num_nodes":1,"tensor_parallel_size":1,"training_type":"sft"}]` | Training options for different fine-tuning methods. |
| customizerConfig.models."meta/llama-3.2-1b-instruct" | object | This object has the following default values for the Llama 3.2 1B Instruct model. | Llama 3.2 1B Instruct model configuration. |
| customizerConfig.models."meta/llama-3.2-1b-instruct".enabled | bool | `false` | Whether to enable the model. |
| customizerConfig.models."meta/llama-3.2-1b-instruct".max_seq_length | int | `4096` | Maximum sequence length for input tokens. |
| customizerConfig.models."meta/llama-3.2-1b-instruct".micro_batch_size | int | `1` | Micro batch size for training. |
| customizerConfig.models."meta/llama-3.2-1b-instruct".model_path | string | `"llama32_1b-instruct"` | Path where model files are stored. |
| customizerConfig.models."meta/llama-3.2-1b-instruct".model_uri | string | `"ngc://nvidia/nemo/llama-3_2-1b-instruct:2.0"` | NGC model URI for Llama 3.2 1B Instruct model. |
| customizerConfig.models."meta/llama-3.2-1b-instruct".num_parameters | int | `1000000000` | Number of model parameters. |
| customizerConfig.models."meta/llama-3.2-1b-instruct".precision | string | `"bf16-mixed"` | Model precision format. |
| customizerConfig.models."meta/llama-3.2-1b-instruct".prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. |
| customizerConfig.models."meta/llama-3.2-1b-instruct".training_options | list | `[{"finetuning_type":"lora","num_gpus":1,"num_nodes":1,"tensor_parallel_size":1,"training_type":"sft"},{"finetuning_type":"all_weights","num_gpus":1,"num_nodes":1,"tensor_parallel_size":1,"training_type":"sft"}]` | Training options for different fine-tuning methods. |
| customizerConfig.models."meta/llama-3.2-3b-instruct" | object | This object has the following default values for the Llama 3.2 3B Instruct model. | Llama 3.2 3B Instruct model configuration. |
| customizerConfig.models."meta/llama-3.2-3b-instruct".enabled | bool | `false` | Whether to enable the model. |
| customizerConfig.models."meta/llama-3.2-3b-instruct".max_seq_length | int | `4096` | Maximum sequence length for input tokens. |
| customizerConfig.models."meta/llama-3.2-3b-instruct".micro_batch_size | int | `1` | Micro batch size for training. |
| customizerConfig.models."meta/llama-3.2-3b-instruct".model_path | string | `"llama32_3b-instruct"` | Path where model files are stored. |
| customizerConfig.models."meta/llama-3.2-3b-instruct".model_uri | string | `"ngc://nvidia/nemo/llama-3_2-3b-instruct:2.0"` | NGC model URI. |
| customizerConfig.models."meta/llama-3.2-3b-instruct".num_parameters | int | `3000000000` | Number of model parameters. |
| customizerConfig.models."meta/llama-3.2-3b-instruct".precision | string | `"bf16-mixed"` | Model precision format. |
| customizerConfig.models."meta/llama-3.2-3b-instruct".prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. |
| customizerConfig.models."meta/llama-3.2-3b-instruct".training_options | list | `[{"finetuning_type":"lora","num_gpus":1,"num_nodes":1,"tensor_parallel_size":1,"training_type":"sft"}]` | Training options for different fine-tuning methods. |
| customizerConfig.models."meta/llama-3.3-70b-instruct" | object | This object has the following default values for the Llama 3.3 70B Instruct model. | Llama 3.3 70B Instruct model configuration. |
| customizerConfig.models."meta/llama-3.3-70b-instruct".enabled | bool | `false` | Whether to enable the model. |
| customizerConfig.models."meta/llama-3.3-70b-instruct".max_seq_length | int | `4096` | Maximum sequence length for input tokens. |
| customizerConfig.models."meta/llama-3.3-70b-instruct".micro_batch_size | int | `1` | Micro batch size for training. |
| customizerConfig.models."meta/llama-3.3-70b-instruct".model_path | string | `"llama-3_3-70b-instruct_0_0_1"` | Path where model files are stored. |
| customizerConfig.models."meta/llama-3.3-70b-instruct".model_uri | string | `"ngc://nvidia/nemo/llama-3_3-70b-instruct:2.0"` | NGC model URI for Llama 3.3 70B Instruct model. |
| customizerConfig.models."meta/llama-3.3-70b-instruct".num_parameters | int | `70000000000` | Number of model parameters. |
| customizerConfig.models."meta/llama-3.3-70b-instruct".precision | string | `"bf16-mixed"` | Model precision format. |
| customizerConfig.models."meta/llama-3.3-70b-instruct".prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. |
| customizerConfig.models."meta/llama-3.3-70b-instruct".training_options | list | `[{"finetuning_type":"lora","num_gpus":4,"num_nodes":1,"tensor_parallel_size":4,"training_type":"sft"}]` | Training options for different fine-tuning methods. |
| customizerConfig.models.meta/llama3-70b-instruct | object | This object has the following default values for the Llama 3 70B Instruct model. | Llama 3 70B Instruct model configuration. |
| customizerConfig.models.meta/llama3-70b-instruct.enabled | bool | `false` | Whether to enable the model. |
| customizerConfig.models.meta/llama3-70b-instruct.max_seq_length | int | `4096` | Maximum sequence length for input tokens. |
| customizerConfig.models.meta/llama3-70b-instruct.micro_batch_size | int | `1` | Micro batch size for training. |
| customizerConfig.models.meta/llama3-70b-instruct.model_path | string | `"llama-3-70b-bf16"` | Path where model files are stored. |
| customizerConfig.models.meta/llama3-70b-instruct.model_uri | string | `"ngc://nvidia/nemo/llama-3-70b-instruct-nemo:2.0"` | NGC model URI for Llama 3 70B Instruct model. |
| customizerConfig.models.meta/llama3-70b-instruct.num_parameters | int | `70000000000` | Number of model parameters. |
| customizerConfig.models.meta/llama3-70b-instruct.precision | string | `"bf16-mixed"` | Model precision format. |
| customizerConfig.models.meta/llama3-70b-instruct.prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. |
| customizerConfig.models.meta/llama3-70b-instruct.training_options | list | `[{"finetuning_type":"lora","num_gpus":4,"num_nodes":1,"tensor_parallel_size":4,"training_type":"sft"}]` | Training options for different fine-tuning methods. |
| customizerConfig.models.microsoft/phi-4 | object | This object has the following default values for the Phi-4. | Phi-4 model configuration. |
| customizerConfig.models.microsoft/phi-4.enabled | bool | `false` | Whether to enable the model. |
| customizerConfig.models.microsoft/phi-4.max_seq_length | int | `4096` | Maximum sequence length for input tokens. |
| customizerConfig.models.microsoft/phi-4.micro_batch_size | int | `1` | Micro batch size for training. |
| customizerConfig.models.microsoft/phi-4.model_path | string | `"phi-4"` | Path where model files are stored. |
| customizerConfig.models.microsoft/phi-4.model_uri | string | `"ngc://nvidia/nemo/phi-4:1.0"` | NGC model URI for Phi-4 model. |
| customizerConfig.models.microsoft/phi-4.num_parameters | int | `14659507200` | Number of model parameters. |
| customizerConfig.models.microsoft/phi-4.precision | string | `"bf16"` | Model precision format. |
| customizerConfig.models.microsoft/phi-4.prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. |
| customizerConfig.models.microsoft/phi-4.training_options | list | `[{"finetuning_type":"lora","num_gpus":1,"num_nodes":1,"training_type":"sft"},{"finetuning_type":"all_weights","num_gpus":4,"num_nodes":2,"tensor_parallel_size":8,"training_type":"sft"}]` | Training options for different fine-tuning methods. |
| customizerConfig.nemoDataStoreURL | string | `"http://nds-datastore-http.nds-v2.svc.cluster.local:3000"` | URL for the NeMo Data Store microservice. |
| customizerConfig.openTelemetry | object | This object has the following default values for the OpenTelemetry settings. | OpenTelemetry settings. |
| customizerConfig.openTelemetry.enabled | bool | `true` | Whether to enable OpenTelemetry. |
| customizerConfig.openTelemetry.exporterOtlpEndpoint | string | `""` | Endpoint to access a custom OTLP collector listening on port 4317. Example: "http://$(HOST_IP):4317". |
| customizerConfig.openTelemetry.logsExporter | string | `"otlp"` | Sets the logs exporter type (otlp, console, none). |
| customizerConfig.openTelemetry.metricsExporter | string | `"otlp"` | Sets the metrics exporter type (otlp, console, none). |
| customizerConfig.openTelemetry.tracesExporter | string | `"otlp"` | Sets the traces exporter type (otlp, console, none). |
| customizerConfig.tolerations | list | `[]` | Tolerations on the customization job pods. |
| customizerConfig.training | object | This object has the following default values for the training configuration. | Training configuration for customization jobs. |
| customizerConfig.training.container_defaults | object | `{"env":null,"imagePullPolicy":"IfNotPresent"}` | Default container configuration for training jobs. |
| customizerConfig.training.container_defaults.env | string | `nil` | Environment variables for the training container. Cannot override env variables reserved by NeMo Customizer. |
| customizerConfig.training.pvc.size | string | `"5Gi"` | Size of the training job PVC. |
| customizerConfig.training.pvc.storageClass | string | `"local-nfs"` | Storage class for the training job PVC. |
| customizerConfig.training.pvc.volumeAccessMode | string | `"ReadWriteMany"` | Volume access mode for the training job PVC. |
| customizerConfig.training.queue | string | `"default"` | Queue name used by the underlying scheduler of NemoTrainingJob. Maps to "resourceGroup" in NemoTrainingJob. |
| customizerConfig.training.workspace_dir | string | `"/pvc/workspace"` | Directory path for training workspace. |
| customizerConfig.trainingNetworking | list | `[{"name":"NCCL_IB_SL","value":0},{"name":"NCCL_IB_TC","value":41},{"name":"NCCL_IB_QPS_PER_CONNECTION","value":4},{"name":"UCX_TLS","value":"TCP"},{"name":"UCX_NET_DEVICES","value":"eth0"},{"name":"HCOLL_ENABLE_MCAST_ALL","value":0},{"name":"NCCL_IB_GID_INDEX","value":3}]` | Network configuration for training jobs on Oracle Kubernetes Engine (OKE) on Oracle Cloud Infrastructure (OCI). |
| customizerConfig.wandb | object | `{"entity":null,"project":"nvidia-nemo-customizer"}` | Weights and Biases (WandB) Python SDK intialization configuration for logging and monitoring training jobs in WandB. |
| customizerConfig.wandb.entity | string | `nil` | Reference: https://docs.wandb.ai/ref/python/init/ |
| env | object | `{}` | A map of environment variables to inject into the NeMo Customizer app container. Example: `{HOST_IP: valueFrom: fieldRef: fieldPath: status.hostIP OTEL_EXPORTER_OTLP_ENDPOINT: "http://$(HOST_IP):4317"}` |
| externalDatabase | object | This object has the following default values for the external PostgreSQL configuration. | External PostgreSQL configuration. |
| externalDatabase.database | string | `"finetuning"` | Name of the database to use. |
| externalDatabase.existingSecret | string | `""` | Name of an existing secret resource containing the database credentials. |
| externalDatabase.existingSecretPasswordKey | string | `""` | Name of an existing secret key containing the database credentials. |
| externalDatabase.host | string | `"localhost"` | External database host address. |
| externalDatabase.password | string | `"nemo"` | Password for the non-root username for the NeMo Customizer microservice. |
| externalDatabase.port | int | `5432` | External database port number. |
| externalDatabase.user | string | `"nemo"` | Non-root username for the NeMo Customizer microservice. |
| fullnameOverride | string | `""` | String to fully override the chart and release name on resulting objects when deployed. |
| image | object | This object has the following default values for the NeMo Customizer microservice image. | NeMo Customizer image that supports training and standalone mode. |
| image.imagePullPolicy | string | `"IfNotPresent"` | Image pull policy for the NeMo Customizer image. |
| image.registry | string | `"nvcr.io"` | Registry for the NeMo Customizer image. |
| image.repository | string | `"nvidia/nemo-microservices/customizer"` | Repository for the NeMo Customizer image. |
| imagePullSecrets | list | `[{"name":"nvcrimagepullsecret"}]` | Image pull secrets configuration. |
| ingress | object | This object has the following default values for the Ingress configuration. | Ingress configuration. |
| ingress.annotations | object | `{}` | Additional annotations for the Ingress resource. |
| ingress.className | string | `""` | Ingress class name. |
| ingress.enabled | bool | `false` | Whether to enable the ingress resource. |
| ingress.hostname | string | `""` | Hostname for the ingress resource. |
| ingress.hosts | list | [] | A list of maps, each containing the keys `host` and `paths` for the ingress resource. You must specify a list for configuring ingress for the microservice. |
| ingress.tls | list | `[]` | TLS configuration for the ingress resource. |
| logging | object | `{"logHealthEndpoints":false,"logLevel":"INFO"}` | Logging configuration. |
| logging.logHealthEndpoints | bool | `false` | Enable logging for health endpoints. |
| logging.logLevel | string | `"INFO"` | Log level for the application. |
| modelDownloader | object | This object has the following default values for the model downloader. | Download models to PVC model cache configuration. |
| modelDownloader.pollIntervalSeconds | int | `15` | Interval in seconds to poll for model download status. |
| modelDownloader.securityContext | object | `{"fsGroup":1000,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}` | Security context for the model downloader. |
| modelDownloader.ttlSecondsAfterFinished | int | `7200` | Time to live in seconds after the job finishes. |
| modelsStorage | object | `{"accessModes":["ReadWriteMany"],"enabled":true,"size":"1Ti","storageClassName":""}` | Configure the PVC for models mount, where we store the parent/base models. |
| modelsStorage.accessModes | list | `["ReadWriteMany"]` | Access modes for the persistent volume. |
| modelsStorage.enabled | bool | `true` | Enable persistent volume for model storage. |
| modelsStorage.size | string | `"1Ti"` | Size of the persistent volume. |
| modelsStorage.storageClassName | string | `""` | Storage class name for the models PVC. Empty string uses the default storage class. |
| nameOverride | string | `""` | String to override chart name on resulting objects when deployed. |
| nemoDataStoreTools | object | This object has the following default values for the NeMo Data Store tools image. | Tools configuration for downloading and uploading entities to NeMo Data Store. |
| nemoDataStoreTools.imagePullSecret | string | `"nvcrimagepullsecret"` | Image pull secret for the NeMo Data Store tools image. |
| nemoDataStoreTools.registry | string | `"nvcr.io"` | Registry for the NeMo Data Store tools image. |
| nemoDataStoreTools.repository | string | `"nvidia/nemo-microservices/nds-v2-huggingface-cli"` | Repository for the NeMo Data Store tools image. |
| nemoDataStoreTools.tag | string | `""` | Tag for the NeMo Data Store tools image. |
| ngcAPISecret | string | `"ngc-api"` | Secret used for auto hydrating the model cache from NGC for enabled models. |
| ngcAPISecretKey | string | `"NGC_API_KEY"` | Key in the NGC API secret containing the API key. |
| opentelemetry-collector | object | This object has the following default values for the Open Telemetry Collector configuration. | Open Telemetry Collector configuration. |
| opentelemetry-collector.config | object | `{"exporters":{"debug":{"verbosity":"detailed"}},"extensions":{"health_check":{},"zpages":{"endpoint":"0.0.0.0:55679"}},"processors":{"batch":{}},"receivers":{"otlp":{"protocols":{"grpc":{},"http":{"cors":{"allowed_origins":["*"]}}}}},"service":{"extensions":["zpages","health_check"],"pipelines":{"logs":{"exporters":["debug"],"processors":["batch"],"receivers":["otlp"]},"metrics":{"exporters":["debug"],"processors":["batch"],"receivers":["otlp"]},"traces":{"exporters":["debug"],"processors":["batch"],"receivers":["otlp"]}}}}` | Base collector configuration for Open Telemetry Collector. |
| opentelemetry-collector.enabled | bool | `true` | Switch to enable or disable Open Telemetry Collector. |
| opentelemetry-collector.image.repository | string | `"otel/opentelemetry-collector-k8s"` | Repository for Open Telemetry Collector image. |
| opentelemetry-collector.image.tag | string | `"0.102.1"` | Overrides the image tag whose default is the chart appVersion. |
| opentelemetry-collector.mode | string | `"deployment"` | Deployment mode for Open Telemetry Collector. Valid values are "daemonset", "deployment", and "statefulset". |
| postgresql | object | This object has the following default values for the PostgreSQL configuration. | PostgreSQL configuration for the NeMo Customizer microservice. |
| postgresql.architecture | string | `"standalone"` | PostgreSQL architecture (`standalone` or `replication`). |
| postgresql.auth.database | string | `"finetuning"` | Name for a custom database to create. |
| postgresql.auth.enablePostgresUser | bool | `true` | Whether to assign a password to the "postgres" admin user. Otherwise, remote access will be blocked for this user. |
| postgresql.auth.existingSecret | string | `""` | Name of existing secret to use for PostgreSQL credentials. |
| postgresql.auth.password | string | `"nemo"` | Password for the custom user to create. |
| postgresql.auth.username | string | `"nemo"` | Name for a custom user to create. |
| postgresql.enabled | bool | `true` | Whether to enable or disable the PostgreSQL helm chart. |
| replicaCount | int | `1` | Number of replicas to deploy. |
| service | object | `{"internalPort":9009,"port":8000,"type":"ClusterIP"}` | Service configuration. |
| service.internalPort | int | `9009` | Internal port for the service. |
| service.port | int | `8000` | External port for the service. |
| service.type | string | `"ClusterIP"` | Type of Kubernetes service to create. |
| serviceAccount | object | `{"annotations":{},"automountServiceAccountToken":true,"create":true,"name":""}` | Service account configuration. |
| serviceAccount.annotations | object | `{}` | Annotations to add to the service account. |
| serviceAccount.automountServiceAccountToken | bool | `true` | Automatically mount a ServiceAccount's API credentials. |
| serviceAccount.create | bool | `true` | Specifies whether a service account should be created. |
| serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated. |
| useRunAIExecutor | bool | `false` | Enable or disable RunAI executor. |
| wandb | object | This object has the following default values for the WandB configuration. | WandB configuration. |
| wandb.existingSecret | string | `""` | Name of an existing Kubernetes secret resource for the WandB encryption secret. |
| wandb.existingSecretKey | string | `""` | Name of the key in the existing WandB secret containing the secret value. The secret value must be exactly 32 alphanumeric characters: ^[a-zA-Z0-9]{32}$ |
| wandb.secretValue | string | `"ec60d96b639764ccf9859bc10d4363d1"` | WandB secret value. Must contain exactly 32 alphanumeric characters. Creates a new Kubernetes secret named "wandb-secret" with key-value pair "encryption_key=<wandb.secretValue>". Ignored if wandb.existingSecret is set. |
@@ -0,0 +1,23 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
@@ -0,0 +1,10 @@
apiVersion: v2
appVersion: latest
description: Library chart for common elements across NeMo Microservices helm charts
keywords:
- library
- common
kubeVersion: '>=v1.23.0-0'
name: nemo-common
type: library
version: 0.1.0
@@ -0,0 +1,5 @@
# NeMo Common Library Chart
This is a library helm chart of templates that are useful for consistency across helm charts and because defined templates exist in the global scope.
All charts should use it's templates if they have relevant objects defined. When importing the chart dependency, use the version ">=0.1.0-0" to ensure all versions are captured and it stays up to date.
@@ -0,0 +1,121 @@
{{/*
nemo-common.env generates an env var array out of a dict or array of supported values
*/}}
{{- define "nemo-common.env" -}}
{{- if and .Values.env (kindIs "slice" .Values.env) -}}
{{- toYaml .Values.env -}}
{{- else if and .Values.env (kindIs "map" .Values.env) -}}
{{- range $k, $v := .Values.env }}
- name: {{ $k }}
{{- if kindIs "map" $v }}
valueFrom:
{{ toYaml $v.valueFrom | nindent 4 | trim }}
{{- else }}
value: {{ $v | quote }}
{{- end }}
{{- end }}
{{- else if .Values.envVars }}
{{- range $k, $v := .Values.envVars }}
- name: {{ $k }}
value: {{ $v | quote }}
{{- end }}
{{- end -}}
{{- end -}}
{{/*
nemo-common.postgresql.uri provides a template for constructing a postgresql database
connection string as an environment variable value
*/}}
{{- define "nemo-common.postgresql.uri" -}}
{{- if and .Values.externalDatabase.uriSecret .Values.externalDatabase.uriSecret.name .Values.externalDatabase.uriSecret.key }}
valueFrom:
secretKeyRef:
name: {{ .Values.externalDatabase.uriSecret.name }}
key: {{ .Values.externalDatabase.uriSecret.key }}
{{- else }}
value: {{ printf "postgresql://%s:$(POSTGRES_DB_PASSWORD)@%s:%s/%s" (include "nemo-common.postgresql.user" .) (include "nemo-common.postgresql.host" .) (include "nemo-common.postgresql.port" .) (include "nemo-common.postgresql.name" .) | quote }}
{{- end }}
{{- end -}}
{{/*
PostgreSQL Hostname
*/}}
{{- define "nemo-common.postgresql.host" -}}
{{- if .Values.postgresql.enabled -}}
{{ template "postgresql.v1.primary.fullname" .Subcharts.postgresql }}
{{- else -}}
{{ .Values.externalDatabase.host }}
{{- end -}}
{{- end -}}
{{/*
nemo-common.database.port chooses between externalDatabase and the bitnami postgresql port values
*/}}
{{- define "nemo-common.postgresql.port" -}}
{{- if .Values.postgresql.enabled -}}
{{- printf "%d" (.Values.postgresql.primary.service.ports.postgresql | int) -}}
{{- else -}}
{{- printf "%d" (.Values.externalDatabase.port | int) -}}
{{- end -}}
{{- end -}}
{{/*
nemo-common.database.user chooses between externalDatabase and the bitnami postgresql user values
*/}}
{{- define "nemo-common.postgresql.user" -}}
{{- if .Values.postgresql.enabled -}}
{{- print .Values.postgresql.auth.username -}}
{{- else -}}
{{- print .Values.externalDatabase.user -}}
{{- end -}}
{{- end -}}
{{/*
nemo-common.database.name chooses between externalDatabase and the bitnami postgresql db name values
*/}}
{{- define "nemo-common.postgresql.name" -}}
{{- if .Values.postgresql.enabled -}}
{{- print .Values.postgresql.auth.database -}}
{{- else -}}
{{- print .Values.externalDatabase.database -}}
{{- end -}}
{{- end -}}
{{/*
nemo-common.database.secret-name chooses between externalDatabase and the bitnami postgresql existing secret values
*/}}
{{- define "nemo-common.postgresql.secret-name" -}}
{{- if .Values.postgresql.enabled -}}
{{- if .Values.postgresql.auth.existingSecret -}}
{{- print .Values.postgresql.auth.existingSecret -}}
{{- else -}}
{{- template "postgresql.v1.primary.fullname" .Subcharts.postgresql -}}
{{- end -}}
{{- else if .Values.externalDatabase.existingSecret -}}
{{- print .Values.externalDatabase.existingSecret -}}
{{- end -}}
{{- end -}}
{{/*
nemo-common.database.password-key chooses between externalDatabase and the bitnami postgresql existing secret key values
*/}}
{{- define "nemo-common.postgresql.password-key" -}}
{{- if or .Values.postgresql.enabled (not .Values.externalDatabase.existingSecret) -}}
{{- print "password" -}}
{{- else -}}
{{- print .Values.externalDatabase.existingSecretPasswordKey -}}
{{- end -}}
{{- end -}}
{{/*
nemo-common.database.password generates a POSTGRES_DB_PASSWORD environment value if a full URI isn't used
*/}}
{{- define "nemo-common.postgresql.password" -}}
{{- if not (and .Values.externalDatabase.uriSecret .Values.externalDatabase.uriSecret.name .Values.externalDatabase.uriSecret.key) }}
- name: POSTGRES_DB_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "nemo-common.postgresql.secret-name" .}}
key: {{ include "nemo-common.postgresql.password-key" .}}
{{- end }}
{{- end -}}
@@ -0,0 +1,27 @@
{{/*
Image Definition Parsing
Favor not using a separate registry because it is confusing, but support it.
*/}}
{{- define "nemo-common.mainimage" -}}
{{- if .Values.image.registry -}}
{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}
{{- else -}}
{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}
{{- end }}
{{- end }}
{{/*
Image Pull Secrets
Use globals if they are there, but define sensibly.
*/}}
{{- define "nemo-common.imagepullsecrets" -}}
{{- if and .Values.global .Values.global.imagePullSecrets }}
{{- range .Values.global.imagePullSecrets }}
- {{ . | toYaml }}
{{- end }}
{{- else }}
{{- range .Values.imagePullSecrets }}
- {{ . | toYaml }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,21 @@
{{/*
Global level service name definition.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
This will allow top level overrides of service names when using the platform helm chart
*/}}
{{- define "nemo-common.servicename" -}}
{{- if .Values.serviceName }}
{{- .Values.serviceName | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,27 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
# Ignore unittest
tests/
*/__snapshot__/*
@@ -0,0 +1,12 @@
# Collector Chart Contributing Guide
All changes to the chart require a bump to the version in `chart.yaml`. See the [Contributing Guide](https://github.com/open-telemetry/opentelemetry-helm-charts/blob/main/CONTRIBUTING.md#versioning) for our versioning requirements.
Once the chart version is bumped, the examples must be regenerated. You can regenerate examples by running `make generate-examples CHARTS=opentelemetry-collector`.
## Bumping Default Collector Version
1. Increase the minor version of the chart by one and set the patch version to zero.
2. Update the chart's `appVersion` to match the new collector version. This version will be used as the image tag by default.
3. Review the corresponding release notes in [Collector Core](https://github.com/open-telemetry/opentelemetry-collector/releases), [Collector Contrib](https://github.com/open-telemetry/opentelemetry-collector-contrib/releases), and [Collector Releases](https://github.com/open-telemetry/opentelemetry-collector-releases/releases). If any changes affect the helm charts, adjust the helm chart accordingly.
4. Run `make generate-examples CHARTS=opentelemetry-collector`.
@@ -0,0 +1,14 @@
apiVersion: v2
appVersion: 0.102.1
description: OpenTelemetry Collector Helm chart for Kubernetes
home: https://opentelemetry.io/
icon: https://opentelemetry.io/img/logos/opentelemetry-logo-nav.png
maintainers:
- name: dmitryax
- name: TylerHelmuth
name: opentelemetry-collector
sources:
- https://github.com/open-telemetry/opentelemetry-collector
- https://github.com/open-telemetry/opentelemetry-collector-contrib
type: application
version: 0.93.3
@@ -0,0 +1,251 @@
# OpenTelemetry Collector Helm Chart
The helm chart installs [OpenTelemetry Collector](https://github.com/open-telemetry/opentelemetry-collector)
in kubernetes cluster.
## Prerequisites
- Kubernetes 1.24+
- Helm 3.9+
## Installing the Chart
Add OpenTelemetry Helm repository:
```console
helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
```
To install the chart with the release name my-opentelemetry-collector, run the following command:
```console
helm install my-opentelemetry-collector open-telemetry/opentelemetry-collector --set mode=<value> --set image.repository="otel/opentelemetry-collector-k8s" --set command.name="otelcol-k8s"
```
Where the `mode` value needs to be set to one of `daemonset`, `deployment` or `statefulset`.
For an in-depth walk through getting started in Kubernetes using this helm chart, see [OpenTelemetry Kubernetes Getting Started](https://opentelemetry.io/docs/kubernetes/getting-started/).
## Upgrading
See [UPGRADING.md](UPGRADING.md).
## Security Considerations
OpenTelemetry Collector recommends to bind receivers' servers to addresses that limit connections to authorized users.
For this reason, by default the chart binds all the Collector's endpoints to the pod's IP.
More info is available in the [Security Best Practices docummentation](https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks)
Some care must be taken when using `hostNetwork: true`, as then OpenTelemetry Collector will listen on all the addresses in the host network namespace.
## Configuration
### Default configuration
By default this chart will deploy an OpenTelemetry Collector with three pipelines (logs, metrics and traces)
and debug exporter enabled by default. The collector can be installed either as daemonset (agent), deployment or stateful set.
*Example*: Install collector as a deployment.
```yaml
mode: deployment
```
By default collector has the following receivers enabled:
- **metrics**: OTLP and prometheus. Prometheus is configured only for scraping collector's own metrics.
- **traces**: OTLP, zipkin and jaeger (thrift and grpc).
- **logs**: OTLP (to enable container logs, see [Configuration for Kubernetes container logs](#configuration-for-kubernetes-container-logs)).
### Basic Top Level Configuration
The Collector's configuration is set via the `config` section. Default components can be removed with `null`. Remember that lists in helm are not merged, so if you want to modify any default list you must specify all items, including any default items you want to keep.
*Example*: Disable metrics and logs pipelines and non-otlp receivers:
```yaml
config:
receivers:
jaeger: null
prometheus: null
zipkin: null
service:
pipelines:
traces:
receivers:
- otlp
metrics: null
logs: null
```
The chart also provides several presets, detailed below, to help configure important Kubernetes components. For more details on each component, see [Kubernetes Collector Components](https://opentelemetry.io/docs/kubernetes/collector/components/).
### Configuration for Kubernetes Container Logs
The collector can be used to collect logs sent to standard output by Kubernetes containers.
This feature is disabled by default. It has the following requirements:
- It needs agent collector to be deployed.
- It requires the [Filelog receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#filelog-receiver) to be included in the collector, such as [k8s](https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s) version of the collector image.
To enable this feature, set the `presets.logsCollection.enabled` property to `true`.
Here is an example `values.yaml`:
```yaml
mode: daemonset
presets:
logsCollection:
enabled: true
includeCollectorLogs: true
```
The way this feature works is it adds a `filelog` receiver on the `logs` pipeline. This receiver is preconfigured
to read the files where Kubernetes container runtime writes all containers' console output to.
#### :warning: Warning: Risk of looping the exported logs back into the receiver, causing "log explosion"
The container logs pipeline uses the `debug` exporter by default.
Paired with the default `filelog` receiver that receives all containers' console output,
it is easy to accidentally feed the exported logs back into the receiver.
Also note that using the `--verbosity=detailed` option for the `debug` exporter causes it to output
multiple lines per single received log, which when looped, would amplify the logs exponentially.
To prevent the looping, the default configuration of the receiver excludes logs from the collector's containers.
If you want to include the collector's logs, make sure to replace the `debug` exporter
with an exporter that does not send logs to collector's standard output.
Here's an example `values.yaml` file that replaces the default `debug` exporter on the `logs` pipeline
with an `otlphttp` exporter that sends the container logs to `https://example.com:55681` endpoint.
It also clears the `filelog` receiver's `exclude` property, for collector logs to be included in the pipeline.
```yaml
mode: daemonset
presets:
logsCollection:
enabled: true
includeCollectorLogs: true
config:
exporters:
otlphttp:
endpoint: https://example.com:55681
service:
pipelines:
logs:
exporters:
- otlphttp
```
### Configuration for Kubernetes Attributes Processor
The collector can be configured to add Kubernetes metadata, such as pod name and namespace name, as resource attributes to incoming logs, metrics and traces.
This feature is disabled by default. It has the following requirements:
- It requires the [Kubernetes Attributes processor](https://opentelemetry.io/docs/kubernetes/collector/components/#kubernetes-attributes-processor) to be included in the collector, such as [k8s](https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s) version of the collector image.
To enable this feature, set the `presets.kubernetesAttributes.enabled` property to `true`.
Here is an example `values.yaml`:
```yaml
mode: daemonset
presets:
kubernetesAttributes:
enabled: true
# You can also configure the preset to add all of the associated pod's labels and annotations to you telemetry.
# The label/annotation name will become the resource attribute's key.
extractAllPodLabels: true
extractAllPodAnnotations: true
```
### Configuration for Retrieving Kubelet Metrics
The collector can be configured to collect node, pod, and container metrics from the API server on a kubelet.
This feature is disabled by default. It has the following requirements:
- It requires the [Kubeletstats receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#kubeletstats-receiver) to be included in the collector, such as [k8s](https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s) version of the collector image.
To enable this feature, set the `presets.kubeletMetrics.enabled` property to `true`.
Here is an example `values.yaml`:
```yaml
mode: daemonset
presets:
kubeletMetrics:
enabled: true
```
### Configuration for Kubernetes Cluster Metrics
The collector can be configured to collects cluster-level metrics from the Kubernetes API server. A single instance of this receiver can be used to monitor a cluster.
This feature is disabled by default. It has the following requirements:
- It requires the [Kubernetes Cluster receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#kubernetes-cluster-receiver) to be included in the collector, such as [k8s](https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s) version of the collector image.
- It requires statefulset or deployment mode with a single replica.
To enable this feature, set the `presets.clusterMetrics.enabled` property to `true`.
Here is an example `values.yaml`:
```yaml
mode: deployment
replicaCount: 1
presets:
clusterMetrics:
enabled: true
```
### Configuration for Retrieving Kubernetes Events
The collector can be configured to collect Kubernetes events.
This feature is disabled by default. It has the following requirements:
- It requires [Kubernetes Objects receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#kubernetes-objects-receiver) to be included in the collector, such as [k8s](https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s) version of the collector image.
To enable this feature, set the `presets.kubernetesEvents.enabled` property to `true`.
Here is an example `values.yaml`:
```yaml
mode: deployment
replicaCount: 1
presets:
kubernetesEvents:
enabled: true
```
### Configuration for Host Metrics
The collector can be configured to collect host metrics for Kubernetes nodes.
This feature is disabled by default. It has the following requirements:
- It requires [Host Metrics receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#host-metrics-receiver) to be included in the collector, such as [k8s](https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s) version of the collector image.
To enable this feature, set the `presets.hostMetrics.enabled` property to `true`.
Here is an example `values.yaml`:
```yaml
mode: daemonset
presets:
hostMetrics:
enabled: true
```
## CRDs
At this time, Prometheus CRDs are supported but other CRDs are not.
### Other configuration options
The [values.yaml](./values.yaml) file contains information about all other configuration
options for this chart.
For more examples see [Examples](examples).
@@ -0,0 +1,424 @@
# Upgrade guidelines
These upgrade guidelines only contain instructions for version upgrades which require manual modifications on the user's side.
If the version you want to upgrade to is not listed here, then there is nothing to do for you.
Just upgrade and enjoy.
## Feature Flags
### useGOMEMLIMIT
A new flag, `useGOMEMLIMIT` has been added that allows specifying whether or not the chart should use the `GOMEMLIMIT` environment variable or the Memory Ballast Extension.
When enabled, the chart will remove the Memory Ballast Extension from the collector's configuration AND will setup a `GOMEMLIMIT` environment variable that is set to 80%
of the configured `resources.limits.memory`. If no `resources.limits.memory` are set when `useGOMEMLIMIT` is enabled then a `GOMEMLIMIT` environment variable WILL NOT be
created but the Memory Ballast Extension will still be removed.
Depending on the progress made in [Issue 891](https://github.com/open-telemetry/opentelemetry-helm-charts/issues/891),
the use of `GOMEMLIMIT` may completely replace the Memory Ballast Extension in the future.
## 0.88.0 to 0.89.0
> [!WARNING]
> Critical content demanding immediate user attention due to potential risks.
As part of working towards using the [OpenTelemetry Collector Kubernetes Distro](https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s) by default, the chart now requires users to explicitly set an image repository. If you are already explicitly setting an image repository this breaking change does not affect you.
If you are using a OpenTelemetry Community distribution of the Collector we recommend you use `otel/opentelemetry-collector-k8s`, but carefully review the [components included in this distribution](https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/distributions/otelcol-k8s/manifest.yaml) to make sure it includes all the components you use in your configuration. In the future this distribution will become the default image used for the chart.
You can use the OpenTelemetry Collector Kubernetes Distro by adding these lines to your values.yaml:
```yaml
image:
repository: "otel/opentelemetry-collector-k8s"
```
If you want to stick with using the Contrib distribution, add these lines to your values.yaml:
```yaml
image:
repository: "otel/opentelemetry-collector-contrib"
```
For more details see [#1135](https://github.com/open-telemetry/opentelemetry-helm-charts/issues/1135).
## 0.84.0 to 0.85.0
The `loggingexporter` has been removed from the default configuration. Use the `debugexporter` instead.
## 0.78.2 to 0.78.3
[Update Health Check Extension's endpoints to use Pod IP Instead of 0.0.0.0](https://github.com/open-telemetry/opentelemetry-helm-charts/pull/1012)
The [Collector's security guidelines were updated](https://github.com/open-telemetry/opentelemetry-collector/pull/6959) to include containerized environments when discussing safeguards against denial of service attacks.
To be in compliance with the Collector's security best practices the chart has been updated to use the Collector's pod IP in place of `0.0.0.0`.
The chart will continue to allow complete configuration of the Collector via the `config` field in the values.yaml. If pod IP does not suite your needs you can use `config` to set something different.
See [Security Best Practices docummentation](https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks) for more details.
## 0.75.1 to 0.76.0
Enable the `useGOMEMLIMIT` feature flag by default. This means by default the chart now does not use the Memory Ballast Extension and any custom configuraiton applied to the Memory Ballast Extension is ignored.
**If you're still interested in using the Memory Ballast Extension set this back to false.**
## 0.69.3 to 0.70.0
The following deprecated fields have been removed. Please use the new values:
- `extraConfigMapMounts` -> `extraVolumes`
- `extraHostPathMounts` -> `extraVolumes`
- `secretMounts` -> `extraVolumes`
- `containerLogs` -> `presets.logsCollection`
## 0.69.0 to 0.69.1 & 0.69.2
The `loggingexporter` was replaced with the `debugexporter`. This ended up being an accidental breaking change for any user that depended on the default logging exporter config when explicitly listing the logging exporter in an exporter list.
When using versions `0.69.1` or `0.69.2` you should explicitly list the debugging exporter instead of the logging exporter. You other option is to skip these version and use `0.69.3` or newer, which includes the logging exporter configuration.
**The logging exporter will be removed in a future version.** We highly recommend switching to the debug exporter.
## 0.67 to 0.68
The `preset.kubernetesEvents` preset now excludes `DELETED` watch types so that an log is not ingested when Kubernetes deletes an event.
The intention behind this change is to cleanup the data ingested by the preset as the `DELETED` updated for a Kubernetes Events is
uninteresting. If you want to keep ingesting `DELETED` updates for Kubernetes Events you will need to configure the `k8sobjectsreceiver` manually.
## 0.62 to 0.63
The `kubernetesAttributes` preset now respects order of processors in logs, metrics and traces pipelines.
This implicitly might break your pipelines if you relied on having the `k8sAttributes` processor rendered as the first processor but also explicitly listed it in the signal's pipeline somewhere else.
## 0.55.2 to 0.56
The `tpl` function has been added to references of pod labels and ingress hosts. This adds the ability to add some reusability in
charts values through referencing global values. If you are currently using any `{{ }}` syntax in pod labels or ingress hosts it will now be rendered. To escape existing instances of {{ }}, use {{` <original content> `}}.
```yaml
global:
region: us-east-1
environment: stage
# Tests `tpl` function reference used in pod labels and
# ingress.hosts[*]
podLabels:
environment: "{{ .Values.global.environment }}"
ingress:
enabled: true
hosts:
- host: "otlp-collector-{{ .Values.global.region }}-{{ .Values.global.environment }}-example.dev"
paths:
- path: /
pathType: Prefix
port: 4318
```
Note that only global Helm values can be referenced as the Helm Chart schema currently does not allow `additionalValues`.
## 0.55.0 to 0.55.1
As of v0.55.1 Collector chart use `${env:ENV}` style syntax when getting environment variables and that $`{env:ENV}` syntax is not supported before collector 0.71. If you upgrade collector chart to v0.55.1, you need to make sure your collector version is after than 0.71 (default is v0.76.1).
## 0.53.1 to 0.54.0
As of v0.54.0 Collector chart, the default resource limits are removed. If you want to keep old values you can use the following configuration:
```
resources:
limits:
# CPU units are in fractions of 1000; memory in powers of 2
cpu: 250m
memory: 512Mi
```
See [the 644 issue](https://github.com/open-telemetry/opentelemetry-helm-charts/issues/644) for more information.
## 0.46.0 to 0.47.0
[Update Collector Endpoints to use Pod IP Instead of 0.0.0.0](https://github.com/open-telemetry/opentelemetry-helm-charts/pull/603)
The [Collector's security guidelines were updated](https://github.com/open-telemetry/opentelemetry-collector/pull/6959) to include containerized environments when discussing safeguards against denial of service attacks.
To be in compliance with the Collector's security best practices the chart has been updated to use the Collector's pod IP in place of `0.0.0.0`.
The chart will continue to allow complete configuration of the Collector via the `config` field in the values.yaml. If pod IP does not suite your needs you can use `config` to set something different.
See [Security Best Practices docummentation](https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks) for more details.
The new default of binding to the pod IP, rather than `0.0.0.0`, will cause `kubectl port-forward` to fail. If port-forwarding is desired, the following `value.yaml` snippet will allow the Collector bind to `127.0.0.1` inside the pod, in addition to the pod's IP:
```yaml
config:
receivers:
jaeger/local:
protocols:
grpc:
endpoint: 127.0.0.1:14250
thrift_compact:
endpoint: 127.0.0.1:6831
thrift_http:
endpoint: 127.0.0.1:14268
otlp/local:
protocols:
grpc:
endpoint: 127.0.0.1:4317
http:
endpoint: 127.0.0.1:4318
zipkin/local:
endpoint: 127.0.0.1:9411
service:
pipelines:
traces:
receivers:
- otlp
- otlp/local
- jaeger
- jaeger/local
- zipkin
- zipkin/local
```
## 0.40.7 to 0.41.0
[Require Kubernetes version 1.23 or later](https://github.com/open-telemetry/opentelemetry-helm-charts/pull/541)
If you enable use of a _HorizontalPodAutoscaler_ for the collector when running in the "deployment" mode by way of `.Values.autoscaling.enabled`, the manifest now uses the "autoscaling/v2" API group version, which [is available only as recently as Kubernetes version 1.23](https://kubernetes.io/blog/2021/12/07/kubernetes-1-23-release-announcement/#horizontalpodautoscaler-v2-graduates-to-ga). As [all previous versions of this API group are deprecated and removed as of Kubernetes version 1.26](https://kubernetes.io/docs/reference/using-api/deprecation-guide/#horizontalpodautoscaler-v126), we don't offer support for Kubernetes versions older than 1.23.
## 0.34.0 to 0.34.0
[config supports templating](TBD)
The chart now supports templating in `.Values.config`. If you are currently using any `{{ }}` syntax in `.Values.yaml` it will now be rendered. To escape existing instances of `{{ }}`, use ``` {{` <original content> `}} ```. For example, `{{ REDACTED_EMAIL }}` becomes ``` {{` {{ REDACTED_EMAIL }} `}} ```.
## 0.28.0 to 0.29.0
[Reduce requested resources](https://github.com/open-telemetry/opentelemetry-helm-charts/pull/273)
Resource `limits` have been reduced. Upgrades/installs of chart 0.29.0 will now use fewer resources. In order to set the resources back to what they were, you will need to override the `resources` section in the `values.yaml`.
*Example*:
```yaml
resources:
limits:
cpu: 1
memory: 2Gi
```
## 0.23.1 to 0.24.0
[Remove containerLogs in favor of presets.logsCollection]()
The ability to enable logs collection from the collector has been moved from `containerLogs.enabled` to `presets.logsCollection.enabled`. If you are currently using `containerLogs.enabled`, you should instead use the preset:
```yaml
presets:
logsCollection:
enabled: true
```
If you are using `containerLogs.enabled` and also enabling collection of the collector logs you can use `includeCollectorLogs`
```yaml
presets:
logsCollection:
enabled: true
includeCollectorLogs: true
```
You no longer need to update `config.service.pipelines.logs` to include the filelog receiver yourself as the preset will automatically update the logs pipeline to include the filelog receiver.
The filelog's preset configuration can modified by `config.receivers`, but preset configuration cannot be removed. If you need to remove any filelog receiver configuration generated by the preset you should not use the preset. Instead, configure the filelog receiver manually in `config.receivers` and set any other necessary fields in the values.yaml to modify k8s as needed.
See the [daemonset-collector-logs example](https://github.com/open-telemetry/opentelemetry-helm-charts/tree/main/charts/opentelemetry-collector/examples/daemonset-collector-logs) to see an example of the preset in action.
## 0.18.0 to 0.19.0
[Remove agentCollector and standaloneCollector settings](https://github.com/open-telemetry/opentelemetry-helm-charts/pull/216)
The `agentCollector` and `standaloneCollector` config sections have been removed. Upgrades/installs of chart 0.19.0 will fail if `agentCollector` or `standaloneCollector` are in the values.yaml. See the [Migrate to mode](#migrate-to-mode) steps for instructions on how to replace `agentCollector` and `standaloneCollector` with `mode`.
## 0.13.0 to 0.14.0
[Remove two-deployment mode](https://github.com/open-telemetry/opentelemetry-helm-charts/pull/159)
The ability to install both the agent and standalone collectors simultaneous with the chart has been removed. Installs/upgrades where both `.Values.agentCollector.enabled` and `.Values.standloneCollector.enables` are true will fail. `agentCollector` and `standloneCollector` have also be deprecated, but backward compatibility has been maintained.
### To run both a deployment and daemonset
Install a deployment version of the collector. This is done by setting `.Values.mode` to `deployment`
```yaml
mode: deployment
```
Next, install an daemonset version of the collector that is configured to send traffic to the previously installed deployment. This is done by setting `.Values.mode` to `daemonset` and updating `.Values.config` so that data is exported to the deployment.
```yaml
mode: daemonset
config:
exporters:
otlp:
endpoint: example-opentelemetry-collector:4317
tls:
insecure: true
service:
pipelines:
logs:
exporters:
- otlp
- logging
metrics:
exporters:
- otlp
- logging
traces:
exporters:
- otlp
- logging
```
See the [daemonset-and-deployment](examples/daemonset-and-deployment) example to see the rendered config.
### Migrate to `mode`:
The `agentCollector` and `standaloneCollector` sections in values.yaml have been deprecated. Instead there is a new field, `mode`, that determines if the collector is being installed as a daemonset or deployment.
```yaml
# Valid values are "daemonset" and "deployment".
# If set, agentCollector and standaloneCollector are ignored.
mode: <daemonset|deployment>
```
The following fields have also been added to the root-level to replace the depracated `agentCollector` and `standaloneCollector` settings.
```yaml
containerLogs:
enabled: false
resources:
limits:
cpu: 1
memory: 2Gi
podAnnotations: {}
podLabels: {}
# Host networking requested for this pod. Use the host's network namespace.
hostNetwork: false
# only used with deployment mode
replicaCount: 1
annotations: {}
```
When using `mode`, these settings should be used instead of their counterparts in `agentCollector` and `standaloneCollector`.
Set `mode` to `daemonset` if `agentCollector` was being used. Move all `agentCollector` settings to the corresponding root-level setting. If `agentCollector.configOverride` was being used, merge the settings with `.Values.config`.
Example agentCollector values.yaml:
```yaml
agentCollector:
resources:
limits:
cpu: 3
memory: 6Gi
configOverride:
receivers:
hostmetrics:
scrapers:
cpu:
disk:
filesystem:
service:
pipelines:
metrics:
receivers: [otlp, prometheus, hostmetrics]
```
Example mode values.yaml:
```yaml
mode: daemonset
resources:
limits:
cpu: 3
memory: 6Gi
config:
receivers:
hostmetrics:
scrapers:
cpu:
disk:
filesystem:
service:
pipelines:
metrics:
receivers: [otlp, prometheus, hostmetrics]
```
Set `mode` to `deployment` if `standaloneCollector` was being used. Move all `standaloneCollector` settings to the corresponding root-level setting. If `standaloneCollector.configOverride` was being used, merge the settings with `.Values.config`.
Example standaloneCollector values.yaml:
```yaml
standaloneCollector:
enabled: true
replicaCount: 2
configOverride:
receivers:
podman_stats:
endpoint: unix://run/podman/podman.sock
timeout: 10s
collection_interval: 10s
service:
pipelines:
metrics:
receivers: [otlp, prometheus, podman_stats]
```
Example mode values.yaml:
```yaml
mode: deployment
replicaCount: 2
config:
receivers:
receivers:
podman_stats:
endpoint: unix://run/podman/podman.sock
timeout: 10s
collection_interval: 10s
service:
pipelines:
metrics:
receivers: [otlp, prometheus, podman_stats]
```
Default configuration in `.Values.config` can now be removed with `null`. When changing a pipeline, you must explicitly list all the components that are in the pipeline, including any default components.
*Example*: Disable metrics and logging pipelines and non-otlp receivers:
```yaml
config:
receivers:
jaeger: null
prometheus: null
zipkin: null
service:
pipelines:
traces:
receivers:
- otlp
metrics: null
logs: null
```
@@ -0,0 +1,9 @@
mode: deployment
image:
repository: "otel/opentelemetry-collector-k8s"
command:
name: "otelcol-k8s"
useGOMEMLIMIT: true
@@ -0,0 +1,27 @@
mode: daemonset
image:
repository: "otel/opentelemetry-collector-k8s"
command:
name: "otelcol-k8s"
clusterRole:
create: true
name: "testing-clusterrole"
rules:
- apiGroups:
- ''
resources:
- 'pods'
- 'nodes'
verbs:
- 'get'
- 'list'
- 'watch'
clusterRoleBinding:
name: "testing-clusterrolebinding"
resources:
limits:
cpu: 100m
memory: 200M
@@ -0,0 +1,26 @@
mode: daemonset
image:
repository: "otel/opentelemetry-collector-k8s"
command:
name: "otelcol-k8s"
config:
receivers:
jaeger: null
otlp: null
zipkin: null
hostmetrics:
scrapers:
cpu:
disk:
filesystem:
service:
pipelines:
metrics:
receivers:
- prometheus
- hostmetrics
traces: null
logs: null
@@ -0,0 +1,12 @@
mode: daemonset
image:
repository: "otel/opentelemetry-collector-k8s"
command:
name: "otelcol-k8s"
resources:
limits:
cpu: 100m
memory: 200M
@@ -0,0 +1,29 @@
global:
test: templated-value
mode: deployment
image:
repository: "otel/opentelemetry-collector-k8s"
command:
name: "otelcol-k8s"
resources:
limits:
cpu: 100m
memory: 200M
# Tests `tpl` function reference used in pod labels and
# ingress.hosts[*]
podLabels:
testLabel: "{{ .Values.global.test }}"
ingress:
enabled: true
hosts:
- host: "{{ .Values.global.test }}"
paths:
- path: /
pathType: Prefix
port: 4318
@@ -0,0 +1,21 @@
mode: deployment
image:
repository: "otel/opentelemetry-collector-k8s"
command:
name: "otelcol-k8s"
ports:
jaeger-compact:
enabled: false
jaeger-thrift:
enabled: false
jaeger-grpc:
enabled: false
zipkin:
enabled: false
resources:
limits:
cpu: 100m
memory: 200M
@@ -0,0 +1,15 @@
mode: deployment
image:
repository: "otel/opentelemetry-collector-k8s"
command:
name: "otelcol-k8s"
autoscaling:
enabled: true
minReplicas: 1
maxReplicas: 10
behavior: {}
targetCPUUtilizationPercentage: 80
targetMemoryUtilizationPercentage: 80
@@ -0,0 +1,15 @@
mode: statefulset
image:
repository: "otel/opentelemetry-collector-k8s"
command:
name: "otelcol-k8s"
autoscaling:
enabled: true
minReplicas: 1
maxReplicas: 10
behavior: {}
targetCPUUtilizationPercentage: 80
targetMemoryUtilizationPercentage: 80
@@ -0,0 +1,49 @@
mode: deployment
image:
repository: "otel/opentelemetry-collector-k8s"
command:
name: "otelcol-k8s"
resources:
limits:
cpu: 100m
memory: 200M
ingress:
enabled: true
ingressClassName: nginx
annotations:
test.io/collector: default
hosts:
- host: defaultcollector.example.com
paths:
- path: /
pathType: Prefix
port: 4318
additionalIngresses:
- name: additional-basic
hosts:
- host: additional-basic.example.com
paths:
- path: /
pathType: Prefix
port: 4318
- name: additional-advanced
ingressClassName: nginx
annotations:
test.io/ingress: additional-advanced
hosts:
- host: additional-advanced.example.com
paths:
- path: /
pathType: Exact
port: 4318
tls:
- secretName: somesecret
hosts:
- additional-advanced.example.com
@@ -0,0 +1,37 @@
mode: daemonset
image:
repository: "otel/opentelemetry-collector-k8s"
command:
name: "otelcol-k8s"
resources:
limits:
cpu: 100m
memory: 200M
networkPolicy:
enabled: true
allowIngressFrom:
- namespaceSelector: {}
- ipBlock:
cidr: 127.0.0.1/32
extraIngressRules:
- ports:
- port: metrics
protocol: TCP
from:
- ipBlock:
cidr: 127.0.0.1/32
egressRules:
- to:
- podSelector:
matchLabels:
app: jaeger
ports:
- port: 4317
protocol: TCP
@@ -0,0 +1,15 @@
mode: deployment
image:
repository: "otel/opentelemetry-collector-k8s"
command:
name: "otelcol-k8s"
resources:
limits:
cpu: 100m
memory: 200M
networkPolicy:
enabled: true
@@ -0,0 +1,16 @@
mode: deployment
image:
repository: "otel/opentelemetry-collector-k8s"
command:
name: "otelcol-k8s"
presets:
clusterMetrics:
enabled: true
resources:
limits:
cpu: 100m
memory: 200M
@@ -0,0 +1,16 @@
mode: daemonset
image:
repository: "otel/opentelemetry-collector-k8s"
command:
name: "otelcol-k8s"
presets:
hostMetrics:
enabled: true
resources:
limits:
cpu: 100m
memory: 200M
@@ -0,0 +1,16 @@
mode: deployment
image:
repository: "otel/opentelemetry-collector-k8s"
command:
name: "otelcol-k8s"
presets:
kubernetesEvents:
enabled: true
resources:
limits:
cpu: 100m
memory: 200M
@@ -0,0 +1,16 @@
mode: daemonset
image:
repository: "otel/opentelemetry-collector-k8s"
command:
name: "otelcol-k8s"
presets:
kubeletMetrics:
enabled: true
resources:
limits:
cpu: 100m
memory: 200M
@@ -0,0 +1,16 @@
mode: daemonset
image:
repository: "otel/opentelemetry-collector-k8s"
command:
name: "otelcol-k8s"
presets:
kubernetesAttributes:
enabled: true
resources:
limits:
cpu: 100m
memory: 200M
@@ -0,0 +1,17 @@
mode: daemonset
image:
repository: "otel/opentelemetry-collector-k8s"
command:
name: "otelcol-k8s"
presets:
logsCollection:
enabled: true
includeCollectorLogs: true
resources:
limits:
cpu: 100m
memory: 200M
@@ -0,0 +1,27 @@
mode: daemonset
image:
repository: "otel/opentelemetry-collector-k8s"
command:
name: "otelcol-k8s"
livenessProbe:
initialDelaySeconds: 10
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 2
terminationGracePeriodSeconds: 40
httpGet:
port: 8989
path: /healthz
readinessProbe:
initialDelaySeconds: 10
periodSeconds: 5
timeoutSeconds: 3
successThreshold: 2
failureThreshold: 2
httpGet:
port: 8989
path: /healthz
@@ -0,0 +1,13 @@
mode: statefulset
image:
repository: "otel/opentelemetry-collector-k8s"
command:
name: "otelcol-k8s"
replicaCount: 2
resources:
limits:
cpu: 100m
memory: 200M
@@ -0,0 +1,17 @@
# Examples of chart configuration
Here is a collection of common configurations for the OpenTelemetry collector. Each folder contains an example `values.yaml` and the resulting configurations that are generated by the opentelemetry-collector helm charts.
- [Daemonset only](daemonset-only)
- [Deployment only](deployment-only)
- [Daemonset and deployment](daemonset-and-deployment)
- [Log collection, including collector logs](daemonset-collector-logs)
- [Add component (hostmetrics)](daemonset-hostmetrics)
The manifests are rendered using the `helm template` command and the specific example folder's values.yaml.
Examples are generated by (from root of the repo):
```sh
make generate-examples CHARTS=opentelemetry-collector
```
@@ -0,0 +1,34 @@
mode: daemonset
image:
repository: "otel/opentelemetry-collector-k8s"
command:
name: "otelcol-k8s"
config:
exporters:
otlp:
endpoint: example-opentelemetry-collector:4317
tls:
insecure: true
service:
pipelines:
logs:
exporters:
- otlp
- debug
metrics:
exporters:
- otlp
- debug
traces:
exporters:
- otlp
- debug
resources:
limits:
cpu: 100m
memory: 200M
@@ -0,0 +1,13 @@
mode: deployment
image:
repository: "otel/opentelemetry-collector-k8s"
command:
name: "otelcol-k8s"
resources:
limits:
cpu: 100m
memory: 200M
@@ -0,0 +1,93 @@
---
# Source: opentelemetry-collector/templates/configmap-agent.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: example-opentelemetry-collector-agent
namespace: default
labels:
helm.sh/chart: opentelemetry-collector-0.93.3
app.kubernetes.io/name: opentelemetry-collector
app.kubernetes.io/instance: example
app.kubernetes.io/version: "0.102.1"
app.kubernetes.io/managed-by: Helm
data:
relay: |
exporters:
debug: {}
otlp:
endpoint: example-opentelemetry-collector:4317
tls:
insecure: true
extensions:
health_check:
endpoint: ${env:MY_POD_IP}:13133
processors:
batch: {}
memory_limiter:
check_interval: 5s
limit_percentage: 80
spike_limit_percentage: 25
receivers:
jaeger:
protocols:
grpc:
endpoint: ${env:MY_POD_IP}:14250
thrift_compact:
endpoint: ${env:MY_POD_IP}:6831
thrift_http:
endpoint: ${env:MY_POD_IP}:14268
otlp:
protocols:
grpc:
endpoint: ${env:MY_POD_IP}:4317
http:
endpoint: ${env:MY_POD_IP}:4318
prometheus:
config:
scrape_configs:
- job_name: opentelemetry-collector
scrape_interval: 10s
static_configs:
- targets:
- ${env:MY_POD_IP}:8888
zipkin:
endpoint: ${env:MY_POD_IP}:9411
service:
extensions:
- health_check
pipelines:
logs:
exporters:
- otlp
- debug
processors:
- memory_limiter
- batch
receivers:
- otlp
metrics:
exporters:
- otlp
- debug
processors:
- memory_limiter
- batch
receivers:
- otlp
- prometheus
traces:
exporters:
- otlp
- debug
processors:
- memory_limiter
- batch
receivers:
- otlp
- jaeger
- zipkin
telemetry:
metrics:
address: ${env:MY_POD_IP}:8888

Some files were not shown because too many files have changed in this diff Show More