init nemo chart
This commit is contained in:
@@ -0,0 +1,39 @@
|
|||||||
|
dependencies:
|
||||||
|
- name: nemo-guardrails
|
||||||
|
repository: file://components/guardrails
|
||||||
|
version: 0.2.0
|
||||||
|
- name: nemo-entity-store
|
||||||
|
repository: file://components/entity-store
|
||||||
|
version: 0.2.0
|
||||||
|
- name: nemo-datastore
|
||||||
|
repository: file://components/datastore
|
||||||
|
version: 0.5.1
|
||||||
|
- name: nemo-customizer
|
||||||
|
repository: file://components/customizer
|
||||||
|
version: 0.11.0-alpha.13
|
||||||
|
- name: nemo-evaluator
|
||||||
|
repository: file://components/evaluator
|
||||||
|
version: 0.2.4
|
||||||
|
- name: nemo-operator
|
||||||
|
repository: file://components/nemo-operator
|
||||||
|
version: 0.2.0
|
||||||
|
- name: nemo-deployment-management
|
||||||
|
repository: file://components/deployment-management
|
||||||
|
version: 0.2.1
|
||||||
|
- name: nemo-nim-proxy
|
||||||
|
repository: file://components/nim-proxy
|
||||||
|
version: 0.2.0
|
||||||
|
- name: nim-llm
|
||||||
|
repository: https://helm.ngc.nvidia.com/nvidian/nemo-llm
|
||||||
|
version: 1.7.0-rc1
|
||||||
|
- name: k8s-nim-operator
|
||||||
|
repository: https://helm.ngc.nvidia.com/nvidia/
|
||||||
|
version: 1.0.1
|
||||||
|
- name: dgxc-admission-controller
|
||||||
|
repository: https://helm.ngc.nvidia.com/nvidian/nemo-llm
|
||||||
|
version: 1.642.0
|
||||||
|
- name: volcano
|
||||||
|
repository: https://volcano-sh.github.io/helm-charts
|
||||||
|
version: 1.9.0
|
||||||
|
digest: sha256:a182a437e6ff1efa1a0a6b56cf7533e612fe9a6ee087ad7f813193f63bcf0bd4
|
||||||
|
generated: "2025-04-16T22:58:40.429920984Z"
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
apiVersion: v2
|
||||||
|
dependencies:
|
||||||
|
- alias: guardrails
|
||||||
|
condition: guardrails.enabled
|
||||||
|
name: nemo-guardrails
|
||||||
|
repository: file://components/guardrails
|
||||||
|
version: '>=0.1.0-0'
|
||||||
|
- alias: entity-store
|
||||||
|
condition: entity-store.enabled
|
||||||
|
name: nemo-entity-store
|
||||||
|
repository: file://components/entity-store
|
||||||
|
version: '>=0.1.0-0'
|
||||||
|
- alias: data-store
|
||||||
|
condition: data-store.enabled
|
||||||
|
name: nemo-datastore
|
||||||
|
repository: file://components/datastore
|
||||||
|
version: '>=0.1.0-0'
|
||||||
|
- alias: customizer
|
||||||
|
condition: customizer.enabled
|
||||||
|
name: nemo-customizer
|
||||||
|
repository: file://components/customizer
|
||||||
|
version: '>=0.1.0-0'
|
||||||
|
- alias: evaluator
|
||||||
|
condition: evaluator.enabled
|
||||||
|
name: nemo-evaluator
|
||||||
|
repository: file://components/evaluator
|
||||||
|
version: '>=0.1.0-0'
|
||||||
|
- alias: nemo-operator
|
||||||
|
condition: nemo-operator.enabled
|
||||||
|
name: nemo-operator
|
||||||
|
repository: file://components/nemo-operator
|
||||||
|
version: '>=0.1.0-0'
|
||||||
|
- alias: deployment-management
|
||||||
|
condition: deployment-management.enabled
|
||||||
|
name: nemo-deployment-management
|
||||||
|
repository: file://components/deployment-management
|
||||||
|
version: '>=0.1.0-0'
|
||||||
|
- alias: nim-proxy
|
||||||
|
condition: nim-proxy.enabled
|
||||||
|
name: nemo-nim-proxy
|
||||||
|
repository: file://components/nim-proxy
|
||||||
|
version: '>=0.1.0-0'
|
||||||
|
- alias: nim
|
||||||
|
condition: nim.enabled
|
||||||
|
name: nim-llm
|
||||||
|
repository: https://helm.ngc.nvidia.com/nvidian/nemo-llm
|
||||||
|
version: 1.7.0-rc1
|
||||||
|
- alias: nim-operator
|
||||||
|
condition: nim-operator.enabled
|
||||||
|
name: k8s-nim-operator
|
||||||
|
repository: https://helm.ngc.nvidia.com/nvidia/
|
||||||
|
version: 1.0.1
|
||||||
|
- condition: dgxc-admission-controller.enabled
|
||||||
|
name: dgxc-admission-controller
|
||||||
|
repository: https://helm.ngc.nvidia.com/nvidian/nemo-llm
|
||||||
|
version: 1.642.0
|
||||||
|
- condition: volcano.enabled
|
||||||
|
name: volcano
|
||||||
|
repository: https://volcano-sh.github.io/helm-charts
|
||||||
|
version: 1.9.0
|
||||||
|
description: NeMo Microservices Helm Chart
|
||||||
|
home: https://nvidia.com
|
||||||
|
maintainers:
|
||||||
|
- name: NVIDIA, Inc.
|
||||||
|
url: https://www.nvidia.com
|
||||||
|
name: nemo-microservices-helm-chart
|
||||||
|
type: application
|
||||||
|
version: 25.4.0
|
||||||
@@ -0,0 +1,152 @@
|
|||||||
|
# NeMo Microservices Helm Chart
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
For deployment guide, see [Admin Setup](https://docs.nvidia.com/nemo/microservices/latest/set-up/index.html) in the NeMo Microservices documentation.
|
||||||
|
|
||||||
|
## Values
|
||||||
|
|
||||||
|
| Key | Type | Default | Description |
|
||||||
|
|-----|------|---------|-------------|
|
||||||
|
| customizer.customizerConfig.entityStoreURL | string | `"http://nemo-entity-store:8000"` | Specifies the internal K8s DNS record for the NeMo Entity Store service. |
|
||||||
|
| customizer.customizerConfig.mlflowURL | string | `""` | Specifies the internal K8s DNS record for the MLflow service. |
|
||||||
|
| customizer.customizerConfig.models | object | `{"meta/llama-3.1-8b-instruct":{"enabled":true}}` | Specifies the model information to configure with the NeMo Customizer microservice. To find a complete list of models that NeMo Customizer supports, see [Model Catalog](https://docs.nvidia.com/nemo/microservices/latest/fine-tune/models/index.html). |
|
||||||
|
| customizer.customizerConfig.models."meta/llama-3.1-8b-instruct".enabled | bool | `true` | Specifies whether to use the `meta/llama-3.1-8b-instruct` model for a customization job. |
|
||||||
|
| customizer.customizerConfig.nemoDataStoreURL | string | `"http://nemo-data-store:3000"` | Specifies the internal K8s DNS record for the NeMo Data Store service. |
|
||||||
|
| customizer.customizerConfig.training.pvc | object | `{"storageClass":""}` | Specifies where to store the customization job information during training. |
|
||||||
|
| customizer.enabled | bool | `true` | Specifies whether to install the NeMo Customizer microservice. |
|
||||||
|
| customizer.modelsStorage | object | `{"accessModes":["ReadWriteOnce"],"enabled":true,"size":"20Gi","storageClassName":""}` | Specifies whether to enable the PVC for storing base models. |
|
||||||
|
| customizer.modelsStorage.accessModes | list | `["ReadWriteOnce"]` | Specifies the access modes for the PVC. |
|
||||||
|
| customizer.modelsStorage.size | string | `"20Gi"` | Specifies the size of the PVC. |
|
||||||
|
| customizer.modelsStorage.storageClassName | string | `""` | Specifies the storage class to use for the PVC. |
|
||||||
|
| customizer.nemoDataStoreTools | object | `{"imagePullSecret":"nvcrimagepullsecret"}` | Specifies the image pull secret for the NeMo Data Store tools. |
|
||||||
|
| customizer.postgresql.nameOverride | string | `"customizerdb"` | The name override for the PostgreSQL database. |
|
||||||
|
| customizer.postgresql.serviceAccount.create | bool | `true` | Specifies whether to create a new service account for PostgreSQL. |
|
||||||
|
| customizer.postgresql.serviceAccount.name | string | `"customizer-postgresql"` | |
|
||||||
|
| customizer.serviceName | string | `"nemo-customizer"` | |
|
||||||
|
| customizer.workspaceStorage.enabled | bool | `false` | Specifies whether to enable the PVC for storing a training job's workspace. |
|
||||||
|
| data-store.enabled | bool | `true` | Specifies whether to install the NeMo Data Store microservice. |
|
||||||
|
| data-store.objectStore.enabled | bool | `false` | Specifies whether to enable object storage for the NeMo Data Store microservice. |
|
||||||
|
| data-store.persistence.accessModes | list | `["ReadWriteOnce"]` | The access modes for the persistent volume. |
|
||||||
|
| data-store.persistence.claimName | string | `"datastore-shared-storage"` | The name of the persistent volume claim. |
|
||||||
|
| data-store.persistence.enabled | bool | `true` | Specifies whether to enable persistent volume for the NeMo Data Store microservice. |
|
||||||
|
| data-store.persistence.size | string | `"1Gi"` | The size of the persistent volume. |
|
||||||
|
| data-store.persistence.storageClass | string | `""` | The storage class to use for the persistent volume. |
|
||||||
|
| data-store.replicaCount | int | `1` | The number of replicas for the NeMo Data Store microservice deployment. |
|
||||||
|
| data-store.serviceAccount.create | bool | `true` | Specifies whether to create a service account for the NeMo Data Store microservice. |
|
||||||
|
| data-store.serviceAccount.name | string | `"gitea"` | The service account name for the NeMo Data Store microservice. |
|
||||||
|
| data-store.serviceName | string | `"nemo-data-store"` | The service name for the NeMo Data Store microservice. |
|
||||||
|
| data-store.strategy.type | string | `"Recreate"` | The type of deployment strategy to use for the NeMo Data Store microservice. |
|
||||||
|
| deployment-management.deployments.entityStoreUrl | string | `"http://nemo-entity-store:8000"` | The URL for the NeMo Entity Store service. |
|
||||||
|
| deployment-management.deployments.modelSyncPeriod | string | `"30"` | The period in seconds for model synchronization. |
|
||||||
|
| deployment-management.deployments.nimImagePullSecrets | list | `["nvcrimagepullsecret"]` | The image pull secrets for the NeMo Deployment Management microservice. |
|
||||||
|
| deployment-management.deployments.nimPeftSource | string | `"http://nemo-entity-store:8000"` | The URL for the NIM PEFT source. |
|
||||||
|
| deployment-management.enabled | bool | `true` | Specifies whether to install the NeMo Deployment Management microservice. |
|
||||||
|
| deployment-management.image.pullPolicy | string | `"IfNotPresent"` | The pull policy for the NeMo Deployment Management container image. |
|
||||||
|
| deployment-management.serviceName | string | `"nemo-deployment-management"` | The service name for the NeMo Deployment Management microservice. |
|
||||||
|
| dgxc-admission-controller.cloudProviderSpec.dummy | object | `{"type":"dummy"}` | The type of the dummy cloud provider. |
|
||||||
|
| dgxc-admission-controller.dgxcController.image | string | `"nvcr.io/nvidia/nemo-microservices/dgxc-admission-controller:1.642.0"` | The image for the DGX Cloud Admission Controller. |
|
||||||
|
| dgxc-admission-controller.enableDGXCRbac | bool | `false` | |
|
||||||
|
| dgxc-admission-controller.enableIntegrationsSupport | bool | `false` | Specifies whether to enable integrations support. |
|
||||||
|
| dgxc-admission-controller.enableLabeler | bool | `false` | Specifies whether to enable the labeler. |
|
||||||
|
| dgxc-admission-controller.enableNetworkPolicies | bool | `false` | Specifies whether to enable network policies. |
|
||||||
|
| dgxc-admission-controller.enabled | bool | `false` | Specifies whether to install the DGX Cloud Admission Controller. |
|
||||||
|
| dgxc-admission-controller.imagePullSecret | string | `"nvcrimagepullsecret"` | The image pull secret for the DGX Cloud Admission Controller. |
|
||||||
|
| dgxc-admission-controller.kubernetesAffinity.enabled | bool | `false` | Specifies whether to enable Kubernetes affinity. |
|
||||||
|
| entity-store.appConfig.BASE_URL_DATASTORE | string | `"http://nemo-data-store:3000/v1/hf"` | The base URL for the NeMo Data Store microservice. |
|
||||||
|
| entity-store.appConfig.BASE_URL_NIM | string | `"http://nemo-nim-proxy:8000"` | The base URL for the NIM service used by the NeMo Entity Store microservice. |
|
||||||
|
| entity-store.enabled | bool | `true` | Specifies whether to install the NeMo Entity Store microservice. |
|
||||||
|
| entity-store.postgresql.nameOverride | string | `"entity-storedb"` | The name override for the PostgreSQL database. |
|
||||||
|
| entity-store.postgresql.serviceAccount.create | bool | `true` | Specifies whether to create a new service account for PostgreSQL. |
|
||||||
|
| entity-store.postgresql.serviceAccount.name | string | `"entity-store-postgresql"` | The service account name for PostgreSQL. |
|
||||||
|
| entity-store.serviceName | string | `"nemo-entity-store"` | The service name for the NeMo Entity Store microservice. |
|
||||||
|
| evaluator.argoWorkflows.controller.serviceAccount.name | string | `"argo-workflows-controller"` | The name of the service account for the Argo Workflows controller. |
|
||||||
|
| evaluator.argoWorkflows.crds.install | bool | `true` | Specifies whether to install the CRDs for Argo Workflows. |
|
||||||
|
| evaluator.argoWorkflows.enabled | bool | `true` | Specifies whether to enable the default Argo Workflows integration for the NeMo Evaluator microservice. |
|
||||||
|
| evaluator.argoWorkflows.server.authModes[0] | string | `"server"` | |
|
||||||
|
| evaluator.argoWorkflows.server.secure | bool | `true` | Specifies whether to secure the connection to the Argo Workflows server. |
|
||||||
|
| evaluator.argoWorkflows.server.serviceAccount.name | string | `"argo-workflows-server"` | The name of the service account for the Argo Workflows server. |
|
||||||
|
| evaluator.argoWorkflows.server.servicePort | int | `2746` | The service port for the Argo Workflows server. |
|
||||||
|
| evaluator.argoWorkflows.serviceName | string | `"argo-workflows-server"` | The Argo Workflows service name for the NeMo Evaluator microservice. |
|
||||||
|
| evaluator.enabled | bool | `true` | Specifies whether to install the NeMo Evaluator microservice. |
|
||||||
|
| evaluator.external.dataStore | object | `{"endpoint":"http://nemo-data-store:3000/v1/hf"}` | The external URL of the NeMo Data Store microservice. |
|
||||||
|
| evaluator.milvus.standalone.persistence.persistentVolumeClaim.storageClass | string | `""` | Specifies the storage class to use for the Milvus PVC. |
|
||||||
|
| evaluator.postgresql.nameOverride | string | `"evaluatordb"` | The name override for the default PostgreSQL database. |
|
||||||
|
| evaluator.postgresql.primary.networkPolicy.enabled | bool | `false` | Specifies whether to enable the network policy for the PostgreSQL service. |
|
||||||
|
| evaluator.postgresql.serviceAccount.name | string | `"evaluator-postgresql"` | The name of the service account for PostgreSQL. |
|
||||||
|
| evaluator.serviceName | string | `"nemo-evaluator"` | |
|
||||||
|
| existingImagePullSecret | string | `""` | You can specify an existing Kubernetes image pull secret for pulling images from the container registry. The chart doesn't reference any image pull secret if you leave this empty. |
|
||||||
|
| existingSecret | string | `""` | You can use an existing Kubernetes secret for pulling images. The chart uses the `ngcAPIKey` value to generate the secret if you leave this empty. |
|
||||||
|
| global.imagePullSecrets | list | `[{"name":"nvcrimagepullsecret"}]` | The name of the image pull secret to use globally across all services. |
|
||||||
|
| guardrails.enabled | bool | `true` | Specifies whether to install the NeMo Guardrails microservice. |
|
||||||
|
| guardrails.env | object | `{"NIM_ENDPOINT_URL":"http://nemo-nim-proxy:8000/v1"}` | The environment variables for the NeMo Guardrails microservice. |
|
||||||
|
| guardrails.env.NIM_ENDPOINT_URL | string | `"http://nemo-nim-proxy:8000/v1"` | The NIM endpoint URL for the NeMo Guardrails microservice. |
|
||||||
|
| guardrails.serviceName | string | `"nemo-guardrails"` | The service name for the NeMo Guardrails microservice. |
|
||||||
|
| imagePullSecrets | list | `[{"name":"nvcrimagepullsecret","password":"YOUR-NGC-API-KEY","registry":"nvcr.io","username":"$$oauthtoken"}]` | List of image pull secrets. Existing secrets override these values if you specify them. Use this only for experimentation when you want to hardcode a secret in your values file. |
|
||||||
|
| ingress.annotations | object | `{}` | Annotations for the ingress resource. |
|
||||||
|
| ingress.className | string | `""` | The ingress class to use if your cluster has more than one class. |
|
||||||
|
| ingress.enabled | bool | `false` | Specifies whether to enable the ingress. |
|
||||||
|
| ingress.hosts | object | `{"dataStore":{"name":"data-store.test","paths":[{"path":"/","pathType":"Prefix","port":3000,"service":"nemo-data-store"}]},"default":{"name":"","paths":[{"path":"/v1/namespaces","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/projects","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/datasets","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/repos","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/models","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/customization","pathType":"Prefix","port":8000,"service":"nemo-customizer"},{"path":"/v1/evaluation","pathType":"Prefix","port":7331,"service":"nemo-evaluator"},{"path":"/v1/guardrail","pathType":"Prefix","port":7331,"service":"nemo-guardrails"},{"path":"/v1/deployment","pathType":"Prefix","port":8000,"service":"nemo-deployment-management"}]},"nimProxy":{"name":"nim.test","paths":[{"path":"/v1/completions","pathType":"Prefix","port":8000,"service":"nemo-nim-proxy"},{"path":"/v1/chat","pathType":"Prefix","port":8000,"service":"nemo-nim-proxy"},{"path":"/v1/models","pathType":"Prefix","port":8000,"service":"nemo-nim-proxy"}]}}` | A map of hosts and their corresponding paths for the ingress. |
|
||||||
|
| ingress.hosts.dataStore.name | string | `"data-store.test"` | The host name for the third ingress host for the NeMo Data Store microservice. |
|
||||||
|
| ingress.hosts.dataStore.paths | list | `[{"path":"/","pathType":"Prefix","port":3000,"service":"nemo-data-store"}]` | The path rules for the third ingress host. |
|
||||||
|
| ingress.hosts.default.name | string | `""` | The host name for the default ingress host. |
|
||||||
|
| ingress.hosts.default.paths | list | `[{"path":"/v1/namespaces","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/projects","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/datasets","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/repos","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/models","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/customization","pathType":"Prefix","port":8000,"service":"nemo-customizer"},{"path":"/v1/evaluation","pathType":"Prefix","port":7331,"service":"nemo-evaluator"},{"path":"/v1/guardrail","pathType":"Prefix","port":7331,"service":"nemo-guardrails"},{"path":"/v1/deployment","pathType":"Prefix","port":8000,"service":"nemo-deployment-management"}]` | The path rules for the default ingress host. |
|
||||||
|
| ingress.hosts.nimProxy.name | string | `"nim.test"` | The host name for the second ingress host for the NIM Proxy microservice. |
|
||||||
|
| ingress.hosts.nimProxy.paths | list | `[{"path":"/v1/completions","pathType":"Prefix","port":8000,"service":"nemo-nim-proxy"},{"path":"/v1/chat","pathType":"Prefix","port":8000,"service":"nemo-nim-proxy"},{"path":"/v1/models","pathType":"Prefix","port":8000,"service":"nemo-nim-proxy"}]` | The path rules for the second ingress host. |
|
||||||
|
| ingress.tls | list | `[]` | TLS configurations. |
|
||||||
|
| nemo-operator.controllerManager.manager.scheduler | string | `"volcano"` | The scheduler to use for the NeMo Operator. Available options: `volcano` and `runai`. |
|
||||||
|
| nemo-operator.enabled | bool | `true` | Specifies whether to install the NeMo Operator microservice. |
|
||||||
|
| ngcAPIKey | string | `"YOUR-NGC-API-KEY"` | Your NVIDIA GPU Cloud (NGC) API key authenticates and enables pulling images from the NGC container registry. The existing secret overrides this key if you provide one to the `existingSecret` key. |
|
||||||
|
| nim-operator.enabled | bool | `true` | Specifies whether to install the NIM Operator. |
|
||||||
|
| nim-operator.nfd.nodeFeatureRules.deviceID | bool | `false` | Specifies whether to enable device ID feature rules. |
|
||||||
|
| nim-proxy.enabled | bool | `true` | Specifies whether to install the NIM Proxy microservice. |
|
||||||
|
| nim-proxy.monitoring.enabled | bool | `false` | Specifies whether to enable monitoring for the NIM Proxy microservice. |
|
||||||
|
| nim-proxy.serviceName | string | `"nemo-nim-proxy"` | The service name for the NIM Proxy microservice. |
|
||||||
|
| nim.enabled | bool | `false` | Specifies whether to deploy a NIM for LLM during the Helm installation of the chart. You can deploy a single static NIM by enabling this object and its values. When enabled, the chart deploys `meta/llama-3.1-8b-instruct` as the default NIM. |
|
||||||
|
| nim.env | list | `[{"name":"NIM_PEFT_SOURCE","value":"http://nemo-entity-store:8000"},{"name":"NIM_PEFT_REFRESH_INTERVAL","value":"30"},{"name":"NIM_MAX_CPU_LORAS","value":"16"},{"name":"NIM_MAX_GPU_LORAS","value":"8"}]` | Environment variables for the NIM service. |
|
||||||
|
| nim.image.repository | string | `"nvcr.io/nim/meta/llama-3.1-8b-instruct"` | |
|
||||||
|
| nim.image.tag | string | `"1.8.3"` | |
|
||||||
|
| nim.imagePullSecrets[0].name | string | `"nvcrimagepullsecret"` | |
|
||||||
|
| nim.model.name | string | `"meta/llama-3.1-8b-instruct"` | The name of the model to deploy as NIM. |
|
||||||
|
| nim.model.ngcAPISecret | string | `"ngc-api"` | The NGC API secret for model access. |
|
||||||
|
| nim.persistence.annotations | object | `{"helm.sh/resource-policy":"keep"}` | Annotations for the PVC. |
|
||||||
|
| nim.persistence.enabled | bool | `true` | Specifies whether to enable persistence volume claim (PVC) for the NIM service. |
|
||||||
|
| nim.persistence.storageClass | string | `""` | Specifies the storage class to use for the PVC. |
|
||||||
|
| nim.resources | object | `{"limits":{"nvidia.com/gpu":1},"requests":{"nvidia.com/gpu":1}}` | Specifies resources for the NIM service. |
|
||||||
|
| nim.resources.requests | object | `{"nvidia.com/gpu":1}` | Specifies requests for the NIM service. |
|
||||||
|
| nim.service.labels | object | `{"app.nvidia.com/nim-type":"inference"}` | Labels for the NIM service. |
|
||||||
|
| nim.statefulSet.enabled | bool | `false` | Specifies whether to enable a stateful set for the NIM service. |
|
||||||
|
| virtualService | object | A virtual service configuration template. | Specifies whether to enable the virtual service. If you are not using istio and virtualservices, it can be useful to create some virtual services for the NeMo Microservices system. Don't enable unless you use istio. |
|
||||||
|
| virtualService.additional.data-store | object | `{"entries":{"data-store":{"corsPolicy":{},"match":[{"uri":{"prefix":"/"}}],"route":[{"destination":{"host":"nemo-data-store","port":{"number":3000}}}]}},"gateways":[],"hosts":[]}` | Additional virtual service configurations. |
|
||||||
|
| virtualService.additional.data-store.entries | object | `{"data-store":{"corsPolicy":{},"match":[{"uri":{"prefix":"/"}}],"route":[{"destination":{"host":"nemo-data-store","port":{"number":3000}}}]}}` | The entries for the virtual NeMo Data Store service. |
|
||||||
|
| virtualService.additional.data-store.entries.data-store.corsPolicy | object | `{}` | The CORS policy for the virtual NeMo Data Store service. |
|
||||||
|
| virtualService.additional.data-store.entries.data-store.match | list | `[{"uri":{"prefix":"/"}}]` | The match for the virtual NeMo Data Store service. |
|
||||||
|
| virtualService.additional.data-store.entries.data-store.route | list | `[{"destination":{"host":"nemo-data-store","port":{"number":3000}}}]` | The route for the virtual NeMo Data Store service. |
|
||||||
|
| virtualService.additional.data-store.gateways | list | `[]` | The gateways for the virtual NeMo Data Store service. |
|
||||||
|
| virtualService.additional.data-store.hosts | list | `[]` | The hosts for the virtual NeMo Data Store service. |
|
||||||
|
| virtualService.additional.nim-proxy.entries.nim-proxy.corsPolicy | object | `{}` | The CORS policy for the virtual NIM Proxy service. |
|
||||||
|
| virtualService.additional.nim-proxy.entries.nim-proxy.match | list | `[{"uri":{"prefix":"/"}}]` | The match for the virtual NIM Proxy service. |
|
||||||
|
| virtualService.additional.nim-proxy.entries.nim-proxy.route | list | `[{"destination":{"host":"nemo-nim-proxy","port":{"number":8000}}}]` | The route for the virtual NIM Proxy service. |
|
||||||
|
| virtualService.additional.nim-proxy.gateways | list | `[]` | The gateways for the virtual NIM Proxy service. |
|
||||||
|
| virtualService.additional.nim-proxy.hosts | list | `[]` | The hosts for the virtual NIM Proxy service. |
|
||||||
|
| virtualService.annotations | object | `{}` | Annotations for the virtual service. |
|
||||||
|
| virtualService.enabled | bool | `false` | Specifies whether to enable the virtual service. |
|
||||||
|
| virtualService.labels | object | `{}` | Labels for the virtual service. |
|
||||||
|
| virtualService.main.entries.customizer.corsPolicy | object | `{}` | The CORS policy for the virtual NeMo Customizer service. |
|
||||||
|
| virtualService.main.entries.customizer.match | list | `[{"uri":{"prefix":"/v1/customization"}}]` | The match for the virtual NeMo Customizer service. |
|
||||||
|
| virtualService.main.entries.customizer.route | list | `[{"destination":{"host":"nemo-customizer","port":{"number":8000}}}]` | The route for the virtual NeMo Customizer service. |
|
||||||
|
| virtualService.main.entries.deployment-management.corsPolicy | object | `{}` | The CORS policy for the virtual NeMo Deployment Management service. |
|
||||||
|
| virtualService.main.entries.deployment-management.match | list | `[{"uri":{"prefix":"/v1/deployment"}}]` | The match for the virtual NeMo Deployment Management service. |
|
||||||
|
| virtualService.main.entries.deployment-management.route | list | `[{"destination":{"host":"nemo-deployment-management","port":{"number":8000}}}]` | The route for the virtual NeMo Deployment Management service. |
|
||||||
|
| virtualService.main.entries.entity-store.corsPolicy | object | `{}` | The CORS policy for the virtual NeMo Entity Store service. |
|
||||||
|
| virtualService.main.entries.entity-store.match | list | `[{"uri":{"prefix":"/v1/namespaces"}},{"uri":{"prefix":"/v1/projects"}},{"uri":{"prefix":"/v1/datasets"}},{"uri":{"prefix":"/v1/repos"}},{"uri":{"prefix":"/v1/models"}}]` | The match for the virtual NeMo Entity Store service. |
|
||||||
|
| virtualService.main.entries.entity-store.route | list | `[{"destination":{"host":"nemo-entity-store","port":{"number":8000}}}]` | The route for the virtual NeMo Entity Store service. |
|
||||||
|
| virtualService.main.entries.evaluator.corsPolicy | object | `{}` | The CORS policy for the virtual NeMo Evaluator service. |
|
||||||
|
| virtualService.main.entries.evaluator.match | list | `[{"uri":{"prefix":"/v1/evaluation"}}]` | The match for the virtual NeMo Evaluator service. |
|
||||||
|
| virtualService.main.entries.evaluator.route | list | `[{"destination":{"host":"nemo-evaluator","port":{"number":7331}}}]` | The route for the virtual NeMo Evaluator service. |
|
||||||
|
| virtualService.main.entries.guardrails.corsPolicy | object | `{}` | The CORS policy for the virtual NeMo Guardrails service. |
|
||||||
|
| virtualService.main.entries.guardrails.match | list | `[{"uri":{"prefix":"/v1/guardrail"}}]` | The match for the virtual NeMo Guardrails service. |
|
||||||
|
| virtualService.main.entries.guardrails.route | list | `[{"destination":{"host":"nemo-guardrails","port":{"number":7331}}}]` | The route for the virtual NeMo Guardrails service. |
|
||||||
|
| virtualService.main.gateways | list | `[]` | A list of gateways for the virtual service. |
|
||||||
|
| virtualService.main.hosts | list | `[]` | A list of hosts for the virtual service. |
|
||||||
|
| volcano.enabled | bool | `false` | Specifies whether to enable the default Volcano scheduler installation. To learn more, see [Volcano](https://docs.nvidia.com/nemo/microservices/latest/set-up/deploy-as-microservices/customizer.html#volcano). |
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
appVersion: 0.0.1
|
||||||
|
description: A Helm chart for Kubernetes
|
||||||
|
name: dgxc-admission-controller
|
||||||
|
version: 1.642.0
|
||||||
+220
@@ -0,0 +1,220 @@
|
|||||||
|
---
|
||||||
|
apiVersion: apiextensions.k8s.io/v1
|
||||||
|
kind: CustomResourceDefinition
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
controller-gen.kubebuilder.io/version: v0.16.3
|
||||||
|
name: nvstoragelocations.storage.dgxc.nvidia.com
|
||||||
|
spec:
|
||||||
|
group: storage.dgxc.nvidia.com
|
||||||
|
names:
|
||||||
|
kind: NvStorageLocation
|
||||||
|
listKind: NvStorageLocationList
|
||||||
|
plural: nvstoragelocations
|
||||||
|
singular: nvstoragelocation
|
||||||
|
scope: Namespaced
|
||||||
|
versions:
|
||||||
|
- additionalPrinterColumns:
|
||||||
|
- description: Bucket
|
||||||
|
jsonPath: .spec.bucket
|
||||||
|
name: Bucket
|
||||||
|
type: string
|
||||||
|
name: v1beta1
|
||||||
|
schema:
|
||||||
|
openAPIV3Schema:
|
||||||
|
description: NvStorageLocation defines storage for DGXC Run:ai accelerated
|
||||||
|
compute environments.
|
||||||
|
properties:
|
||||||
|
apiVersion:
|
||||||
|
description: |-
|
||||||
|
APIVersion defines the versioned schema of this representation of an object.
|
||||||
|
Servers should convert recognized schemas to the latest internal value, and
|
||||||
|
may reject unrecognized values.
|
||||||
|
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
|
||||||
|
type: string
|
||||||
|
kind:
|
||||||
|
description: |-
|
||||||
|
Kind is a string value representing the REST resource this object represents.
|
||||||
|
Servers may infer this from the endpoint the client submits requests to.
|
||||||
|
Cannot be updated.
|
||||||
|
In CamelCase.
|
||||||
|
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
|
||||||
|
type: string
|
||||||
|
metadata:
|
||||||
|
type: object
|
||||||
|
spec:
|
||||||
|
properties:
|
||||||
|
description:
|
||||||
|
type: string
|
||||||
|
mountPath:
|
||||||
|
type: string
|
||||||
|
pvc:
|
||||||
|
description: '`pvc` represents a PVC to manage.'
|
||||||
|
properties:
|
||||||
|
capacity:
|
||||||
|
description: Requested size of the volume (required for a managed
|
||||||
|
volume)
|
||||||
|
format: ByteSize
|
||||||
|
type: string
|
||||||
|
mountOptions:
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
type: array
|
||||||
|
objectSpec:
|
||||||
|
description: '`object` represents a volume that uses object storage.'
|
||||||
|
properties:
|
||||||
|
bucket:
|
||||||
|
type: string
|
||||||
|
endpointUrl:
|
||||||
|
type: string
|
||||||
|
required:
|
||||||
|
- bucket
|
||||||
|
- endpointUrl
|
||||||
|
type: object
|
||||||
|
parameters:
|
||||||
|
additionalProperties:
|
||||||
|
type: string
|
||||||
|
description: |-
|
||||||
|
Parameters contains an arbitrary set of key-value pairs
|
||||||
|
to supply to whatever storage backend we're using
|
||||||
|
according to our rules for that object storage backend.
|
||||||
|
Currently the only thing supported is the GCS OIDC ConfigMap
|
||||||
|
this parameter is "gcs-oidc-configmap" with the value being
|
||||||
|
the name of the configmap
|
||||||
|
type: object
|
||||||
|
required:
|
||||||
|
- capacity
|
||||||
|
- objectSpec
|
||||||
|
type: object
|
||||||
|
volumeName:
|
||||||
|
description: volumeName will be used to name the PVC and if needed
|
||||||
|
the PVs
|
||||||
|
type: string
|
||||||
|
required:
|
||||||
|
- description
|
||||||
|
- mountPath
|
||||||
|
type: object
|
||||||
|
status:
|
||||||
|
properties:
|
||||||
|
conditions:
|
||||||
|
items:
|
||||||
|
description: Condition contains details for one aspect of the current
|
||||||
|
state of this API Resource.
|
||||||
|
properties:
|
||||||
|
lastTransitionTime:
|
||||||
|
description: |-
|
||||||
|
lastTransitionTime is the last time the condition transitioned from one status to another.
|
||||||
|
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
|
||||||
|
format: date-time
|
||||||
|
type: string
|
||||||
|
message:
|
||||||
|
description: |-
|
||||||
|
message is a human readable message indicating details about the transition.
|
||||||
|
This may be an empty string.
|
||||||
|
maxLength: 32768
|
||||||
|
type: string
|
||||||
|
observedGeneration:
|
||||||
|
description: |-
|
||||||
|
observedGeneration represents the .metadata.generation that the condition was set based upon.
|
||||||
|
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
|
||||||
|
with respect to the current state of the instance.
|
||||||
|
format: int64
|
||||||
|
minimum: 0
|
||||||
|
type: integer
|
||||||
|
reason:
|
||||||
|
description: |-
|
||||||
|
reason contains a programmatic identifier indicating the reason for the condition's last transition.
|
||||||
|
Producers of specific condition types may define expected values and meanings for this field,
|
||||||
|
and whether the values are considered a guaranteed API.
|
||||||
|
The value should be a CamelCase string.
|
||||||
|
This field may not be empty.
|
||||||
|
maxLength: 1024
|
||||||
|
minLength: 1
|
||||||
|
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
|
||||||
|
type: string
|
||||||
|
status:
|
||||||
|
description: status of the condition, one of True, False, Unknown.
|
||||||
|
enum:
|
||||||
|
- "True"
|
||||||
|
- "False"
|
||||||
|
- Unknown
|
||||||
|
type: string
|
||||||
|
type:
|
||||||
|
description: type of condition in CamelCase or in foo.example.com/CamelCase.
|
||||||
|
maxLength: 316
|
||||||
|
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
|
||||||
|
type: string
|
||||||
|
required:
|
||||||
|
- lastTransitionTime
|
||||||
|
- message
|
||||||
|
- reason
|
||||||
|
- status
|
||||||
|
- type
|
||||||
|
type: object
|
||||||
|
type: array
|
||||||
|
pendingCount:
|
||||||
|
description: The number of PVCs monitored by this NVStorageLocation
|
||||||
|
that are not in a ready state (e.g. missing, error state).
|
||||||
|
type: integer
|
||||||
|
pvcs:
|
||||||
|
description: |-
|
||||||
|
A list of PVC statuses. This is a list because we expect to support multiple namespaces,
|
||||||
|
but it may very well be a list of one object.
|
||||||
|
items:
|
||||||
|
properties:
|
||||||
|
externalReferences:
|
||||||
|
items:
|
||||||
|
properties:
|
||||||
|
clusterName:
|
||||||
|
type: string
|
||||||
|
externalID:
|
||||||
|
type: string
|
||||||
|
platform:
|
||||||
|
type: string
|
||||||
|
required:
|
||||||
|
- clusterName
|
||||||
|
- externalID
|
||||||
|
- platform
|
||||||
|
type: object
|
||||||
|
type: array
|
||||||
|
namespace:
|
||||||
|
description: The namespace in which the PersistentVolumeClaim
|
||||||
|
exists.
|
||||||
|
type: string
|
||||||
|
pv:
|
||||||
|
type: string
|
||||||
|
state:
|
||||||
|
description: The current state of this volume.
|
||||||
|
type: string
|
||||||
|
volumeName:
|
||||||
|
description: The full name of the PersistentVolumeClaim.
|
||||||
|
type: string
|
||||||
|
required:
|
||||||
|
- namespace
|
||||||
|
- pv
|
||||||
|
- state
|
||||||
|
- volumeName
|
||||||
|
type: object
|
||||||
|
type: array
|
||||||
|
readyCount:
|
||||||
|
description: The number of PVCs monitored by this NVStorageLocation
|
||||||
|
that are in a ready state.
|
||||||
|
type: integer
|
||||||
|
summary:
|
||||||
|
description: |-
|
||||||
|
A short human-readable summary of the status of all managed and monitored PVCs.
|
||||||
|
Machine-readable data should be sourced from the `Ready` and `NotReady` counts, or
|
||||||
|
from pvcs[].State for detailed information.
|
||||||
|
type: string
|
||||||
|
required:
|
||||||
|
- pendingCount
|
||||||
|
- pvcs
|
||||||
|
- readyCount
|
||||||
|
type: object
|
||||||
|
required:
|
||||||
|
- spec
|
||||||
|
type: object
|
||||||
|
served: true
|
||||||
|
storage: true
|
||||||
|
subresources:
|
||||||
|
status: {}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
No notes
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
{{/* vim: set filetype=mustache: */}}
|
||||||
|
{{/*
|
||||||
|
Expand the name of the chart.
|
||||||
|
*/}}
|
||||||
|
|
||||||
|
{{- define "dgxcController.name" -}}
|
||||||
|
{{- default "controller" .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Create a default fully qualified app name.
|
||||||
|
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
|
||||||
|
If release name contains chart name it will be used as a full name.
|
||||||
|
*/}}
|
||||||
|
|
||||||
|
{{- define "dgxcController.fullname" -}}
|
||||||
|
{{- if .Values.dgxcController.deploymentFullNameOverride -}}
|
||||||
|
{{- .Values.dgxcController.deploymentFullNameOverride | trunc 63 | trimSuffix "-" -}}
|
||||||
|
{{- else -}}
|
||||||
|
{{- $name := default .Chart.Name .Values.nameOverride -}}
|
||||||
|
{{- if contains $name .Release.Name -}}
|
||||||
|
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
|
||||||
|
{{- else -}}
|
||||||
|
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
|
||||||
|
{{- end -}}
|
||||||
|
{{- end -}}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Create chart name and version as used by the chart label.
|
||||||
|
*/}}
|
||||||
|
{{- define "dgxcController.chart" -}}
|
||||||
|
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{- define "dgx-application-deploy" -}}
|
||||||
|
{{ .Release.Name }}-{{ .Release.Revision }}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{- define "kubernetesAffinityConfig" }}
|
||||||
|
affinity:
|
||||||
|
nodeAffinity:
|
||||||
|
requiredDuringSchedulingIgnoredDuringExecution:
|
||||||
|
nodeSelectorTerms:
|
||||||
|
- matchExpressions:
|
||||||
|
- key: {{ .Values.kubernetesAffinity.antiAffinityKey | default "reason-to-avoid" }}
|
||||||
|
operator: DoesNotExist
|
||||||
|
podAntiAffinity:
|
||||||
|
requiredDuringSchedulingIgnoredDuringExecution:
|
||||||
|
- labelSelector:
|
||||||
|
matchExpressions:
|
||||||
|
- key: "nvidia.com/dgx-application-deploy"
|
||||||
|
operator: In
|
||||||
|
values:
|
||||||
|
- {{ include "dgx-application-deploy" . }}
|
||||||
|
topologyKey: {{ .Values.kubernetesAffinity.topologyKey | default "kubernetes.io/hostname" }}
|
||||||
|
{{- if .Values.kubernetesAffinity.runOnMasters }}
|
||||||
|
tolerations:
|
||||||
|
- effect: NoSchedule
|
||||||
|
key: node-role.kubernetes.io/master
|
||||||
|
operator: Exists
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
@@ -0,0 +1,250 @@
|
|||||||
|
{{- if .Values.enabledComponents.dgxcAdmissionController }}
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: {{ .Values.dgxcController.name}}-config
|
||||||
|
data:
|
||||||
|
config.yaml: |-
|
||||||
|
apiVersion: runai.dgxc.nvidia.com/v1
|
||||||
|
kind: DGXCRunaiMutatingWebhook
|
||||||
|
certs:
|
||||||
|
namespace: dgxc-admission-controller
|
||||||
|
service: {{ .Values.serviceName }}
|
||||||
|
port: 9443
|
||||||
|
|
||||||
|
organization: {{ .Values.webhooks.organization }}
|
||||||
|
|
||||||
|
mountPath: {{ .Values.dgxcController.certs.mountPath }}
|
||||||
|
certName: {{ .Values.dgxcController.certs.certName }}
|
||||||
|
keyName: {{ .Values.dgxcController.certs.keyName }}
|
||||||
|
|
||||||
|
podMutatingConfig:
|
||||||
|
name: dgxc-create-pod-mwh
|
||||||
|
kind: {{ .Values.webhooks.podMutatingWebhook.kind }}
|
||||||
|
fqdn: {{ .Values.webhooks.podMutatingWebhook.fqdn }}
|
||||||
|
path: {{ .Values.webhooks.podMutatingWebhook.path }}
|
||||||
|
failurePolicy: {{ .Values.webhooks.podMutatingWebhook.failurePolicy }}
|
||||||
|
timeoutSeconds: {{ .Values.webhooks.podMutatingWebhook.timeoutSeconds }}
|
||||||
|
fqdn: {{ .Values.webhooks.podMutatingWebhook.fqdn }}
|
||||||
|
sideEffects: {{ .Values.webhooks.podMutatingWebhook.sideEffects | default "NoneOnDryRun" }}
|
||||||
|
admissionReviewVersions:
|
||||||
|
{{- range $i, $value := .Values.webhooks.podMutatingWebhook.admissionReviewVersions }}
|
||||||
|
- {{ $value }}
|
||||||
|
{{- end }}
|
||||||
|
rules:
|
||||||
|
{{- range $i, $value := .Values.webhooks.podMutatingWebhook.rules }}
|
||||||
|
- apiGroups:
|
||||||
|
{{- range $j, $value2 := $value.apiGroups }}
|
||||||
|
- {{ $value2 | quote }}
|
||||||
|
{{- end }}
|
||||||
|
apiVersions:
|
||||||
|
{{- range $j, $value2 := $value.apiVersions }}
|
||||||
|
- {{ $value2 | quote }}
|
||||||
|
{{- end }}
|
||||||
|
operations:
|
||||||
|
{{- range $j, $value2 := $value.operations }}
|
||||||
|
- {{ $value2 | quote }}
|
||||||
|
{{- end }}
|
||||||
|
resources:
|
||||||
|
{{- range $j, $value2 := $value.resources }}
|
||||||
|
- {{ $value2 | quote }}
|
||||||
|
{{- end }}
|
||||||
|
scope: {{ $value.scope | quote }}
|
||||||
|
{{- end }}
|
||||||
|
nsExcludeLabels:
|
||||||
|
- key: kubernetes.io/metadata.name
|
||||||
|
values:
|
||||||
|
- dgxc-admission-controller
|
||||||
|
- argocd
|
||||||
|
- nvdiag
|
||||||
|
objectMatchLabels:
|
||||||
|
{{- range $i, $value := .Values.webhooks.podMutatingWebhook.objectLabelSelector }}
|
||||||
|
- key: {{ $value.key}}
|
||||||
|
values:
|
||||||
|
{{- range $j, $value2 := $value.values }}
|
||||||
|
- {{ $value2 | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
podMutatingSpec:
|
||||||
|
{{- with (first .Values.webhooks.podMutatingWebhook.customSettings) }}
|
||||||
|
addRdmaEnabled: {{ .addRdmaEnabled }}
|
||||||
|
rdmaResourcePerGpu: {{ .rdmaResourcePerGpu }}
|
||||||
|
networkAttachmentDefinition: {{ .networkAttachmentDefinition }}
|
||||||
|
rdmaResourceName: {{ .rdmaResourceName }}
|
||||||
|
addScratchEnabled: {{ .scratchEnabled }}
|
||||||
|
scratchVolName: {{ .scratchVolName | quote }}
|
||||||
|
scratchMountPath: {{ .scratchMountpath | quote }}
|
||||||
|
scratchVolSize: {{ .scratchVolSize | quote }}
|
||||||
|
scratchVolSizePerGpu: {{ .scratchVolSizePerGpu }}
|
||||||
|
scratchVolEnforceLimit: {{ .scratchVolEnforceLimit }}
|
||||||
|
{{- if .scratchVolSource }}
|
||||||
|
scratchVolSource:
|
||||||
|
{{ .scratchVolSource | toYaml | nindent 8 }}
|
||||||
|
{{- end}}
|
||||||
|
scratchOnCpu: {{ .scratchOnCpu | default false }}
|
||||||
|
addTcpxSidecarEnabled: {{ .addTcpxSidecarEnabled }}
|
||||||
|
addTcpxoSidecarEnabled: {{ .addTcpxoSidecarEnabled }}
|
||||||
|
addTcpxNcclVars: {{ .addTcpxNCCLVars }}
|
||||||
|
addTcpxoNcclVars: {{ .addTcpxoNCCLVars }}
|
||||||
|
addNcclVars: {{ .addNcclVars }}
|
||||||
|
csp: {{ .csp}}
|
||||||
|
addGcpFuseAnnotation: {{ .addGcpFuseAnnotation}}
|
||||||
|
gcpFuseAnnotationKey: {{ .gcpFuseAnnotationKey | quote }}
|
||||||
|
gcpFuseWICredentialConfigMapAnnotationKey: {{ .gcpFuseWICredentialConfigMapAnnotationKey | quote }}
|
||||||
|
gcpFuseStorageClasses:
|
||||||
|
{{- range .gcpFuseStorageClasses}}
|
||||||
|
- {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
clusterName: {{ .clusterName }}
|
||||||
|
injectRunaiJob: true
|
||||||
|
{{- end}}
|
||||||
|
systemNamespaceNames:
|
||||||
|
{{- range .Values.clusterScopedKinds }}
|
||||||
|
{{- if eq .kind "Namespace" }}
|
||||||
|
{{- range .systemResources}}
|
||||||
|
- {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
|
||||||
|
pvcMutatingConfig:
|
||||||
|
name: dgxc-create-pvc-mwh
|
||||||
|
kind: {{ .Values.webhooks.pvcMutatingWebhook.kind }}
|
||||||
|
fqdn: {{ .Values.webhooks.pvcMutatingWebhook.fqdn }}
|
||||||
|
path: {{ .Values.webhooks.pvcMutatingWebhook.path }}
|
||||||
|
failurePolicy: {{ .Values.webhooks.pvcMutatingWebhook.failurePolicy }}
|
||||||
|
timeoutSeconds: {{ .Values.webhooks.pvcMutatingWebhook.timeoutSeconds }}
|
||||||
|
fqdn: {{ .Values.webhooks.pvcMutatingWebhook.fqdn }}
|
||||||
|
sideEffects: {{ .Values.webhooks.pvcMutatingWebhook.sideEffects | default "NoneOnDryRun" }}
|
||||||
|
admissionReviewVersions:
|
||||||
|
{{- range $i, $value := .Values.webhooks.pvcMutatingWebhook.admissionReviewVersions }}
|
||||||
|
- {{ $value }}
|
||||||
|
{{- end }}
|
||||||
|
rules:
|
||||||
|
{{- range $i, $value := .Values.webhooks.pvcMutatingWebhook.rules }}
|
||||||
|
- apiGroups:
|
||||||
|
{{- range $j, $value2 := $value.apiGroups }}
|
||||||
|
- {{ $value2 | quote }}
|
||||||
|
{{- end }}
|
||||||
|
apiVersions:
|
||||||
|
{{- range $j, $value2 := $value.apiVersions }}
|
||||||
|
- {{ $value2 | quote }}
|
||||||
|
{{- end }}
|
||||||
|
operations:
|
||||||
|
{{- range $j, $value2 := $value.operations }}
|
||||||
|
- {{ $value2 | quote }}
|
||||||
|
{{- end }}
|
||||||
|
resources:
|
||||||
|
{{- range $j, $value2 := $value.resources }}
|
||||||
|
- {{ $value2 | quote }}
|
||||||
|
{{- end }}
|
||||||
|
scope: {{ $value.scope | quote }}
|
||||||
|
{{- end }}
|
||||||
|
nsMatchLabels:
|
||||||
|
{{- range $i, $value := .Values.webhooks.pvcMutatingWebhook.namespaceLabelSelector }}
|
||||||
|
- key: {{ $value.key}}
|
||||||
|
values:
|
||||||
|
{{- range $j, $value2 := $value.values }}
|
||||||
|
- {{ $value2 | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
objectMatchLabels:
|
||||||
|
{{- range $i, $value := .Values.webhooks.pvcMutatingWebhook.objectLabelSelector }}
|
||||||
|
- key: {{ $value.key}}
|
||||||
|
values:
|
||||||
|
{{- range $j, $value2 := $value.values }}
|
||||||
|
- {{ $value2 | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
pvcMutatingSpec:
|
||||||
|
rules:
|
||||||
|
{{- range $i, $value := .Values.webhooks.pvcMutatingWebhook.customSettings }}
|
||||||
|
- storageClasses:
|
||||||
|
{{- range $j, $value2 := $value.storageClasses }}
|
||||||
|
- {{ $value2 | quote }}
|
||||||
|
{{- end }}
|
||||||
|
incrementAdjustmentEnabled: {{ $value.incrementCheckEnabled }}
|
||||||
|
incrementAdjustmentValue: {{ $value.incrementValue | quote }}
|
||||||
|
unitConversionEnabled: {{ $value.unitConversionEnabled }}
|
||||||
|
unitConversionFormat: {{ $value.unitConversionFormat }}
|
||||||
|
{{- if $value.minSizeValue }}
|
||||||
|
adjustedMinimumSize: {{ $value.minSizeValue | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
pvcValidatingConfig:
|
||||||
|
name: dgxc-create-pvc-vwh
|
||||||
|
kind: {{ .Values.webhooks.pvcValidatingWebhook.kind }}
|
||||||
|
fqdn: {{ .Values.webhooks.pvcValidatingWebhook.fqdn }}
|
||||||
|
path: {{ .Values.webhooks.pvcValidatingWebhook.path }}
|
||||||
|
failurePolicy: {{ .Values.webhooks.pvcValidatingWebhook.failurePolicy }}
|
||||||
|
timeoutSeconds: {{ .Values.webhooks.pvcValidatingWebhook.timeoutSeconds }}
|
||||||
|
fqdn: {{ .Values.webhooks.pvcValidatingWebhook.fqdn }}
|
||||||
|
sideEffects: {{ .Values.webhooks.pvcValidatingWebhook.sideEffects | default "NoneOnDryRun" }}
|
||||||
|
admissionReviewVersions:
|
||||||
|
{{- range $i, $value := .Values.webhooks.pvcValidatingWebhook.admissionReviewVersions }}
|
||||||
|
- {{ $value }}
|
||||||
|
{{- end }}
|
||||||
|
rules:
|
||||||
|
{{- range $i, $value := .Values.webhooks.pvcValidatingWebhook.rules }}
|
||||||
|
- apiGroups:
|
||||||
|
{{- range $j, $value2 := $value.apiGroups }}
|
||||||
|
- {{ $value2 | quote }}
|
||||||
|
{{- end }}
|
||||||
|
apiVersions:
|
||||||
|
{{- range $j, $value2 := $value.apiVersions }}
|
||||||
|
- {{ $value2 | quote }}
|
||||||
|
{{- end }}
|
||||||
|
operations:
|
||||||
|
{{- range $j, $value2 := $value.operations }}
|
||||||
|
- {{ $value2 | quote }}
|
||||||
|
{{- end }}
|
||||||
|
resources:
|
||||||
|
{{- range $j, $value2 := $value.resources }}
|
||||||
|
- {{ $value2 | quote }}
|
||||||
|
{{- end }}
|
||||||
|
scope: {{ $value.scope | quote }}
|
||||||
|
{{- end }}
|
||||||
|
nsMatchLabels:
|
||||||
|
{{- range $i, $value := .Values.webhooks.pvcValidatingWebhook.namespaceLabelSelector }}
|
||||||
|
- key: {{ $value.key}}
|
||||||
|
values:
|
||||||
|
{{- range $j, $value2 := $value.values }}
|
||||||
|
- {{ $value2 | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
objectMatchLabels:
|
||||||
|
{{- range $i, $value := .Values.webhooks.pvcValidatingWebhook.objectLabelSelector }}
|
||||||
|
- key: {{ $value.key}}
|
||||||
|
values:
|
||||||
|
{{- range $j, $value2 := $value.values }}
|
||||||
|
- {{ $value2 | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
pvcValidatingSpec:
|
||||||
|
rules:
|
||||||
|
{{- range $i, $value := .Values.webhooks.pvcValidatingWebhook.customSettings }}
|
||||||
|
- name: {{ $value.name }}
|
||||||
|
storageClasses:
|
||||||
|
{{- range $j, $value2 := $value.storageClasses }}
|
||||||
|
- {{ $value2 | quote }}
|
||||||
|
{{- end }}
|
||||||
|
minsizeCheckEnabled: {{ $value.minSizeCheckEnabled }}
|
||||||
|
minsizeValue: {{ $value.minSizeValue | quote }}
|
||||||
|
maxsizeCheckEnabled: {{ $value.maxSizeCheckEnabled }}
|
||||||
|
maxsizeValue: {{ $value.maxSizeValue | quote }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
systemNamespaceNames:
|
||||||
|
{{- range .Values.clusterScopedKinds }}
|
||||||
|
{{- if eq .kind "Namespace" }}
|
||||||
|
{{- range .systemResources}}
|
||||||
|
- {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{ end }}
|
||||||
@@ -0,0 +1,161 @@
|
|||||||
|
{{ if .Values.enabledComponents.dgxcRunaiController }}
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: dgxc-runai-controller
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
data:
|
||||||
|
config.yaml: |-
|
||||||
|
apiVersion: runai.dgxc.nvidia.com/v1
|
||||||
|
kind: Controller
|
||||||
|
{{- if .Values.prober }}
|
||||||
|
prober:
|
||||||
|
intervalInSeconds: {{ .Values.prober.intervalInSeconds }}
|
||||||
|
timeoutInSeconds: {{ .Values.prober.timeoutInSeconds }}
|
||||||
|
endpoints:
|
||||||
|
{{- range .Values.prober.endpoints }}
|
||||||
|
- name: {{ .name }}
|
||||||
|
url: {{ .url }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- if .Values.sso }}
|
||||||
|
cluster: {{ .Values.sso.cluster }}
|
||||||
|
sso:
|
||||||
|
controlPlaneDomain: {{ .Values.sso.controlPlaneDomain }}
|
||||||
|
realm: {{ .Values.sso.realm }}
|
||||||
|
pollIntervalSeconds: {{ .Values.sso.pollIntervalSeconds }}
|
||||||
|
pollAttempts: {{ .Values.sso.pollAttempts }}
|
||||||
|
{{- end }}
|
||||||
|
tenantMetadata:
|
||||||
|
cluster: {{ .Values.tenantMetadata.cluster }}
|
||||||
|
platform: {{ .Values.tenantMetadata.platform }}
|
||||||
|
orgName: {{ .Values.tenantMetadata.orgName }}
|
||||||
|
tenantName: {{ .Values.tenantMetadata.tenantName }}
|
||||||
|
tenantType: {{ .Values.tenantMetadata.tenantType }}
|
||||||
|
startDate: {{ .Values.tenantMetadata.startDate }}
|
||||||
|
endDate: {{ .Values.tenantMetadata.endDate }}
|
||||||
|
CSP: {{ .Values.tenantMetadata.CSP }}
|
||||||
|
allocatedCapacityGPU: {{ .Values.tenantMetadata.allocatedCapacityGPU }}
|
||||||
|
allocatedCapacityCPU: {{ .Values.tenantMetadata.allocatedCapacityCPU }}
|
||||||
|
committedCapacityGPU: {{ .Values.tenantMetadata.committedCapacityGPU }}
|
||||||
|
committedCapacityCPU: {{ .Values.tenantMetadata.committedCapacityCPU }}
|
||||||
|
CSPProject: {{ .Values.tenantMetadata.CSPProject }}
|
||||||
|
dataCenterRegion: {{ .Values.tenantMetadata.dataCenterRegion }}
|
||||||
|
storageType: {{ .Values.tenantMetadata.storageType }}
|
||||||
|
storageQuota: {{ .Values.tenantMetadata.storageQuota }}
|
||||||
|
{{- if .Values.initSettings }}
|
||||||
|
appID: {{ .Values.initSettings.appID }}
|
||||||
|
tenant: {{ .Values.initSettings.tenant }}
|
||||||
|
controlPlane: {{ .Values.initSettings.controlPlane }}
|
||||||
|
initSettings:
|
||||||
|
supportEmailAddress: {{ .Values.initSettings.supportEmailAddress }}
|
||||||
|
customerEmailAddress: {{ .Values.initSettings.customerEmailAddress }}
|
||||||
|
bootstrapEmailAddress: {{ .Values.initSettings.bootstrapEmailAddress }}
|
||||||
|
{{- end }}
|
||||||
|
dgxcAdmins:
|
||||||
|
{{- range .Values.dgxcAdmins }}
|
||||||
|
- {{ . }}
|
||||||
|
{{- end }}
|
||||||
|
tenantSubjects:
|
||||||
|
{{- range .Values.tenantSubjects.User }}
|
||||||
|
- {{ . }}
|
||||||
|
{{- end }}
|
||||||
|
reservedCPU: {{ .Values.reserved.CPU }}
|
||||||
|
reservedMemory: {{ .Values.reserved.Memory }}
|
||||||
|
maxCPUJobs: {{ .Values.maxCPUJobs }}
|
||||||
|
cloudProviderSpec:
|
||||||
|
type: {{ .Values.cloudProviderSpec.type }}
|
||||||
|
{{- if eq .Values.cloudProviderSpec.type "gcp" }}
|
||||||
|
gcp:
|
||||||
|
projectId: {{ .Values.cloudProviderSpec.gcp.projectId }}
|
||||||
|
projectNumber: {{ .Values.cloudProviderSpec.gcp.projectNumber | int64 }}
|
||||||
|
securityPolicyRuleName: {{ .Values.cloudProviderSpec.gcp.securityPolicyRuleName }}
|
||||||
|
region: {{ .Values.cloudProviderSpec.gcp.region }}
|
||||||
|
{{- else if eq .Values.cloudProviderSpec.type "aws" }}
|
||||||
|
aws:
|
||||||
|
accountName: {{ .Values.cloudProviderSpec.aws.accountName }}
|
||||||
|
clusterName: {{ .Values.cloudProviderSpec.aws.clusterName }}
|
||||||
|
region: {{ .Values.cloudProviderSpec.aws.region }}
|
||||||
|
securityGroupARN: {{ .Values.cloudProviderSpec.aws.securityGroupARN }}
|
||||||
|
statefulRuleGroupName: {{ .Values.cloudProviderSpec.aws.statefulRuleGroupName }}
|
||||||
|
{{- else if eq .Values.cloudProviderSpec.type "oci" }}
|
||||||
|
oci:
|
||||||
|
compartmentID: {{ .Values.cloudProviderSpec.oci.compartmentID }}
|
||||||
|
clusterID: {{ .Values.cloudProviderSpec.oci.clusterID }}
|
||||||
|
fssAvailabilityDomain: "{{ .Values.cloudProviderSpec.oci.fssAvailabilityDomain }}"
|
||||||
|
publicSubnetFirewallID: "{{ .Values.cloudProviderSpec.oci.publicSubnetFirewallID }}"
|
||||||
|
privateSubnetFirewallID: "{{ .Values.cloudProviderSpec.oci.privateSubnetFirewallID }}"
|
||||||
|
baseFirewallPolicyID: "{{ .Values.cloudProviderSpec.oci.baseFirewallPolicyID }}"
|
||||||
|
{{- else if eq .Values.cloudProviderSpec.type "azure" }}
|
||||||
|
azure:
|
||||||
|
subscriptionId: {{ .Values.cloudProviderSpec.azure.subscriptionId }}
|
||||||
|
resourceGroupName: {{ .Values.cloudProviderSpec.azure.resourceGroupName }}
|
||||||
|
ingressNsgName: {{ .Values.cloudProviderSpec.azure.ingressNsgName }}
|
||||||
|
egressNsgName: {{ .Values.cloudProviderSpec.azure.egressNsgName }}
|
||||||
|
standardStorageAccountName: {{ .Values.cloudProviderSpec.azure.standardStorageAccountName }}
|
||||||
|
premiumStorageAccountName: {{ .Values.cloudProviderSpec.azure.premiumStorageAccountName }}
|
||||||
|
logExporterAnalyticsWorkspaceId: {{ .Values.cloudProviderSpec.azure.logExporterAnalyticsWorkspaceId }}
|
||||||
|
{{- end }}
|
||||||
|
logExporterControllerSpec:
|
||||||
|
enabled: {{ .Values.logExporterControllerSpec.enabled }}
|
||||||
|
exportFrequency: {{ .Values.logExporterControllerSpec.exportFrequency }}
|
||||||
|
exportBuffer: {{ .Values.logExporterControllerSpec.exportBuffer }}
|
||||||
|
fetchTimeout: {{ .Values.logExporterControllerSpec.fetchTimeout }}
|
||||||
|
systemResources:
|
||||||
|
{{- range $i, $csk := .Values.clusterScopedKinds }}
|
||||||
|
{{- $group := $csk.apiGroup | default "core" -}}
|
||||||
|
{{- range $j, $kind := $csk.resources }}
|
||||||
|
{{- range $csk.systemResources}}
|
||||||
|
{{- if and (eq $kind "namespaces") (eq . "dgxc-tenant-cluster-policies") | not }}
|
||||||
|
- {{ print $group "/" $kind "/" . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
networkControllerSpec:
|
||||||
|
enabled: {{ .Values.networkControllerSpec.enabled }}
|
||||||
|
defaultIngressCIDRs:
|
||||||
|
{{- .Values.bootstrapIngressCIDR | toYaml | nindent 6 }}
|
||||||
|
{{- if eq .Values.cloudProviderSpec.type "gcp" }}
|
||||||
|
storageControllerSpec:
|
||||||
|
enabled: {{ .Values.storageControllerSpecGCP.enabled }}
|
||||||
|
defaultCustomerPVReclaimPolicy: {{ .Values.storageControllerSpecGCP.defaultCustomerPVReclaimPolicy | default "Retain" }}
|
||||||
|
defaultStorageTypes:
|
||||||
|
{{- .Values.storageControllerSpecGCP.defaultStorageTypes | toYaml | nindent 6 }}
|
||||||
|
{{- else if eq .Values.cloudProviderSpec.type "aws" }}
|
||||||
|
storageControllerSpec:
|
||||||
|
enabled: {{ .Values.storageControllerSpecAWS.enabled }}
|
||||||
|
defaultCustomerPVReclaimPolicy: {{ .Values.storageControllerSpecAWS.defaultCustomerPVReclaimPolicy | default "Retain" }}
|
||||||
|
defaultStorageTypes:
|
||||||
|
{{- .Values.storageControllerSpecAWS.defaultStorageTypes | toYaml | nindent 6 }}
|
||||||
|
{{- else if eq .Values.cloudProviderSpec.type "oci" }}
|
||||||
|
storageControllerSpec:
|
||||||
|
enabled: {{ .Values.storageControllerSpecOCI.enabled }}
|
||||||
|
defaultCustomerPVReclaimPolicy: {{ .Values.storageControllerSpecOCI.defaultCustomerPVReclaimPolicy | default "Retain" }}
|
||||||
|
defaultStorageTypes:
|
||||||
|
{{- .Values.storageControllerSpecOCI.defaultStorageTypes | toYaml | nindent 6 }}
|
||||||
|
{{- else if eq .Values.cloudProviderSpec.type "azure" }}
|
||||||
|
storageControllerSpec:
|
||||||
|
enabled: {{ .Values.storageControllerSpecAzure.enabled }}
|
||||||
|
defaultCustomerPVReclaimPolicy: {{ .Values.storageControllerSpecAzure.defaultCustomerPVReclaimPolicy | default "Retain" }}
|
||||||
|
defaultStorageTypes:
|
||||||
|
{{- .Values.storageControllerSpecAzure.defaultStorageTypes | toYaml | nindent 6 }}
|
||||||
|
{{- end }}
|
||||||
|
nvStorageControllerSpec:
|
||||||
|
enabled: {{ .Values.nvStorage.enabled }}
|
||||||
|
storageClass: {{ .Values.nvStorage.storageClass }}
|
||||||
|
{{- if .Values.nvStorage.platform }}
|
||||||
|
platform:
|
||||||
|
provider: {{ .Values.nvStorage.platform.provider }}
|
||||||
|
clusterName: {{ .Values.nvStorage.platform.clusterName }}
|
||||||
|
config:
|
||||||
|
{{- .Values.nvStorage.platform.config | toYaml | nindent 10 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- if .Values.nvStorage.csi }}
|
||||||
|
csi:
|
||||||
|
driver: {{ .Values.nvStorage.csi.driver }}
|
||||||
|
volumeAttributes:
|
||||||
|
{{- .Values.nvStorage.csi.volumeAttributes | toYaml | nindent 10 }}
|
||||||
|
{{- end}}
|
||||||
|
|
||||||
|
{{ end}}
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
{{ if .Values.enabledComponents.dgxcRunaiController }}
|
||||||
|
kind: ClusterRole
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: dgxc-runai-controller
|
||||||
|
rules:
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["services", "configmaps", "secrets"]
|
||||||
|
verbs: ["*"]
|
||||||
|
- apiGroups: ["authentication.gke.io"]
|
||||||
|
resources: ["clientconfigs"]
|
||||||
|
verbs: ["*"]
|
||||||
|
- apiGroups: ["run.ai"]
|
||||||
|
resources: ["runaiconfigs"]
|
||||||
|
verbs: ["*"]
|
||||||
|
- apiGroups: ["runai.dgxc.nvidia.com", "storage.dgxc.nvidia.com", "kyverno.io"]
|
||||||
|
resources: ["*"]
|
||||||
|
verbs: ["*"]
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["persistentvolumes", "persistentvolumeclaims"]
|
||||||
|
verbs: ["*"]
|
||||||
|
- apiGroups: ["rbac.authorization.k8s.io"]
|
||||||
|
resources: ["clusterrolebindings", "clusterroles", "roles", "rolebindings"]
|
||||||
|
verbs: ["*"]
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["namespaces"]
|
||||||
|
verbs: ["get", "list", "watch"]
|
||||||
|
{{ end }}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
{{ if .Values.enabledComponents.dgxcRunaiController }}
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: dgxc-runai-controller
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: dgxc-runai-controller
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
roleRef:
|
||||||
|
kind: ClusterRole
|
||||||
|
name: dgxc-runai-controller
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
{{ end }}
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
{{ if .Values.enabledComponents.dgxcRunaiController }}
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: "dgxc-runai-controller"
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: "dgxc-runai-controller"
|
||||||
|
helm.sh/chart: {{ include "dgxcController.chart" . }}
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
|
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: "dgxc-runai-controller"
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: "dgxc-runai-controller"
|
||||||
|
app.kubernetes.io/name: "dgxc-runai-controller"
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
|
{{- with .Values.extraControllerPodLabels }}
|
||||||
|
{{ toYaml . }}
|
||||||
|
{{- end }}
|
||||||
|
spec:
|
||||||
|
serviceAccountName: dgxc-runai-controller
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: {{ .Values.imagePullSecret | required ".Values.imagePullSecret" }}
|
||||||
|
containers:
|
||||||
|
- name: controller
|
||||||
|
image: {{ .Values.dgxcController.image | required "dgxcController.image is required." | quote }}
|
||||||
|
args:
|
||||||
|
- controller
|
||||||
|
- --zap-log-level=debug
|
||||||
|
imagePullPolicy: Always
|
||||||
|
{{- if eq .Values.cloudProviderSpec.type "oci" }}
|
||||||
|
env:
|
||||||
|
- name: OCI_RESOURCE_PRINCIPAL_VERSION
|
||||||
|
value: {{ .Values.cloudProviderSpec.oci.resourcePrincipalVersion | quote }}
|
||||||
|
- name: OCI_RESOURCE_PRINCIPAL_REGION
|
||||||
|
value: {{ .Values.cloudProviderSpec.oci.region }}
|
||||||
|
{{ end }}
|
||||||
|
ports:
|
||||||
|
- containerPort: 8080
|
||||||
|
{{- if and (hasKey .Values.nvStorage "platform") (eq .Values.nvStorage.platform.provider "runai" )}}
|
||||||
|
volumeMounts:
|
||||||
|
- name: run-ai-secret
|
||||||
|
mountPath: {{ .Values.nvStorage.platform.config.secret_path | required "config.secret_path is required for runai setup" | quote }}
|
||||||
|
readOnly: true
|
||||||
|
{{- end }}
|
||||||
|
tolerations:
|
||||||
|
- effect: NoSchedule
|
||||||
|
key: "{{ .Values.systemTolerationKeyName }}"
|
||||||
|
operator: Equal
|
||||||
|
value: "{{ .Values.systemTolerationValueName }}"
|
||||||
|
- effect: NoExecute
|
||||||
|
key: "{{ .Values.systemTolerationKeyName }}"
|
||||||
|
operator: Equal
|
||||||
|
value: "{{ .Values.systemTolerationValueName }}"
|
||||||
|
{{- if and (hasKey .Values.nvStorage "platform") (eq .Values.nvStorage.platform.provider "runai" )}}
|
||||||
|
volumes:
|
||||||
|
- name: run-ai-secret
|
||||||
|
secret:
|
||||||
|
secretName: run-ai-secret
|
||||||
|
{{- end}}
|
||||||
|
{{ end }}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
{{ if .Values.enabledComponents.dgxcRunaiController }}
|
||||||
|
{{- if .Values.controllerServiceAccount.create }}
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: {{ .Values.controllerServiceAccount.name }}
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
{{- if .Values.controllerServiceAccount.annotations }}
|
||||||
|
annotations:
|
||||||
|
{{ toYaml .Values.controllerServiceAccount.annotations }}
|
||||||
|
{{- end }}
|
||||||
|
{{ end }}
|
||||||
|
{{ end }}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
{{ if .Values.enabledComponents.dgxcAdmissionController }}
|
||||||
|
kind: ClusterRole
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: {{ .Values.dgxcController.name }}
|
||||||
|
rules:
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["pods", "secrets", "configmaps", "persistentvolumeclaims", "storageclass"]
|
||||||
|
verbs: ["*"]
|
||||||
|
- apiGroups: ["admissionregistration.k8s.io"]
|
||||||
|
resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"]
|
||||||
|
verbs: ["*"]
|
||||||
|
{{ end }}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
{{ if .Values.enabledComponents.dgxcAdmissionController }}
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: {{ .Values.dgxcController.name }}
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: {{ .Values.serviceAccount.name }}
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
roleRef:
|
||||||
|
kind: ClusterRole
|
||||||
|
name: {{ .Values.dgxcController.name }}
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
{{ end }}
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
{{- if eq .Values.cloudProviderSpec.type "gcp" }}
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: DaemonSet
|
||||||
|
metadata:
|
||||||
|
name: fix-hostname
|
||||||
|
namespace: kube-system
|
||||||
|
labels:
|
||||||
|
nodeGroup: customer-gpu
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
nodeGroup: customer-gpu
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
nodeGroup: customer-gpu
|
||||||
|
spec:
|
||||||
|
nodeSelector:
|
||||||
|
nodeGroup: customer-gpu
|
||||||
|
hostNetwork: true
|
||||||
|
containers:
|
||||||
|
- image: gcr.io/google-containers/pause:3.2
|
||||||
|
name: pause
|
||||||
|
initContainers:
|
||||||
|
- name: fix-hostname
|
||||||
|
image: ubuntu
|
||||||
|
env: [ {name: NODE_NAME, valueFrom: { fieldRef: {fieldPath: spec.nodeName } } } ]
|
||||||
|
command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- |
|
||||||
|
set -xeuo pipefail
|
||||||
|
hostname $(NODE_NAME)
|
||||||
|
cat << EOF > /host/etc/systemd/network/97-temp.network
|
||||||
|
[Match]
|
||||||
|
Name=eth* !eth0
|
||||||
|
|
||||||
|
[Network]
|
||||||
|
DHCP=yes
|
||||||
|
IPv6AcceptRA=yes
|
||||||
|
|
||||||
|
# non-eth0 modification
|
||||||
|
[DHCPv4]
|
||||||
|
RoutesToDNS=false
|
||||||
|
RoutesToNTP=false
|
||||||
|
|
||||||
|
[DHCP]
|
||||||
|
UseMTU=true
|
||||||
|
UseDomains=true
|
||||||
|
|
||||||
|
# non-eth0 modification
|
||||||
|
UseHostname=false
|
||||||
|
EOF
|
||||||
|
chroot /host networkctl reload
|
||||||
|
echo "Success"
|
||||||
|
|
||||||
|
|
||||||
|
securityContext: { privileged: true, capabilities: { add: [ NET_ADMIN ] } }
|
||||||
|
volumeMounts:
|
||||||
|
- name: host
|
||||||
|
mountPath: /host
|
||||||
|
tolerations:
|
||||||
|
- effect: NoExecute
|
||||||
|
operator: Exists
|
||||||
|
- effect: NoSchedule
|
||||||
|
operator: Exists
|
||||||
|
volumes:
|
||||||
|
- name: host
|
||||||
|
hostPath:
|
||||||
|
path: /
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,97 @@
|
|||||||
|
{{ if .Values.enabledComponents.dgxcAdmissionController }}
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: {{ include "dgxcController.fullname" . }}
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: {{ include "dgxcController.name" . }}
|
||||||
|
helm.sh/chart: {{ include "dgxcController.chart" . }}
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
|
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||||
|
spec:
|
||||||
|
replicas: {{ .Values.dgxcController.replicas | required ".Values.dgxcController.replicas required" }}
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: {{ include "dgxcController.name" . }}
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
|
strategy:
|
||||||
|
rollingUpdate:
|
||||||
|
maxSurge: 1
|
||||||
|
maxUnavailable: 50%
|
||||||
|
type: RollingUpdate
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: {{ .Values.dgxcController.name }}
|
||||||
|
app.kubernetes.io/name: {{ include "dgxcController.name" . }}
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
|
nvidia.com/dgx-application-deploy: {{ include "dgx-application-deploy" . }}
|
||||||
|
{{- with .Values.extraPodLabels }}
|
||||||
|
{{ toYaml . }}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.podAnnotations }}
|
||||||
|
annotations:
|
||||||
|
{{ toYaml . }}
|
||||||
|
{{- end }}
|
||||||
|
spec:
|
||||||
|
serviceAccountName: {{ .Values.serviceAccount.name }}
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: {{ .Values.imagePullSecret | required ".Values.imagePullSecret" }}
|
||||||
|
initContainers:
|
||||||
|
- name: init
|
||||||
|
image: {{ .Values.dgxcController.image | required ".Values.dgxcController.image is required." | quote }}
|
||||||
|
args: [ "admission-init", "--config", "/etc/{{ .Values.dgxcController.name}}/config.yaml", "--zap-log-level=debug" ]
|
||||||
|
imagePullPolicy: Always
|
||||||
|
volumeMounts:
|
||||||
|
- name: {{ .Values.dgxcController.name}}-config
|
||||||
|
mountPath: /etc/{{ .Values.dgxcController.name}}/
|
||||||
|
readOnly: true
|
||||||
|
- name: {{ .Values.dgxcController.name}}-certs
|
||||||
|
mountPath: {{ .Values.dgxcController.certs.mountPath }}
|
||||||
|
containers:
|
||||||
|
- name: controller
|
||||||
|
image: {{ .Values.dgxcController.image | required "dgxcController.image is required." | quote }}
|
||||||
|
args: ["admission", "--config", "/etc/{{ .Values.dgxcController.name}}/config.yaml", "--zap-log-level=debug"]
|
||||||
|
imagePullPolicy: Always
|
||||||
|
ports: []
|
||||||
|
volumeMounts:
|
||||||
|
- name: {{ .Values.dgxcController.name}}-certs
|
||||||
|
mountPath: {{ .Values.dgxcController.certs.mountPath }}
|
||||||
|
readOnly: true
|
||||||
|
- name: {{ .Values.dgxcController.name}}-config
|
||||||
|
mountPath: /etc/{{ .Values.dgxcController.name}}/
|
||||||
|
readOnly: true
|
||||||
|
{{/* - name: app-share-ns*/}}
|
||||||
|
{{/* command:*/}}
|
||||||
|
{{/* - sleep*/}}
|
||||||
|
{{/* - infinity*/}}
|
||||||
|
{{/* image: nvcr.io/nvidian/nsvis-corestorage/ubuntu:latest*/}}
|
||||||
|
{{/* imagePullPolicy: IfNotPresent*/}}
|
||||||
|
{{/* terminationMessagePath: /dev/termination-log*/}}
|
||||||
|
{{/* terminationMessagePolicy: File*/}}
|
||||||
|
{{/* volumeMounts:*/}}
|
||||||
|
{{/* - name: {{ .Values.dgxcController.name}}-certs*/}}
|
||||||
|
{{/* mountPath: {{ .Values.dgxcController.certs.mountPath }}*/}}
|
||||||
|
{{/* readOnly: true*/}}
|
||||||
|
{{/* - name: {{ .Values.dgxcController.name}}-config*/}}
|
||||||
|
{{/* mountPath: /etc/{{ .Values.dgxcController.name}}-/*/}}
|
||||||
|
{{/* readOnly: true*/}}
|
||||||
|
{{ if .Values.kubernetesAffinity.enabled }}
|
||||||
|
{{- include "kubernetesAffinityConfig" . | nindent 6 }}
|
||||||
|
{{ end }}
|
||||||
|
tolerations:
|
||||||
|
- effect: NoSchedule
|
||||||
|
key: "{{ .Values.systemTolerationKeyName }}"
|
||||||
|
operator: Equal
|
||||||
|
value: "{{ .Values.systemTolerationValueName }}"
|
||||||
|
- effect: NoExecute
|
||||||
|
key: "{{ .Values.systemTolerationKeyName }}"
|
||||||
|
operator: Equal
|
||||||
|
value: "{{ .Values.systemTolerationValueName }}"
|
||||||
|
volumes:
|
||||||
|
- name: {{ .Values.dgxcController.name}}-certs
|
||||||
|
emptyDir: {}
|
||||||
|
- name: {{ .Values.dgxcController.name}}-config
|
||||||
|
configMap:
|
||||||
|
name: {{ .Values.dgxcController.name}}-config
|
||||||
|
{{ end }}
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
{{ if .Values.enabledComponents.dgxcExporter }}
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: dgxc-exporter
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: dgxc-exporter
|
||||||
|
helm.sh/chart: {{ include "dgxcController.chart" . }}
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
|
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: dgxc-exporter
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: dgxc-exporter
|
||||||
|
app.kubernetes.io/name: dgxc-exporter
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
|
spec:
|
||||||
|
serviceAccount: dgxc-exporter
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: {{ .Values.imagePullSecret | required ".Values.imagePullSecret" }}
|
||||||
|
containers:
|
||||||
|
- name: exporter
|
||||||
|
image: {{ .Values.dgxcController.image | required "dgxcController.image is required." | quote }}
|
||||||
|
args:
|
||||||
|
- exporter
|
||||||
|
- --port={{ .Values.exporter.metricsPort }}
|
||||||
|
- --zap-log-level=debug
|
||||||
|
imagePullPolicy: Always
|
||||||
|
{{- if eq .Values.cloudProviderSpec.type "oci" }}
|
||||||
|
env:
|
||||||
|
- name: OCI_RESOURCE_PRINCIPAL_VERSION
|
||||||
|
value: {{ .Values.cloudProviderSpec.oci.resourcePrincipalVersion | quote }}
|
||||||
|
- name: OCI_RESOURCE_PRINCIPAL_REGION
|
||||||
|
value: {{ .Values.cloudProviderSpec.oci.region }}
|
||||||
|
{{ end }}
|
||||||
|
ports:
|
||||||
|
- containerPort: {{ .Values.exporter.metricsPort }}
|
||||||
|
protocol: TCP
|
||||||
|
tolerations:
|
||||||
|
- effect: NoSchedule
|
||||||
|
key: "{{ .Values.systemTolerationKeyName }}"
|
||||||
|
operator: Equal
|
||||||
|
value: "{{ .Values.systemTolerationValueName }}"
|
||||||
|
- effect: NoExecute
|
||||||
|
key: "{{ .Values.systemTolerationKeyName }}"
|
||||||
|
operator: Equal
|
||||||
|
value: "{{ .Values.systemTolerationValueName }}"
|
||||||
|
{{ end }}
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
{{ if .Values.enabledComponents.dgxcExporter }}
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: dgxc-exporter
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
---
|
||||||
|
kind: ClusterRole
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: dgxc-exporter
|
||||||
|
rules:
|
||||||
|
- apiGroups: [kyverno.io]
|
||||||
|
resources: [updaterequests]
|
||||||
|
verbs: [get, list, watch]
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: [configmaps]
|
||||||
|
verbs: [get, list, watch]
|
||||||
|
---
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: dgxc-exporter
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: dgxc-exporter
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
roleRef:
|
||||||
|
kind: ClusterRole
|
||||||
|
name: dgxc-exporter
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
{{ end }}
|
||||||
@@ -0,0 +1,416 @@
|
|||||||
|
{{- if .Values.enableIntegrationsSupport }}
|
||||||
|
# cluster roles required by kyverno to label cluster scoped resources
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: kyverno:dgxc-integrations
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/component: background-controller
|
||||||
|
app.kubernetes.io/instance: kyverno
|
||||||
|
app.kubernetes.io/part-of: kyverno
|
||||||
|
rules:
|
||||||
|
{{- range .Values.clusterScopedKinds }}
|
||||||
|
{{- if gt (len .systemResources) 0 }}
|
||||||
|
- apiGroups: [ {{ .apiGroup | quote }} ]
|
||||||
|
resources:
|
||||||
|
{{- range .resources }}
|
||||||
|
- {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
verbs: ['create', 'delete', 'deletecollection', 'get', 'list', 'patch', 'update', 'watch']
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
- apiGroups: ['rbac.authorization.k8s.io']
|
||||||
|
resources: ['clusterroles']
|
||||||
|
verbs: ['escalate', 'bind']
|
||||||
|
---
|
||||||
|
# label system namespaces with dgxc/system-managed: "true"
|
||||||
|
apiVersion: kyverno.io/v1
|
||||||
|
kind: ClusterPolicy
|
||||||
|
metadata:
|
||||||
|
name: dgxc-integrations-set-tolerations
|
||||||
|
labels:
|
||||||
|
dgxc/system-managed: "true"
|
||||||
|
spec:
|
||||||
|
background: false # must be done as part of request
|
||||||
|
validationFailureAction: {{ .Values.validationFailureAction }}
|
||||||
|
rules:
|
||||||
|
- name: inject-system-pod-tolerations # inject system node tolerations on system pods
|
||||||
|
preconditions: # pod doesn't already have system toleration
|
||||||
|
any:
|
||||||
|
- key:
|
||||||
|
- NoExecute
|
||||||
|
- NoSchedule
|
||||||
|
operator: AnyNotIn
|
||||||
|
value: |
|
||||||
|
{{`{{`}} request.object.spec.tolerations[?(key == '{{ .Values.tolerations.system.key }}' && value == '{{ .Values.tolerations.system.value }}')].effect {{`}}`}}
|
||||||
|
match:
|
||||||
|
all:
|
||||||
|
- resources: # system pods
|
||||||
|
kinds: [ Pod ]
|
||||||
|
namespaceSelector:
|
||||||
|
matchExpressions: [ { key: kubernetes.io/metadata.name , operator: In, values: [ "runai" ] } ]
|
||||||
|
mutate:
|
||||||
|
patchesJson6902: |-
|
||||||
|
- op: add
|
||||||
|
path: "/spec/tolerations/-"
|
||||||
|
value:
|
||||||
|
key: {{ .Values.tolerations.system.key | quote }}
|
||||||
|
operator: Equal
|
||||||
|
value: {{ .Values.tolerations.system.value | quote }}
|
||||||
|
effect: NoSchedule
|
||||||
|
- op: add
|
||||||
|
path: "/spec/tolerations/-"
|
||||||
|
value:
|
||||||
|
key: {{ .Values.tolerations.system.key | quote }}
|
||||||
|
operator: Equal
|
||||||
|
value: {{ .Values.tolerations.system.value | quote }}
|
||||||
|
effect: NoExecute
|
||||||
|
- name: inject-customer-pod-tolerations
|
||||||
|
preconditions: # pod doesn't already have customer toleration
|
||||||
|
any:
|
||||||
|
- key:
|
||||||
|
- NoExecute
|
||||||
|
- NoSchedule
|
||||||
|
operator: AnyNotIn
|
||||||
|
value: |
|
||||||
|
{{`{{`}} request.object.spec.tolerations[?(key == '{{ .Values.tolerations.customer.key }}' && value == '{{ .Values.tolerations.customer.value }}')].effect {{`}}`}}
|
||||||
|
match:
|
||||||
|
all:
|
||||||
|
- resources:
|
||||||
|
kinds: [ Pod ]
|
||||||
|
exclude:
|
||||||
|
all:
|
||||||
|
- resources:
|
||||||
|
namespaceSelector:
|
||||||
|
matchExpressions: [ { key: dgxc/system-managed , operator: In, values: [ "true" ] } ]
|
||||||
|
mutate:
|
||||||
|
patchesJson6902: |-
|
||||||
|
- op: add
|
||||||
|
path: "/spec/tolerations/-"
|
||||||
|
value:
|
||||||
|
key: {{ .Values.tolerations.customer.key | quote }}
|
||||||
|
operator: Equal
|
||||||
|
value: {{ .Values.tolerations.customer.value | quote }}
|
||||||
|
effect: NoSchedule
|
||||||
|
- op: add
|
||||||
|
path: "/spec/tolerations/-"
|
||||||
|
value:
|
||||||
|
key: {{ .Values.tolerations.customer.key | quote }}
|
||||||
|
operator: Equal
|
||||||
|
value: {{ .Values.tolerations.customer.value | quote }}
|
||||||
|
effect: NoExecute
|
||||||
|
- name: inject-customer-ds-tolerations
|
||||||
|
match:
|
||||||
|
all:
|
||||||
|
- resources:
|
||||||
|
kinds: [ DaemonSet ]
|
||||||
|
exclude:
|
||||||
|
all:
|
||||||
|
- resources:
|
||||||
|
namespaceSelector:
|
||||||
|
matchExpressions: [ { key: dgxc/system-managed , operator: In, values: [ "true" ] } ]
|
||||||
|
preconditions:
|
||||||
|
any:
|
||||||
|
- key:
|
||||||
|
- NoExecute
|
||||||
|
- NoSchedule
|
||||||
|
operator: AnyNotIn
|
||||||
|
value: |
|
||||||
|
{{`{{`}} request.object.spec.template.spec.tolerations[?(key=='{{ .Values.tolerations.customer.key }}' && value=='{{ .Values.tolerations.customer.value }}')].effect {{`}}`}}
|
||||||
|
mutate:
|
||||||
|
patchesJson6902: |-
|
||||||
|
- op: add
|
||||||
|
path: "/spec/template/spec/tolerations/-"
|
||||||
|
value:
|
||||||
|
key: {{ .Values.tolerations.customer.key | quote }}
|
||||||
|
operator: Equal
|
||||||
|
value: {{ .Values.tolerations.customer.value | quote }}
|
||||||
|
effect: NoSchedule
|
||||||
|
- op: add
|
||||||
|
path: "/spec/template/spec/tolerations/-"
|
||||||
|
value:
|
||||||
|
key: {{ .Values.tolerations.customer.key | quote }}
|
||||||
|
operator: Equal
|
||||||
|
value: {{ .Values.tolerations.customer.value | quote }}
|
||||||
|
effect: NoExecute
|
||||||
|
- name: deny-customer-pod-tolerations
|
||||||
|
match:
|
||||||
|
all:
|
||||||
|
- resources:
|
||||||
|
kinds: [ Pod ]
|
||||||
|
exclude:
|
||||||
|
all:
|
||||||
|
- resources:
|
||||||
|
namespaceSelector: # resources in system namespaces
|
||||||
|
matchExpressions: [ { key: dgxc/system-managed , operator: In, values: [ "true" ] } ]
|
||||||
|
preconditions:
|
||||||
|
any:
|
||||||
|
- key:
|
||||||
|
- NoExecute
|
||||||
|
- NoSchedule
|
||||||
|
operator: AnyIn
|
||||||
|
value: |
|
||||||
|
{{`{{ request.object.spec.tolerations[?(key==`}}'{{ .Values.tolerations.system.key }}'{{` && value==`}}'{{ .Values.tolerations.system.value }}'{{` || key=='' || key == null)].effect }}`}}
|
||||||
|
- key: "true" # explicitly bound to system node
|
||||||
|
operator: Equals
|
||||||
|
value: |
|
||||||
|
{{`{{`}} spec.nodename && contains(spec.nodename, 'system') {{`}}`}}
|
||||||
|
validate:
|
||||||
|
message: "Customer pods may not have tolerations with an empty key or the key \"{{ .Values.tolerations.system.key }}\" or a nodeName containing 'system' as these allow them to schedule on system pods."
|
||||||
|
deny: {}
|
||||||
|
- name: deny-customer-ds-blanket-tolerations
|
||||||
|
match:
|
||||||
|
all:
|
||||||
|
- resources:
|
||||||
|
kinds: [ DaemonSet ]
|
||||||
|
exclude:
|
||||||
|
all:
|
||||||
|
- resources:
|
||||||
|
namespaceSelector: # resources in system namespaces
|
||||||
|
matchExpressions: [ { key: dgxc/system-managed , operator: In, values: [ "true" ] } ]
|
||||||
|
validate:
|
||||||
|
message: "Customer daemonsets may not have tolerations with an empty key or empty value."
|
||||||
|
pattern:
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
=(tolerations):
|
||||||
|
- key: "?*"
|
||||||
|
value: "?*"
|
||||||
|
---
|
||||||
|
apiVersion: kyverno.io/v1
|
||||||
|
kind: ClusterPolicy
|
||||||
|
metadata:
|
||||||
|
name: dgxc-integrations-deny-changes-to-system-resources
|
||||||
|
labels:
|
||||||
|
dgxc/system-managed: "true"
|
||||||
|
spec:
|
||||||
|
background: false
|
||||||
|
validationFailureAction: {{ .Values.validationFailureAction }}
|
||||||
|
rules:
|
||||||
|
# Deny all writes to resources to cluster scoped system resources not made by system accounts
|
||||||
|
- name: deny-cluster-scoped
|
||||||
|
match:
|
||||||
|
any:
|
||||||
|
# Only below resources are considered dgxc/system-managed
|
||||||
|
{{- range .Values.clusterScopedKinds }}
|
||||||
|
{{- if gt (len .systemResources) 0 }}
|
||||||
|
- resources:
|
||||||
|
kinds: [ {{ .kind | quote }} ]
|
||||||
|
names:
|
||||||
|
{{- range .systemResources }}
|
||||||
|
- {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
exclude: # allow writes by system accounts
|
||||||
|
any:
|
||||||
|
- subjects:
|
||||||
|
# allow writes from service accounts in system namespaces
|
||||||
|
{{- range .Values.clusterScopedKinds }}
|
||||||
|
{{- if eq .kind "Namespace" }}
|
||||||
|
{{- range .systemResources}}
|
||||||
|
- kind: ServiceAccount
|
||||||
|
namespace: "{{ . }}"
|
||||||
|
name: "*"
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
# allow writes from system users and groups
|
||||||
|
{{- range $k, $v := .Values.systemSubjects }}
|
||||||
|
{{- range $v }}
|
||||||
|
- kind: {{ $k | quote }}
|
||||||
|
name: {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
# allow writes from system CSP service accounts
|
||||||
|
{{- range $k, $v := .Values.cspSubjects }}
|
||||||
|
{{- range $v }}
|
||||||
|
- kind: {{ $k | quote }}
|
||||||
|
name: {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
validate:
|
||||||
|
message: "Changes to system resources are not allowed."
|
||||||
|
deny: {}
|
||||||
|
# Deny all writes to resources in system namespaces not made by system accounts
|
||||||
|
- name: deny-namespace-scoped
|
||||||
|
match:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
namespaceSelector: # resources in system namespaces
|
||||||
|
matchExpressions: [ { key: dgxc/system-managed , operator: In, values: [ "true" ] } ]
|
||||||
|
kinds:
|
||||||
|
{{- $kinds := dict }}
|
||||||
|
{{- range .Values.namespacedKinds }}
|
||||||
|
{{- if .kind }}
|
||||||
|
{{- if not (hasKey $kinds .kind) }}
|
||||||
|
- {{ .kind | quote }}
|
||||||
|
{{ $_ := set $kinds .kind .kind }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
- 'ServiceAccount/token'
|
||||||
|
exclude: # allow writes by system accounts
|
||||||
|
any:
|
||||||
|
- subjects:
|
||||||
|
# allow writes from service accounts in system namespaces
|
||||||
|
{{- range .Values.clusterScopedKinds }}
|
||||||
|
{{- if eq .kind "Namespace" }}
|
||||||
|
{{- range .systemResources}}
|
||||||
|
- kind: ServiceAccount
|
||||||
|
namespace: "{{ . }}"
|
||||||
|
name: "*"
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
# allow writes from system users and groups
|
||||||
|
{{- range $k, $v := .Values.systemSubjects }}
|
||||||
|
{{- range $v }}
|
||||||
|
- kind: {{ $k | quote }}
|
||||||
|
name: {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
# allow writes from system CSP service accounts
|
||||||
|
{{- range $k, $v := .Values.cspSubjects }}
|
||||||
|
{{- range $v }}
|
||||||
|
- kind: {{ $k | quote }}
|
||||||
|
name: {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
validate:
|
||||||
|
message: "Changes to system resources are not allowed."
|
||||||
|
deny: {}
|
||||||
|
# Deny customer from applying the dgxc/system-managed label
|
||||||
|
- name: deny-system-label-on-user-namespaces
|
||||||
|
match:
|
||||||
|
any:
|
||||||
|
{{- range .Values.clusterScopedKinds }}
|
||||||
|
{{- if gt (len .systemResources) 0 }}
|
||||||
|
{{- if eq .kind "Namespace" }}
|
||||||
|
- resources:
|
||||||
|
kinds: [ {{ .kind | quote }} ]
|
||||||
|
names:
|
||||||
|
{{- range .systemResources }}
|
||||||
|
- {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
exclude: # allow writes by system accounts
|
||||||
|
any:
|
||||||
|
- subjects:
|
||||||
|
# allow writes from service accounts in system namespaces
|
||||||
|
{{- range .Values.clusterScopedKinds }}
|
||||||
|
{{- if eq .kind "Namespace" }}
|
||||||
|
{{- range .systemResources}}
|
||||||
|
- kind: ServiceAccount
|
||||||
|
namespace: "{{ . }}"
|
||||||
|
name: "*"
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
# allow writes from system users and groups
|
||||||
|
{{- range $k, $v := .Values.systemSubjects }}
|
||||||
|
{{- range $v }}
|
||||||
|
- kind: {{ $k | quote }}
|
||||||
|
name: {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
# allow writes from system CSP service accounts
|
||||||
|
{{- range $k, $v := .Values.cspSubjects }}
|
||||||
|
{{- range $v }}
|
||||||
|
- kind: {{ $k | quote }}
|
||||||
|
name: {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
validate:
|
||||||
|
message: "Setting label dgxc/system-managed not allowed"
|
||||||
|
deny:
|
||||||
|
conditions:
|
||||||
|
- key: "{{`{{ request.object.metadata.labels.\"dgxc/system-managed\" || '' }}`}}"
|
||||||
|
operator: Equals
|
||||||
|
value: "true"
|
||||||
|
- name: deny-system-connect
|
||||||
|
match:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
kinds: [ 'Pod/exec', 'Pod/portforward', 'Pod/attach', 'Pod/ephemeralcontainers', 'Pod/proxy' ]
|
||||||
|
namespaceSelector: # resources in system namespaces
|
||||||
|
matchExpressions: [ { key: dgxc/system-managed , operator: In, values: [ "true" ] } ]
|
||||||
|
exclude: # allow writes by system accounts
|
||||||
|
any:
|
||||||
|
- subjects:
|
||||||
|
# allow writes from service accounts in system namespaces
|
||||||
|
{{- range .Values.clusterScopedKinds }}
|
||||||
|
{{- if eq .kind "Namespace" }}
|
||||||
|
{{- range .systemResources}}
|
||||||
|
- kind: ServiceAccount
|
||||||
|
namespace: "{{ . }}"
|
||||||
|
name: "*"
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
# allow writes from system users and groups
|
||||||
|
{{- range $k, $v := .Values.systemSubjects }}
|
||||||
|
{{- range $v }}
|
||||||
|
- kind: {{ $k | quote }}
|
||||||
|
name: {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
# allow writes from system CSP service accounts
|
||||||
|
{{- range $k, $v := .Values.cspSubjects }}
|
||||||
|
{{- range $v }}
|
||||||
|
- kind: {{ $k | quote }}
|
||||||
|
name: {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
preconditions:
|
||||||
|
all:
|
||||||
|
- key: "{{`{{ request.operation || 'BACKGROUND' }}`}}"
|
||||||
|
operator: Equals
|
||||||
|
value: CONNECT
|
||||||
|
validate:
|
||||||
|
message: "Changes to system resources are not allowed."
|
||||||
|
deny: {}
|
||||||
|
---
|
||||||
|
apiVersion: kyverno.io/v1
|
||||||
|
kind: ClusterPolicy
|
||||||
|
metadata:
|
||||||
|
name: dgxc-integrations-generate-rbac
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-options: Force=true,Replace=true
|
||||||
|
labels:
|
||||||
|
dgxc/system-managed: "true"
|
||||||
|
spec:
|
||||||
|
rules:
|
||||||
|
- name: generate-rbac-for-crd
|
||||||
|
match: { any: [ resources: { kinds: [ "CustomResourceDefinition" ] } ] }
|
||||||
|
exclude:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
names:
|
||||||
|
{{- range .Values.clusterScopedKinds }}
|
||||||
|
{{- if eq .kind "CustomResourceDefinition" }}
|
||||||
|
{{- range .systemResources }}
|
||||||
|
- {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
generate:
|
||||||
|
synchronize: true
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
name: "dgxc-customer-app-admin-{{ `{{ request.object.spec.names.plural }}` }}"
|
||||||
|
data:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
dgxc/customer-app-admin: "true"
|
||||||
|
dgxc/system-managed: "true"
|
||||||
|
dgxc/generated: "true"
|
||||||
|
rules:
|
||||||
|
- apiGroups: [ "{{ `{{request.object.spec.group}}` }}" ]
|
||||||
|
resources: [ "{{ `{{request.object.spec.names.plural}}` }}", "{{ `{{request.object.spec.names.plural}}` }}/finalizers", "{{ `{{request.object.spec.names.plural}}` }}/status", "{{ `{{request.object.spec.names.plural}}` }}/scale" ]
|
||||||
|
verbs: ['create', 'delete', 'deletecollection', 'get', 'list', 'patch', 'update', 'watch']
|
||||||
|
---
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
{{- if .Values.enableIntegrationsSupport }}
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: dgxc-customer-app-admin-base
|
||||||
|
labels:
|
||||||
|
dgxc/customer-app-admin: "true"
|
||||||
|
dgxc/system-managed: "true"
|
||||||
|
rules:
|
||||||
|
{{- range .Values.namespacedKinds }}
|
||||||
|
{{- if gt (len .verbs) 0 }}
|
||||||
|
- apiGroups: [ {{ .apiGroup | quote }} ]
|
||||||
|
resources:
|
||||||
|
{{- range .resources }}
|
||||||
|
- {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
verbs:
|
||||||
|
{{- range .verbs }}
|
||||||
|
- {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- range .Values.clusterScopedKinds }}
|
||||||
|
{{- if gt (len .verbs) 0 }}
|
||||||
|
- apiGroups: [ {{ .apiGroup | quote }} ]
|
||||||
|
resources:
|
||||||
|
{{- range .resources }}
|
||||||
|
- {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
verbs:
|
||||||
|
{{- range .verbs }}
|
||||||
|
- {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
---
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
{{- if .Values.enableLabeler }}
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: kyverno:watch-update-namespaces
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/component: background-controller
|
||||||
|
app.kubernetes.io/instance: kyverno
|
||||||
|
app.kubernetes.io/part-of: kyverno
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- namespaces
|
||||||
|
verbs: ["get", "watch", "list", "patch", "update"]
|
||||||
|
---
|
||||||
|
# label system namespaces with dgxc/system-managed: "true"
|
||||||
|
apiVersion: kyverno.io/v1
|
||||||
|
kind: ClusterPolicy
|
||||||
|
metadata:
|
||||||
|
name: dgxc-label-system-resources
|
||||||
|
labels:
|
||||||
|
dgxc/system-managed: "true"
|
||||||
|
spec:
|
||||||
|
mutateExistingOnPolicyUpdate: true
|
||||||
|
rules:
|
||||||
|
- name: label-cluster-scoped-system-resources
|
||||||
|
mutate:
|
||||||
|
targets:
|
||||||
|
- apiVersion: "{{`{{request.object.apiVersion}}`}}"
|
||||||
|
kind: "{{`{{request.object.kind}}`}}"
|
||||||
|
name: "{{`{{request.object.metadata.name}}`}}"
|
||||||
|
patchStrategicMerge:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
"dgxc/system-managed": "true"
|
||||||
|
match:
|
||||||
|
any:
|
||||||
|
{{- range .Values.clusterScopedKinds }}
|
||||||
|
{{- if gt (len .systemResources) 0 }}
|
||||||
|
{{- if eq .kind "Namespace" }}
|
||||||
|
- resources:
|
||||||
|
kinds: [ {{ .kind | quote }} ]
|
||||||
|
names:
|
||||||
|
{{- range .systemResources }}
|
||||||
|
- {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
---
|
||||||
|
apiVersion: kyverno.io/v1
|
||||||
|
kind: ClusterPolicy
|
||||||
|
metadata:
|
||||||
|
name: dgxc-copy-node-group-to-node-role
|
||||||
|
spec:
|
||||||
|
rules:
|
||||||
|
- name: cleanup-node-roles
|
||||||
|
match:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
kinds:
|
||||||
|
- Node
|
||||||
|
context:
|
||||||
|
- name: nodeGroup
|
||||||
|
variable:
|
||||||
|
jmesPath: request.object.metadata.labels.nodeGroup
|
||||||
|
default: ""
|
||||||
|
mutate:
|
||||||
|
foreach:
|
||||||
|
- list: keys(request.object.metadata.labels)[?starts_with(@, 'node-role.kubernetes.io')]
|
||||||
|
patchesJson6902: |-
|
||||||
|
- op: remove
|
||||||
|
path: "/metadata/labels/{{`{{ replace_all('{{ element }}', '/', '~1') }}`}}"
|
||||||
|
preconditions:
|
||||||
|
all:
|
||||||
|
- key: "{{`{{ element }}`}}"
|
||||||
|
operator: NotEquals
|
||||||
|
value: "{{`{{ nodeGroup }}`}}"
|
||||||
|
- name: copy-node-role
|
||||||
|
match:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
kinds:
|
||||||
|
- Node
|
||||||
|
context:
|
||||||
|
- name: nodeGroup
|
||||||
|
variable:
|
||||||
|
jmesPath: request.object.metadata.labels.nodeGroup
|
||||||
|
default: ""
|
||||||
|
mutate:
|
||||||
|
patchStrategicMerge:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
node-role.kubernetes.io/{{`{{ nodeGroup }}`}}: ""
|
||||||
|
preconditions:
|
||||||
|
all:
|
||||||
|
- key: "{{`{{ nodeGroup }}`}}"
|
||||||
|
operator: NotEquals
|
||||||
|
value: ""
|
||||||
|
---
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
{{- if .Values.enableServiceMonitor }}
|
||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: ServiceMonitor
|
||||||
|
metadata:
|
||||||
|
name: dgxc-admission-controller
|
||||||
|
labels:
|
||||||
|
app: {{ .Values.dgxcController.name }}-metrics
|
||||||
|
spec:
|
||||||
|
endpoints:
|
||||||
|
- interval: 30s
|
||||||
|
path: /metrics
|
||||||
|
port: metrics
|
||||||
|
namespaceSelector:
|
||||||
|
matchNames:
|
||||||
|
- {{ .Release.Namespace }}
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: {{ .Values.dgxcController.name }}-metrics
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,400 @@
|
|||||||
|
{{- if and (.Values.cloudProviderSpec) (eq .Values.cloudProviderSpec.type "aws") ((.Values.cloudProviderSpec.aws).efa)}}
|
||||||
|
# EFA config changes:
|
||||||
|
# add volume /opt/amazon-efa-ofi to pod, env vars and volumeMount to applicable containers in pod
|
||||||
|
# Applies to:
|
||||||
|
# 1) Worker pods resulting from MPIJob or PyTorchJob, and containers only when requested GPUs == gpuAllocatable
|
||||||
|
# 2) Batch Jobs - these act as launchers for MPIJob and PyTorchJob. Generally no GPU requested.
|
||||||
|
apiVersion: kyverno.io/v1
|
||||||
|
kind: ClusterPolicy
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
dgxc/system-managed: "true"
|
||||||
|
name: dgxc-eks-efa-configs
|
||||||
|
spec:
|
||||||
|
background: false
|
||||||
|
rules:
|
||||||
|
# attach hostpath volume to Pytorch and MPI workers that are full GPU
|
||||||
|
- match:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
kinds:
|
||||||
|
- Pod
|
||||||
|
operations:
|
||||||
|
- CREATE
|
||||||
|
exclude:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
annotations:
|
||||||
|
disable-auto-efa: "true"
|
||||||
|
mutate:
|
||||||
|
patchStrategicMerge:
|
||||||
|
spec:
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /opt/amazon-efa-ofi
|
||||||
|
type: Directory
|
||||||
|
name: amazon-efa
|
||||||
|
preconditions:
|
||||||
|
all:
|
||||||
|
- key: "{{`{{ request.object.spec.containers[].resources.requests.\"nvidia.com/gpu\" || '' }}`}}"
|
||||||
|
operator: AnyIn
|
||||||
|
value: [ "{{ .Values.cloudProviderSpec.aws.gpuNode.gpuAllocatable }}" ]
|
||||||
|
- key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}"
|
||||||
|
operator: AnyIn
|
||||||
|
value: [ "PyTorchJob", "MPIJob", "Job", "RunaiJob" ]
|
||||||
|
- key: "{{`{{ request.object.metadata.ownerReferences[].apiVersion.split(@, '/')[0] }}`}}"
|
||||||
|
operator: AnyIn
|
||||||
|
value: [ "batch.volcano.sh", "kubeflow.org", "run.ai" ]
|
||||||
|
name: efa-pod-volumes-workers
|
||||||
|
# PyTorch and MPI launchers can be CPU-only Job, still need the hostpath volume
|
||||||
|
- match:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
kinds:
|
||||||
|
- Pod
|
||||||
|
operations:
|
||||||
|
- CREATE
|
||||||
|
exclude:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
annotations:
|
||||||
|
disable-auto-efa: "true"
|
||||||
|
mutate:
|
||||||
|
patchStrategicMerge:
|
||||||
|
spec:
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /opt/amazon-efa-ofi
|
||||||
|
type: Directory
|
||||||
|
name: amazon-efa
|
||||||
|
preconditions:
|
||||||
|
all:
|
||||||
|
- key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}"
|
||||||
|
operator: AnyIn
|
||||||
|
value: [ "Job" ]
|
||||||
|
name: efa-pod-volumes-launchers
|
||||||
|
# Make container changes for EFA - mount and resources
|
||||||
|
- match:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
kinds:
|
||||||
|
- Pod
|
||||||
|
operations:
|
||||||
|
- CREATE
|
||||||
|
exclude:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
annotations:
|
||||||
|
disable-auto-efa: "true"
|
||||||
|
preconditions:
|
||||||
|
all:
|
||||||
|
- key: "{{`{{ request.object.spec.containers[].resources.requests.\"nvidia.com/gpu\" || '' }}`}}"
|
||||||
|
operator: AnyIn
|
||||||
|
value: [ "{{ .Values.cloudProviderSpec.aws.gpuNode.gpuAllocatable }}" ]
|
||||||
|
- key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}"
|
||||||
|
operator: AnyIn
|
||||||
|
value: [ "PyTorchJob", "MPIJob", "Job", "RunaiJob" ]
|
||||||
|
- key: "{{`{{ request.object.metadata.ownerReferences[].apiVersion.split(@, '/')[0] }}`}}"
|
||||||
|
operator: AnyIn
|
||||||
|
value: [ "batch.volcano.sh", "kubeflow.org", "run.ai" ]
|
||||||
|
mutate:
|
||||||
|
foreach:
|
||||||
|
- list: "request.object.spec.containers"
|
||||||
|
patchStrategicMerge:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- (name): "{{`{{ element.name }}`}}"
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /opt/amazon-efa-ofi
|
||||||
|
name: amazon-efa
|
||||||
|
readOnly: true
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
"vpc.amazonaws.com/efa": "{{ .Values.cloudProviderSpec.aws.gpuNode.efaAllocatable }}"
|
||||||
|
hugepages-2Mi: "{{`{{ element.resources.requests.\"hugepages-2Mi\" || '`}}{{ .Values.cloudProviderSpec.aws.gpuNode.hugePagesAllocatable }}{{`' }}`}}"
|
||||||
|
cpu: "{{`{{ element.resources.requests.cpu || '100m' }}`}}"
|
||||||
|
limits:
|
||||||
|
"vpc.amazonaws.com/efa": "{{ .Values.cloudProviderSpec.aws.gpuNode.efaAllocatable }}"
|
||||||
|
hugepages-2Mi: "{{`{{ element.resources.limits.\"hugepages-2Mi\" || '`}}{{ .Values.cloudProviderSpec.aws.gpuNode.hugePagesAllocatable }}{{`' }}`}}"
|
||||||
|
preconditions:
|
||||||
|
any:
|
||||||
|
- key: "{{`{{ element.resources.requests.\"nvidia.com/gpu\" || '' }}`}}"
|
||||||
|
operator: Equals
|
||||||
|
value: "{{ .Values.cloudProviderSpec.aws.gpuNode.gpuAllocatable }}"
|
||||||
|
name: container-efa-volume-mounts-worker
|
||||||
|
# We treat the launcher differently, doesn't have GPU
|
||||||
|
- match:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
kinds:
|
||||||
|
- Pod
|
||||||
|
operations:
|
||||||
|
- CREATE
|
||||||
|
exclude:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
annotations:
|
||||||
|
disable-auto-efa: "true"
|
||||||
|
preconditions:
|
||||||
|
all:
|
||||||
|
- key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}"
|
||||||
|
operator: AnyIn
|
||||||
|
value: [ "Job" ]
|
||||||
|
mutate:
|
||||||
|
foreach:
|
||||||
|
- list: "request.object.spec.containers"
|
||||||
|
patchStrategicMerge:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- (name): "{{`{{ element.name }}`}}"
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /opt/amazon-efa-ofi
|
||||||
|
name: amazon-efa
|
||||||
|
readOnly: true
|
||||||
|
name: container-efa-volume-mounts-launcher
|
||||||
|
# worker pods need LD_LIBRARY_PATH updated, but we limit this to containers that have gpuAllocatable GPU
|
||||||
|
- match:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
kinds:
|
||||||
|
- Pod
|
||||||
|
operations:
|
||||||
|
- CREATE
|
||||||
|
exclude:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
annotations:
|
||||||
|
disable-auto-efa: "true"
|
||||||
|
preconditions:
|
||||||
|
all:
|
||||||
|
- key: "{{`{{ request.object.spec.containers[].resources.requests.\"nvidia.com/gpu\" || '' }}`}}"
|
||||||
|
operator: AnyIn
|
||||||
|
value: [ "{{ .Values.cloudProviderSpec.aws.gpuNode.gpuAllocatable }}" ]
|
||||||
|
- key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}"
|
||||||
|
operator: AnyIn
|
||||||
|
value: [ "PyTorchJob", "MPIJob", "Job", "RunaiJob" ]
|
||||||
|
- key: "{{`{{ request.object.metadata.ownerReferences[].apiVersion.split(@, '/')[0] }}`}}"
|
||||||
|
operator: AnyIn
|
||||||
|
value: [ "batch.volcano.sh", "kubeflow.org", "run.ai" ]
|
||||||
|
mutate:
|
||||||
|
foreach:
|
||||||
|
- list: "request.object.spec.containers"
|
||||||
|
context:
|
||||||
|
- name: baselibpath
|
||||||
|
variable:
|
||||||
|
jmesPath: element.env[?(name=='LD_LIBRARY_PATH')].value | [0]
|
||||||
|
default: "/usr/lib:/usr/lib64"
|
||||||
|
- name: extralibpathprefix
|
||||||
|
variable:
|
||||||
|
value: "/opt/amazon-efa-ofi/ofi/lib:/opt/amazon-efa-ofi/efa/lib:/opt/amazon-efa-ofi/openmpi/lib:/usr/local/nvidia/lib64"
|
||||||
|
- name: extralibpathsuffix
|
||||||
|
variable:
|
||||||
|
value: "/usr/lib/x86_64-linux-gnu"
|
||||||
|
patchStrategicMerge:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- (name): "{{`{{ element.name }}`}}"
|
||||||
|
env:
|
||||||
|
- name: LD_LIBRARY_PATH
|
||||||
|
value: "{{`{{ extralibpathprefix }}:{{ baselibpath }}:{{ extralibpathsuffix }}`}}"
|
||||||
|
preconditions:
|
||||||
|
all:
|
||||||
|
- key: "{{`{{ element.resources.requests.\"nvidia.com/gpu\" || '' }}`}}"
|
||||||
|
operator: Equals
|
||||||
|
value: "{{ .Values.cloudProviderSpec.aws.gpuNode.gpuAllocatable }}"
|
||||||
|
- key: "{{`{{ element.[env[?(name=='LD_LIBRARY_PATH' && contains(value, 'amazon'))] || `}}`[]`{{` ][] | length(@) }}`}}"
|
||||||
|
operator: Equals
|
||||||
|
value: 0
|
||||||
|
name: container-worker-library-path
|
||||||
|
# launcher pods need LD_LIBRARY_PATH updated
|
||||||
|
- match:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
kinds:
|
||||||
|
- Pod
|
||||||
|
operations:
|
||||||
|
- CREATE
|
||||||
|
exclude:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
annotations:
|
||||||
|
disable-auto-efa: "true"
|
||||||
|
preconditions:
|
||||||
|
all:
|
||||||
|
- key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}"
|
||||||
|
operator: AnyIn
|
||||||
|
value: [ "Job" ]
|
||||||
|
- key: "{{`{{ request.object.metadata.ownerReferences[].apiVersion.split(@, '/')[0] }}`}}"
|
||||||
|
operator: AnyIn
|
||||||
|
value: [ "batch" ]
|
||||||
|
mutate:
|
||||||
|
foreach:
|
||||||
|
- list: "request.object.spec.containers"
|
||||||
|
context:
|
||||||
|
- name: baselibpath
|
||||||
|
variable:
|
||||||
|
jmesPath: element.env[?(name=='LD_LIBRARY_PATH')].value | [0]
|
||||||
|
default: "/usr/lib:/usr/lib64"
|
||||||
|
- name: extralibpathprefix
|
||||||
|
variable:
|
||||||
|
value: "/opt/amazon-efa-ofi/ofi/lib:/opt/amazon-efa-ofi/efa/lib:/opt/amazon-efa-ofi/openmpi/lib:/usr/local/nvidia/lib64"
|
||||||
|
- name: extralibpathsuffix
|
||||||
|
variable:
|
||||||
|
value: "/usr/lib/x86_64-linux-gnu"
|
||||||
|
patchStrategicMerge:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- (name): "{{`{{ element.name }}`}}"
|
||||||
|
env:
|
||||||
|
- name: LD_LIBRARY_PATH
|
||||||
|
value: "{{`{{ extralibpathprefix }}:{{ baselibpath }}:{{ extralibpathsuffix }}`}}"
|
||||||
|
preconditions:
|
||||||
|
all:
|
||||||
|
- key: "{{`{{ element.[env[?(name=='LD_LIBRARY_PATH' && contains(value, 'amazon'))] || `}}`[]`{{` ][] | length(@) }}`}}"
|
||||||
|
operator: Equals
|
||||||
|
value: 0
|
||||||
|
name: container-launcher-library-path
|
||||||
|
{{- if and ((.Values.cloudProviderSpec.aws).efa) (((.Values.envVars).user).efa)}}
|
||||||
|
- match:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
kinds:
|
||||||
|
- Pod
|
||||||
|
operations:
|
||||||
|
- CREATE
|
||||||
|
exclude:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
annotations:
|
||||||
|
disable-auto-efa: "true"
|
||||||
|
preconditions:
|
||||||
|
all:
|
||||||
|
- key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}"
|
||||||
|
operator: AnyIn
|
||||||
|
value: [ "PyTorchJob", "MPIJob", "Job", "RunaiJob" ]
|
||||||
|
- key: "{{`{{ request.object.metadata.ownerReferences[].apiVersion.split(@, '/')[0] }}`}}"
|
||||||
|
operator: AnyIn
|
||||||
|
value: [ "batch.volcano.sh", "kubeflow.org", "run.ai" ]
|
||||||
|
mutate:
|
||||||
|
foreach:
|
||||||
|
{{- range $e := .Values.envVars.user.efa }}
|
||||||
|
- list: "request.object.spec.containers"
|
||||||
|
patchStrategicMerge:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- (name): "{{`{{ element.name }}`}}"
|
||||||
|
env:
|
||||||
|
- name: {{ $e.name }}
|
||||||
|
value: "{{ $e.value }}"
|
||||||
|
preconditions:
|
||||||
|
all:
|
||||||
|
- key: {{ $e.name }}
|
||||||
|
operator: AnyNotIn
|
||||||
|
value: "{{`{{ element.env[].name || '' }}`}}"
|
||||||
|
- key: "{{`{{ element.resources.requests.\"nvidia.com/gpu\" || '' }}`}}"
|
||||||
|
operator: Equals
|
||||||
|
value: "{{ $.Values.cloudProviderSpec.aws.gpuNode.gpuAllocatable }}"
|
||||||
|
{{- end }}
|
||||||
|
name: extra-gpu-env-vars-workers
|
||||||
|
- match:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
kinds:
|
||||||
|
- Pod
|
||||||
|
operations:
|
||||||
|
- CREATE
|
||||||
|
exclude:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
annotations:
|
||||||
|
disable-auto-efa: "true"
|
||||||
|
preconditions:
|
||||||
|
all:
|
||||||
|
- key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}"
|
||||||
|
operator: AnyIn
|
||||||
|
value: [ "Job" ]
|
||||||
|
- key: "{{`{{ request.object.metadata.ownerReferences[].apiVersion.split(@, '/')[0] }}`}}"
|
||||||
|
operator: AnyIn
|
||||||
|
value: [ "batch" ]
|
||||||
|
mutate:
|
||||||
|
foreach:
|
||||||
|
{{- range $e := .Values.envVars.user.efa }}
|
||||||
|
- list: "request.object.spec.containers"
|
||||||
|
patchStrategicMerge:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- (name): "{{`{{ element.name }}`}}"
|
||||||
|
env:
|
||||||
|
- name: {{ $e.name }}
|
||||||
|
value: "{{ $e.value }}"
|
||||||
|
preconditions:
|
||||||
|
all:
|
||||||
|
- key: {{ $e.name }}
|
||||||
|
operator: AnyNotIn
|
||||||
|
value: "{{`{{ element.env[].name || '' }}`}}"
|
||||||
|
{{- end }}
|
||||||
|
name: extra-gpu-env-vars-launchers
|
||||||
|
{{- end}}
|
||||||
|
---
|
||||||
|
{{- end}}
|
||||||
|
|
||||||
|
{{- if and (.Values.cloudProviderSpec) (eq .Values.cloudProviderSpec.type "azure") (eq .Values.cloudProviderSpec.azure.instanceType "Standard_ND96amsr_A100_v4")}}
|
||||||
|
apiVersion: kyverno.io/v1
|
||||||
|
kind: ClusterPolicy
|
||||||
|
metadata:
|
||||||
|
name: nccl-topo-configmap-policy
|
||||||
|
spec:
|
||||||
|
generateExisting: true
|
||||||
|
rules:
|
||||||
|
- name: generate-nccl-topo-configmap
|
||||||
|
match:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
kinds:
|
||||||
|
- Namespace
|
||||||
|
name: "{{ .Values.ncclTopoConfigMapNamespaces }}"
|
||||||
|
generate:
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
name: nccl-topo
|
||||||
|
synchronize: true
|
||||||
|
namespace: "{{`{{request.object.metadata.name}}`}}"
|
||||||
|
data:
|
||||||
|
data:
|
||||||
|
nccl-topo.xml: |
|
||||||
|
<system version="1">
|
||||||
|
<cpu numaid="0" affinity="0000ffff,0000ffff" arch="x86_64" vendor="AuthenticAMD" familyid="23" modelid="49">
|
||||||
|
<pci busid="ffff:ff:01.0" class="0x060400" link_speed="16 GT/s" link_width="16">
|
||||||
|
<pci busid="0003:00:00.0" class="0x030200" link_speed="16 GT/s" link_width="16"/>
|
||||||
|
<pci busid="0103:00:00.0" class="0x020700" link_speed="16 GT/s" link_width="16"/>
|
||||||
|
<pci busid="0004:00:00.0" class="0x030200" link_speed="16 GT/s" link_width="16"/>
|
||||||
|
<pci busid="0104:00:00.0" class="0x020700" link_speed="16 GT/s" link_width="16"/>
|
||||||
|
</pci>
|
||||||
|
</cpu>
|
||||||
|
<cpu numaid="1" affinity="0000ffff,0000ffff" arch="x86_64" vendor="AuthenticAMD" familyid="23" modelid="49">
|
||||||
|
<pci busid="ffff:ff:02.0" class="0x060400" link_speed="16 GT/s" link_width="16">
|
||||||
|
<pci busid="0001:00:00.0" class="0x030200" link_speed="16 GT/s" link_width="16"/>
|
||||||
|
<pci busid="0101:00:00.0" class="0x020700" link_speed="16 GT/s" link_width="16"/>
|
||||||
|
<pci busid="0002:00:00.0" class="0x030200" link_speed="16 GT/s" link_width="16"/>
|
||||||
|
<pci busid="0102:00:00.0" class="0x020700" link_speed="16 GT/s" link_width="16"/>
|
||||||
|
</pci>
|
||||||
|
</cpu>
|
||||||
|
<cpu numaid="2" affinity="0000ffff,0000ffff" arch="x86_64" vendor="AuthenticAMD" familyid="23" modelid="49">
|
||||||
|
<pci busid="ffff:ff:03.0" class="0x060400" link_speed="16 GT/s" link_width="16">
|
||||||
|
<pci busid="000d:00:00.0" class="0x030200" link_speed="16 GT/s" link_width="16"/>
|
||||||
|
<pci busid="0107:00:00.0" class="0x020700" link_speed="16 GT/s" link_width="16"/>
|
||||||
|
<pci busid="000e:00:00.0" class="0x030200" link_speed="16 GT/s" link_width="16"/>
|
||||||
|
<pci busid="0108:00:00.0" class="0x020700" link_speed="16 GT/s" link_width="16"/>
|
||||||
|
</pci>
|
||||||
|
</cpu>
|
||||||
|
<cpu numaid="3" affinity="0000ffff,0000ffff" arch="x86_64" vendor="AuthenticAMD" familyid="23" modelid="49">
|
||||||
|
<pci busid="ffff:ff:04.0" class="0x060400" link_speed="16 GT/s" link_width="16">
|
||||||
|
<pci busid="000b:00:00.0" class="0x030200" link_speed="16 GT/s" link_width="16"/>
|
||||||
|
<pci busid="0105:00:00.0" class="0x020700" link_speed="16 GT/s" link_width="16"/>
|
||||||
|
<pci busid="000c:00:00.0" class="0x030200" link_speed="16 GT/s" link_width="16"/>
|
||||||
|
<pci busid="0106:00:00.0" class="0x020700" link_speed="16 GT/s" link_width="16"/>
|
||||||
|
</pci>
|
||||||
|
</cpu>
|
||||||
|
</system>
|
||||||
|
---
|
||||||
|
{{- end}}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
apiVersion: policy/v1
|
||||||
|
kind: PodDisruptionBudget
|
||||||
|
metadata:
|
||||||
|
name: {{ include "dgxcController.fullname" . }}
|
||||||
|
spec:
|
||||||
|
minAvailable: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: {{ include "dgxcController.name" . }}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
{{- if .Values.enableRunAIRbac }}
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: dgxc-customer-app-admin
|
||||||
|
labels:
|
||||||
|
dgxc/system-managed: "true"
|
||||||
|
aggregationRule:
|
||||||
|
clusterRoleSelectors:
|
||||||
|
- matchLabels:
|
||||||
|
dgxc/customer-app-admin: "true"
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
metadata:
|
||||||
|
name: dgxc-customer-app-admin
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: ClusterRole
|
||||||
|
name: dgxc-customer-app-admin
|
||||||
|
subjects:
|
||||||
|
{{- range $k, $v := .Values.tenantSubjects }}
|
||||||
|
{{- range $v }}
|
||||||
|
- apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: {{ $k | quote }}
|
||||||
|
name: {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
+152
@@ -0,0 +1,152 @@
|
|||||||
|
---
|
||||||
|
apiVersion: apiextensions.k8s.io/v1
|
||||||
|
kind: CustomResourceDefinition
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
controller-gen.kubebuilder.io/version: v0.16.3
|
||||||
|
name: runaidgxclogexporters.runai.dgxc.nvidia.com
|
||||||
|
spec:
|
||||||
|
group: runai.dgxc.nvidia.com
|
||||||
|
names:
|
||||||
|
kind: RunaiDGXCLogExporter
|
||||||
|
listKind: RunaiDGXCLogExporterList
|
||||||
|
plural: runaidgxclogexporters
|
||||||
|
singular: runaidgxclogexporter
|
||||||
|
scope: Namespaced
|
||||||
|
versions:
|
||||||
|
- additionalPrinterColumns:
|
||||||
|
- jsonPath: .spec.destination
|
||||||
|
name: Destination
|
||||||
|
type: string
|
||||||
|
- jsonPath: .status.watermark
|
||||||
|
name: Watermark
|
||||||
|
type: date
|
||||||
|
name: v1beta1
|
||||||
|
schema:
|
||||||
|
openAPIV3Schema:
|
||||||
|
description: RunaiDGXCLogExporter exports audit logs to an object storage
|
||||||
|
bucket.
|
||||||
|
properties:
|
||||||
|
apiVersion:
|
||||||
|
description: |-
|
||||||
|
APIVersion defines the versioned schema of this representation of an object.
|
||||||
|
Servers should convert recognized schemas to the latest internal value, and
|
||||||
|
may reject unrecognized values.
|
||||||
|
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
|
||||||
|
type: string
|
||||||
|
kind:
|
||||||
|
description: |-
|
||||||
|
Kind is a string value representing the REST resource this object represents.
|
||||||
|
Servers may infer this from the endpoint the client submits requests to.
|
||||||
|
Cannot be updated.
|
||||||
|
In CamelCase.
|
||||||
|
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
|
||||||
|
type: string
|
||||||
|
metadata:
|
||||||
|
type: object
|
||||||
|
spec:
|
||||||
|
properties:
|
||||||
|
destination:
|
||||||
|
description: |-
|
||||||
|
Destination is the cloud storage bucket to write logs to.
|
||||||
|
|
||||||
|
If set, logs of each type specified in LogTypes will be exported to the
|
||||||
|
bucket periodically. The destination is created and managed by the user.
|
||||||
|
The Status.WriterPrincipal field will specify the principal that must be
|
||||||
|
granted write access to the destination.
|
||||||
|
type: string
|
||||||
|
logTypes:
|
||||||
|
default:
|
||||||
|
- apiserver-audit
|
||||||
|
description: LogTypes specifies the types of logs to export.
|
||||||
|
items:
|
||||||
|
enum:
|
||||||
|
- apiserver-audit
|
||||||
|
type: string
|
||||||
|
minItems: 1
|
||||||
|
type: array
|
||||||
|
required:
|
||||||
|
- destination
|
||||||
|
- logTypes
|
||||||
|
type: object
|
||||||
|
status:
|
||||||
|
properties:
|
||||||
|
conditions:
|
||||||
|
items:
|
||||||
|
description: Condition contains details for one aspect of the current
|
||||||
|
state of this API Resource.
|
||||||
|
properties:
|
||||||
|
lastTransitionTime:
|
||||||
|
description: |-
|
||||||
|
lastTransitionTime is the last time the condition transitioned from one status to another.
|
||||||
|
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
|
||||||
|
format: date-time
|
||||||
|
type: string
|
||||||
|
message:
|
||||||
|
description: |-
|
||||||
|
message is a human readable message indicating details about the transition.
|
||||||
|
This may be an empty string.
|
||||||
|
maxLength: 32768
|
||||||
|
type: string
|
||||||
|
observedGeneration:
|
||||||
|
description: |-
|
||||||
|
observedGeneration represents the .metadata.generation that the condition was set based upon.
|
||||||
|
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
|
||||||
|
with respect to the current state of the instance.
|
||||||
|
format: int64
|
||||||
|
minimum: 0
|
||||||
|
type: integer
|
||||||
|
reason:
|
||||||
|
description: |-
|
||||||
|
reason contains a programmatic identifier indicating the reason for the condition's last transition.
|
||||||
|
Producers of specific condition types may define expected values and meanings for this field,
|
||||||
|
and whether the values are considered a guaranteed API.
|
||||||
|
The value should be a CamelCase string.
|
||||||
|
This field may not be empty.
|
||||||
|
maxLength: 1024
|
||||||
|
minLength: 1
|
||||||
|
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
|
||||||
|
type: string
|
||||||
|
status:
|
||||||
|
description: status of the condition, one of True, False, Unknown.
|
||||||
|
enum:
|
||||||
|
- "True"
|
||||||
|
- "False"
|
||||||
|
- Unknown
|
||||||
|
type: string
|
||||||
|
type:
|
||||||
|
description: type of condition in CamelCase or in foo.example.com/CamelCase.
|
||||||
|
maxLength: 316
|
||||||
|
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
|
||||||
|
type: string
|
||||||
|
required:
|
||||||
|
- lastTransitionTime
|
||||||
|
- message
|
||||||
|
- reason
|
||||||
|
- status
|
||||||
|
- type
|
||||||
|
type: object
|
||||||
|
type: array
|
||||||
|
x-kubernetes-list-map-keys:
|
||||||
|
- type
|
||||||
|
x-kubernetes-list-type: map
|
||||||
|
watermark:
|
||||||
|
description: Watermark is the latest timestamp up until which logs
|
||||||
|
are exported.
|
||||||
|
format: date-time
|
||||||
|
type: string
|
||||||
|
writerPrincipal:
|
||||||
|
description: |-
|
||||||
|
WriterPrincipal is the cloud provider-specific identifier of the log writer.
|
||||||
|
|
||||||
|
The principal must be granted write access to the destination in order for log
|
||||||
|
writes to succeed.
|
||||||
|
type: string
|
||||||
|
type: object
|
||||||
|
required:
|
||||||
|
- spec
|
||||||
|
type: object
|
||||||
|
served: true
|
||||||
|
storage: true
|
||||||
|
subresources:
|
||||||
|
status: {}
|
||||||
+151
@@ -0,0 +1,151 @@
|
|||||||
|
---
|
||||||
|
apiVersion: apiextensions.k8s.io/v1
|
||||||
|
kind: CustomResourceDefinition
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
controller-gen.kubebuilder.io/version: v0.16.3
|
||||||
|
name: runaidgxcnetworkpolicies.runai.dgxc.nvidia.com
|
||||||
|
spec:
|
||||||
|
group: runai.dgxc.nvidia.com
|
||||||
|
names:
|
||||||
|
kind: RunaiDGXCNetworkPolicy
|
||||||
|
listKind: RunaiDGXCNetworkPolicyList
|
||||||
|
plural: runaidgxcnetworkpolicies
|
||||||
|
singular: runaidgxcnetworkpolicy
|
||||||
|
scope: Namespaced
|
||||||
|
versions:
|
||||||
|
- name: v1beta1
|
||||||
|
schema:
|
||||||
|
openAPIV3Schema:
|
||||||
|
description: RunaiDGXCNetworkPolicy defines tenant cluster scoped network
|
||||||
|
policies.
|
||||||
|
properties:
|
||||||
|
apiVersion:
|
||||||
|
description: |-
|
||||||
|
APIVersion defines the versioned schema of this representation of an object.
|
||||||
|
Servers should convert recognized schemas to the latest internal value, and
|
||||||
|
may reject unrecognized values.
|
||||||
|
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
|
||||||
|
type: string
|
||||||
|
kind:
|
||||||
|
description: |-
|
||||||
|
Kind is a string value representing the REST resource this object represents.
|
||||||
|
Servers may infer this from the endpoint the client submits requests to.
|
||||||
|
Cannot be updated.
|
||||||
|
In CamelCase.
|
||||||
|
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
|
||||||
|
type: string
|
||||||
|
metadata:
|
||||||
|
type: object
|
||||||
|
spec:
|
||||||
|
properties:
|
||||||
|
clusterAccessPolicy:
|
||||||
|
description: ClusterAccessPolicy defines how the cluster may be accessed
|
||||||
|
through the apiserver.
|
||||||
|
properties:
|
||||||
|
permittedCIDR:
|
||||||
|
description: PermittedCIDR defines IP blocks that may access the
|
||||||
|
cluster through the apiserver.
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
type: array
|
||||||
|
required:
|
||||||
|
- permittedCIDR
|
||||||
|
type: object
|
||||||
|
egressPolicy:
|
||||||
|
description: EgressPolicy if defined limits the egress NetworkPolicies
|
||||||
|
that may be defined per-namespace. Applies to all namespaces.
|
||||||
|
properties:
|
||||||
|
permittedCIDR:
|
||||||
|
description: PermittedCIDR defines permitted egress CIDR ranges
|
||||||
|
items:
|
||||||
|
properties:
|
||||||
|
cidr:
|
||||||
|
description: CIDR defines the CIDR range
|
||||||
|
type: string
|
||||||
|
port:
|
||||||
|
description: Port defines the allowed port
|
||||||
|
type: string
|
||||||
|
required:
|
||||||
|
- cidr
|
||||||
|
- port
|
||||||
|
type: object
|
||||||
|
type: array
|
||||||
|
required:
|
||||||
|
- permittedCIDR
|
||||||
|
type: object
|
||||||
|
ingressPolicy:
|
||||||
|
description: IngressPolicy if defined limits the ingress NetworkPolicies
|
||||||
|
that may be defined per-namespace. Applies to all namespaces.
|
||||||
|
properties:
|
||||||
|
permittedCIDR:
|
||||||
|
description: PermittedCIDR defines permitted ingress CIDR ranges
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
type: array
|
||||||
|
required:
|
||||||
|
- permittedCIDR
|
||||||
|
type: object
|
||||||
|
type: object
|
||||||
|
status:
|
||||||
|
properties:
|
||||||
|
clusterAccessPolicy:
|
||||||
|
description: ClusterAccessPolicy defines how the cluster may be accessed
|
||||||
|
through the apiserver.
|
||||||
|
properties:
|
||||||
|
permittedCIDR:
|
||||||
|
description: PermittedCIDR defines IP blocks that may access the
|
||||||
|
cluster through the apiserver.
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
type: array
|
||||||
|
required:
|
||||||
|
- permittedCIDR
|
||||||
|
type: object
|
||||||
|
egressPolicy:
|
||||||
|
description: EgressPolicy is the egressPolicy for the last fully reconciled
|
||||||
|
spec
|
||||||
|
properties:
|
||||||
|
permittedCIDR:
|
||||||
|
description: PermittedCIDR defines permitted egress CIDR ranges
|
||||||
|
items:
|
||||||
|
properties:
|
||||||
|
cidr:
|
||||||
|
description: CIDR defines the CIDR range
|
||||||
|
type: string
|
||||||
|
port:
|
||||||
|
description: Port defines the allowed port
|
||||||
|
type: string
|
||||||
|
required:
|
||||||
|
- cidr
|
||||||
|
- port
|
||||||
|
type: object
|
||||||
|
type: array
|
||||||
|
required:
|
||||||
|
- permittedCIDR
|
||||||
|
type: object
|
||||||
|
ingressPolicy:
|
||||||
|
description: IngressPolicy is the ingressPolicy for the last fully
|
||||||
|
reconciled spec
|
||||||
|
properties:
|
||||||
|
permittedCIDR:
|
||||||
|
description: PermittedCIDR defines permitted ingress CIDR ranges
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
type: array
|
||||||
|
required:
|
||||||
|
- permittedCIDR
|
||||||
|
type: object
|
||||||
|
reconcileErrors:
|
||||||
|
description: ReconcileErrors are errors seen during the last reconciliation.
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
type: array
|
||||||
|
type: object
|
||||||
|
required:
|
||||||
|
- spec
|
||||||
|
type: object
|
||||||
|
served: true
|
||||||
|
storage: true
|
||||||
|
subresources:
|
||||||
|
status: {}
|
||||||
+236
@@ -0,0 +1,236 @@
|
|||||||
|
---
|
||||||
|
apiVersion: apiextensions.k8s.io/v1
|
||||||
|
kind: CustomResourceDefinition
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
controller-gen.kubebuilder.io/version: v0.16.3
|
||||||
|
name: runaidgxcstorages.runai.dgxc.nvidia.com
|
||||||
|
spec:
|
||||||
|
group: runai.dgxc.nvidia.com
|
||||||
|
names:
|
||||||
|
kind: RunaiDGXCStorage
|
||||||
|
listKind: RunaiDGXCStorageList
|
||||||
|
plural: runaidgxcstorages
|
||||||
|
singular: runaidgxcstorage
|
||||||
|
scope: Namespaced
|
||||||
|
versions:
|
||||||
|
- additionalPrinterColumns:
|
||||||
|
- description: Storage classes
|
||||||
|
jsonPath: .status.storageClassesDisplayString
|
||||||
|
name: StorageClasses
|
||||||
|
type: string
|
||||||
|
- description: Percent of storage quota used for volumes
|
||||||
|
jsonPath: .status.storageUsedPct
|
||||||
|
name: QuotaUsedPct
|
||||||
|
type: string
|
||||||
|
- description: Storage quota
|
||||||
|
jsonPath: .status.storageQuota
|
||||||
|
name: Quota
|
||||||
|
type: string
|
||||||
|
- description: Storage quota used by PVs
|
||||||
|
jsonPath: .status.storageRequested
|
||||||
|
name: QuotaUsed
|
||||||
|
type: string
|
||||||
|
- description: Storage utilized by PVs
|
||||||
|
jsonPath: .status.storageConsumed
|
||||||
|
name: StorageConsumed
|
||||||
|
type: string
|
||||||
|
name: v1beta1
|
||||||
|
schema:
|
||||||
|
openAPIV3Schema:
|
||||||
|
description: RunaiDGXCStorage defines storage for DGXC Run:ai accelerated
|
||||||
|
compute environments.
|
||||||
|
properties:
|
||||||
|
apiVersion:
|
||||||
|
description: |-
|
||||||
|
APIVersion defines the versioned schema of this representation of an object.
|
||||||
|
Servers should convert recognized schemas to the latest internal value, and
|
||||||
|
may reject unrecognized values.
|
||||||
|
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
|
||||||
|
type: string
|
||||||
|
kind:
|
||||||
|
description: |-
|
||||||
|
Kind is a string value representing the REST resource this object represents.
|
||||||
|
Servers may infer this from the endpoint the client submits requests to.
|
||||||
|
Cannot be updated.
|
||||||
|
In CamelCase.
|
||||||
|
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
|
||||||
|
type: string
|
||||||
|
metadata:
|
||||||
|
type: object
|
||||||
|
spec:
|
||||||
|
description: RunaiDGXCStorageSpec defines the desired storage state
|
||||||
|
properties:
|
||||||
|
instances:
|
||||||
|
additionalProperties:
|
||||||
|
properties:
|
||||||
|
name:
|
||||||
|
description: Name of the PV
|
||||||
|
type: string
|
||||||
|
persistentVolumeReclaimPolicy:
|
||||||
|
description: PersistentVolumeReclaimPolicy describes a policy
|
||||||
|
for end-of-life maintenance of persistent volumes.
|
||||||
|
type: string
|
||||||
|
type: object
|
||||||
|
type: object
|
||||||
|
quota:
|
||||||
|
type: string
|
||||||
|
quotaID:
|
||||||
|
type: string
|
||||||
|
x-kubernetes-validations:
|
||||||
|
- message: quotaID is immutable
|
||||||
|
rule: self == oldSelf
|
||||||
|
tiers:
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
type: array
|
||||||
|
required:
|
||||||
|
- quotaID
|
||||||
|
type: object
|
||||||
|
status:
|
||||||
|
properties:
|
||||||
|
altNames:
|
||||||
|
description: Alternative names of the storage type
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
type: array
|
||||||
|
conditions:
|
||||||
|
description: Conditions contains storage conditions
|
||||||
|
items:
|
||||||
|
description: Condition contains details for one aspect of the current
|
||||||
|
state of this API Resource.
|
||||||
|
properties:
|
||||||
|
lastTransitionTime:
|
||||||
|
description: |-
|
||||||
|
lastTransitionTime is the last time the condition transitioned from one status to another.
|
||||||
|
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
|
||||||
|
format: date-time
|
||||||
|
type: string
|
||||||
|
message:
|
||||||
|
description: |-
|
||||||
|
message is a human readable message indicating details about the transition.
|
||||||
|
This may be an empty string.
|
||||||
|
maxLength: 32768
|
||||||
|
type: string
|
||||||
|
observedGeneration:
|
||||||
|
description: |-
|
||||||
|
observedGeneration represents the .metadata.generation that the condition was set based upon.
|
||||||
|
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
|
||||||
|
with respect to the current state of the instance.
|
||||||
|
format: int64
|
||||||
|
minimum: 0
|
||||||
|
type: integer
|
||||||
|
reason:
|
||||||
|
description: |-
|
||||||
|
reason contains a programmatic identifier indicating the reason for the condition's last transition.
|
||||||
|
Producers of specific condition types may define expected values and meanings for this field,
|
||||||
|
and whether the values are considered a guaranteed API.
|
||||||
|
The value should be a CamelCase string.
|
||||||
|
This field may not be empty.
|
||||||
|
maxLength: 1024
|
||||||
|
minLength: 1
|
||||||
|
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
|
||||||
|
type: string
|
||||||
|
status:
|
||||||
|
description: status of the condition, one of True, False, Unknown.
|
||||||
|
enum:
|
||||||
|
- "True"
|
||||||
|
- "False"
|
||||||
|
- Unknown
|
||||||
|
type: string
|
||||||
|
type:
|
||||||
|
description: type of condition in CamelCase or in foo.example.com/CamelCase.
|
||||||
|
maxLength: 316
|
||||||
|
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
|
||||||
|
type: string
|
||||||
|
required:
|
||||||
|
- lastTransitionTime
|
||||||
|
- message
|
||||||
|
- reason
|
||||||
|
- status
|
||||||
|
- type
|
||||||
|
type: object
|
||||||
|
type: array
|
||||||
|
description:
|
||||||
|
description: Short description for the quota
|
||||||
|
type: string
|
||||||
|
docs:
|
||||||
|
description: Documentation for the storage class
|
||||||
|
type: string
|
||||||
|
instances:
|
||||||
|
additionalProperties:
|
||||||
|
properties:
|
||||||
|
capacity:
|
||||||
|
description: Capacity of the PV
|
||||||
|
type: string
|
||||||
|
creationTime:
|
||||||
|
description: When the volume was created
|
||||||
|
format: date-time
|
||||||
|
type: string
|
||||||
|
fileShares:
|
||||||
|
additionalProperties:
|
||||||
|
properties:
|
||||||
|
name:
|
||||||
|
description: Name of the fileshare
|
||||||
|
type: string
|
||||||
|
orphaned:
|
||||||
|
description: Orphaned is true if the volume is not mounted
|
||||||
|
in any cluster
|
||||||
|
type: boolean
|
||||||
|
values:
|
||||||
|
additionalProperties:
|
||||||
|
type: string
|
||||||
|
description: Usage metrics
|
||||||
|
type: object
|
||||||
|
required:
|
||||||
|
- name
|
||||||
|
type: object
|
||||||
|
description: Fileshare used for the volume
|
||||||
|
type: object
|
||||||
|
location:
|
||||||
|
description: Region where the resource is provisioned
|
||||||
|
type: string
|
||||||
|
name:
|
||||||
|
description: Name of the PV
|
||||||
|
type: string
|
||||||
|
persistentVolumeReclaimPolicy:
|
||||||
|
description: PersistentVolumeReclaimPolicy describes a policy
|
||||||
|
for end-of-life maintenance of persistent volumes.
|
||||||
|
type: string
|
||||||
|
state:
|
||||||
|
description: Current state of the resource
|
||||||
|
type: string
|
||||||
|
tier:
|
||||||
|
description: Storage tier
|
||||||
|
type: string
|
||||||
|
type: object
|
||||||
|
description: PVs consuming the quota
|
||||||
|
type: object
|
||||||
|
storageClasses:
|
||||||
|
description: Storage classes that are bound to this quota
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
type: array
|
||||||
|
storageClassesDisplayString:
|
||||||
|
description: StorageClassesDisplayString has a display string for
|
||||||
|
printing the storage classes from kubectl
|
||||||
|
type: string
|
||||||
|
storageConsumed:
|
||||||
|
description: Amount of storage consumed from the requested
|
||||||
|
type: string
|
||||||
|
storageQuota:
|
||||||
|
description: Amount of quota allocated
|
||||||
|
type: string
|
||||||
|
storageRequested:
|
||||||
|
description: Amount of storage requested from the quota
|
||||||
|
type: string
|
||||||
|
storageUsedPct:
|
||||||
|
type: string
|
||||||
|
type: object
|
||||||
|
required:
|
||||||
|
- spec
|
||||||
|
type: object
|
||||||
|
served: true
|
||||||
|
storage: true
|
||||||
|
subresources:
|
||||||
|
status: {}
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
{{ if .Values.enabledComponents.dgxcAdmissionController }}
|
||||||
|
{{- if .Values.serviceAccount.create }}
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: {{ .Values.serviceAccount.name }}
|
||||||
|
{{- if .Values.serviceAccount.annotations }}
|
||||||
|
annotations:
|
||||||
|
{{ toYaml .Values.serviceAccount.annotations }}
|
||||||
|
{{- end }}
|
||||||
|
{{ end }}
|
||||||
|
{{ end }}
|
||||||
@@ -0,0 +1,308 @@
|
|||||||
|
{{- if .Values.enableNetworkPolicies }}
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: kyverno:watch-update-services
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/component: background-controller
|
||||||
|
app.kubernetes.io/instance: kyverno
|
||||||
|
app.kubernetes.io/part-of: kyverno
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- services
|
||||||
|
verbs: ["get", "watch", "list", "patch", "update"]
|
||||||
|
---
|
||||||
|
apiVersion: kyverno.io/v1
|
||||||
|
kind: ClusterPolicy
|
||||||
|
metadata:
|
||||||
|
name: add-runai-cluster-redis-networkpolicy
|
||||||
|
spec:
|
||||||
|
generateExisting: true
|
||||||
|
rules:
|
||||||
|
- generate:
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
data:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
dgxc/generated: "true"
|
||||||
|
spec:
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchExpressions:
|
||||||
|
- key: runai/queue
|
||||||
|
operator: Exists
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
run.ai/control-plane-managed: "true"
|
||||||
|
ports:
|
||||||
|
- port: 6379
|
||||||
|
protocol: TCP
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app: cluster-redis
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
kind: NetworkPolicy
|
||||||
|
name: allow-runai-cluster-redis-ingress
|
||||||
|
namespace: "{{`{{request.object.metadata.name}}`}}"
|
||||||
|
synchronize: true
|
||||||
|
match:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
kinds:
|
||||||
|
- Namespace
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: "runai"
|
||||||
|
name: allow-runai-cluster-redis-ingress
|
||||||
|
---
|
||||||
|
apiVersion: kyverno.io/v1
|
||||||
|
kind: ClusterPolicy
|
||||||
|
metadata:
|
||||||
|
name: add-networkpolicy-allow-system-ingress
|
||||||
|
spec:
|
||||||
|
generateExisting: true
|
||||||
|
rules:
|
||||||
|
{{- if and (.Values.cloudProviderSpec) (eq .Values.cloudProviderSpec.type "oci") }}
|
||||||
|
- name: allow-system-ingress-via-ciliumnetworkpolicy
|
||||||
|
match:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
kinds:
|
||||||
|
- Namespace
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
dgxc/system-managed: "true"
|
||||||
|
generate:
|
||||||
|
apiVersion: cilium.io/v2
|
||||||
|
kind: CiliumNetworkPolicy
|
||||||
|
name: allow-system-ingress
|
||||||
|
synchronize: true
|
||||||
|
namespace: "{{`{{request.object.metadata.name}}`}}"
|
||||||
|
data:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
dgxc/generated: "true"
|
||||||
|
spec:
|
||||||
|
endpointSelector: {}
|
||||||
|
ingress:
|
||||||
|
- fromEntities:
|
||||||
|
- remote-node
|
||||||
|
{{- end }}
|
||||||
|
- name: allow-system-ingress-via-clusterpolicy
|
||||||
|
match:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
kinds:
|
||||||
|
- Namespace
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
dgxc/system-managed: "true"
|
||||||
|
generate:
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
name: allow-system-ingress
|
||||||
|
synchronize: true
|
||||||
|
namespace: "{{`{{request.object.metadata.name}}`}}"
|
||||||
|
data:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
dgxc/generated: "true"
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
{{- range .Values.network.systemCIDRs }}
|
||||||
|
- ipBlock:
|
||||||
|
cidr: {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
- namespaceSelector:
|
||||||
|
matchExpressions:
|
||||||
|
- key: dgxc/system-managed
|
||||||
|
operator: In
|
||||||
|
values: ["true"]
|
||||||
|
- name: default-deny-ingress-via-clusterpolicy
|
||||||
|
match:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
kinds:
|
||||||
|
- Namespace
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
dgxc/system-managed: "true"
|
||||||
|
generate:
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
name: default-deny
|
||||||
|
namespace: "{{`{{request.object.metadata.name}}`}}"
|
||||||
|
synchronize: true
|
||||||
|
data:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
dgxc/generated: "true"
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
---
|
||||||
|
apiVersion: kyverno.io/v1
|
||||||
|
kind: ClusterPolicy
|
||||||
|
metadata:
|
||||||
|
name: manage-loadbalancer-networkpolicy
|
||||||
|
spec:
|
||||||
|
generateExisting: true
|
||||||
|
rules:
|
||||||
|
- name: allow-internet-access-for-loadbalancer
|
||||||
|
match:
|
||||||
|
resources:
|
||||||
|
kinds:
|
||||||
|
- Service
|
||||||
|
preconditions:
|
||||||
|
any:
|
||||||
|
- key: "{{`{{request.object.spec.type}}`}}"
|
||||||
|
operator: Equals
|
||||||
|
value: "LoadBalancer"
|
||||||
|
- key: "{{`{{request.object.spec.type}}`}}"
|
||||||
|
operator: Equals
|
||||||
|
value: "NodePort"
|
||||||
|
generate:
|
||||||
|
kind: NetworkPolicy
|
||||||
|
name: allow-internet-access
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
namespace: "{{`{{request.object.metadata.namespace}}`}}"
|
||||||
|
synchronize: true
|
||||||
|
data:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
dgxc/generated: "true"
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels: "{{`{{request.object.spec.selector}}`}}"
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
ingress:
|
||||||
|
- {}
|
||||||
|
---
|
||||||
|
apiVersion: kyverno.io/v1
|
||||||
|
kind: ClusterPolicy
|
||||||
|
metadata:
|
||||||
|
name: deny-metadata-access-policy
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-options: Force=true,Replace=true
|
||||||
|
spec:
|
||||||
|
generateExisting: true
|
||||||
|
rules:
|
||||||
|
- name: deny-metadata-access-rule
|
||||||
|
match:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
kinds:
|
||||||
|
- Namespace
|
||||||
|
exclude:
|
||||||
|
any:
|
||||||
|
{{- range .Values.clusterScopedKinds }}
|
||||||
|
{{- if gt (len .systemResources) 0 }}
|
||||||
|
{{- if eq .kind "Namespace" }}
|
||||||
|
- resources:
|
||||||
|
kinds: [ {{ .kind | quote }} ]
|
||||||
|
names:
|
||||||
|
{{- range .systemResources }}
|
||||||
|
- {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
generate:
|
||||||
|
kind: NetworkPolicy
|
||||||
|
name: deny-metadata-access
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
namespace: "{{`{{request.object.metadata.name}}`}}"
|
||||||
|
synchronize: true
|
||||||
|
data:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
dgxc/generated: "true"
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Egress
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 0.0.0.0/0
|
||||||
|
{{- if not (and
|
||||||
|
(hasKey .Values "cloudProviderSpec")
|
||||||
|
(hasKey .Values.cloudProviderSpec "type")
|
||||||
|
(eq .Values.cloudProviderSpec.type "gcp")
|
||||||
|
(hasKey .Values.cloudProviderSpec "gcp")
|
||||||
|
(hasKey .Values.cloudProviderSpec.gcp "workloadIdentityEnabled")
|
||||||
|
(.Values.cloudProviderSpec.gcp.workloadIdentityEnabled)
|
||||||
|
) }}
|
||||||
|
except:
|
||||||
|
- 169.254.0.0/16
|
||||||
|
{{- end }}
|
||||||
|
{{- if and (.Values.cloudProviderSpec) (or (eq .Values.cloudProviderSpec.type "gcp") (eq .Values.cloudProviderSpec.type "oci") (eq .Values.cloudProviderSpec.type "azure")) }}
|
||||||
|
- namespaceSelector: {}
|
||||||
|
- podSelector: {}
|
||||||
|
{{- end }}
|
||||||
|
{{- if and (.Values.cloudProviderSpec) (eq .Values.cloudProviderSpec.type "aws")}}
|
||||||
|
---
|
||||||
|
apiVersion: kyverno.io/v1
|
||||||
|
kind: ClusterPolicy
|
||||||
|
metadata:
|
||||||
|
name: deny-node-traffic
|
||||||
|
spec:
|
||||||
|
generateExisting: true
|
||||||
|
rules:
|
||||||
|
- name: deny-node-traffic
|
||||||
|
match:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
kinds:
|
||||||
|
- Namespace
|
||||||
|
exclude:
|
||||||
|
any:
|
||||||
|
{{- range .Values.clusterScopedKinds }}
|
||||||
|
{{- if gt (len .systemResources) 0 }}
|
||||||
|
{{- if eq .kind "Namespace" }}
|
||||||
|
- resources:
|
||||||
|
kinds: [ {{ .kind | quote }} ]
|
||||||
|
names:
|
||||||
|
{{- range .systemResources }}
|
||||||
|
- {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
context:
|
||||||
|
- name: nodesIPs
|
||||||
|
apiCall:
|
||||||
|
urlPath: "/api/v1/nodes"
|
||||||
|
jmesPath: |
|
||||||
|
map(&join('', [@, '/32']), items[].status[].addresses[?type=='InternalIP'].address | [])
|
||||||
|
generate:
|
||||||
|
synchronize: true
|
||||||
|
kind: NetworkPolicy
|
||||||
|
name: deny-node-access
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
namespace: "{{`{{request.object.metadata.name}}`}}"
|
||||||
|
data:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
dgxc/generated: "true"
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Egress
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 0.0.0.0/0
|
||||||
|
except: "{{`{{ nodesIPs }}`}}"
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
{{- if .Values.enableNetworkPolicies }}
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: allow-dns
|
||||||
|
namespace: kube-system
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: "kube-dns"
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
ingress:
|
||||||
|
- from: []
|
||||||
|
{{- if and (.Values.cloudProviderSpec) (eq .Values.cloudProviderSpec.type "oci") }}
|
||||||
|
---
|
||||||
|
apiVersion: cilium.io/v2
|
||||||
|
kind: CiliumClusterwideNetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: allow-egress-to-all
|
||||||
|
spec:
|
||||||
|
endpointSelector: {}
|
||||||
|
egress:
|
||||||
|
- toEntities:
|
||||||
|
- all
|
||||||
|
{{- end }}
|
||||||
|
---
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: {{ .Values.serviceName }}
|
||||||
|
labels:
|
||||||
|
app: {{ .Values.dgxcController.name }}
|
||||||
|
spec:
|
||||||
|
ports:
|
||||||
|
- port: 443
|
||||||
|
targetPort: 9443
|
||||||
|
selector:
|
||||||
|
app: {{ .Values.dgxcController.name }}
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: dgx-controller-metrics
|
||||||
|
labels:
|
||||||
|
app: {{ .Values.dgxcController.name }}-metrics
|
||||||
|
spec:
|
||||||
|
ports:
|
||||||
|
- name: metrics
|
||||||
|
port: 80
|
||||||
|
protocol: TCP
|
||||||
|
targetPort: 8080
|
||||||
|
selector:
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
@@ -0,0 +1,222 @@
|
|||||||
|
{{- if .Values.enableDGXCRbac }}
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: sre-cluster-scoped-access
|
||||||
|
rules:
|
||||||
|
- apiGroups: [ "" ]
|
||||||
|
resources:
|
||||||
|
- nodes
|
||||||
|
- nodes/proxy
|
||||||
|
- nodes/status
|
||||||
|
- componentstatuses
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "apiregistration.k8s.io" ]
|
||||||
|
resources:
|
||||||
|
- apiservices
|
||||||
|
- apiservices/status
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "authentication.k8s.io" ]
|
||||||
|
resources:
|
||||||
|
- selfsubjectreviews
|
||||||
|
- tokenreviews
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "authorization.k8s.io" ]
|
||||||
|
resources:
|
||||||
|
- selfsubjectaccessreviews
|
||||||
|
- selfsubjectrulesreviews
|
||||||
|
- subjectaccessreviews
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "certificates.k8s.io" ]
|
||||||
|
resources:
|
||||||
|
- certificatesigningrequests
|
||||||
|
- certificatesigningrequests/approval
|
||||||
|
- certificatesigningrequests/status
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "networking.k8s.io" ]
|
||||||
|
resources:
|
||||||
|
- ingressclasses
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "rbac.authorization.k8s.io" ]
|
||||||
|
resources:
|
||||||
|
- clusterrolebindings
|
||||||
|
- clusterroles
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "storage.k8s.io" ]
|
||||||
|
resources:
|
||||||
|
- csidrivers
|
||||||
|
- csinodes
|
||||||
|
- storageclasses
|
||||||
|
- volumeattachments
|
||||||
|
- volumeattachments/status
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "admissionregistration.k8s.io" ]
|
||||||
|
resources:
|
||||||
|
- mutatingwebhookconfigurations
|
||||||
|
- validatingwebhookconfigurations
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "apiextensions.k8s.io" ]
|
||||||
|
resources:
|
||||||
|
- customresourcedefinitions
|
||||||
|
- customresourcedefinitions/status
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "scheduling.k8s.io" ]
|
||||||
|
resources:
|
||||||
|
- priorityclasses
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "node.k8s.io" ]
|
||||||
|
resources:
|
||||||
|
- runtimeclasses
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "flowcontrol.apiserver.k8s.io" ]
|
||||||
|
resources:
|
||||||
|
- flowschemas
|
||||||
|
- flowschemas/status
|
||||||
|
- prioritylevelconfigurations
|
||||||
|
- prioritylevelconfigurations/status
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "auto.gke.io" ]
|
||||||
|
resources:
|
||||||
|
- allowlistedworkloads
|
||||||
|
- allowlistedv2workloads
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "cilium.io" ]
|
||||||
|
resources:
|
||||||
|
- ciliumexternalworkloads
|
||||||
|
- ciliumexternalworkloads/status
|
||||||
|
- ciliumidentities
|
||||||
|
- ciliumidentities/status
|
||||||
|
- ciliumnodes
|
||||||
|
- ciliumnodes/status
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "engine.run.ai" ]
|
||||||
|
resources:
|
||||||
|
- configs
|
||||||
|
- configs/status
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "external-secrets.io" ]
|
||||||
|
resources:
|
||||||
|
- clusterexternalsecrets
|
||||||
|
- clusterexternalsecrets/status
|
||||||
|
- clustersecretstores
|
||||||
|
- clustersecretstores/status
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "hub.gke.io" ]
|
||||||
|
resources:
|
||||||
|
- memberships
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "kyverno.io" ]
|
||||||
|
resources:
|
||||||
|
- clusterpolicies
|
||||||
|
- clusterpolicies/status
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "monitoring.googleapis.com" ]
|
||||||
|
resources:
|
||||||
|
- clusterpodmonitorings
|
||||||
|
- clusterpodmonitorings/status
|
||||||
|
- clusterrules
|
||||||
|
- clusterrules/status
|
||||||
|
- globalrules
|
||||||
|
- globalrules/status
|
||||||
|
- clusternodemonitorings
|
||||||
|
- clusternodemonitorings/status
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "networking.gke.io" ]
|
||||||
|
resources:
|
||||||
|
- gkenetworkparamsets
|
||||||
|
- gkenetworkparamsets/status
|
||||||
|
- networks
|
||||||
|
- networks/status
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "nfd.k8s-sigs.io" ]
|
||||||
|
resources:
|
||||||
|
- nodefeaturerules
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "reports.kyverno.io" ]
|
||||||
|
resources:
|
||||||
|
- clusterephemeralreports
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "skyhook.nvidia.com" ]
|
||||||
|
resources:
|
||||||
|
- skyhooks
|
||||||
|
- skyhooks/status
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "snapshot.storage.k8s.io" ]
|
||||||
|
resources:
|
||||||
|
- volumesnapshotcontents
|
||||||
|
- volumesnapshotcontents/status
|
||||||
|
- volumesnapshotclasses
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "warden.gke.io" ]
|
||||||
|
resources:
|
||||||
|
- audits
|
||||||
|
- audits/status
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "wgpolicyk8s.io" ]
|
||||||
|
resources:
|
||||||
|
- clusterpolicyreports
|
||||||
|
verbs: [ "*" ]
|
||||||
|
- apiGroups: [ "metrics.k8s.io" ]
|
||||||
|
resources:
|
||||||
|
- nodes
|
||||||
|
verbs: [ "*" ]
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
metadata:
|
||||||
|
name: sre-cluster-scoped-access
|
||||||
|
subjects:
|
||||||
|
- kind: Group
|
||||||
|
name: {{ .Values.sre.group.default }}
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
roleRef:
|
||||||
|
kind: ClusterRole
|
||||||
|
name: sre-cluster-scoped-access
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
---
|
||||||
|
apiVersion: kyverno.io/v1
|
||||||
|
kind: ClusterPolicy
|
||||||
|
metadata:
|
||||||
|
name: dgxc-sre-generate-namespace-rolebindings
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-options: Force=true,Replace=true
|
||||||
|
labels:
|
||||||
|
dgxc/system-managed: "true"
|
||||||
|
spec:
|
||||||
|
generateExisting: true
|
||||||
|
rules:
|
||||||
|
- name: generate-sre-namespace-rolebinding
|
||||||
|
match:
|
||||||
|
any:
|
||||||
|
- resources:
|
||||||
|
kinds:
|
||||||
|
- Namespace
|
||||||
|
names:
|
||||||
|
{{- range .Values.clusterScopedKinds }}
|
||||||
|
{{- if eq .kind "Namespace" }}
|
||||||
|
{{- range .systemResources }}
|
||||||
|
- {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
generate:
|
||||||
|
synchronize: true
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: RoleBinding
|
||||||
|
name: "sre-ns-{{ `{{ request.object.metadata.name }}` }}-binding"
|
||||||
|
namespace: "{{ `{{ request.object.metadata.name }}` }}"
|
||||||
|
data:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
dgxc/customer-app-admin: "true"
|
||||||
|
dgxc/system-managed: "true"
|
||||||
|
dgxc/generated: "true"
|
||||||
|
subjects:
|
||||||
|
- kind: Group
|
||||||
|
name: {{ .Values.sre.group.default }}
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
roleRef:
|
||||||
|
kind: ClusterRole
|
||||||
|
name: cluster-admin
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
{{- if .Values.enableDGXCRbac }}
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
metadata:
|
||||||
|
name: sre-full-access-binding
|
||||||
|
subjects:
|
||||||
|
- kind: Group
|
||||||
|
name: {{ .Values.sre.group.privileged }}
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
roleRef:
|
||||||
|
kind: ClusterRole
|
||||||
|
name: cluster-admin
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
{{- end }}
|
||||||
+3
@@ -0,0 +1,3 @@
|
|||||||
|
{{- if .Values.nvStorage.enabled }}
|
||||||
|
{{ $.Files.Get "files/storage.dgxc.nvidia.com_nvstoragelocations.yaml" }}
|
||||||
|
{{- end }}
|
||||||
+26
@@ -0,0 +1,26 @@
|
|||||||
|
{{- if .Values.enableDGXCAPIBootstrap }}
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: dgxc-customer-network-policy
|
||||||
|
labels:
|
||||||
|
dgxc/customer-app-admin: "true"
|
||||||
|
dgxc/system-managed: "true"
|
||||||
|
rules:
|
||||||
|
- apiGroups: [ "runai.dgxc.nvidia.com" ]
|
||||||
|
resources:
|
||||||
|
- runaidgxcnetworkpolicies
|
||||||
|
verbs: ["get", "list", "watch", "update", "patch"]
|
||||||
|
- apiGroups: [ "runai.dgxc.nvidia.com" ]
|
||||||
|
resources:
|
||||||
|
- runaidgxcstorages
|
||||||
|
verbs: ["get", "list", "watch", "update", "patch"]
|
||||||
|
- apiGroups: [ "runai.dgxc.nvidia.com" ]
|
||||||
|
resources:
|
||||||
|
- runaidgxclogexporters
|
||||||
|
verbs: ["get", "list", "watch", "update", "patch"]
|
||||||
|
- apiGroups: [ "storage.dgxc.nvidia.com" ]
|
||||||
|
resources:
|
||||||
|
- nvstoragelocations
|
||||||
|
verbs: ["*"]
|
||||||
|
{{- end }}
|
||||||
+6
@@ -0,0 +1,6 @@
|
|||||||
|
{{- if .Values.enableDGXCAPIBootstrap }}
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: dgxc-tenant-cluster-policies
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
---
|
||||||
|
kubernetesAffinity:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
systemTolerationKeyName: dedicated
|
||||||
|
systemTolerationValueName: "system-workload"
|
||||||
|
|
||||||
|
tolerations:
|
||||||
|
customer:
|
||||||
|
key: "dedicated"
|
||||||
|
value: "user-workload"
|
||||||
|
system:
|
||||||
|
key: "dedicated"
|
||||||
|
value: "system-workload"
|
||||||
|
|
||||||
|
cloudProviderSpec:
|
||||||
|
type: aws
|
||||||
|
aws:
|
||||||
|
accountName: nv-dgxc-non-prod-runai-dev
|
||||||
|
clusterName: t11111111-dgxc-k8s-runai-aws-us-east-1-dev
|
||||||
|
efa: true
|
||||||
|
region: us-east-1
|
||||||
|
securityGroupARN: arn:aws:ec2:us-east-1:111111111111:security-group/sg-11111111111111111
|
||||||
|
statefulRuleGroupName: dgxc-runai-dev-fw-nacls
|
||||||
|
gpuNode:
|
||||||
|
gpuAllocatable: "8"
|
||||||
|
hugePagesAllocatable: 42242Mi
|
||||||
|
efaAllocatable: "32"
|
||||||
|
|
||||||
|
envVars:
|
||||||
|
user:
|
||||||
|
efa:
|
||||||
|
- name: FI_EFA_USE_DEVICE_RDMA
|
||||||
|
value: "1"
|
||||||
|
- name: OPAL_PREFIX
|
||||||
|
value: "/opt/amazon-efa-ofi/openmpi"
|
||||||
|
- name: NVIDIA_GDRCOPY
|
||||||
|
value: "enabled"
|
||||||
|
|
||||||
|
storageControllerSpecAWS:
|
||||||
|
enabled: false
|
||||||
|
defaultStorageTypes:
|
||||||
|
- apiVersion: runai.dgxc.nvidia.com/v1beta1
|
||||||
|
kind: RunaiDGXCStorage
|
||||||
|
metadata:
|
||||||
|
name: amazon-fsx-for-lustre
|
||||||
|
namespace: dgxc-tenant-cluster-policies
|
||||||
|
spec:
|
||||||
|
quotaID: L-C8640C82
|
||||||
|
status:
|
||||||
|
altNames: [ "fsx" ]
|
||||||
|
storageClasses:
|
||||||
|
- lustre-sc
|
||||||
|
- dgxc-enterprise-file
|
||||||
|
- dgxc-standard-file
|
||||||
|
storageClassesDisplayString: "lustre-sc"
|
||||||
|
docs: https://aws.amazon.com/fsx/when-to-choose-fsx/
|
||||||
|
|
||||||
|
nvStorage:
|
||||||
|
# NVStorageLocation CRD management
|
||||||
|
enabled: false
|
||||||
|
storageClass: "dgxc-standard-object"
|
||||||
|
csi:
|
||||||
|
driver: "s3.csi.aws.com"
|
||||||
|
volumeAttributes:
|
||||||
|
authenticationSource: pod
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,78 @@
|
|||||||
|
kubernetesAffinity:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
systemTolerationKeyName: dedicated
|
||||||
|
systemTolerationValueName: "system-workload"
|
||||||
|
|
||||||
|
tolerations:
|
||||||
|
customer:
|
||||||
|
key: "dedicated"
|
||||||
|
value: "user-workload"
|
||||||
|
system:
|
||||||
|
key: "dedicated"
|
||||||
|
value: "system-workload"
|
||||||
|
|
||||||
|
cloudProviderSpec:
|
||||||
|
type: azure
|
||||||
|
azure:
|
||||||
|
subscriptionId: 99e004ca-5389-45c5-9e7b-402c0a46550c
|
||||||
|
subscriptionName: sub-dgxc-runai-non-prod-test
|
||||||
|
resourceGroupName: rg-runai-scus-dev-test
|
||||||
|
logExporterAnalyticsWorkspaceId: workspace-id
|
||||||
|
ingressNsgName: ingress-name
|
||||||
|
egressNsgName: egress-name
|
||||||
|
standardStorageAccountName: standardtz990000
|
||||||
|
premiumStorageAccountName: premiumtz990000
|
||||||
|
instanceType: Standard_ND96amsr_A100_v4
|
||||||
|
|
||||||
|
storageControllerSpecAzure:
|
||||||
|
enabled: false
|
||||||
|
defaultStorageTypes:
|
||||||
|
- apiVersion: runai.dgxc.nvidia.com/v1beta1
|
||||||
|
kind: RunaiDGXCStorage
|
||||||
|
metadata:
|
||||||
|
name: dgxc-azurefile
|
||||||
|
namespace: dgxc-tenant-cluster-policies
|
||||||
|
spec:
|
||||||
|
quotaID: AzureFileStandard
|
||||||
|
status:
|
||||||
|
altNames: [ "dgxc-azurefile" ]
|
||||||
|
storageClasses:
|
||||||
|
- dgxc-azurefile
|
||||||
|
storageClassesDisplayString: "dgxc-azurefile"
|
||||||
|
docs: https://learn.microsoft.com/en-us/azure/storage/files/storage-files-planning
|
||||||
|
- apiVersion: runai.dgxc.nvidia.com/v1beta1
|
||||||
|
kind: RunaiDGXCStorage
|
||||||
|
metadata:
|
||||||
|
name: dgxc-azurefile-premium-pool-1
|
||||||
|
namespace: dgxc-tenant-cluster-policies
|
||||||
|
spec:
|
||||||
|
quotaID: AzureFilePremiumPool1
|
||||||
|
status:
|
||||||
|
altNames: [ "dgxc-azurefile-premium-pool-1" ]
|
||||||
|
storageClasses:
|
||||||
|
- dgxc-azurefile-premium-pool-1
|
||||||
|
storageClassesDisplayString: "dgxc-azurefile-premium-pool-1"
|
||||||
|
- apiVersion: runai.dgxc.nvidia.com/v1beta1
|
||||||
|
kind: RunaiDGXCStorage
|
||||||
|
metadata:
|
||||||
|
name: dgxc-azurefile-premium-pool-2
|
||||||
|
namespace: dgxc-tenant-cluster-policies
|
||||||
|
spec:
|
||||||
|
quotaID: AzureFilePremiumPool2
|
||||||
|
status:
|
||||||
|
altNames: [ "dgxc-azurefile-premium-pool-2" ]
|
||||||
|
storageClasses:
|
||||||
|
- dgxc-azurefile-premium-pool-2
|
||||||
|
storageClassesDisplayString: "dgxc-azurefile-premium-pool-2"
|
||||||
|
|
||||||
|
podMutatingWebhook:
|
||||||
|
customSettings:
|
||||||
|
- name: modify-pod-rule
|
||||||
|
addRdmaEnabled: true
|
||||||
|
rdmaResourcePerGpu: 1
|
||||||
|
rdmaResourceName: nvidia.com/mlnxnics
|
||||||
|
addNcclVars: true
|
||||||
|
csp: "azure"
|
||||||
|
|
||||||
|
ncclTopoConfigMapNamespaces: runai-*
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,107 @@
|
|||||||
|
---
|
||||||
|
kubernetesAffinity:
|
||||||
|
enabled: true
|
||||||
|
# we will make an effort to not take up valuable GPU resources
|
||||||
|
# so try and run on a CPU node that is not reserved for the Kubernetes Master.
|
||||||
|
|
||||||
|
# matchExpressions:
|
||||||
|
#- key: "node-role.kubernetes.io/fss-workload"
|
||||||
|
# operator: Exists
|
||||||
|
# matchExpressions:
|
||||||
|
#- key: "kubernetes.io/role"
|
||||||
|
# operator: In
|
||||||
|
# values: ["fss-workload"]
|
||||||
|
#
|
||||||
|
# The below configurationa replaces the match expressions.
|
||||||
|
# The default setting is the handle legacy clusters without the above
|
||||||
|
# matching updated label.
|
||||||
|
antiAffinityKey: cloud.google.com/gke-gpu
|
||||||
|
antiAffinityValue: true
|
||||||
|
# The below is used for pod anti-affinity to ensure pods are running on
|
||||||
|
# different nodes
|
||||||
|
topologyKey: kubernetes.io/hostname
|
||||||
|
# Run on Masters should remain false unless there is a very specific need
|
||||||
|
# (ie. if you have been given a minimal dev cluster, with _only_ masters)
|
||||||
|
runOnMasters: false
|
||||||
|
systemTolerationKeyName: components.gke.io/gke-managed-components
|
||||||
|
systemTolerationValueName: "true"
|
||||||
|
|
||||||
|
tolerations:
|
||||||
|
customer:
|
||||||
|
key: "dedicated"
|
||||||
|
value: "user-workload"
|
||||||
|
system:
|
||||||
|
key: "components.gke.io/gke-managed-components"
|
||||||
|
value: "true"
|
||||||
|
|
||||||
|
cloudProviderSpec:
|
||||||
|
type: gcp
|
||||||
|
gcp:
|
||||||
|
projectId: proj-dgxc-runai-np-dev-mega
|
||||||
|
projectNumber: 574718269555
|
||||||
|
securityPolicyRuleName: dgxc-runai-us-east5-ngc-00546849-system-lb-backend-policy
|
||||||
|
region: us-east5
|
||||||
|
# GCP Workload Identity requires access to the metadata server
|
||||||
|
# Setting this flag, disables the kyverno security policy preventing
|
||||||
|
# Pods for accessing the metadata server.
|
||||||
|
workloadIdentityEnabled: false
|
||||||
|
|
||||||
|
storageControllerSpecGCP:
|
||||||
|
enabled: false
|
||||||
|
defaultStorageTypes:
|
||||||
|
- apiVersion: runai.dgxc.nvidia.com/v1beta1
|
||||||
|
kind: RunaiDGXCStorage
|
||||||
|
metadata:
|
||||||
|
name: "filestore-enterprise"
|
||||||
|
namespace: dgxc-tenant-cluster-policies
|
||||||
|
spec:
|
||||||
|
quotaID: "EnterpriseStorageGibPerRegion"
|
||||||
|
status:
|
||||||
|
altNames: [ "enterprise" ]
|
||||||
|
storageClasses:
|
||||||
|
- enterprise-multishare-rwx
|
||||||
|
- enterprise-rwx
|
||||||
|
storageClassesDisplayString: "enterprise-multishare-rwx,enterprise-rwx"
|
||||||
|
docs: https://cloud.google.com/filestore/docs/service-tiers
|
||||||
|
- apiVersion: runai.dgxc.nvidia.com/v1beta1
|
||||||
|
kind: RunaiDGXCStorage
|
||||||
|
metadata:
|
||||||
|
name: "filestore-premium"
|
||||||
|
namespace: dgxc-tenant-cluster-policies
|
||||||
|
spec:
|
||||||
|
quotaID: "PremiumStorageGbPerRegion"
|
||||||
|
status:
|
||||||
|
altNames: [ "premium" ]
|
||||||
|
storageClasses:
|
||||||
|
- premium-rwo
|
||||||
|
- premium-rwx
|
||||||
|
storageClassesDisplayString: "premium-rwo,premium-rwx"
|
||||||
|
docs: https://cloud.google.com/filestore/docs/service-tiers
|
||||||
|
- apiVersion: runai.dgxc.nvidia.com/v1beta1
|
||||||
|
kind: RunaiDGXCStorage
|
||||||
|
metadata:
|
||||||
|
name: "filestore-standard"
|
||||||
|
namespace: dgxc-tenant-cluster-policies
|
||||||
|
spec:
|
||||||
|
quotaID: "StandardStorageGbPerRegion"
|
||||||
|
status:
|
||||||
|
altNames: [ "filestore-standard" ]
|
||||||
|
storageClasses:
|
||||||
|
- standard
|
||||||
|
- standard-rwo
|
||||||
|
- standard-rwx
|
||||||
|
storageClassesDisplayString: "standard,standard-rwo,standard-rwx"
|
||||||
|
docs: https://cloud.google.com/filestore/docs/service-tiers
|
||||||
|
- apiVersion: runai.dgxc.nvidia.com/v1beta1
|
||||||
|
kind: RunaiDGXCStorage
|
||||||
|
metadata:
|
||||||
|
name: "filestore-high-scale"
|
||||||
|
namespace: dgxc-tenant-cluster-policies
|
||||||
|
spec:
|
||||||
|
quotaID: "HighScaleSSDStorageGibPerRegion"
|
||||||
|
status:
|
||||||
|
altNames: [ "zonal" ]
|
||||||
|
storageClasses:
|
||||||
|
- zonal-rwx
|
||||||
|
storageClassesDisplayString: "zonal-rwx"
|
||||||
|
docs: https://cloud.google.com/filestore/docs/service-tiers
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,61 @@
|
|||||||
|
---
|
||||||
|
kubernetesAffinity:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
systemTolerationKeyName: CriticalAddonsOnly
|
||||||
|
systemTolerationValueName: "true"
|
||||||
|
|
||||||
|
tolerations:
|
||||||
|
customer:
|
||||||
|
key: "dedicated"
|
||||||
|
value: "user-workload"
|
||||||
|
system:
|
||||||
|
key: "CriticalAddonsOnly"
|
||||||
|
value: "true"
|
||||||
|
|
||||||
|
cloudProviderSpec:
|
||||||
|
type: oci
|
||||||
|
oci:
|
||||||
|
compartmentID: ocid1.compartment.oc1..aaaaaaaadt4vviaydftj2dknihhot2dhpmb4cvjn6lr57idmp4q2tbnhj6kq
|
||||||
|
clusterID: ocid1.cluster.oc1.iad.aaaaaaaalrxcu6og5prnj4hm2kcp7qvkyu26spioeic34i5ytc5bnhgiz67q
|
||||||
|
fssAvailabilityDomain: VSwX:US-ASHBURN-AD-1
|
||||||
|
publicSubnetFirewallID: ocid1.networkfirewall.oc1.iad.amaaaaaaaqwjupqaedtz64tuorazbidw36a647cyluuqs2gf4kwev4ayf3yq
|
||||||
|
privateSubnetFirewallID: ocid1.networkfirewall.oc1.iad.amaaaaaaaqwjupqadoaciovfm5lpledoh4vbnx3iuytj2gwhzu74hlpfcfha
|
||||||
|
baseFirewallPolicyID: ocid1.fpid.oc1.iad.aaaaaaaalrxcu6og5prnj4hm2kcp7qvkyu26spioeic34i5ytc5bnhgiz67q
|
||||||
|
resourcePrincipalVersion: "2.2"
|
||||||
|
region: us-ashburn-1
|
||||||
|
|
||||||
|
storageControllerSpecOCI:
|
||||||
|
enabled: true
|
||||||
|
defaultStorageTypes:
|
||||||
|
- apiVersion: runai.dgxc.nvidia.com/v1beta1
|
||||||
|
kind: RunaiDGXCStorage
|
||||||
|
metadata:
|
||||||
|
name: fss
|
||||||
|
namespace: dgxc-tenant-cluster-policies
|
||||||
|
spec:
|
||||||
|
quotaID: unlimited
|
||||||
|
status:
|
||||||
|
altNames: [ "fss" ]
|
||||||
|
storageClasses:
|
||||||
|
- fss
|
||||||
|
storageClassesDisplayString: "fss"
|
||||||
|
|
||||||
|
webhooks:
|
||||||
|
registeredWebhooks:
|
||||||
|
- name: dgxc-create-pvc-vwh
|
||||||
|
enabled: false
|
||||||
|
- name: dgxc-create-pvc-mwh
|
||||||
|
enabled: false
|
||||||
|
- name: dgxc-create-pod-mwh
|
||||||
|
enabled: true
|
||||||
|
|
||||||
|
podMutatingWebhook:
|
||||||
|
customSettings:
|
||||||
|
- name: modify-pod-rule
|
||||||
|
addRdmaEnabled: true
|
||||||
|
rdmaResourcePerGpu: 2
|
||||||
|
networkAttachmentDefinition: network-operator/sriov-net
|
||||||
|
rdmaResourceName: nvidia.com/mlnxnics
|
||||||
|
addNcclVars: true
|
||||||
|
csp: "oci"
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,282 @@
|
|||||||
|
---
|
||||||
|
imagePullSecret: "nvidia-ngcuser-pull-secret"
|
||||||
|
|
||||||
|
enabledComponents:
|
||||||
|
dgxcAdmissionController: true
|
||||||
|
dgxcRunaiController: true
|
||||||
|
dgxcExporter: true
|
||||||
|
|
||||||
|
# The storage controller is the thing we're running
|
||||||
|
dgxcController:
|
||||||
|
name: dgxc-admission-controller
|
||||||
|
replicas: 3
|
||||||
|
image: "nvcr.io/nv-ngc-devops/dgxc-admission-controller:latest"
|
||||||
|
# shared between the init container and the controller container
|
||||||
|
certs:
|
||||||
|
mountPath: "/tmp/k8s-webhook-server/serving-certs"
|
||||||
|
certName: tls.crt
|
||||||
|
keyName: tls.key
|
||||||
|
|
||||||
|
nvStorage:
|
||||||
|
# NVStorageLocation CRD management
|
||||||
|
enabled: false
|
||||||
|
storageClass: ""
|
||||||
|
csi: {}
|
||||||
|
|
||||||
|
serviceName: dgx-controller-svc
|
||||||
|
serviceAccount:
|
||||||
|
create: true
|
||||||
|
name: dgxc-admission-controller-sa
|
||||||
|
annotations: {}
|
||||||
|
|
||||||
|
controllerServiceAccount:
|
||||||
|
create: true
|
||||||
|
name: dgxc-runai-controller
|
||||||
|
annotations: {}
|
||||||
|
|
||||||
|
webhooks:
|
||||||
|
organization: dgxc.nvidia.com
|
||||||
|
registeredWebhooks:
|
||||||
|
- name: dgxc-create-pvc-vwh
|
||||||
|
type: PvcValidatingWebhook
|
||||||
|
cmName: dgxc-create-pvc-vwh-config
|
||||||
|
enabled: true
|
||||||
|
- name: dgxc-create-pvc-mwh
|
||||||
|
type: PvcMutatingWebhook
|
||||||
|
cmName: dgxc-create-pvc-mwh-config
|
||||||
|
enabled: true
|
||||||
|
- name: dgxc-create-pod-mwh
|
||||||
|
type: PodMutatingWebhook
|
||||||
|
cmName: dgxc-create-pod-mwh-config
|
||||||
|
enabled: true
|
||||||
|
|
||||||
|
pvcValidatingWebhook:
|
||||||
|
cmName: dgxc-create-pvc-vwh-config
|
||||||
|
type: PvcValidatingWebhook
|
||||||
|
kind: ValidatingWebhookConfiguration
|
||||||
|
fqdn: dgxc-create-pvc-vwh.dgxc.nvidia.com
|
||||||
|
path: "/pvccreate/v1/validation"
|
||||||
|
failurePolicy: Fail
|
||||||
|
timeoutSeconds: 10
|
||||||
|
sideEffects: NoneOnDryRun
|
||||||
|
admissionReviewVersions:
|
||||||
|
- v1
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
apiVersions:
|
||||||
|
- "v1"
|
||||||
|
operations:
|
||||||
|
- CREATE
|
||||||
|
resources:
|
||||||
|
- persistentvolumeclaims
|
||||||
|
scope: "*"
|
||||||
|
namespaceLabelSelector:
|
||||||
|
- key: runai/namespace-version
|
||||||
|
values:
|
||||||
|
- "v1"
|
||||||
|
- "v2"
|
||||||
|
objectLabelSelector:
|
||||||
|
customSettings:
|
||||||
|
- name: filestore-pvc-premium-rule
|
||||||
|
storageClasses:
|
||||||
|
- zonal-rwx
|
||||||
|
minSizeCheckEnabled: false
|
||||||
|
minSizeValue: 10Ti
|
||||||
|
maxSizeCheckEnabled: false
|
||||||
|
maxSizeValue: 100Ti
|
||||||
|
- name: filestore-pvc-standard-rule
|
||||||
|
storageClasses:
|
||||||
|
- standard-rwx
|
||||||
|
minSizeCheckEnabled: false
|
||||||
|
minSizeValue: 1Ti
|
||||||
|
maxSizeCheckEnabled: false
|
||||||
|
maxSizeValue: 10Ti
|
||||||
|
pvcMutatingWebhook:
|
||||||
|
cmName: dgxc-create-pvc-mwh-config
|
||||||
|
type: PvcMutatingWebhook
|
||||||
|
kind: MutatingWebhookConfiguration
|
||||||
|
fqdn: dgxc-create-pvc-mwh.dgxc.nvidia.com
|
||||||
|
path: "/pvccreate/v1/mutation"
|
||||||
|
failurePolicy: Fail
|
||||||
|
timeoutSeconds: 10
|
||||||
|
sideEffects: NoneOnDryRun
|
||||||
|
admissionReviewVersions:
|
||||||
|
- v1
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
apiVersions:
|
||||||
|
- "v1"
|
||||||
|
operations:
|
||||||
|
- CREATE
|
||||||
|
resources:
|
||||||
|
- persistentvolumeclaims
|
||||||
|
scope: "*"
|
||||||
|
namespaceLabelSelector:
|
||||||
|
- key: runai/namespace-version
|
||||||
|
values:
|
||||||
|
- "v1"
|
||||||
|
- "v2"
|
||||||
|
objectLabelSelector:
|
||||||
|
|
||||||
|
podMutatingWebhook:
|
||||||
|
cmName: dgxc-create-pod-mwh-config
|
||||||
|
kind: MutatingWebhookConfiguration
|
||||||
|
type: PodMutatingWebhook
|
||||||
|
fqdn: dgxc-create-pod-mwh.dgxc.mvp.nvidia.com
|
||||||
|
path: "/podcreate/v1/mutation"
|
||||||
|
failurePolicy: Fail
|
||||||
|
timeoutSeconds: 10
|
||||||
|
sideEffects: NoneOnDryRun
|
||||||
|
admissionReviewVersions:
|
||||||
|
- v1
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
apiVersions:
|
||||||
|
- "v1"
|
||||||
|
operations:
|
||||||
|
- CREATE
|
||||||
|
resources:
|
||||||
|
- pods
|
||||||
|
scope: "*"
|
||||||
|
namespaceLabelSelector:
|
||||||
|
objectLabelSelector:
|
||||||
|
customSettings:
|
||||||
|
- name: modify-pod-rule
|
||||||
|
scratchEnabled: true
|
||||||
|
scratchVolName: "scratch"
|
||||||
|
scratchMountpath: "/scratch"
|
||||||
|
scratchVolSize: "200Gi"
|
||||||
|
scratchVolSizePerGpu: true
|
||||||
|
scratchVolEnforceLimit: true
|
||||||
|
addTcpxSidecarEnabled: true
|
||||||
|
addTcpxoSidecarEnabled: false
|
||||||
|
clusterName: "test"
|
||||||
|
addTcpxNCCLVars: true
|
||||||
|
addTcpxoNCCLVars: false
|
||||||
|
addRdmaEnabled: false
|
||||||
|
addRoceOciNcclVars: false
|
||||||
|
addGcpFuseAnnotation: true
|
||||||
|
gcpFuseAnnotationKey: "gke-gcsfuse/volumes"
|
||||||
|
gcpFuseWICredentialConfigMapAnnotationKey: "gke-gcsfuse/workload-identity-credential-configmap" # This is the annotation key for the ConfigMap that contains the GCP Workload Identity credentials.
|
||||||
|
gcpFuseStorageClasses:
|
||||||
|
- dgxc-standard-object
|
||||||
|
prober:
|
||||||
|
intervalInSeconds: 60
|
||||||
|
timeoutInSeconds: 5
|
||||||
|
endpoints:
|
||||||
|
- name: dev-cluster
|
||||||
|
url: https://dgxc-runai-us-east5-ngc-11918940.dgxc.nonprod-nvkong.com
|
||||||
|
|
||||||
|
sso:
|
||||||
|
controlPlaneDomain: run.ai
|
||||||
|
cluster: "TODO"
|
||||||
|
realm: dgxc
|
||||||
|
pollIntervalSeconds: 10
|
||||||
|
pollAttempts: 180 # 30 minutes
|
||||||
|
|
||||||
|
initSettings:
|
||||||
|
enabled: false
|
||||||
|
appID: dgxc
|
||||||
|
tenant: dgxc
|
||||||
|
controlPlane: run.ai
|
||||||
|
supportEmailAddress: support@nvidia.com
|
||||||
|
customerEmailAddress: customer@nvidia.com
|
||||||
|
bootstrapEmailAddress: bootstrap@nvidia.com
|
||||||
|
|
||||||
|
crdRBACName: "{{request.object.spec.names.plural}}"
|
||||||
|
|
||||||
|
validationFailureAction: Enforce
|
||||||
|
|
||||||
|
tenantSubjects:
|
||||||
|
User:
|
||||||
|
- ebalduf@test.nvidia.com
|
||||||
|
- kelhussein@test.nvidia.com
|
||||||
|
- ladithyav@test.nvidia.com
|
||||||
|
- pwittrock@test.nvidia.com
|
||||||
|
|
||||||
|
tolerations:
|
||||||
|
customer:
|
||||||
|
key: "dedicated"
|
||||||
|
value: "user-workload"
|
||||||
|
system:
|
||||||
|
key: "components.gke.io/gke-managed-components"
|
||||||
|
value: "true"
|
||||||
|
|
||||||
|
bootstrapIngressCIDR: [ "216.228.125.128/30", "12.97.180.32/27", "216.228.127.128/30", "202.164.25.0/27", "120.31.162.32/28", "203.18.50.0/27", "103.74.125.160/28", "216.228.112.0/26" ]
|
||||||
|
|
||||||
|
dgxcAdmins:
|
||||||
|
- Run:ai
|
||||||
|
- svctdcsqa1@nvidia.com
|
||||||
|
|
||||||
|
reserved:
|
||||||
|
CPU: 2
|
||||||
|
Memory: 5Gi
|
||||||
|
|
||||||
|
maxCPUJobs: 10
|
||||||
|
|
||||||
|
logExporterControllerSpec:
|
||||||
|
enabled: false
|
||||||
|
exportFrequency: 5m
|
||||||
|
exportBuffer: 15m
|
||||||
|
fetchTimeout: 5m
|
||||||
|
|
||||||
|
networkControllerSpec:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
storageControllerSpecGCP:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
storageControllerSpecAWS:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
storageControllerSpecOCI:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
storageControllerSpecAzure:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
exporter:
|
||||||
|
metricsPort: 8080
|
||||||
|
|
||||||
|
sre:
|
||||||
|
group:
|
||||||
|
default: "non-existing-engineer@nvidia.com"
|
||||||
|
privileged: "non-existing-admin@nvidia.com"
|
||||||
|
|
||||||
|
tenantMetadata:
|
||||||
|
cluster: TODO
|
||||||
|
platform: runai
|
||||||
|
orgName: nvidian
|
||||||
|
tenantName: test-mega
|
||||||
|
tenantType: Internal
|
||||||
|
startDate: '8/02/24'
|
||||||
|
endDate: '01/01/99'
|
||||||
|
CSP: gcp
|
||||||
|
allocatedCapacityGPU: 2
|
||||||
|
allocatedCapacityCPU: 2
|
||||||
|
committedCapacityGPU: 2
|
||||||
|
committedCapacityCPU: 2
|
||||||
|
CSPProject: proj-dgxc-runai-np-test-mega
|
||||||
|
dataCenterRegion: us-central1
|
||||||
|
storageType: GCPFilestore
|
||||||
|
storageQuota: 1
|
||||||
|
|
||||||
|
cspSubjects:
|
||||||
|
User:
|
||||||
|
- "107878228028626601324"
|
||||||
|
|
||||||
|
network:
|
||||||
|
systemCIDRs:
|
||||||
|
- "10.0.3.0/24"
|
||||||
|
- "10.0.4.0/24"
|
||||||
|
|
||||||
|
enableIntegrationsSupport: true
|
||||||
|
enableLabeler: true
|
||||||
|
enableRunAIRbac: true
|
||||||
|
enableNetworkPolicies: true
|
||||||
|
enableDGXCRbac: true
|
||||||
|
enableDGXCAPIBootstrap: true
|
||||||
|
enableServiceMonitor: true
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
# Patterns to ignore when building packages.
|
||||||
|
# This supports shell glob matching, relative path matching, and
|
||||||
|
# negation (prefixed with !). Only one pattern per line.
|
||||||
|
.DS_Store
|
||||||
|
# Common VCS dirs
|
||||||
|
.git/
|
||||||
|
.gitignore
|
||||||
|
.bzr/
|
||||||
|
.bzrignore
|
||||||
|
.hg/
|
||||||
|
.hgignore
|
||||||
|
.svn/
|
||||||
|
# Common backup files
|
||||||
|
*.swp
|
||||||
|
*.bak
|
||||||
|
*.tmp
|
||||||
|
*.orig
|
||||||
|
*~
|
||||||
|
# Various IDEs
|
||||||
|
.project
|
||||||
|
.idea/
|
||||||
|
*.tmproj
|
||||||
|
.vscode/
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
apiVersion: v2
|
||||||
|
appVersion: 1.0.1
|
||||||
|
description: NVIDIA NIM Operator creates/configures/manages NVIDIA NIM microservices
|
||||||
|
in Kubernetes
|
||||||
|
home: https://docs.nvidia.com/nim-operator/latest/index.html
|
||||||
|
icon: https://assets.nvidiagrid.net/ngc/logos/NIMOperator.png
|
||||||
|
keywords:
|
||||||
|
- gpu
|
||||||
|
- nim
|
||||||
|
- nemo
|
||||||
|
- operator
|
||||||
|
- ai
|
||||||
|
name: k8s-nim-operator
|
||||||
|
sources:
|
||||||
|
- https://github.com/NVIDIA/k8s-nim-operator
|
||||||
|
type: application
|
||||||
|
version: 1.0.1
|
||||||
@@ -0,0 +1,500 @@
|
|||||||
|
---
|
||||||
|
apiVersion: apiextensions.k8s.io/v1
|
||||||
|
kind: CustomResourceDefinition
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
controller-gen.kubebuilder.io/version: v0.16.2
|
||||||
|
name: nimcaches.apps.nvidia.com
|
||||||
|
spec:
|
||||||
|
group: apps.nvidia.com
|
||||||
|
names:
|
||||||
|
kind: NIMCache
|
||||||
|
listKind: NIMCacheList
|
||||||
|
plural: nimcaches
|
||||||
|
singular: nimcache
|
||||||
|
scope: Namespaced
|
||||||
|
versions:
|
||||||
|
- additionalPrinterColumns:
|
||||||
|
- jsonPath: .status.state
|
||||||
|
name: Status
|
||||||
|
type: string
|
||||||
|
- jsonPath: .status.pvc
|
||||||
|
name: PVC
|
||||||
|
type: string
|
||||||
|
- format: date-time
|
||||||
|
jsonPath: .metadata.creationTimestamp
|
||||||
|
name: Age
|
||||||
|
type: date
|
||||||
|
name: v1alpha1
|
||||||
|
schema:
|
||||||
|
openAPIV3Schema:
|
||||||
|
description: NIMCache is the Schema for the nimcaches API
|
||||||
|
properties:
|
||||||
|
apiVersion:
|
||||||
|
description: |-
|
||||||
|
APIVersion defines the versioned schema of this representation of an object.
|
||||||
|
Servers should convert recognized schemas to the latest internal value, and
|
||||||
|
may reject unrecognized values.
|
||||||
|
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
|
||||||
|
type: string
|
||||||
|
kind:
|
||||||
|
description: |-
|
||||||
|
Kind is a string value representing the REST resource this object represents.
|
||||||
|
Servers may infer this from the endpoint the client submits requests to.
|
||||||
|
Cannot be updated.
|
||||||
|
In CamelCase.
|
||||||
|
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
|
||||||
|
type: string
|
||||||
|
metadata:
|
||||||
|
type: object
|
||||||
|
spec:
|
||||||
|
description: NIMCacheSpec defines the desired state of NIMCache
|
||||||
|
properties:
|
||||||
|
certConfig:
|
||||||
|
description: |-
|
||||||
|
CertConfig is the name of the ConfigMap containing the custom certificates.
|
||||||
|
for secure communication.
|
||||||
|
properties:
|
||||||
|
mountPath:
|
||||||
|
description: MountPath is the path where the certificates should
|
||||||
|
be mounted in the container.
|
||||||
|
type: string
|
||||||
|
name:
|
||||||
|
description: Name of the ConfigMap containing the certificate
|
||||||
|
data.
|
||||||
|
type: string
|
||||||
|
required:
|
||||||
|
- mountPath
|
||||||
|
- name
|
||||||
|
type: object
|
||||||
|
env:
|
||||||
|
description: Env are the additional custom environment variabes for
|
||||||
|
the caching job
|
||||||
|
items:
|
||||||
|
description: EnvVar represents an environment variable present in
|
||||||
|
a Container.
|
||||||
|
properties:
|
||||||
|
name:
|
||||||
|
description: Name of the environment variable. Must be a C_IDENTIFIER.
|
||||||
|
type: string
|
||||||
|
value:
|
||||||
|
description: |-
|
||||||
|
Variable references $(VAR_NAME) are expanded
|
||||||
|
using the previously defined environment variables in the container and
|
||||||
|
any service environment variables. If a variable cannot be resolved,
|
||||||
|
the reference in the input string will be unchanged. Double $$ are reduced
|
||||||
|
to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e.
|
||||||
|
"$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)".
|
||||||
|
Escaped references will never be expanded, regardless of whether the variable
|
||||||
|
exists or not.
|
||||||
|
Defaults to "".
|
||||||
|
type: string
|
||||||
|
valueFrom:
|
||||||
|
description: Source for the environment variable's value. Cannot
|
||||||
|
be used if value is not empty.
|
||||||
|
properties:
|
||||||
|
configMapKeyRef:
|
||||||
|
description: Selects a key of a ConfigMap.
|
||||||
|
properties:
|
||||||
|
key:
|
||||||
|
description: The key to select.
|
||||||
|
type: string
|
||||||
|
name:
|
||||||
|
default: ""
|
||||||
|
description: |-
|
||||||
|
Name of the referent.
|
||||||
|
This field is effectively required, but due to backwards compatibility is
|
||||||
|
allowed to be empty. Instances of this type with an empty value here are
|
||||||
|
almost certainly wrong.
|
||||||
|
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
|
||||||
|
type: string
|
||||||
|
optional:
|
||||||
|
description: Specify whether the ConfigMap or its key
|
||||||
|
must be defined
|
||||||
|
type: boolean
|
||||||
|
required:
|
||||||
|
- key
|
||||||
|
type: object
|
||||||
|
x-kubernetes-map-type: atomic
|
||||||
|
fieldRef:
|
||||||
|
description: |-
|
||||||
|
Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['<KEY>']`, `metadata.annotations['<KEY>']`,
|
||||||
|
spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.
|
||||||
|
properties:
|
||||||
|
apiVersion:
|
||||||
|
description: Version of the schema the FieldPath is
|
||||||
|
written in terms of, defaults to "v1".
|
||||||
|
type: string
|
||||||
|
fieldPath:
|
||||||
|
description: Path of the field to select in the specified
|
||||||
|
API version.
|
||||||
|
type: string
|
||||||
|
required:
|
||||||
|
- fieldPath
|
||||||
|
type: object
|
||||||
|
x-kubernetes-map-type: atomic
|
||||||
|
resourceFieldRef:
|
||||||
|
description: |-
|
||||||
|
Selects a resource of the container: only resources limits and requests
|
||||||
|
(limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.
|
||||||
|
properties:
|
||||||
|
containerName:
|
||||||
|
description: 'Container name: required for volumes,
|
||||||
|
optional for env vars'
|
||||||
|
type: string
|
||||||
|
divisor:
|
||||||
|
anyOf:
|
||||||
|
- type: integer
|
||||||
|
- type: string
|
||||||
|
description: Specifies the output format of the exposed
|
||||||
|
resources, defaults to "1"
|
||||||
|
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
|
||||||
|
x-kubernetes-int-or-string: true
|
||||||
|
resource:
|
||||||
|
description: 'Required: resource to select'
|
||||||
|
type: string
|
||||||
|
required:
|
||||||
|
- resource
|
||||||
|
type: object
|
||||||
|
x-kubernetes-map-type: atomic
|
||||||
|
secretKeyRef:
|
||||||
|
description: Selects a key of a secret in the pod's namespace
|
||||||
|
properties:
|
||||||
|
key:
|
||||||
|
description: The key of the secret to select from. Must
|
||||||
|
be a valid secret key.
|
||||||
|
type: string
|
||||||
|
name:
|
||||||
|
default: ""
|
||||||
|
description: |-
|
||||||
|
Name of the referent.
|
||||||
|
This field is effectively required, but due to backwards compatibility is
|
||||||
|
allowed to be empty. Instances of this type with an empty value here are
|
||||||
|
almost certainly wrong.
|
||||||
|
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
|
||||||
|
type: string
|
||||||
|
optional:
|
||||||
|
description: Specify whether the Secret or its key must
|
||||||
|
be defined
|
||||||
|
type: boolean
|
||||||
|
required:
|
||||||
|
- key
|
||||||
|
type: object
|
||||||
|
x-kubernetes-map-type: atomic
|
||||||
|
type: object
|
||||||
|
required:
|
||||||
|
- name
|
||||||
|
type: object
|
||||||
|
type: array
|
||||||
|
groupID:
|
||||||
|
description: GroupID is the group ID for the caching job
|
||||||
|
format: int64
|
||||||
|
type: integer
|
||||||
|
nodeSelector:
|
||||||
|
additionalProperties:
|
||||||
|
type: string
|
||||||
|
description: NodeSelector is the node selector labels to schedule
|
||||||
|
the caching job.
|
||||||
|
type: object
|
||||||
|
resources:
|
||||||
|
description: Resources defines the minimum resources required for
|
||||||
|
the caching job to run(cpu, memory, gpu).
|
||||||
|
properties:
|
||||||
|
cpu:
|
||||||
|
anyOf:
|
||||||
|
- type: integer
|
||||||
|
- type: string
|
||||||
|
description: CPU indicates the minimum number of CPUs to use while
|
||||||
|
caching NIM
|
||||||
|
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
|
||||||
|
x-kubernetes-int-or-string: true
|
||||||
|
memory:
|
||||||
|
anyOf:
|
||||||
|
- type: integer
|
||||||
|
- type: string
|
||||||
|
description: |-
|
||||||
|
Memory indicates the minimum amount of memory to use while caching NIM
|
||||||
|
Valid values are numbers followed by one of the suffixes Ki, Mi, Gi, or Ti (e.g. "4Gi", "4096Mi").
|
||||||
|
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
|
||||||
|
x-kubernetes-int-or-string: true
|
||||||
|
type: object
|
||||||
|
runtimeClassName:
|
||||||
|
description: RuntimeClassName is the runtimeclass for the caching
|
||||||
|
job
|
||||||
|
type: string
|
||||||
|
source:
|
||||||
|
description: Source is the NIM model source to cache
|
||||||
|
properties:
|
||||||
|
dataStore:
|
||||||
|
description: NGCSource represents models stored in NVIDIA DataStore
|
||||||
|
service
|
||||||
|
properties:
|
||||||
|
authSecret:
|
||||||
|
description: The name of an existing auth secret containing
|
||||||
|
the AUTH_TOKEN"
|
||||||
|
type: string
|
||||||
|
checkpointName:
|
||||||
|
type: string
|
||||||
|
datasetName:
|
||||||
|
type: string
|
||||||
|
endpoint:
|
||||||
|
description: The endpoint for datastore
|
||||||
|
type: string
|
||||||
|
modelName:
|
||||||
|
description: Name of either model/checkpoint or dataset to
|
||||||
|
download
|
||||||
|
type: string
|
||||||
|
modelPuller:
|
||||||
|
description: ModelPuller is the container image that can pull
|
||||||
|
the model
|
||||||
|
type: string
|
||||||
|
pullSecret:
|
||||||
|
description: PullSecret for the model puller image
|
||||||
|
type: string
|
||||||
|
required:
|
||||||
|
- authSecret
|
||||||
|
- endpoint
|
||||||
|
- modelPuller
|
||||||
|
type: object
|
||||||
|
ngc:
|
||||||
|
description: NGCSource represents models stored in NGC
|
||||||
|
properties:
|
||||||
|
authSecret:
|
||||||
|
description: The name of an existing pull secret containing
|
||||||
|
the NGC_API_KEY
|
||||||
|
type: string
|
||||||
|
model:
|
||||||
|
description: Model spec for caching
|
||||||
|
properties:
|
||||||
|
buildable:
|
||||||
|
description: Buildable indicates generic model profiles
|
||||||
|
that can be optimized with an NVIDIA engine for any
|
||||||
|
GPUs
|
||||||
|
type: boolean
|
||||||
|
engine:
|
||||||
|
description: Engine is the backend engine (tensort_llm,
|
||||||
|
vllm)
|
||||||
|
type: string
|
||||||
|
gpus:
|
||||||
|
description: GPU is the spec for matching GPUs for caching
|
||||||
|
optimized models
|
||||||
|
items:
|
||||||
|
description: GPUSpec is the spec required to cache models
|
||||||
|
for selected gpu type
|
||||||
|
properties:
|
||||||
|
ids:
|
||||||
|
description: IDs are the device-ids for a specific
|
||||||
|
GPU SKU
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
type: array
|
||||||
|
product:
|
||||||
|
description: Product is the GPU product string (h100,
|
||||||
|
a100, l40s)
|
||||||
|
type: string
|
||||||
|
type: object
|
||||||
|
type: array
|
||||||
|
lora:
|
||||||
|
description: Lora indicates a finetuned model with LoRa
|
||||||
|
adapters
|
||||||
|
type: boolean
|
||||||
|
precision:
|
||||||
|
description: Precision is the precision for model quantization
|
||||||
|
type: string
|
||||||
|
profiles:
|
||||||
|
description: Profiles are the specific model profiles
|
||||||
|
to cache. When these are provided, rest of the model
|
||||||
|
parameters for profile selection are ignored
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
type: array
|
||||||
|
qosProfile:
|
||||||
|
description: QoSProfile is the supported QoS profile types
|
||||||
|
for the models (throughput, latency)
|
||||||
|
type: string
|
||||||
|
tensorParallelism:
|
||||||
|
description: TensorParallelism is the minimum GPUs required
|
||||||
|
for the model computations
|
||||||
|
type: string
|
||||||
|
type: object
|
||||||
|
modelPuller:
|
||||||
|
description: ModelPuller is the container image that can pull
|
||||||
|
the model
|
||||||
|
type: string
|
||||||
|
x-kubernetes-validations:
|
||||||
|
- message: modelPuller is an immutable field. Please create
|
||||||
|
a new NIMCache resource instead when you want to change
|
||||||
|
this container.
|
||||||
|
rule: self == oldSelf
|
||||||
|
pullSecret:
|
||||||
|
description: PullSecret to pull the model puller image
|
||||||
|
type: string
|
||||||
|
required:
|
||||||
|
- authSecret
|
||||||
|
- modelPuller
|
||||||
|
type: object
|
||||||
|
type: object
|
||||||
|
storage:
|
||||||
|
description: Storage is the target storage for caching NIM model
|
||||||
|
properties:
|
||||||
|
hostPath:
|
||||||
|
description: HostPath is the host path volume for caching NIM
|
||||||
|
type: string
|
||||||
|
pvc:
|
||||||
|
description: PersistentVolumeClaim is the pvc volume used for
|
||||||
|
caching NIM
|
||||||
|
properties:
|
||||||
|
create:
|
||||||
|
description: Create indicates to create a new PVC
|
||||||
|
type: boolean
|
||||||
|
name:
|
||||||
|
description: Name is the name of the PVC
|
||||||
|
type: string
|
||||||
|
size:
|
||||||
|
description: Size of the NIM cache in Gi, used during PVC
|
||||||
|
creation
|
||||||
|
type: string
|
||||||
|
storageClass:
|
||||||
|
description: StorageClass to be used for PVC creation. Leave
|
||||||
|
it as empty if the PVC is already created.
|
||||||
|
type: string
|
||||||
|
subPath:
|
||||||
|
type: string
|
||||||
|
volumeAccessMode:
|
||||||
|
description: VolumeAccessMode is the volume access mode of
|
||||||
|
the PVC
|
||||||
|
type: string
|
||||||
|
type: object
|
||||||
|
type: object
|
||||||
|
tolerations:
|
||||||
|
description: Tolerations for running the job to cache the NIM model
|
||||||
|
items:
|
||||||
|
description: |-
|
||||||
|
The pod this Toleration is attached to tolerates any taint that matches
|
||||||
|
the triple <key,value,effect> using the matching operator <operator>.
|
||||||
|
properties:
|
||||||
|
effect:
|
||||||
|
description: |-
|
||||||
|
Effect indicates the taint effect to match. Empty means match all taint effects.
|
||||||
|
When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
|
||||||
|
type: string
|
||||||
|
key:
|
||||||
|
description: |-
|
||||||
|
Key is the taint key that the toleration applies to. Empty means match all taint keys.
|
||||||
|
If the key is empty, operator must be Exists; this combination means to match all values and all keys.
|
||||||
|
type: string
|
||||||
|
operator:
|
||||||
|
description: |-
|
||||||
|
Operator represents a key's relationship to the value.
|
||||||
|
Valid operators are Exists and Equal. Defaults to Equal.
|
||||||
|
Exists is equivalent to wildcard for value, so that a pod can
|
||||||
|
tolerate all taints of a particular category.
|
||||||
|
type: string
|
||||||
|
tolerationSeconds:
|
||||||
|
description: |-
|
||||||
|
TolerationSeconds represents the period of time the toleration (which must be
|
||||||
|
of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,
|
||||||
|
it is not set, which means tolerate the taint forever (do not evict). Zero and
|
||||||
|
negative values will be treated as 0 (evict immediately) by the system.
|
||||||
|
format: int64
|
||||||
|
type: integer
|
||||||
|
value:
|
||||||
|
description: |-
|
||||||
|
Value is the taint value the toleration matches to.
|
||||||
|
If the operator is Exists, the value should be empty, otherwise just a regular string.
|
||||||
|
type: string
|
||||||
|
type: object
|
||||||
|
type: array
|
||||||
|
userID:
|
||||||
|
description: UserID is the user ID for the caching job
|
||||||
|
format: int64
|
||||||
|
type: integer
|
||||||
|
required:
|
||||||
|
- source
|
||||||
|
- storage
|
||||||
|
type: object
|
||||||
|
status:
|
||||||
|
description: NIMCacheStatus defines the observed state of NIMCache
|
||||||
|
properties:
|
||||||
|
conditions:
|
||||||
|
items:
|
||||||
|
description: Condition contains details for one aspect of the current
|
||||||
|
state of this API Resource.
|
||||||
|
properties:
|
||||||
|
lastTransitionTime:
|
||||||
|
description: |-
|
||||||
|
lastTransitionTime is the last time the condition transitioned from one status to another.
|
||||||
|
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
|
||||||
|
format: date-time
|
||||||
|
type: string
|
||||||
|
message:
|
||||||
|
description: |-
|
||||||
|
message is a human readable message indicating details about the transition.
|
||||||
|
This may be an empty string.
|
||||||
|
maxLength: 32768
|
||||||
|
type: string
|
||||||
|
observedGeneration:
|
||||||
|
description: |-
|
||||||
|
observedGeneration represents the .metadata.generation that the condition was set based upon.
|
||||||
|
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
|
||||||
|
with respect to the current state of the instance.
|
||||||
|
format: int64
|
||||||
|
minimum: 0
|
||||||
|
type: integer
|
||||||
|
reason:
|
||||||
|
description: |-
|
||||||
|
reason contains a programmatic identifier indicating the reason for the condition's last transition.
|
||||||
|
Producers of specific condition types may define expected values and meanings for this field,
|
||||||
|
and whether the values are considered a guaranteed API.
|
||||||
|
The value should be a CamelCase string.
|
||||||
|
This field may not be empty.
|
||||||
|
maxLength: 1024
|
||||||
|
minLength: 1
|
||||||
|
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
|
||||||
|
type: string
|
||||||
|
status:
|
||||||
|
description: status of the condition, one of True, False, Unknown.
|
||||||
|
enum:
|
||||||
|
- "True"
|
||||||
|
- "False"
|
||||||
|
- Unknown
|
||||||
|
type: string
|
||||||
|
type:
|
||||||
|
description: type of condition in CamelCase or in foo.example.com/CamelCase.
|
||||||
|
maxLength: 316
|
||||||
|
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
|
||||||
|
type: string
|
||||||
|
required:
|
||||||
|
- lastTransitionTime
|
||||||
|
- message
|
||||||
|
- reason
|
||||||
|
- status
|
||||||
|
- type
|
||||||
|
type: object
|
||||||
|
type: array
|
||||||
|
profiles:
|
||||||
|
items:
|
||||||
|
description: NIMProfile defines the profiles that were cached
|
||||||
|
properties:
|
||||||
|
config:
|
||||||
|
additionalProperties:
|
||||||
|
type: string
|
||||||
|
type: object
|
||||||
|
model:
|
||||||
|
type: string
|
||||||
|
name:
|
||||||
|
type: string
|
||||||
|
release:
|
||||||
|
type: string
|
||||||
|
type: object
|
||||||
|
type: array
|
||||||
|
pvc:
|
||||||
|
type: string
|
||||||
|
state:
|
||||||
|
type: string
|
||||||
|
type: object
|
||||||
|
type: object
|
||||||
|
served: true
|
||||||
|
storage: true
|
||||||
|
subresources:
|
||||||
|
status: {}
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,58 @@
|
|||||||
|
{{/*
|
||||||
|
Expand the name of the chart.
|
||||||
|
*/}}
|
||||||
|
{{- define "k8s-nim-operator.name" -}}
|
||||||
|
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Create a default fully qualified app name.
|
||||||
|
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
|
||||||
|
If release name contains chart name it will be used as a full name.
|
||||||
|
*/}}
|
||||||
|
{{- define "k8s-nim-operator.fullname" -}}
|
||||||
|
{{- if .Values.fullnameOverride }}
|
||||||
|
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
|
||||||
|
{{- else }}
|
||||||
|
{{- $name := default .Chart.Name .Values.nameOverride }}
|
||||||
|
{{- if contains $name .Release.Name }}
|
||||||
|
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
|
||||||
|
{{- else }}
|
||||||
|
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Create chart name and version as used by the chart label.
|
||||||
|
*/}}
|
||||||
|
{{- define "k8s-nim-operator.chart" -}}
|
||||||
|
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Common labels
|
||||||
|
*/}}
|
||||||
|
{{- define "k8s-nim-operator.labels" -}}
|
||||||
|
helm.sh/chart: {{ include "k8s-nim-operator.chart" . }}
|
||||||
|
{{ include "k8s-nim-operator.selectorLabels" . }}
|
||||||
|
{{- if .Chart.AppVersion }}
|
||||||
|
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||||
|
{{- end }}
|
||||||
|
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Selector labels
|
||||||
|
*/}}
|
||||||
|
{{- define "k8s-nim-operator.selectorLabels" -}}
|
||||||
|
app.kubernetes.io/name: {{ include "k8s-nim-operator.name" . }}
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Full image name with tag
|
||||||
|
*/}}
|
||||||
|
{{- define "k8s-nim-operator.fullimage" -}}
|
||||||
|
{{- .Values.operator.image.repository -}}:{{- .Values.operator.image.tag | default (printf "v%s" .Chart.AppVersion) -}}
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: {{ include "k8s-nim-operator.fullname" . }}
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/component: manager
|
||||||
|
app.kubernetes.io/created-by: k8s-nim-operator
|
||||||
|
app.kubernetes.io/part-of: k8s-nim-operator
|
||||||
|
control-plane: controller-manager
|
||||||
|
{{- include "k8s-nim-operator.labels" . | nindent 4 }}
|
||||||
|
spec:
|
||||||
|
replicas: {{ .Values.operator.replicas }}
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
control-plane: controller-manager
|
||||||
|
{{- include "k8s-nim-operator.selectorLabels" . | nindent 6 }}
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
control-plane: controller-manager
|
||||||
|
{{- include "k8s-nim-operator.selectorLabels" . | nindent 8 }}
|
||||||
|
annotations:
|
||||||
|
kubectl.kubernetes.io/default-container: manager
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- args: {{- toYaml .Values.operator.args | nindent 8 }}
|
||||||
|
command:
|
||||||
|
- /manager
|
||||||
|
image: {{ include "k8s-nim-operator.fullimage" . }}
|
||||||
|
imagePullPolicy: {{ .Values.operator.image.pullPolicy }}
|
||||||
|
env:
|
||||||
|
- name: WATCH_NAMESPACE
|
||||||
|
value: ""
|
||||||
|
- name: OPERATOR_NAMESPACE
|
||||||
|
valueFrom:
|
||||||
|
fieldRef:
|
||||||
|
fieldPath: metadata.namespace
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /healthz
|
||||||
|
port: 8081
|
||||||
|
initialDelaySeconds: 15
|
||||||
|
periodSeconds: 20
|
||||||
|
name: manager
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /readyz
|
||||||
|
port: 8081
|
||||||
|
initialDelaySeconds: 5
|
||||||
|
periodSeconds: 10
|
||||||
|
resources: {{- toYaml .Values.operator.resources | nindent 10 }}
|
||||||
|
securityContext:
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
capabilities:
|
||||||
|
drop:
|
||||||
|
- ALL
|
||||||
|
imagePullSecrets:
|
||||||
|
{{- range .Values.operator.image.pullSecrets }}
|
||||||
|
- name: {{ . }}
|
||||||
|
{{- end }}
|
||||||
|
securityContext:
|
||||||
|
seccompProfile:
|
||||||
|
type: RuntimeDefault
|
||||||
|
runAsNonRoot: true
|
||||||
|
serviceAccountName: k8s-nim-operator
|
||||||
|
terminationGracePeriodSeconds: 10
|
||||||
|
{{- with .Values.operator.nodeSelector }}
|
||||||
|
nodeSelector:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.operator.affinity }}
|
||||||
|
affinity:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.operator.tolerations }}
|
||||||
|
tolerations:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: Role
|
||||||
|
metadata:
|
||||||
|
name: k8s-nim-operator-leader-election-role
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/component: rbac
|
||||||
|
app.kubernetes.io/created-by: k8s-nim-operator
|
||||||
|
app.kubernetes.io/part-of: k8s-nim-operator
|
||||||
|
{{- include "k8s-nim-operator.labels" . | nindent 4 }}
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- configmaps
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- watch
|
||||||
|
- create
|
||||||
|
- update
|
||||||
|
- patch
|
||||||
|
- delete
|
||||||
|
- apiGroups:
|
||||||
|
- coordination.k8s.io
|
||||||
|
resources:
|
||||||
|
- leases
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- watch
|
||||||
|
- create
|
||||||
|
- update
|
||||||
|
- patch
|
||||||
|
- delete
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- events
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- patch
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: RoleBinding
|
||||||
|
metadata:
|
||||||
|
name: k8s-nim-operator-leader-election-rolebinding
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/component: rbac
|
||||||
|
app.kubernetes.io/created-by: k8s-nim-operator
|
||||||
|
app.kubernetes.io/part-of: k8s-nim-operator
|
||||||
|
{{- include "k8s-nim-operator.labels" . | nindent 4 }}
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: Role
|
||||||
|
name: k8s-nim-operator-leader-election-role
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: k8s-nim-operator
|
||||||
|
namespace: '{{ .Release.Namespace }}'
|
||||||
@@ -0,0 +1,315 @@
|
|||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: k8s-nim-operator-role
|
||||||
|
labels:
|
||||||
|
{{- include "k8s-nim-operator.labels" . | nindent 4 }}
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- nodes
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- configmaps
|
||||||
|
- persistentvolumeclaims
|
||||||
|
- secrets
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- delete
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- events
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- update
|
||||||
|
- patch
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- endpoints
|
||||||
|
- pods
|
||||||
|
- pods/log
|
||||||
|
- pods/eviction
|
||||||
|
- serviceaccounts
|
||||||
|
- services
|
||||||
|
- services/finalizers
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- delete
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- pods
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- delete
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- apps
|
||||||
|
resources:
|
||||||
|
- deployments
|
||||||
|
- statefulsets
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- delete
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- apps.nvidia.com
|
||||||
|
resources:
|
||||||
|
- nimcaches
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- delete
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- apps.nvidia.com
|
||||||
|
resources:
|
||||||
|
- nimcaches/finalizers
|
||||||
|
verbs:
|
||||||
|
- update
|
||||||
|
- apiGroups:
|
||||||
|
- apps.nvidia.com
|
||||||
|
resources:
|
||||||
|
- nimcaches/status
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
- apiGroups:
|
||||||
|
- apps.nvidia.com
|
||||||
|
resources:
|
||||||
|
- nimpipelines
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- delete
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- apps.nvidia.com
|
||||||
|
resources:
|
||||||
|
- nimpipelines/finalizers
|
||||||
|
verbs:
|
||||||
|
- update
|
||||||
|
- apiGroups:
|
||||||
|
- apps.nvidia.com
|
||||||
|
resources:
|
||||||
|
- nimpipelines/status
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
- apiGroups:
|
||||||
|
- apps.nvidia.com
|
||||||
|
resources:
|
||||||
|
- nimservices
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- delete
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- apps.nvidia.com
|
||||||
|
resources:
|
||||||
|
- nimservices/finalizers
|
||||||
|
verbs:
|
||||||
|
- update
|
||||||
|
- apiGroups:
|
||||||
|
- apps.nvidia.com
|
||||||
|
resources:
|
||||||
|
- nimservices/status
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
- apiGroups:
|
||||||
|
- batch
|
||||||
|
resources:
|
||||||
|
- jobs
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- delete
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- config.openshift.io
|
||||||
|
resources:
|
||||||
|
- clusterversions
|
||||||
|
- proxies
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- persistentvolumeclaims
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- delete
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- apiGroups:
|
||||||
|
- monitoring.coreos.com
|
||||||
|
resources:
|
||||||
|
- prometheusrules
|
||||||
|
- servicemonitors
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- delete
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- rbac.authorization.k8s.io
|
||||||
|
resources:
|
||||||
|
- rolebindings
|
||||||
|
- roles
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- delete
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- route.openshift.io
|
||||||
|
resources:
|
||||||
|
- routes
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- networking.k8s.io
|
||||||
|
resources:
|
||||||
|
- ingresses
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
- watch
|
||||||
|
- delete
|
||||||
|
- apiGroups:
|
||||||
|
- autoscaling
|
||||||
|
resources:
|
||||||
|
- horizontalpodautoscalers
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- delete
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- scheduling.k8s.io
|
||||||
|
resources:
|
||||||
|
- priorityclasses
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- security.openshift.io
|
||||||
|
resourceNames:
|
||||||
|
- nonroot
|
||||||
|
resources:
|
||||||
|
- securitycontextconstraints
|
||||||
|
verbs:
|
||||||
|
- use
|
||||||
|
- apiGroups:
|
||||||
|
- security.openshift.io
|
||||||
|
resources:
|
||||||
|
- securitycontextconstraints
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- delete
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- storage.k8s.io
|
||||||
|
resources:
|
||||||
|
- storageclasses
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- authentication.k8s.io
|
||||||
|
resources:
|
||||||
|
- tokenreviews
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- apiGroups:
|
||||||
|
- authorization.k8s.io
|
||||||
|
resources:
|
||||||
|
- subjectaccessreviews
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
metadata:
|
||||||
|
name: k8s-nim-operator-rolebinding
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/component: rbac
|
||||||
|
app.kubernetes.io/created-by: k8s-nim-operator
|
||||||
|
app.kubernetes.io/part-of: k8s-nim-operator
|
||||||
|
{{- include "k8s-nim-operator.labels" . | nindent 4 }}
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: ClusterRole
|
||||||
|
name: k8s-nim-operator-role
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: k8s-nim-operator
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: k8s-nim-operator-metrics-reader
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/component: kube-rbac-proxy
|
||||||
|
app.kubernetes.io/created-by: k8s-nim-operator
|
||||||
|
app.kubernetes.io/part-of: k8s-nim-operator
|
||||||
|
{{- include "k8s-nim-operator.labels" . | nindent 4 }}
|
||||||
|
rules:
|
||||||
|
- nonResourceURLs:
|
||||||
|
- /metrics
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: k8s-nim-operator-metrics-service
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/component: kube-rbac-proxy
|
||||||
|
app.kubernetes.io/created-by: k8s-nim-operator
|
||||||
|
app.kubernetes.io/part-of: k8s-nim-operator
|
||||||
|
control-plane: controller-manager
|
||||||
|
{{- include "k8s-nim-operator.labels" . | nindent 4 }}
|
||||||
|
spec:
|
||||||
|
type: {{ .Values.metricsService.type }}
|
||||||
|
selector:
|
||||||
|
control-plane: controller-manager
|
||||||
|
{{- include "k8s-nim-operator.selectorLabels" . | nindent 4 }}
|
||||||
|
ports:
|
||||||
|
{{- .Values.metricsService.ports | toYaml | nindent 2 -}}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
{{- if .Values.nfd.nodeFeatureRules.deviceID }}
|
||||||
|
kind: NodeFeatureRule
|
||||||
|
apiVersion: nfd.k8s-sigs.io/v1alpha1
|
||||||
|
metadata:
|
||||||
|
name: nfd-pci-device-rule
|
||||||
|
spec:
|
||||||
|
rules:
|
||||||
|
- name: "NVIDIA PCIe Device Label"
|
||||||
|
labelsTemplate: |
|
||||||
|
{{`{{ range .pci.device }}pci-{{ .vendor }}-{{ .device }}.present=true`}}
|
||||||
|
{{`{{ end }}`}}
|
||||||
|
matchFeatures:
|
||||||
|
- feature: pci.device
|
||||||
|
matchExpressions:
|
||||||
|
vendor: {op: In, value: ["10de"]}
|
||||||
|
class: {op: In, value: ["0300", "0302"]}
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: k8s-nim-operator
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/component: rbac
|
||||||
|
app.kubernetes.io/created-by: k8s-nim-operator
|
||||||
|
app.kubernetes.io/part-of: k8s-nim-operator
|
||||||
|
{{- include "k8s-nim-operator.labels" . | nindent 4 }}
|
||||||
|
annotations:
|
||||||
|
{{- toYaml .Values.operator.annotations | nindent 4 }}
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
{{- if .Values.operator.upgradeCRD }}
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: nim-operator-upgrade-crd-hook-sa
|
||||||
|
annotations:
|
||||||
|
helm.sh/hook: pre-upgrade
|
||||||
|
helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation
|
||||||
|
helm.sh/hook-weight: "0"
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: nim-operator-upgrade-crd-hook-role
|
||||||
|
annotations:
|
||||||
|
helm.sh/hook: pre-upgrade
|
||||||
|
helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation
|
||||||
|
helm.sh/hook-weight: "0"
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- apiextensions.k8s.io
|
||||||
|
resources:
|
||||||
|
- customresourcedefinitions
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- watch
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
---
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: nim-operator-upgrade-crd-hook-binding
|
||||||
|
annotations:
|
||||||
|
helm.sh/hook: pre-upgrade
|
||||||
|
helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation
|
||||||
|
helm.sh/hook-weight: "0"
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: nim-operator-upgrade-crd-hook-sa
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
roleRef:
|
||||||
|
kind: ClusterRole
|
||||||
|
name: nim-operator-upgrade-crd-hook-role
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: Job
|
||||||
|
metadata:
|
||||||
|
name: nim-operator-upgrade-crd
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
annotations:
|
||||||
|
"helm.sh/hook": pre-upgrade
|
||||||
|
"helm.sh/hook-weight": "1"
|
||||||
|
"helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation
|
||||||
|
labels:
|
||||||
|
{{- include "k8s-nim-operator.labels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: "nim-operator"
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
name: nim-operator-upgrade-crd
|
||||||
|
labels:
|
||||||
|
{{- include "k8s-nim-operator.labels" . | nindent 8 }}
|
||||||
|
app.kubernetes.io/component: "nim-operator"
|
||||||
|
spec:
|
||||||
|
serviceAccountName: nim-operator-upgrade-crd-hook-sa
|
||||||
|
{{- if .Values.operator.image.pullSecrets }}
|
||||||
|
imagePullSecrets:
|
||||||
|
{{- range .Values.operator.image.pullSecrets }}
|
||||||
|
- name: {{ . }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.operator.tolerations }}
|
||||||
|
securityContext:
|
||||||
|
seccompProfile:
|
||||||
|
type: RuntimeDefault
|
||||||
|
runAsNonRoot: true
|
||||||
|
tolerations:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
containers:
|
||||||
|
- name: upgrade-crd
|
||||||
|
image: {{ include "k8s-nim-operator.fullimage" . }}
|
||||||
|
imagePullPolicy: {{ .Values.operator.image.pullPolicy }}
|
||||||
|
securityContext:
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
capabilities:
|
||||||
|
drop:
|
||||||
|
- ALL
|
||||||
|
command: ["kubectl"]
|
||||||
|
args:
|
||||||
|
- "apply"
|
||||||
|
- "-f"
|
||||||
|
- "/opt/nim-operator/crds/*.yaml"
|
||||||
|
restartPolicy: OnFailure
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
operator:
|
||||||
|
replicas: 1
|
||||||
|
upgradeCRD: true
|
||||||
|
image:
|
||||||
|
repository: nvcr.io/nvidia/cloud-native/k8s-nim-operator
|
||||||
|
tag: v1.0.1
|
||||||
|
pullSecrets: []
|
||||||
|
pullPolicy: Always
|
||||||
|
args:
|
||||||
|
- --health-probe-bind-address=:8081
|
||||||
|
- --metrics-bind-address=:8080
|
||||||
|
- --leader-elect
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 256Mi
|
||||||
|
requests:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 128Mi
|
||||||
|
nodeSelector: {}
|
||||||
|
annotations: {}
|
||||||
|
tolerations:
|
||||||
|
- key: "node-role.kubernetes.io/control-plane"
|
||||||
|
operator: "Equal"
|
||||||
|
value: ""
|
||||||
|
effect: "NoSchedule"
|
||||||
|
affinity:
|
||||||
|
nodeAffinity:
|
||||||
|
preferredDuringSchedulingIgnoredDuringExecution:
|
||||||
|
- weight: 1
|
||||||
|
preference:
|
||||||
|
matchExpressions:
|
||||||
|
- key: "node-role.kubernetes.io/control-plane"
|
||||||
|
operator: In
|
||||||
|
values: [""]
|
||||||
|
|
||||||
|
metricsService:
|
||||||
|
ports:
|
||||||
|
- name: metrics
|
||||||
|
port: 8080
|
||||||
|
protocol: TCP
|
||||||
|
type: ClusterIP
|
||||||
|
|
||||||
|
nfd:
|
||||||
|
nodeFeatureRules:
|
||||||
|
deviceID: true
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
# Patterns to ignore when building packages.
|
||||||
|
# This supports shell glob matching, relative path matching, and
|
||||||
|
# negation (prefixed with !). Only one pattern per line.
|
||||||
|
.DS_Store
|
||||||
|
# Common VCS dirs
|
||||||
|
.git/
|
||||||
|
.gitignore
|
||||||
|
.bzr/
|
||||||
|
.bzrignore
|
||||||
|
.hg/
|
||||||
|
.hgignore
|
||||||
|
.svn/
|
||||||
|
# Common backup files
|
||||||
|
*.swp
|
||||||
|
*.bak
|
||||||
|
*.tmp
|
||||||
|
*.orig
|
||||||
|
*~
|
||||||
|
# Various IDEs
|
||||||
|
.project
|
||||||
|
.idea/
|
||||||
|
*.tmproj
|
||||||
|
.vscode/
|
||||||
|
|
||||||
|
# Our example-config.yaml is just an example, users should provide their own values.
|
||||||
|
example-config.yaml
|
||||||
|
ephemeral.values.yaml
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
dependencies:
|
||||||
|
- name: postgresql
|
||||||
|
repository: oci://dockerhub.nvidia.com/bitnamicharts
|
||||||
|
version: 13.3.1
|
||||||
|
- name: opentelemetry-collector
|
||||||
|
repository: https://open-telemetry.github.io/opentelemetry-helm-charts
|
||||||
|
version: 0.93.3
|
||||||
|
- name: nemo-common
|
||||||
|
repository: file://../../../library/
|
||||||
|
version: 0.1.0
|
||||||
|
digest: sha256:a30bc670003a830025a3cc57257082923e852c35a108629d30768e9aa2e69c5e
|
||||||
|
generated: "2025-04-16T22:06:20.120607586Z"
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
apiVersion: v2
|
||||||
|
appVersion: "25.04"
|
||||||
|
dependencies:
|
||||||
|
- condition: postgresql.enabled
|
||||||
|
name: postgresql
|
||||||
|
repository: oci://dockerhub.nvidia.com/bitnamicharts
|
||||||
|
version: 13.3.1
|
||||||
|
- condition: opentelemetry-collector.enabled
|
||||||
|
name: opentelemetry-collector
|
||||||
|
repository: https://open-telemetry.github.io/opentelemetry-helm-charts
|
||||||
|
version: 0.93.3
|
||||||
|
- name: nemo-common
|
||||||
|
repository: file://../../../library/
|
||||||
|
version: '>=0.1.0-0'
|
||||||
|
description: NeMo Customizer Microservice Helm Chart
|
||||||
|
name: nemo-customizer
|
||||||
|
type: application
|
||||||
|
version: 25.4.0
|
||||||
@@ -0,0 +1,188 @@
|
|||||||
|
# NeMo Customizer Microservice Helm Chart
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
For deployment guide, see [Admin Setup](https://docs.nvidia.com/nemo/microservices/latest/set-up/index.html) in the NeMo Microservices documentation.
|
||||||
|
|
||||||
|
## Values
|
||||||
|
|
||||||
|
| Key | Type | Default | Description |
|
||||||
|
|-----|------|---------|-------------|
|
||||||
|
| apiImage | object | This object has the following default values for the NeMo Customizer API only image. | Customizer API only image configuration. |
|
||||||
|
| apiImage.imagePullPolicy | string | `"IfNotPresent"` | Image pull policy for the NeMo Customizer API image. |
|
||||||
|
| apiImage.registry | string | `"nvcr.io"` | Registry for the NeMo Customizer API image. |
|
||||||
|
| apiImage.repository | string | `"nvidia/nemo-microservices/customizer-api"` | Repository for the NeMo Customizer API image. |
|
||||||
|
| customizerConfig | object | This object has default values for the following fields. | Configuration for the NeMo Customizer microservice. |
|
||||||
|
| customizerConfig.entityStoreURL | string | `"http://nemo-gateway.nemo-gateway.svc.cluster.local:8000"` | URL for the NeMo Entity Store microservice. |
|
||||||
|
| customizerConfig.mlflowURL | string | `"http://mlflow-tracking.mlflow-system.svc.cluster.local:80"` | URL for the MLflow tracking server. |
|
||||||
|
| customizerConfig.models | object | This object has default values for the supported models. | Models configuration. |
|
||||||
|
| customizerConfig.models."meta/llama-3.1-70b-instruct" | object | This object has the following default values for the Llama 3.1 70B Instruct model. | Llama 3.1 70B Instruct model configuration. |
|
||||||
|
| customizerConfig.models."meta/llama-3.1-70b-instruct".enabled | bool | `false` | Whether to enable the model. |
|
||||||
|
| customizerConfig.models."meta/llama-3.1-70b-instruct".max_seq_length | int | `4096` | Maximum sequence length for input tokens. |
|
||||||
|
| customizerConfig.models."meta/llama-3.1-70b-instruct".micro_batch_size | int | `1` | Micro batch size for training. |
|
||||||
|
| customizerConfig.models."meta/llama-3.1-70b-instruct".model_path | string | `"llama-3_1-70b-instruct_0_0_1"` | Path where model files are stored. |
|
||||||
|
| customizerConfig.models."meta/llama-3.1-70b-instruct".model_uri | string | `"ngc://nvidia/nemo/llama-3_1-70b-instruct-nemo:2.0"` | NGC model URI for Llama 3.1 70B Instruct model. |
|
||||||
|
| customizerConfig.models."meta/llama-3.1-70b-instruct".num_parameters | int | `70000000000` | Number of model parameters. |
|
||||||
|
| customizerConfig.models."meta/llama-3.1-70b-instruct".precision | string | `"bf16-mixed"` | Model precision format. |
|
||||||
|
| customizerConfig.models."meta/llama-3.1-70b-instruct".prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. |
|
||||||
|
| customizerConfig.models."meta/llama-3.1-70b-instruct".training_options | list | `[{"finetuning_type":"lora","num_gpus":4,"num_nodes":1,"tensor_parallel_size":4,"training_type":"sft"}]` | Training options for different fine-tuning methods. |
|
||||||
|
| customizerConfig.models."meta/llama-3.1-8b-instruct" | object | This object has the following default values for the Llama 3.1 8B Instruct model. | Llama 3.1 8B Instruct model configuration. |
|
||||||
|
| customizerConfig.models."meta/llama-3.1-8b-instruct".enabled | bool | `false` | Whether to enable the model. |
|
||||||
|
| customizerConfig.models."meta/llama-3.1-8b-instruct".max_seq_length | int | `4096` | Maximum sequence length for input tokens. |
|
||||||
|
| customizerConfig.models."meta/llama-3.1-8b-instruct".micro_batch_size | int | `1` | Micro batch size for training. |
|
||||||
|
| customizerConfig.models."meta/llama-3.1-8b-instruct".model_path | string | `"llama-3_1-8b-instruct_0_0_1"` | Path where model files are stored. |
|
||||||
|
| customizerConfig.models."meta/llama-3.1-8b-instruct".model_uri | string | `"ngc://nvidia/nemo/llama-3_1-8b-instruct-nemo:2.0"` | NGC model URI for Llama 3.1 8B Instruct model. |
|
||||||
|
| customizerConfig.models."meta/llama-3.1-8b-instruct".num_parameters | int | `8000000000` | Number of model parameters. |
|
||||||
|
| customizerConfig.models."meta/llama-3.1-8b-instruct".precision | string | `"bf16-mixed"` | Model precision format. |
|
||||||
|
| customizerConfig.models."meta/llama-3.1-8b-instruct".prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. |
|
||||||
|
| customizerConfig.models."meta/llama-3.1-8b-instruct".training_options | list | `[{"finetuning_type":"lora","num_gpus":1,"training_type":"sft"},{"finetuning_type":"all_weights","num_gpus":8,"num_nodes":1,"tensor_parallel_size":4,"training_type":"sft"}]` | Training options for different fine-tuning methods. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-1b" | object | This object has the following default values for the Llama 3.2 1B model. | Llama 3.2 1B model configuration. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-1b".enabled | bool | `false` | Whether to enable the model. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-1b".max_seq_length | int | `4096` | Maximum sequence length for input tokens. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-1b".micro_batch_size | int | `1` | Micro batch size for training. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-1b".model_path | string | `"llama32_1b"` | Path where model files are stored. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-1b".model_uri | string | `"ngc://nvidia/nemo/llama-3_2-1b:2.0"` | NGC model URI for Llama 3.2 1B model. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-1b".num_parameters | int | `1000000000` | Number of model parameters. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-1b".precision | string | `"bf16-mixed"` | Model precision format. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-1b".prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-1b".training_options | list | `[{"finetuning_type":"lora","num_gpus":1,"num_nodes":1,"tensor_parallel_size":1,"training_type":"sft"},{"finetuning_type":"all_weights","num_gpus":1,"num_nodes":1,"tensor_parallel_size":1,"training_type":"sft"}]` | Training options for different fine-tuning methods. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-1b-instruct" | object | This object has the following default values for the Llama 3.2 1B Instruct model. | Llama 3.2 1B Instruct model configuration. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-1b-instruct".enabled | bool | `false` | Whether to enable the model. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-1b-instruct".max_seq_length | int | `4096` | Maximum sequence length for input tokens. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-1b-instruct".micro_batch_size | int | `1` | Micro batch size for training. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-1b-instruct".model_path | string | `"llama32_1b-instruct"` | Path where model files are stored. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-1b-instruct".model_uri | string | `"ngc://nvidia/nemo/llama-3_2-1b-instruct:2.0"` | NGC model URI for Llama 3.2 1B Instruct model. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-1b-instruct".num_parameters | int | `1000000000` | Number of model parameters. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-1b-instruct".precision | string | `"bf16-mixed"` | Model precision format. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-1b-instruct".prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-1b-instruct".training_options | list | `[{"finetuning_type":"lora","num_gpus":1,"num_nodes":1,"tensor_parallel_size":1,"training_type":"sft"},{"finetuning_type":"all_weights","num_gpus":1,"num_nodes":1,"tensor_parallel_size":1,"training_type":"sft"}]` | Training options for different fine-tuning methods. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-3b-instruct" | object | This object has the following default values for the Llama 3.2 3B Instruct model. | Llama 3.2 3B Instruct model configuration. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-3b-instruct".enabled | bool | `false` | Whether to enable the model. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-3b-instruct".max_seq_length | int | `4096` | Maximum sequence length for input tokens. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-3b-instruct".micro_batch_size | int | `1` | Micro batch size for training. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-3b-instruct".model_path | string | `"llama32_3b-instruct"` | Path where model files are stored. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-3b-instruct".model_uri | string | `"ngc://nvidia/nemo/llama-3_2-3b-instruct:2.0"` | NGC model URI. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-3b-instruct".num_parameters | int | `3000000000` | Number of model parameters. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-3b-instruct".precision | string | `"bf16-mixed"` | Model precision format. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-3b-instruct".prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. |
|
||||||
|
| customizerConfig.models."meta/llama-3.2-3b-instruct".training_options | list | `[{"finetuning_type":"lora","num_gpus":1,"num_nodes":1,"tensor_parallel_size":1,"training_type":"sft"}]` | Training options for different fine-tuning methods. |
|
||||||
|
| customizerConfig.models."meta/llama-3.3-70b-instruct" | object | This object has the following default values for the Llama 3.3 70B Instruct model. | Llama 3.3 70B Instruct model configuration. |
|
||||||
|
| customizerConfig.models."meta/llama-3.3-70b-instruct".enabled | bool | `false` | Whether to enable the model. |
|
||||||
|
| customizerConfig.models."meta/llama-3.3-70b-instruct".max_seq_length | int | `4096` | Maximum sequence length for input tokens. |
|
||||||
|
| customizerConfig.models."meta/llama-3.3-70b-instruct".micro_batch_size | int | `1` | Micro batch size for training. |
|
||||||
|
| customizerConfig.models."meta/llama-3.3-70b-instruct".model_path | string | `"llama-3_3-70b-instruct_0_0_1"` | Path where model files are stored. |
|
||||||
|
| customizerConfig.models."meta/llama-3.3-70b-instruct".model_uri | string | `"ngc://nvidia/nemo/llama-3_3-70b-instruct:2.0"` | NGC model URI for Llama 3.3 70B Instruct model. |
|
||||||
|
| customizerConfig.models."meta/llama-3.3-70b-instruct".num_parameters | int | `70000000000` | Number of model parameters. |
|
||||||
|
| customizerConfig.models."meta/llama-3.3-70b-instruct".precision | string | `"bf16-mixed"` | Model precision format. |
|
||||||
|
| customizerConfig.models."meta/llama-3.3-70b-instruct".prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. |
|
||||||
|
| customizerConfig.models."meta/llama-3.3-70b-instruct".training_options | list | `[{"finetuning_type":"lora","num_gpus":4,"num_nodes":1,"tensor_parallel_size":4,"training_type":"sft"}]` | Training options for different fine-tuning methods. |
|
||||||
|
| customizerConfig.models.meta/llama3-70b-instruct | object | This object has the following default values for the Llama 3 70B Instruct model. | Llama 3 70B Instruct model configuration. |
|
||||||
|
| customizerConfig.models.meta/llama3-70b-instruct.enabled | bool | `false` | Whether to enable the model. |
|
||||||
|
| customizerConfig.models.meta/llama3-70b-instruct.max_seq_length | int | `4096` | Maximum sequence length for input tokens. |
|
||||||
|
| customizerConfig.models.meta/llama3-70b-instruct.micro_batch_size | int | `1` | Micro batch size for training. |
|
||||||
|
| customizerConfig.models.meta/llama3-70b-instruct.model_path | string | `"llama-3-70b-bf16"` | Path where model files are stored. |
|
||||||
|
| customizerConfig.models.meta/llama3-70b-instruct.model_uri | string | `"ngc://nvidia/nemo/llama-3-70b-instruct-nemo:2.0"` | NGC model URI for Llama 3 70B Instruct model. |
|
||||||
|
| customizerConfig.models.meta/llama3-70b-instruct.num_parameters | int | `70000000000` | Number of model parameters. |
|
||||||
|
| customizerConfig.models.meta/llama3-70b-instruct.precision | string | `"bf16-mixed"` | Model precision format. |
|
||||||
|
| customizerConfig.models.meta/llama3-70b-instruct.prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. |
|
||||||
|
| customizerConfig.models.meta/llama3-70b-instruct.training_options | list | `[{"finetuning_type":"lora","num_gpus":4,"num_nodes":1,"tensor_parallel_size":4,"training_type":"sft"}]` | Training options for different fine-tuning methods. |
|
||||||
|
| customizerConfig.models.microsoft/phi-4 | object | This object has the following default values for the Phi-4. | Phi-4 model configuration. |
|
||||||
|
| customizerConfig.models.microsoft/phi-4.enabled | bool | `false` | Whether to enable the model. |
|
||||||
|
| customizerConfig.models.microsoft/phi-4.max_seq_length | int | `4096` | Maximum sequence length for input tokens. |
|
||||||
|
| customizerConfig.models.microsoft/phi-4.micro_batch_size | int | `1` | Micro batch size for training. |
|
||||||
|
| customizerConfig.models.microsoft/phi-4.model_path | string | `"phi-4"` | Path where model files are stored. |
|
||||||
|
| customizerConfig.models.microsoft/phi-4.model_uri | string | `"ngc://nvidia/nemo/phi-4:1.0"` | NGC model URI for Phi-4 model. |
|
||||||
|
| customizerConfig.models.microsoft/phi-4.num_parameters | int | `14659507200` | Number of model parameters. |
|
||||||
|
| customizerConfig.models.microsoft/phi-4.precision | string | `"bf16"` | Model precision format. |
|
||||||
|
| customizerConfig.models.microsoft/phi-4.prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. |
|
||||||
|
| customizerConfig.models.microsoft/phi-4.training_options | list | `[{"finetuning_type":"lora","num_gpus":1,"num_nodes":1,"training_type":"sft"},{"finetuning_type":"all_weights","num_gpus":4,"num_nodes":2,"tensor_parallel_size":8,"training_type":"sft"}]` | Training options for different fine-tuning methods. |
|
||||||
|
| customizerConfig.nemoDataStoreURL | string | `"http://nds-datastore-http.nds-v2.svc.cluster.local:3000"` | URL for the NeMo Data Store microservice. |
|
||||||
|
| customizerConfig.openTelemetry | object | This object has the following default values for the OpenTelemetry settings. | OpenTelemetry settings. |
|
||||||
|
| customizerConfig.openTelemetry.enabled | bool | `true` | Whether to enable OpenTelemetry. |
|
||||||
|
| customizerConfig.openTelemetry.exporterOtlpEndpoint | string | `""` | Endpoint to access a custom OTLP collector listening on port 4317. Example: "http://$(HOST_IP):4317". |
|
||||||
|
| customizerConfig.openTelemetry.logsExporter | string | `"otlp"` | Sets the logs exporter type (otlp, console, none). |
|
||||||
|
| customizerConfig.openTelemetry.metricsExporter | string | `"otlp"` | Sets the metrics exporter type (otlp, console, none). |
|
||||||
|
| customizerConfig.openTelemetry.tracesExporter | string | `"otlp"` | Sets the traces exporter type (otlp, console, none). |
|
||||||
|
| customizerConfig.tolerations | list | `[]` | Tolerations on the customization job pods. |
|
||||||
|
| customizerConfig.training | object | This object has the following default values for the training configuration. | Training configuration for customization jobs. |
|
||||||
|
| customizerConfig.training.container_defaults | object | `{"env":null,"imagePullPolicy":"IfNotPresent"}` | Default container configuration for training jobs. |
|
||||||
|
| customizerConfig.training.container_defaults.env | string | `nil` | Environment variables for the training container. Cannot override env variables reserved by NeMo Customizer. |
|
||||||
|
| customizerConfig.training.pvc.size | string | `"5Gi"` | Size of the training job PVC. |
|
||||||
|
| customizerConfig.training.pvc.storageClass | string | `"local-nfs"` | Storage class for the training job PVC. |
|
||||||
|
| customizerConfig.training.pvc.volumeAccessMode | string | `"ReadWriteMany"` | Volume access mode for the training job PVC. |
|
||||||
|
| customizerConfig.training.queue | string | `"default"` | Queue name used by the underlying scheduler of NemoTrainingJob. Maps to "resourceGroup" in NemoTrainingJob. |
|
||||||
|
| customizerConfig.training.workspace_dir | string | `"/pvc/workspace"` | Directory path for training workspace. |
|
||||||
|
| customizerConfig.trainingNetworking | list | `[{"name":"NCCL_IB_SL","value":0},{"name":"NCCL_IB_TC","value":41},{"name":"NCCL_IB_QPS_PER_CONNECTION","value":4},{"name":"UCX_TLS","value":"TCP"},{"name":"UCX_NET_DEVICES","value":"eth0"},{"name":"HCOLL_ENABLE_MCAST_ALL","value":0},{"name":"NCCL_IB_GID_INDEX","value":3}]` | Network configuration for training jobs on Oracle Kubernetes Engine (OKE) on Oracle Cloud Infrastructure (OCI). |
|
||||||
|
| customizerConfig.wandb | object | `{"entity":null,"project":"nvidia-nemo-customizer"}` | Weights and Biases (WandB) Python SDK intialization configuration for logging and monitoring training jobs in WandB. |
|
||||||
|
| customizerConfig.wandb.entity | string | `nil` | Reference: https://docs.wandb.ai/ref/python/init/ |
|
||||||
|
| env | object | `{}` | A map of environment variables to inject into the NeMo Customizer app container. Example: `{HOST_IP: valueFrom: fieldRef: fieldPath: status.hostIP OTEL_EXPORTER_OTLP_ENDPOINT: "http://$(HOST_IP):4317"}` |
|
||||||
|
| externalDatabase | object | This object has the following default values for the external PostgreSQL configuration. | External PostgreSQL configuration. |
|
||||||
|
| externalDatabase.database | string | `"finetuning"` | Name of the database to use. |
|
||||||
|
| externalDatabase.existingSecret | string | `""` | Name of an existing secret resource containing the database credentials. |
|
||||||
|
| externalDatabase.existingSecretPasswordKey | string | `""` | Name of an existing secret key containing the database credentials. |
|
||||||
|
| externalDatabase.host | string | `"localhost"` | External database host address. |
|
||||||
|
| externalDatabase.password | string | `"nemo"` | Password for the non-root username for the NeMo Customizer microservice. |
|
||||||
|
| externalDatabase.port | int | `5432` | External database port number. |
|
||||||
|
| externalDatabase.user | string | `"nemo"` | Non-root username for the NeMo Customizer microservice. |
|
||||||
|
| fullnameOverride | string | `""` | String to fully override the chart and release name on resulting objects when deployed. |
|
||||||
|
| image | object | This object has the following default values for the NeMo Customizer microservice image. | NeMo Customizer image that supports training and standalone mode. |
|
||||||
|
| image.imagePullPolicy | string | `"IfNotPresent"` | Image pull policy for the NeMo Customizer image. |
|
||||||
|
| image.registry | string | `"nvcr.io"` | Registry for the NeMo Customizer image. |
|
||||||
|
| image.repository | string | `"nvidia/nemo-microservices/customizer"` | Repository for the NeMo Customizer image. |
|
||||||
|
| imagePullSecrets | list | `[{"name":"nvcrimagepullsecret"}]` | Image pull secrets configuration. |
|
||||||
|
| ingress | object | This object has the following default values for the Ingress configuration. | Ingress configuration. |
|
||||||
|
| ingress.annotations | object | `{}` | Additional annotations for the Ingress resource. |
|
||||||
|
| ingress.className | string | `""` | Ingress class name. |
|
||||||
|
| ingress.enabled | bool | `false` | Whether to enable the ingress resource. |
|
||||||
|
| ingress.hostname | string | `""` | Hostname for the ingress resource. |
|
||||||
|
| ingress.hosts | list | [] | A list of maps, each containing the keys `host` and `paths` for the ingress resource. You must specify a list for configuring ingress for the microservice. |
|
||||||
|
| ingress.tls | list | `[]` | TLS configuration for the ingress resource. |
|
||||||
|
| logging | object | `{"logHealthEndpoints":false,"logLevel":"INFO"}` | Logging configuration. |
|
||||||
|
| logging.logHealthEndpoints | bool | `false` | Enable logging for health endpoints. |
|
||||||
|
| logging.logLevel | string | `"INFO"` | Log level for the application. |
|
||||||
|
| modelDownloader | object | This object has the following default values for the model downloader. | Download models to PVC model cache configuration. |
|
||||||
|
| modelDownloader.pollIntervalSeconds | int | `15` | Interval in seconds to poll for model download status. |
|
||||||
|
| modelDownloader.securityContext | object | `{"fsGroup":1000,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}` | Security context for the model downloader. |
|
||||||
|
| modelDownloader.ttlSecondsAfterFinished | int | `7200` | Time to live in seconds after the job finishes. |
|
||||||
|
| modelsStorage | object | `{"accessModes":["ReadWriteMany"],"enabled":true,"size":"1Ti","storageClassName":""}` | Configure the PVC for models mount, where we store the parent/base models. |
|
||||||
|
| modelsStorage.accessModes | list | `["ReadWriteMany"]` | Access modes for the persistent volume. |
|
||||||
|
| modelsStorage.enabled | bool | `true` | Enable persistent volume for model storage. |
|
||||||
|
| modelsStorage.size | string | `"1Ti"` | Size of the persistent volume. |
|
||||||
|
| modelsStorage.storageClassName | string | `""` | Storage class name for the models PVC. Empty string uses the default storage class. |
|
||||||
|
| nameOverride | string | `""` | String to override chart name on resulting objects when deployed. |
|
||||||
|
| nemoDataStoreTools | object | This object has the following default values for the NeMo Data Store tools image. | Tools configuration for downloading and uploading entities to NeMo Data Store. |
|
||||||
|
| nemoDataStoreTools.imagePullSecret | string | `"nvcrimagepullsecret"` | Image pull secret for the NeMo Data Store tools image. |
|
||||||
|
| nemoDataStoreTools.registry | string | `"nvcr.io"` | Registry for the NeMo Data Store tools image. |
|
||||||
|
| nemoDataStoreTools.repository | string | `"nvidia/nemo-microservices/nds-v2-huggingface-cli"` | Repository for the NeMo Data Store tools image. |
|
||||||
|
| nemoDataStoreTools.tag | string | `""` | Tag for the NeMo Data Store tools image. |
|
||||||
|
| ngcAPISecret | string | `"ngc-api"` | Secret used for auto hydrating the model cache from NGC for enabled models. |
|
||||||
|
| ngcAPISecretKey | string | `"NGC_API_KEY"` | Key in the NGC API secret containing the API key. |
|
||||||
|
| opentelemetry-collector | object | This object has the following default values for the Open Telemetry Collector configuration. | Open Telemetry Collector configuration. |
|
||||||
|
| opentelemetry-collector.config | object | `{"exporters":{"debug":{"verbosity":"detailed"}},"extensions":{"health_check":{},"zpages":{"endpoint":"0.0.0.0:55679"}},"processors":{"batch":{}},"receivers":{"otlp":{"protocols":{"grpc":{},"http":{"cors":{"allowed_origins":["*"]}}}}},"service":{"extensions":["zpages","health_check"],"pipelines":{"logs":{"exporters":["debug"],"processors":["batch"],"receivers":["otlp"]},"metrics":{"exporters":["debug"],"processors":["batch"],"receivers":["otlp"]},"traces":{"exporters":["debug"],"processors":["batch"],"receivers":["otlp"]}}}}` | Base collector configuration for Open Telemetry Collector. |
|
||||||
|
| opentelemetry-collector.enabled | bool | `true` | Switch to enable or disable Open Telemetry Collector. |
|
||||||
|
| opentelemetry-collector.image.repository | string | `"otel/opentelemetry-collector-k8s"` | Repository for Open Telemetry Collector image. |
|
||||||
|
| opentelemetry-collector.image.tag | string | `"0.102.1"` | Overrides the image tag whose default is the chart appVersion. |
|
||||||
|
| opentelemetry-collector.mode | string | `"deployment"` | Deployment mode for Open Telemetry Collector. Valid values are "daemonset", "deployment", and "statefulset". |
|
||||||
|
| postgresql | object | This object has the following default values for the PostgreSQL configuration. | PostgreSQL configuration for the NeMo Customizer microservice. |
|
||||||
|
| postgresql.architecture | string | `"standalone"` | PostgreSQL architecture (`standalone` or `replication`). |
|
||||||
|
| postgresql.auth.database | string | `"finetuning"` | Name for a custom database to create. |
|
||||||
|
| postgresql.auth.enablePostgresUser | bool | `true` | Whether to assign a password to the "postgres" admin user. Otherwise, remote access will be blocked for this user. |
|
||||||
|
| postgresql.auth.existingSecret | string | `""` | Name of existing secret to use for PostgreSQL credentials. |
|
||||||
|
| postgresql.auth.password | string | `"nemo"` | Password for the custom user to create. |
|
||||||
|
| postgresql.auth.username | string | `"nemo"` | Name for a custom user to create. |
|
||||||
|
| postgresql.enabled | bool | `true` | Whether to enable or disable the PostgreSQL helm chart. |
|
||||||
|
| replicaCount | int | `1` | Number of replicas to deploy. |
|
||||||
|
| service | object | `{"internalPort":9009,"port":8000,"type":"ClusterIP"}` | Service configuration. |
|
||||||
|
| service.internalPort | int | `9009` | Internal port for the service. |
|
||||||
|
| service.port | int | `8000` | External port for the service. |
|
||||||
|
| service.type | string | `"ClusterIP"` | Type of Kubernetes service to create. |
|
||||||
|
| serviceAccount | object | `{"annotations":{},"automountServiceAccountToken":true,"create":true,"name":""}` | Service account configuration. |
|
||||||
|
| serviceAccount.annotations | object | `{}` | Annotations to add to the service account. |
|
||||||
|
| serviceAccount.automountServiceAccountToken | bool | `true` | Automatically mount a ServiceAccount's API credentials. |
|
||||||
|
| serviceAccount.create | bool | `true` | Specifies whether a service account should be created. |
|
||||||
|
| serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated. |
|
||||||
|
| useRunAIExecutor | bool | `false` | Enable or disable RunAI executor. |
|
||||||
|
| wandb | object | This object has the following default values for the WandB configuration. | WandB configuration. |
|
||||||
|
| wandb.existingSecret | string | `""` | Name of an existing Kubernetes secret resource for the WandB encryption secret. |
|
||||||
|
| wandb.existingSecretKey | string | `""` | Name of the key in the existing WandB secret containing the secret value. The secret value must be exactly 32 alphanumeric characters: ^[a-zA-Z0-9]{32}$ |
|
||||||
|
| wandb.secretValue | string | `"ec60d96b639764ccf9859bc10d4363d1"` | WandB secret value. Must contain exactly 32 alphanumeric characters. Creates a new Kubernetes secret named "wandb-secret" with key-value pair "encryption_key=<wandb.secretValue>". Ignored if wandb.existingSecret is set. |
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
# Patterns to ignore when building packages.
|
||||||
|
# This supports shell glob matching, relative path matching, and
|
||||||
|
# negation (prefixed with !). Only one pattern per line.
|
||||||
|
.DS_Store
|
||||||
|
# Common VCS dirs
|
||||||
|
.git/
|
||||||
|
.gitignore
|
||||||
|
.bzr/
|
||||||
|
.bzrignore
|
||||||
|
.hg/
|
||||||
|
.hgignore
|
||||||
|
.svn/
|
||||||
|
# Common backup files
|
||||||
|
*.swp
|
||||||
|
*.bak
|
||||||
|
*.tmp
|
||||||
|
*.orig
|
||||||
|
*~
|
||||||
|
# Various IDEs
|
||||||
|
.project
|
||||||
|
.idea/
|
||||||
|
*.tmproj
|
||||||
|
.vscode/
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
apiVersion: v2
|
||||||
|
appVersion: latest
|
||||||
|
description: Library chart for common elements across NeMo Microservices helm charts
|
||||||
|
keywords:
|
||||||
|
- library
|
||||||
|
- common
|
||||||
|
kubeVersion: '>=v1.23.0-0'
|
||||||
|
name: nemo-common
|
||||||
|
type: library
|
||||||
|
version: 0.1.0
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
# NeMo Common Library Chart
|
||||||
|
|
||||||
|
This is a library helm chart of templates that are useful for consistency across helm charts and because defined templates exist in the global scope.
|
||||||
|
|
||||||
|
All charts should use it's templates if they have relevant objects defined. When importing the chart dependency, use the version ">=0.1.0-0" to ensure all versions are captured and it stays up to date.
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
{{/*
|
||||||
|
nemo-common.env generates an env var array out of a dict or array of supported values
|
||||||
|
*/}}
|
||||||
|
{{- define "nemo-common.env" -}}
|
||||||
|
{{- if and .Values.env (kindIs "slice" .Values.env) -}}
|
||||||
|
{{- toYaml .Values.env -}}
|
||||||
|
{{- else if and .Values.env (kindIs "map" .Values.env) -}}
|
||||||
|
{{- range $k, $v := .Values.env }}
|
||||||
|
- name: {{ $k }}
|
||||||
|
{{- if kindIs "map" $v }}
|
||||||
|
valueFrom:
|
||||||
|
{{ toYaml $v.valueFrom | nindent 4 | trim }}
|
||||||
|
{{- else }}
|
||||||
|
value: {{ $v | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- else if .Values.envVars }}
|
||||||
|
{{- range $k, $v := .Values.envVars }}
|
||||||
|
- name: {{ $k }}
|
||||||
|
value: {{ $v | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end -}}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
nemo-common.postgresql.uri provides a template for constructing a postgresql database
|
||||||
|
connection string as an environment variable value
|
||||||
|
*/}}
|
||||||
|
{{- define "nemo-common.postgresql.uri" -}}
|
||||||
|
{{- if and .Values.externalDatabase.uriSecret .Values.externalDatabase.uriSecret.name .Values.externalDatabase.uriSecret.key }}
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: {{ .Values.externalDatabase.uriSecret.name }}
|
||||||
|
key: {{ .Values.externalDatabase.uriSecret.key }}
|
||||||
|
{{- else }}
|
||||||
|
value: {{ printf "postgresql://%s:$(POSTGRES_DB_PASSWORD)@%s:%s/%s" (include "nemo-common.postgresql.user" .) (include "nemo-common.postgresql.host" .) (include "nemo-common.postgresql.port" .) (include "nemo-common.postgresql.name" .) | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
PostgreSQL Hostname
|
||||||
|
*/}}
|
||||||
|
{{- define "nemo-common.postgresql.host" -}}
|
||||||
|
{{- if .Values.postgresql.enabled -}}
|
||||||
|
{{ template "postgresql.v1.primary.fullname" .Subcharts.postgresql }}
|
||||||
|
{{- else -}}
|
||||||
|
{{ .Values.externalDatabase.host }}
|
||||||
|
{{- end -}}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
nemo-common.database.port chooses between externalDatabase and the bitnami postgresql port values
|
||||||
|
*/}}
|
||||||
|
{{- define "nemo-common.postgresql.port" -}}
|
||||||
|
{{- if .Values.postgresql.enabled -}}
|
||||||
|
{{- printf "%d" (.Values.postgresql.primary.service.ports.postgresql | int) -}}
|
||||||
|
{{- else -}}
|
||||||
|
{{- printf "%d" (.Values.externalDatabase.port | int) -}}
|
||||||
|
{{- end -}}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
nemo-common.database.user chooses between externalDatabase and the bitnami postgresql user values
|
||||||
|
*/}}
|
||||||
|
{{- define "nemo-common.postgresql.user" -}}
|
||||||
|
{{- if .Values.postgresql.enabled -}}
|
||||||
|
{{- print .Values.postgresql.auth.username -}}
|
||||||
|
{{- else -}}
|
||||||
|
{{- print .Values.externalDatabase.user -}}
|
||||||
|
{{- end -}}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
nemo-common.database.name chooses between externalDatabase and the bitnami postgresql db name values
|
||||||
|
*/}}
|
||||||
|
{{- define "nemo-common.postgresql.name" -}}
|
||||||
|
{{- if .Values.postgresql.enabled -}}
|
||||||
|
{{- print .Values.postgresql.auth.database -}}
|
||||||
|
{{- else -}}
|
||||||
|
{{- print .Values.externalDatabase.database -}}
|
||||||
|
{{- end -}}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
nemo-common.database.secret-name chooses between externalDatabase and the bitnami postgresql existing secret values
|
||||||
|
*/}}
|
||||||
|
{{- define "nemo-common.postgresql.secret-name" -}}
|
||||||
|
{{- if .Values.postgresql.enabled -}}
|
||||||
|
{{- if .Values.postgresql.auth.existingSecret -}}
|
||||||
|
{{- print .Values.postgresql.auth.existingSecret -}}
|
||||||
|
{{- else -}}
|
||||||
|
{{- template "postgresql.v1.primary.fullname" .Subcharts.postgresql -}}
|
||||||
|
{{- end -}}
|
||||||
|
{{- else if .Values.externalDatabase.existingSecret -}}
|
||||||
|
{{- print .Values.externalDatabase.existingSecret -}}
|
||||||
|
{{- end -}}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
nemo-common.database.password-key chooses between externalDatabase and the bitnami postgresql existing secret key values
|
||||||
|
*/}}
|
||||||
|
{{- define "nemo-common.postgresql.password-key" -}}
|
||||||
|
{{- if or .Values.postgresql.enabled (not .Values.externalDatabase.existingSecret) -}}
|
||||||
|
{{- print "password" -}}
|
||||||
|
{{- else -}}
|
||||||
|
{{- print .Values.externalDatabase.existingSecretPasswordKey -}}
|
||||||
|
{{- end -}}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
nemo-common.database.password generates a POSTGRES_DB_PASSWORD environment value if a full URI isn't used
|
||||||
|
*/}}
|
||||||
|
{{- define "nemo-common.postgresql.password" -}}
|
||||||
|
{{- if not (and .Values.externalDatabase.uriSecret .Values.externalDatabase.uriSecret.name .Values.externalDatabase.uriSecret.key) }}
|
||||||
|
- name: POSTGRES_DB_PASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: {{ include "nemo-common.postgresql.secret-name" .}}
|
||||||
|
key: {{ include "nemo-common.postgresql.password-key" .}}
|
||||||
|
{{- end }}
|
||||||
|
{{- end -}}
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
{{/*
|
||||||
|
Image Definition Parsing
|
||||||
|
Favor not using a separate registry because it is confusing, but support it.
|
||||||
|
*/}}
|
||||||
|
{{- define "nemo-common.mainimage" -}}
|
||||||
|
{{- if .Values.image.registry -}}
|
||||||
|
{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}
|
||||||
|
{{- else -}}
|
||||||
|
{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Image Pull Secrets
|
||||||
|
Use globals if they are there, but define sensibly.
|
||||||
|
*/}}
|
||||||
|
{{- define "nemo-common.imagepullsecrets" -}}
|
||||||
|
{{- if and .Values.global .Values.global.imagePullSecrets }}
|
||||||
|
{{- range .Values.global.imagePullSecrets }}
|
||||||
|
- {{ . | toYaml }}
|
||||||
|
{{- end }}
|
||||||
|
{{- else }}
|
||||||
|
{{- range .Values.imagePullSecrets }}
|
||||||
|
- {{ . | toYaml }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
{{/*
|
||||||
|
Global level service name definition.
|
||||||
|
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
|
||||||
|
This will allow top level overrides of service names when using the platform helm chart
|
||||||
|
*/}}
|
||||||
|
{{- define "nemo-common.servicename" -}}
|
||||||
|
{{- if .Values.serviceName }}
|
||||||
|
{{- .Values.serviceName | trunc 63 | trimSuffix "-" }}
|
||||||
|
{{- else }}
|
||||||
|
{{- if .Values.fullnameOverride }}
|
||||||
|
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
|
||||||
|
{{- else }}
|
||||||
|
{{- $name := default .Chart.Name .Values.nameOverride }}
|
||||||
|
{{- if contains $name .Release.Name }}
|
||||||
|
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
|
||||||
|
{{- else }}
|
||||||
|
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
# Patterns to ignore when building packages.
|
||||||
|
# This supports shell glob matching, relative path matching, and
|
||||||
|
# negation (prefixed with !). Only one pattern per line.
|
||||||
|
.DS_Store
|
||||||
|
# Common VCS dirs
|
||||||
|
.git/
|
||||||
|
.gitignore
|
||||||
|
.bzr/
|
||||||
|
.bzrignore
|
||||||
|
.hg/
|
||||||
|
.hgignore
|
||||||
|
.svn/
|
||||||
|
# Common backup files
|
||||||
|
*.swp
|
||||||
|
*.bak
|
||||||
|
*.tmp
|
||||||
|
*.orig
|
||||||
|
*~
|
||||||
|
# Various IDEs
|
||||||
|
.project
|
||||||
|
.idea/
|
||||||
|
*.tmproj
|
||||||
|
.vscode/
|
||||||
|
|
||||||
|
# Ignore unittest
|
||||||
|
tests/
|
||||||
|
*/__snapshot__/*
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
# Collector Chart Contributing Guide
|
||||||
|
|
||||||
|
All changes to the chart require a bump to the version in `chart.yaml`. See the [Contributing Guide](https://github.com/open-telemetry/opentelemetry-helm-charts/blob/main/CONTRIBUTING.md#versioning) for our versioning requirements.
|
||||||
|
|
||||||
|
Once the chart version is bumped, the examples must be regenerated. You can regenerate examples by running `make generate-examples CHARTS=opentelemetry-collector`.
|
||||||
|
|
||||||
|
## Bumping Default Collector Version
|
||||||
|
|
||||||
|
1. Increase the minor version of the chart by one and set the patch version to zero.
|
||||||
|
2. Update the chart's `appVersion` to match the new collector version. This version will be used as the image tag by default.
|
||||||
|
3. Review the corresponding release notes in [Collector Core](https://github.com/open-telemetry/opentelemetry-collector/releases), [Collector Contrib](https://github.com/open-telemetry/opentelemetry-collector-contrib/releases), and [Collector Releases](https://github.com/open-telemetry/opentelemetry-collector-releases/releases). If any changes affect the helm charts, adjust the helm chart accordingly.
|
||||||
|
4. Run `make generate-examples CHARTS=opentelemetry-collector`.
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
apiVersion: v2
|
||||||
|
appVersion: 0.102.1
|
||||||
|
description: OpenTelemetry Collector Helm chart for Kubernetes
|
||||||
|
home: https://opentelemetry.io/
|
||||||
|
icon: https://opentelemetry.io/img/logos/opentelemetry-logo-nav.png
|
||||||
|
maintainers:
|
||||||
|
- name: dmitryax
|
||||||
|
- name: TylerHelmuth
|
||||||
|
name: opentelemetry-collector
|
||||||
|
sources:
|
||||||
|
- https://github.com/open-telemetry/opentelemetry-collector
|
||||||
|
- https://github.com/open-telemetry/opentelemetry-collector-contrib
|
||||||
|
type: application
|
||||||
|
version: 0.93.3
|
||||||
@@ -0,0 +1,251 @@
|
|||||||
|
# OpenTelemetry Collector Helm Chart
|
||||||
|
|
||||||
|
The helm chart installs [OpenTelemetry Collector](https://github.com/open-telemetry/opentelemetry-collector)
|
||||||
|
in kubernetes cluster.
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
|
||||||
|
- Kubernetes 1.24+
|
||||||
|
- Helm 3.9+
|
||||||
|
|
||||||
|
## Installing the Chart
|
||||||
|
|
||||||
|
Add OpenTelemetry Helm repository:
|
||||||
|
|
||||||
|
```console
|
||||||
|
helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
|
||||||
|
```
|
||||||
|
|
||||||
|
To install the chart with the release name my-opentelemetry-collector, run the following command:
|
||||||
|
|
||||||
|
```console
|
||||||
|
helm install my-opentelemetry-collector open-telemetry/opentelemetry-collector --set mode=<value> --set image.repository="otel/opentelemetry-collector-k8s" --set command.name="otelcol-k8s"
|
||||||
|
```
|
||||||
|
|
||||||
|
Where the `mode` value needs to be set to one of `daemonset`, `deployment` or `statefulset`.
|
||||||
|
|
||||||
|
For an in-depth walk through getting started in Kubernetes using this helm chart, see [OpenTelemetry Kubernetes Getting Started](https://opentelemetry.io/docs/kubernetes/getting-started/).
|
||||||
|
|
||||||
|
## Upgrading
|
||||||
|
|
||||||
|
See [UPGRADING.md](UPGRADING.md).
|
||||||
|
|
||||||
|
## Security Considerations
|
||||||
|
|
||||||
|
OpenTelemetry Collector recommends to bind receivers' servers to addresses that limit connections to authorized users.
|
||||||
|
For this reason, by default the chart binds all the Collector's endpoints to the pod's IP.
|
||||||
|
|
||||||
|
More info is available in the [Security Best Practices docummentation](https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks)
|
||||||
|
|
||||||
|
Some care must be taken when using `hostNetwork: true`, as then OpenTelemetry Collector will listen on all the addresses in the host network namespace.
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
### Default configuration
|
||||||
|
|
||||||
|
By default this chart will deploy an OpenTelemetry Collector with three pipelines (logs, metrics and traces)
|
||||||
|
and debug exporter enabled by default. The collector can be installed either as daemonset (agent), deployment or stateful set.
|
||||||
|
|
||||||
|
*Example*: Install collector as a deployment.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
mode: deployment
|
||||||
|
```
|
||||||
|
|
||||||
|
By default collector has the following receivers enabled:
|
||||||
|
|
||||||
|
- **metrics**: OTLP and prometheus. Prometheus is configured only for scraping collector's own metrics.
|
||||||
|
- **traces**: OTLP, zipkin and jaeger (thrift and grpc).
|
||||||
|
- **logs**: OTLP (to enable container logs, see [Configuration for Kubernetes container logs](#configuration-for-kubernetes-container-logs)).
|
||||||
|
|
||||||
|
### Basic Top Level Configuration
|
||||||
|
|
||||||
|
The Collector's configuration is set via the `config` section. Default components can be removed with `null`. Remember that lists in helm are not merged, so if you want to modify any default list you must specify all items, including any default items you want to keep.
|
||||||
|
|
||||||
|
*Example*: Disable metrics and logs pipelines and non-otlp receivers:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
config:
|
||||||
|
receivers:
|
||||||
|
jaeger: null
|
||||||
|
prometheus: null
|
||||||
|
zipkin: null
|
||||||
|
service:
|
||||||
|
pipelines:
|
||||||
|
traces:
|
||||||
|
receivers:
|
||||||
|
- otlp
|
||||||
|
metrics: null
|
||||||
|
logs: null
|
||||||
|
```
|
||||||
|
|
||||||
|
The chart also provides several presets, detailed below, to help configure important Kubernetes components. For more details on each component, see [Kubernetes Collector Components](https://opentelemetry.io/docs/kubernetes/collector/components/).
|
||||||
|
|
||||||
|
### Configuration for Kubernetes Container Logs
|
||||||
|
|
||||||
|
The collector can be used to collect logs sent to standard output by Kubernetes containers.
|
||||||
|
This feature is disabled by default. It has the following requirements:
|
||||||
|
|
||||||
|
- It needs agent collector to be deployed.
|
||||||
|
- It requires the [Filelog receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#filelog-receiver) to be included in the collector, such as [k8s](https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s) version of the collector image.
|
||||||
|
|
||||||
|
To enable this feature, set the `presets.logsCollection.enabled` property to `true`.
|
||||||
|
Here is an example `values.yaml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
mode: daemonset
|
||||||
|
|
||||||
|
presets:
|
||||||
|
logsCollection:
|
||||||
|
enabled: true
|
||||||
|
includeCollectorLogs: true
|
||||||
|
```
|
||||||
|
|
||||||
|
The way this feature works is it adds a `filelog` receiver on the `logs` pipeline. This receiver is preconfigured
|
||||||
|
to read the files where Kubernetes container runtime writes all containers' console output to.
|
||||||
|
|
||||||
|
#### :warning: Warning: Risk of looping the exported logs back into the receiver, causing "log explosion"
|
||||||
|
|
||||||
|
The container logs pipeline uses the `debug` exporter by default.
|
||||||
|
Paired with the default `filelog` receiver that receives all containers' console output,
|
||||||
|
it is easy to accidentally feed the exported logs back into the receiver.
|
||||||
|
|
||||||
|
Also note that using the `--verbosity=detailed` option for the `debug` exporter causes it to output
|
||||||
|
multiple lines per single received log, which when looped, would amplify the logs exponentially.
|
||||||
|
|
||||||
|
To prevent the looping, the default configuration of the receiver excludes logs from the collector's containers.
|
||||||
|
|
||||||
|
If you want to include the collector's logs, make sure to replace the `debug` exporter
|
||||||
|
with an exporter that does not send logs to collector's standard output.
|
||||||
|
|
||||||
|
Here's an example `values.yaml` file that replaces the default `debug` exporter on the `logs` pipeline
|
||||||
|
with an `otlphttp` exporter that sends the container logs to `https://example.com:55681` endpoint.
|
||||||
|
It also clears the `filelog` receiver's `exclude` property, for collector logs to be included in the pipeline.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
mode: daemonset
|
||||||
|
|
||||||
|
presets:
|
||||||
|
logsCollection:
|
||||||
|
enabled: true
|
||||||
|
includeCollectorLogs: true
|
||||||
|
|
||||||
|
config:
|
||||||
|
exporters:
|
||||||
|
otlphttp:
|
||||||
|
endpoint: https://example.com:55681
|
||||||
|
service:
|
||||||
|
pipelines:
|
||||||
|
logs:
|
||||||
|
exporters:
|
||||||
|
- otlphttp
|
||||||
|
```
|
||||||
|
|
||||||
|
### Configuration for Kubernetes Attributes Processor
|
||||||
|
|
||||||
|
The collector can be configured to add Kubernetes metadata, such as pod name and namespace name, as resource attributes to incoming logs, metrics and traces.
|
||||||
|
|
||||||
|
This feature is disabled by default. It has the following requirements:
|
||||||
|
|
||||||
|
- It requires the [Kubernetes Attributes processor](https://opentelemetry.io/docs/kubernetes/collector/components/#kubernetes-attributes-processor) to be included in the collector, such as [k8s](https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s) version of the collector image.
|
||||||
|
|
||||||
|
To enable this feature, set the `presets.kubernetesAttributes.enabled` property to `true`.
|
||||||
|
Here is an example `values.yaml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
mode: daemonset
|
||||||
|
presets:
|
||||||
|
kubernetesAttributes:
|
||||||
|
enabled: true
|
||||||
|
# You can also configure the preset to add all of the associated pod's labels and annotations to you telemetry.
|
||||||
|
# The label/annotation name will become the resource attribute's key.
|
||||||
|
extractAllPodLabels: true
|
||||||
|
extractAllPodAnnotations: true
|
||||||
|
```
|
||||||
|
|
||||||
|
### Configuration for Retrieving Kubelet Metrics
|
||||||
|
|
||||||
|
The collector can be configured to collect node, pod, and container metrics from the API server on a kubelet.
|
||||||
|
|
||||||
|
This feature is disabled by default. It has the following requirements:
|
||||||
|
|
||||||
|
- It requires the [Kubeletstats receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#kubeletstats-receiver) to be included in the collector, such as [k8s](https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s) version of the collector image.
|
||||||
|
|
||||||
|
To enable this feature, set the `presets.kubeletMetrics.enabled` property to `true`.
|
||||||
|
Here is an example `values.yaml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
mode: daemonset
|
||||||
|
presets:
|
||||||
|
kubeletMetrics:
|
||||||
|
enabled: true
|
||||||
|
```
|
||||||
|
|
||||||
|
### Configuration for Kubernetes Cluster Metrics
|
||||||
|
|
||||||
|
The collector can be configured to collects cluster-level metrics from the Kubernetes API server. A single instance of this receiver can be used to monitor a cluster.
|
||||||
|
|
||||||
|
This feature is disabled by default. It has the following requirements:
|
||||||
|
|
||||||
|
- It requires the [Kubernetes Cluster receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#kubernetes-cluster-receiver) to be included in the collector, such as [k8s](https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s) version of the collector image.
|
||||||
|
- It requires statefulset or deployment mode with a single replica.
|
||||||
|
|
||||||
|
To enable this feature, set the `presets.clusterMetrics.enabled` property to `true`.
|
||||||
|
|
||||||
|
Here is an example `values.yaml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
mode: deployment
|
||||||
|
replicaCount: 1
|
||||||
|
presets:
|
||||||
|
clusterMetrics:
|
||||||
|
enabled: true
|
||||||
|
```
|
||||||
|
|
||||||
|
### Configuration for Retrieving Kubernetes Events
|
||||||
|
|
||||||
|
The collector can be configured to collect Kubernetes events.
|
||||||
|
|
||||||
|
This feature is disabled by default. It has the following requirements:
|
||||||
|
|
||||||
|
- It requires [Kubernetes Objects receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#kubernetes-objects-receiver) to be included in the collector, such as [k8s](https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s) version of the collector image.
|
||||||
|
|
||||||
|
To enable this feature, set the `presets.kubernetesEvents.enabled` property to `true`.
|
||||||
|
Here is an example `values.yaml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
mode: deployment
|
||||||
|
replicaCount: 1
|
||||||
|
presets:
|
||||||
|
kubernetesEvents:
|
||||||
|
enabled: true
|
||||||
|
```
|
||||||
|
|
||||||
|
### Configuration for Host Metrics
|
||||||
|
|
||||||
|
The collector can be configured to collect host metrics for Kubernetes nodes.
|
||||||
|
|
||||||
|
This feature is disabled by default. It has the following requirements:
|
||||||
|
|
||||||
|
- It requires [Host Metrics receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#host-metrics-receiver) to be included in the collector, such as [k8s](https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s) version of the collector image.
|
||||||
|
|
||||||
|
To enable this feature, set the `presets.hostMetrics.enabled` property to `true`.
|
||||||
|
Here is an example `values.yaml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
mode: daemonset
|
||||||
|
presets:
|
||||||
|
hostMetrics:
|
||||||
|
enabled: true
|
||||||
|
```
|
||||||
|
|
||||||
|
## CRDs
|
||||||
|
|
||||||
|
At this time, Prometheus CRDs are supported but other CRDs are not.
|
||||||
|
|
||||||
|
### Other configuration options
|
||||||
|
|
||||||
|
The [values.yaml](./values.yaml) file contains information about all other configuration
|
||||||
|
options for this chart.
|
||||||
|
|
||||||
|
For more examples see [Examples](examples).
|
||||||
@@ -0,0 +1,424 @@
|
|||||||
|
# Upgrade guidelines
|
||||||
|
|
||||||
|
These upgrade guidelines only contain instructions for version upgrades which require manual modifications on the user's side.
|
||||||
|
If the version you want to upgrade to is not listed here, then there is nothing to do for you.
|
||||||
|
Just upgrade and enjoy.
|
||||||
|
|
||||||
|
## Feature Flags
|
||||||
|
|
||||||
|
### useGOMEMLIMIT
|
||||||
|
|
||||||
|
A new flag, `useGOMEMLIMIT` has been added that allows specifying whether or not the chart should use the `GOMEMLIMIT` environment variable or the Memory Ballast Extension.
|
||||||
|
When enabled, the chart will remove the Memory Ballast Extension from the collector's configuration AND will setup a `GOMEMLIMIT` environment variable that is set to 80%
|
||||||
|
of the configured `resources.limits.memory`. If no `resources.limits.memory` are set when `useGOMEMLIMIT` is enabled then a `GOMEMLIMIT` environment variable WILL NOT be
|
||||||
|
created but the Memory Ballast Extension will still be removed.
|
||||||
|
|
||||||
|
Depending on the progress made in [Issue 891](https://github.com/open-telemetry/opentelemetry-helm-charts/issues/891),
|
||||||
|
the use of `GOMEMLIMIT` may completely replace the Memory Ballast Extension in the future.
|
||||||
|
|
||||||
|
## 0.88.0 to 0.89.0
|
||||||
|
|
||||||
|
> [!WARNING]
|
||||||
|
> Critical content demanding immediate user attention due to potential risks.
|
||||||
|
|
||||||
|
As part of working towards using the [OpenTelemetry Collector Kubernetes Distro](https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s) by default, the chart now requires users to explicitly set an image repository. If you are already explicitly setting an image repository this breaking change does not affect you.
|
||||||
|
|
||||||
|
If you are using a OpenTelemetry Community distribution of the Collector we recommend you use `otel/opentelemetry-collector-k8s`, but carefully review the [components included in this distribution](https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/distributions/otelcol-k8s/manifest.yaml) to make sure it includes all the components you use in your configuration. In the future this distribution will become the default image used for the chart.
|
||||||
|
|
||||||
|
You can use the OpenTelemetry Collector Kubernetes Distro by adding these lines to your values.yaml:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
```
|
||||||
|
|
||||||
|
If you want to stick with using the Contrib distribution, add these lines to your values.yaml:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-contrib"
|
||||||
|
```
|
||||||
|
|
||||||
|
For more details see [#1135](https://github.com/open-telemetry/opentelemetry-helm-charts/issues/1135).
|
||||||
|
|
||||||
|
## 0.84.0 to 0.85.0
|
||||||
|
|
||||||
|
The `loggingexporter` has been removed from the default configuration. Use the `debugexporter` instead.
|
||||||
|
|
||||||
|
## 0.78.2 to 0.78.3
|
||||||
|
|
||||||
|
[Update Health Check Extension's endpoints to use Pod IP Instead of 0.0.0.0](https://github.com/open-telemetry/opentelemetry-helm-charts/pull/1012)
|
||||||
|
|
||||||
|
The [Collector's security guidelines were updated](https://github.com/open-telemetry/opentelemetry-collector/pull/6959) to include containerized environments when discussing safeguards against denial of service attacks.
|
||||||
|
To be in compliance with the Collector's security best practices the chart has been updated to use the Collector's pod IP in place of `0.0.0.0`.
|
||||||
|
|
||||||
|
The chart will continue to allow complete configuration of the Collector via the `config` field in the values.yaml. If pod IP does not suite your needs you can use `config` to set something different.
|
||||||
|
|
||||||
|
See [Security Best Practices docummentation](https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks) for more details.
|
||||||
|
|
||||||
|
## 0.75.1 to 0.76.0
|
||||||
|
|
||||||
|
Enable the `useGOMEMLIMIT` feature flag by default. This means by default the chart now does not use the Memory Ballast Extension and any custom configuraiton applied to the Memory Ballast Extension is ignored.
|
||||||
|
|
||||||
|
**If you're still interested in using the Memory Ballast Extension set this back to false.**
|
||||||
|
|
||||||
|
## 0.69.3 to 0.70.0
|
||||||
|
|
||||||
|
The following deprecated fields have been removed. Please use the new values:
|
||||||
|
|
||||||
|
- `extraConfigMapMounts` -> `extraVolumes`
|
||||||
|
- `extraHostPathMounts` -> `extraVolumes`
|
||||||
|
- `secretMounts` -> `extraVolumes`
|
||||||
|
- `containerLogs` -> `presets.logsCollection`
|
||||||
|
|
||||||
|
## 0.69.0 to 0.69.1 & 0.69.2
|
||||||
|
|
||||||
|
The `loggingexporter` was replaced with the `debugexporter`. This ended up being an accidental breaking change for any user that depended on the default logging exporter config when explicitly listing the logging exporter in an exporter list.
|
||||||
|
|
||||||
|
When using versions `0.69.1` or `0.69.2` you should explicitly list the debugging exporter instead of the logging exporter. You other option is to skip these version and use `0.69.3` or newer, which includes the logging exporter configuration.
|
||||||
|
|
||||||
|
**The logging exporter will be removed in a future version.** We highly recommend switching to the debug exporter.
|
||||||
|
|
||||||
|
## 0.67 to 0.68
|
||||||
|
|
||||||
|
The `preset.kubernetesEvents` preset now excludes `DELETED` watch types so that an log is not ingested when Kubernetes deletes an event.
|
||||||
|
The intention behind this change is to cleanup the data ingested by the preset as the `DELETED` updated for a Kubernetes Events is
|
||||||
|
uninteresting. If you want to keep ingesting `DELETED` updates for Kubernetes Events you will need to configure the `k8sobjectsreceiver` manually.
|
||||||
|
|
||||||
|
## 0.62 to 0.63
|
||||||
|
|
||||||
|
The `kubernetesAttributes` preset now respects order of processors in logs, metrics and traces pipelines.
|
||||||
|
This implicitly might break your pipelines if you relied on having the `k8sAttributes` processor rendered as the first processor but also explicitly listed it in the signal's pipeline somewhere else.
|
||||||
|
|
||||||
|
## 0.55.2 to 0.56
|
||||||
|
|
||||||
|
The `tpl` function has been added to references of pod labels and ingress hosts. This adds the ability to add some reusability in
|
||||||
|
charts values through referencing global values. If you are currently using any `{{ }}` syntax in pod labels or ingress hosts it will now be rendered. To escape existing instances of {{ }}, use {{` <original content> `}}.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
global:
|
||||||
|
region: us-east-1
|
||||||
|
environment: stage
|
||||||
|
|
||||||
|
# Tests `tpl` function reference used in pod labels and
|
||||||
|
# ingress.hosts[*]
|
||||||
|
podLabels:
|
||||||
|
environment: "{{ .Values.global.environment }}"
|
||||||
|
|
||||||
|
ingress:
|
||||||
|
enabled: true
|
||||||
|
hosts:
|
||||||
|
- host: "otlp-collector-{{ .Values.global.region }}-{{ .Values.global.environment }}-example.dev"
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
port: 4318
|
||||||
|
```
|
||||||
|
|
||||||
|
Note that only global Helm values can be referenced as the Helm Chart schema currently does not allow `additionalValues`.
|
||||||
|
|
||||||
|
## 0.55.0 to 0.55.1
|
||||||
|
|
||||||
|
As of v0.55.1 Collector chart use `${env:ENV}` style syntax when getting environment variables and that $`{env:ENV}` syntax is not supported before collector 0.71. If you upgrade collector chart to v0.55.1, you need to make sure your collector version is after than 0.71 (default is v0.76.1).
|
||||||
|
|
||||||
|
## 0.53.1 to 0.54.0
|
||||||
|
|
||||||
|
As of v0.54.0 Collector chart, the default resource limits are removed. If you want to keep old values you can use the following configuration:
|
||||||
|
|
||||||
|
```
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
# CPU units are in fractions of 1000; memory in powers of 2
|
||||||
|
cpu: 250m
|
||||||
|
memory: 512Mi
|
||||||
|
```
|
||||||
|
|
||||||
|
See [the 644 issue](https://github.com/open-telemetry/opentelemetry-helm-charts/issues/644) for more information.
|
||||||
|
|
||||||
|
## 0.46.0 to 0.47.0
|
||||||
|
|
||||||
|
[Update Collector Endpoints to use Pod IP Instead of 0.0.0.0](https://github.com/open-telemetry/opentelemetry-helm-charts/pull/603)
|
||||||
|
|
||||||
|
The [Collector's security guidelines were updated](https://github.com/open-telemetry/opentelemetry-collector/pull/6959) to include containerized environments when discussing safeguards against denial of service attacks.
|
||||||
|
To be in compliance with the Collector's security best practices the chart has been updated to use the Collector's pod IP in place of `0.0.0.0`.
|
||||||
|
|
||||||
|
The chart will continue to allow complete configuration of the Collector via the `config` field in the values.yaml. If pod IP does not suite your needs you can use `config` to set something different.
|
||||||
|
|
||||||
|
See [Security Best Practices docummentation](https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks) for more details.
|
||||||
|
|
||||||
|
The new default of binding to the pod IP, rather than `0.0.0.0`, will cause `kubectl port-forward` to fail. If port-forwarding is desired, the following `value.yaml` snippet will allow the Collector bind to `127.0.0.1` inside the pod, in addition to the pod's IP:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
config:
|
||||||
|
receivers:
|
||||||
|
jaeger/local:
|
||||||
|
protocols:
|
||||||
|
grpc:
|
||||||
|
endpoint: 127.0.0.1:14250
|
||||||
|
thrift_compact:
|
||||||
|
endpoint: 127.0.0.1:6831
|
||||||
|
thrift_http:
|
||||||
|
endpoint: 127.0.0.1:14268
|
||||||
|
otlp/local:
|
||||||
|
protocols:
|
||||||
|
grpc:
|
||||||
|
endpoint: 127.0.0.1:4317
|
||||||
|
http:
|
||||||
|
endpoint: 127.0.0.1:4318
|
||||||
|
zipkin/local:
|
||||||
|
endpoint: 127.0.0.1:9411
|
||||||
|
service:
|
||||||
|
pipelines:
|
||||||
|
traces:
|
||||||
|
receivers:
|
||||||
|
- otlp
|
||||||
|
- otlp/local
|
||||||
|
- jaeger
|
||||||
|
- jaeger/local
|
||||||
|
- zipkin
|
||||||
|
- zipkin/local
|
||||||
|
```
|
||||||
|
|
||||||
|
## 0.40.7 to 0.41.0
|
||||||
|
|
||||||
|
[Require Kubernetes version 1.23 or later](https://github.com/open-telemetry/opentelemetry-helm-charts/pull/541)
|
||||||
|
|
||||||
|
If you enable use of a _HorizontalPodAutoscaler_ for the collector when running in the "deployment" mode by way of `.Values.autoscaling.enabled`, the manifest now uses the "autoscaling/v2" API group version, which [is available only as recently as Kubernetes version 1.23](https://kubernetes.io/blog/2021/12/07/kubernetes-1-23-release-announcement/#horizontalpodautoscaler-v2-graduates-to-ga). As [all previous versions of this API group are deprecated and removed as of Kubernetes version 1.26](https://kubernetes.io/docs/reference/using-api/deprecation-guide/#horizontalpodautoscaler-v126), we don't offer support for Kubernetes versions older than 1.23.
|
||||||
|
|
||||||
|
## 0.34.0 to 0.34.0
|
||||||
|
|
||||||
|
[config supports templating](TBD)
|
||||||
|
|
||||||
|
The chart now supports templating in `.Values.config`. If you are currently using any `{{ }}` syntax in `.Values.yaml` it will now be rendered. To escape existing instances of `{{ }}`, use ``` {{` <original content> `}} ```. For example, `{{ REDACTED_EMAIL }}` becomes ``` {{` {{ REDACTED_EMAIL }} `}} ```.
|
||||||
|
|
||||||
|
## 0.28.0 to 0.29.0
|
||||||
|
|
||||||
|
[Reduce requested resources](https://github.com/open-telemetry/opentelemetry-helm-charts/pull/273)
|
||||||
|
|
||||||
|
Resource `limits` have been reduced. Upgrades/installs of chart 0.29.0 will now use fewer resources. In order to set the resources back to what they were, you will need to override the `resources` section in the `values.yaml`.
|
||||||
|
|
||||||
|
*Example*:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 1
|
||||||
|
memory: 2Gi
|
||||||
|
```
|
||||||
|
|
||||||
|
## 0.23.1 to 0.24.0
|
||||||
|
|
||||||
|
[Remove containerLogs in favor of presets.logsCollection]()
|
||||||
|
|
||||||
|
The ability to enable logs collection from the collector has been moved from `containerLogs.enabled` to `presets.logsCollection.enabled`. If you are currently using `containerLogs.enabled`, you should instead use the preset:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
presets:
|
||||||
|
logsCollection:
|
||||||
|
enabled: true
|
||||||
|
```
|
||||||
|
|
||||||
|
If you are using `containerLogs.enabled` and also enabling collection of the collector logs you can use `includeCollectorLogs`
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
presets:
|
||||||
|
logsCollection:
|
||||||
|
enabled: true
|
||||||
|
includeCollectorLogs: true
|
||||||
|
```
|
||||||
|
|
||||||
|
You no longer need to update `config.service.pipelines.logs` to include the filelog receiver yourself as the preset will automatically update the logs pipeline to include the filelog receiver.
|
||||||
|
|
||||||
|
The filelog's preset configuration can modified by `config.receivers`, but preset configuration cannot be removed. If you need to remove any filelog receiver configuration generated by the preset you should not use the preset. Instead, configure the filelog receiver manually in `config.receivers` and set any other necessary fields in the values.yaml to modify k8s as needed.
|
||||||
|
|
||||||
|
See the [daemonset-collector-logs example](https://github.com/open-telemetry/opentelemetry-helm-charts/tree/main/charts/opentelemetry-collector/examples/daemonset-collector-logs) to see an example of the preset in action.
|
||||||
|
|
||||||
|
## 0.18.0 to 0.19.0
|
||||||
|
|
||||||
|
[Remove agentCollector and standaloneCollector settings](https://github.com/open-telemetry/opentelemetry-helm-charts/pull/216)
|
||||||
|
|
||||||
|
The `agentCollector` and `standaloneCollector` config sections have been removed. Upgrades/installs of chart 0.19.0 will fail if `agentCollector` or `standaloneCollector` are in the values.yaml. See the [Migrate to mode](#migrate-to-mode) steps for instructions on how to replace `agentCollector` and `standaloneCollector` with `mode`.
|
||||||
|
|
||||||
|
## 0.13.0 to 0.14.0
|
||||||
|
|
||||||
|
[Remove two-deployment mode](https://github.com/open-telemetry/opentelemetry-helm-charts/pull/159)
|
||||||
|
|
||||||
|
The ability to install both the agent and standalone collectors simultaneous with the chart has been removed. Installs/upgrades where both `.Values.agentCollector.enabled` and `.Values.standloneCollector.enables` are true will fail. `agentCollector` and `standloneCollector` have also be deprecated, but backward compatibility has been maintained.
|
||||||
|
|
||||||
|
### To run both a deployment and daemonset
|
||||||
|
|
||||||
|
Install a deployment version of the collector. This is done by setting `.Values.mode` to `deployment`
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
mode: deployment
|
||||||
|
```
|
||||||
|
|
||||||
|
Next, install an daemonset version of the collector that is configured to send traffic to the previously installed deployment. This is done by setting `.Values.mode` to `daemonset` and updating `.Values.config` so that data is exported to the deployment.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
mode: daemonset
|
||||||
|
|
||||||
|
config:
|
||||||
|
exporters:
|
||||||
|
otlp:
|
||||||
|
endpoint: example-opentelemetry-collector:4317
|
||||||
|
tls:
|
||||||
|
insecure: true
|
||||||
|
service:
|
||||||
|
pipelines:
|
||||||
|
logs:
|
||||||
|
exporters:
|
||||||
|
- otlp
|
||||||
|
- logging
|
||||||
|
metrics:
|
||||||
|
exporters:
|
||||||
|
- otlp
|
||||||
|
- logging
|
||||||
|
traces:
|
||||||
|
exporters:
|
||||||
|
- otlp
|
||||||
|
- logging
|
||||||
|
```
|
||||||
|
|
||||||
|
See the [daemonset-and-deployment](examples/daemonset-and-deployment) example to see the rendered config.
|
||||||
|
|
||||||
|
### Migrate to `mode`:
|
||||||
|
|
||||||
|
The `agentCollector` and `standaloneCollector` sections in values.yaml have been deprecated. Instead there is a new field, `mode`, that determines if the collector is being installed as a daemonset or deployment.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# Valid values are "daemonset" and "deployment".
|
||||||
|
# If set, agentCollector and standaloneCollector are ignored.
|
||||||
|
mode: <daemonset|deployment>
|
||||||
|
```
|
||||||
|
|
||||||
|
The following fields have also been added to the root-level to replace the depracated `agentCollector` and `standaloneCollector` settings.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
containerLogs:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 1
|
||||||
|
memory: 2Gi
|
||||||
|
|
||||||
|
podAnnotations: {}
|
||||||
|
|
||||||
|
podLabels: {}
|
||||||
|
|
||||||
|
# Host networking requested for this pod. Use the host's network namespace.
|
||||||
|
hostNetwork: false
|
||||||
|
|
||||||
|
# only used with deployment mode
|
||||||
|
replicaCount: 1
|
||||||
|
|
||||||
|
annotations: {}
|
||||||
|
```
|
||||||
|
|
||||||
|
When using `mode`, these settings should be used instead of their counterparts in `agentCollector` and `standaloneCollector`.
|
||||||
|
|
||||||
|
Set `mode` to `daemonset` if `agentCollector` was being used. Move all `agentCollector` settings to the corresponding root-level setting. If `agentCollector.configOverride` was being used, merge the settings with `.Values.config`.
|
||||||
|
|
||||||
|
Example agentCollector values.yaml:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
agentCollector:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 3
|
||||||
|
memory: 6Gi
|
||||||
|
configOverride:
|
||||||
|
receivers:
|
||||||
|
hostmetrics:
|
||||||
|
scrapers:
|
||||||
|
cpu:
|
||||||
|
disk:
|
||||||
|
filesystem:
|
||||||
|
service:
|
||||||
|
pipelines:
|
||||||
|
metrics:
|
||||||
|
receivers: [otlp, prometheus, hostmetrics]
|
||||||
|
```
|
||||||
|
|
||||||
|
Example mode values.yaml:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
mode: daemonset
|
||||||
|
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 3
|
||||||
|
memory: 6Gi
|
||||||
|
|
||||||
|
config:
|
||||||
|
receivers:
|
||||||
|
hostmetrics:
|
||||||
|
scrapers:
|
||||||
|
cpu:
|
||||||
|
disk:
|
||||||
|
filesystem:
|
||||||
|
service:
|
||||||
|
pipelines:
|
||||||
|
metrics:
|
||||||
|
receivers: [otlp, prometheus, hostmetrics]
|
||||||
|
```
|
||||||
|
|
||||||
|
Set `mode` to `deployment` if `standaloneCollector` was being used. Move all `standaloneCollector` settings to the corresponding root-level setting. If `standaloneCollector.configOverride` was being used, merge the settings with `.Values.config`.
|
||||||
|
|
||||||
|
Example standaloneCollector values.yaml:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
standaloneCollector:
|
||||||
|
enabled: true
|
||||||
|
replicaCount: 2
|
||||||
|
configOverride:
|
||||||
|
receivers:
|
||||||
|
podman_stats:
|
||||||
|
endpoint: unix://run/podman/podman.sock
|
||||||
|
timeout: 10s
|
||||||
|
collection_interval: 10s
|
||||||
|
service:
|
||||||
|
pipelines:
|
||||||
|
metrics:
|
||||||
|
receivers: [otlp, prometheus, podman_stats]
|
||||||
|
```
|
||||||
|
|
||||||
|
Example mode values.yaml:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
mode: deployment
|
||||||
|
|
||||||
|
replicaCount: 2
|
||||||
|
|
||||||
|
config:
|
||||||
|
receivers:
|
||||||
|
receivers:
|
||||||
|
podman_stats:
|
||||||
|
endpoint: unix://run/podman/podman.sock
|
||||||
|
timeout: 10s
|
||||||
|
collection_interval: 10s
|
||||||
|
service:
|
||||||
|
pipelines:
|
||||||
|
metrics:
|
||||||
|
receivers: [otlp, prometheus, podman_stats]
|
||||||
|
```
|
||||||
|
|
||||||
|
Default configuration in `.Values.config` can now be removed with `null`. When changing a pipeline, you must explicitly list all the components that are in the pipeline, including any default components.
|
||||||
|
|
||||||
|
*Example*: Disable metrics and logging pipelines and non-otlp receivers:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
config:
|
||||||
|
receivers:
|
||||||
|
jaeger: null
|
||||||
|
prometheus: null
|
||||||
|
zipkin: null
|
||||||
|
service:
|
||||||
|
pipelines:
|
||||||
|
traces:
|
||||||
|
receivers:
|
||||||
|
- otlp
|
||||||
|
metrics: null
|
||||||
|
logs: null
|
||||||
|
```
|
||||||
+9
@@ -0,0 +1,9 @@
|
|||||||
|
mode: deployment
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
|
||||||
|
command:
|
||||||
|
name: "otelcol-k8s"
|
||||||
|
|
||||||
|
useGOMEMLIMIT: true
|
||||||
+27
@@ -0,0 +1,27 @@
|
|||||||
|
mode: daemonset
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
|
||||||
|
command:
|
||||||
|
name: "otelcol-k8s"
|
||||||
|
|
||||||
|
clusterRole:
|
||||||
|
create: true
|
||||||
|
name: "testing-clusterrole"
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- ''
|
||||||
|
resources:
|
||||||
|
- 'pods'
|
||||||
|
- 'nodes'
|
||||||
|
verbs:
|
||||||
|
- 'get'
|
||||||
|
- 'list'
|
||||||
|
- 'watch'
|
||||||
|
clusterRoleBinding:
|
||||||
|
name: "testing-clusterrolebinding"
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 200M
|
||||||
+26
@@ -0,0 +1,26 @@
|
|||||||
|
mode: daemonset
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
|
||||||
|
command:
|
||||||
|
name: "otelcol-k8s"
|
||||||
|
|
||||||
|
config:
|
||||||
|
receivers:
|
||||||
|
jaeger: null
|
||||||
|
otlp: null
|
||||||
|
zipkin: null
|
||||||
|
hostmetrics:
|
||||||
|
scrapers:
|
||||||
|
cpu:
|
||||||
|
disk:
|
||||||
|
filesystem:
|
||||||
|
service:
|
||||||
|
pipelines:
|
||||||
|
metrics:
|
||||||
|
receivers:
|
||||||
|
- prometheus
|
||||||
|
- hostmetrics
|
||||||
|
traces: null
|
||||||
|
logs: null
|
||||||
+12
@@ -0,0 +1,12 @@
|
|||||||
|
mode: daemonset
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
|
||||||
|
command:
|
||||||
|
name: "otelcol-k8s"
|
||||||
|
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 200M
|
||||||
+29
@@ -0,0 +1,29 @@
|
|||||||
|
global:
|
||||||
|
test: templated-value
|
||||||
|
|
||||||
|
mode: deployment
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
|
||||||
|
command:
|
||||||
|
name: "otelcol-k8s"
|
||||||
|
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 200M
|
||||||
|
|
||||||
|
# Tests `tpl` function reference used in pod labels and
|
||||||
|
# ingress.hosts[*]
|
||||||
|
podLabels:
|
||||||
|
testLabel: "{{ .Values.global.test }}"
|
||||||
|
|
||||||
|
ingress:
|
||||||
|
enabled: true
|
||||||
|
hosts:
|
||||||
|
- host: "{{ .Values.global.test }}"
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
port: 4318
|
||||||
+21
@@ -0,0 +1,21 @@
|
|||||||
|
mode: deployment
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
|
||||||
|
command:
|
||||||
|
name: "otelcol-k8s"
|
||||||
|
|
||||||
|
ports:
|
||||||
|
jaeger-compact:
|
||||||
|
enabled: false
|
||||||
|
jaeger-thrift:
|
||||||
|
enabled: false
|
||||||
|
jaeger-grpc:
|
||||||
|
enabled: false
|
||||||
|
zipkin:
|
||||||
|
enabled: false
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 200M
|
||||||
+15
@@ -0,0 +1,15 @@
|
|||||||
|
mode: deployment
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
|
||||||
|
command:
|
||||||
|
name: "otelcol-k8s"
|
||||||
|
|
||||||
|
autoscaling:
|
||||||
|
enabled: true
|
||||||
|
minReplicas: 1
|
||||||
|
maxReplicas: 10
|
||||||
|
behavior: {}
|
||||||
|
targetCPUUtilizationPercentage: 80
|
||||||
|
targetMemoryUtilizationPercentage: 80
|
||||||
+15
@@ -0,0 +1,15 @@
|
|||||||
|
mode: statefulset
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
|
||||||
|
command:
|
||||||
|
name: "otelcol-k8s"
|
||||||
|
|
||||||
|
autoscaling:
|
||||||
|
enabled: true
|
||||||
|
minReplicas: 1
|
||||||
|
maxReplicas: 10
|
||||||
|
behavior: {}
|
||||||
|
targetCPUUtilizationPercentage: 80
|
||||||
|
targetMemoryUtilizationPercentage: 80
|
||||||
+49
@@ -0,0 +1,49 @@
|
|||||||
|
mode: deployment
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
|
||||||
|
command:
|
||||||
|
name: "otelcol-k8s"
|
||||||
|
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 200M
|
||||||
|
|
||||||
|
ingress:
|
||||||
|
enabled: true
|
||||||
|
|
||||||
|
ingressClassName: nginx
|
||||||
|
annotations:
|
||||||
|
test.io/collector: default
|
||||||
|
hosts:
|
||||||
|
- host: defaultcollector.example.com
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
port: 4318
|
||||||
|
|
||||||
|
additionalIngresses:
|
||||||
|
- name: additional-basic
|
||||||
|
hosts:
|
||||||
|
- host: additional-basic.example.com
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
port: 4318
|
||||||
|
|
||||||
|
- name: additional-advanced
|
||||||
|
ingressClassName: nginx
|
||||||
|
annotations:
|
||||||
|
test.io/ingress: additional-advanced
|
||||||
|
hosts:
|
||||||
|
- host: additional-advanced.example.com
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Exact
|
||||||
|
port: 4318
|
||||||
|
tls:
|
||||||
|
- secretName: somesecret
|
||||||
|
hosts:
|
||||||
|
- additional-advanced.example.com
|
||||||
+37
@@ -0,0 +1,37 @@
|
|||||||
|
mode: daemonset
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
|
||||||
|
command:
|
||||||
|
name: "otelcol-k8s"
|
||||||
|
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 200M
|
||||||
|
|
||||||
|
networkPolicy:
|
||||||
|
enabled: true
|
||||||
|
|
||||||
|
allowIngressFrom:
|
||||||
|
- namespaceSelector: {}
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 127.0.0.1/32
|
||||||
|
|
||||||
|
extraIngressRules:
|
||||||
|
- ports:
|
||||||
|
- port: metrics
|
||||||
|
protocol: TCP
|
||||||
|
from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 127.0.0.1/32
|
||||||
|
|
||||||
|
egressRules:
|
||||||
|
- to:
|
||||||
|
- podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app: jaeger
|
||||||
|
ports:
|
||||||
|
- port: 4317
|
||||||
|
protocol: TCP
|
||||||
+15
@@ -0,0 +1,15 @@
|
|||||||
|
mode: deployment
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
|
||||||
|
command:
|
||||||
|
name: "otelcol-k8s"
|
||||||
|
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 200M
|
||||||
|
|
||||||
|
networkPolicy:
|
||||||
|
enabled: true
|
||||||
+16
@@ -0,0 +1,16 @@
|
|||||||
|
mode: deployment
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
|
||||||
|
command:
|
||||||
|
name: "otelcol-k8s"
|
||||||
|
|
||||||
|
presets:
|
||||||
|
clusterMetrics:
|
||||||
|
enabled: true
|
||||||
|
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 200M
|
||||||
+16
@@ -0,0 +1,16 @@
|
|||||||
|
mode: daemonset
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
|
||||||
|
command:
|
||||||
|
name: "otelcol-k8s"
|
||||||
|
|
||||||
|
presets:
|
||||||
|
hostMetrics:
|
||||||
|
enabled: true
|
||||||
|
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 200M
|
||||||
+16
@@ -0,0 +1,16 @@
|
|||||||
|
mode: deployment
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
|
||||||
|
command:
|
||||||
|
name: "otelcol-k8s"
|
||||||
|
|
||||||
|
presets:
|
||||||
|
kubernetesEvents:
|
||||||
|
enabled: true
|
||||||
|
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 200M
|
||||||
+16
@@ -0,0 +1,16 @@
|
|||||||
|
mode: daemonset
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
|
||||||
|
command:
|
||||||
|
name: "otelcol-k8s"
|
||||||
|
|
||||||
|
presets:
|
||||||
|
kubeletMetrics:
|
||||||
|
enabled: true
|
||||||
|
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 200M
|
||||||
+16
@@ -0,0 +1,16 @@
|
|||||||
|
mode: daemonset
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
|
||||||
|
command:
|
||||||
|
name: "otelcol-k8s"
|
||||||
|
|
||||||
|
presets:
|
||||||
|
kubernetesAttributes:
|
||||||
|
enabled: true
|
||||||
|
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 200M
|
||||||
+17
@@ -0,0 +1,17 @@
|
|||||||
|
mode: daemonset
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
|
||||||
|
command:
|
||||||
|
name: "otelcol-k8s"
|
||||||
|
|
||||||
|
presets:
|
||||||
|
logsCollection:
|
||||||
|
enabled: true
|
||||||
|
includeCollectorLogs: true
|
||||||
|
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 200M
|
||||||
+27
@@ -0,0 +1,27 @@
|
|||||||
|
mode: daemonset
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
|
||||||
|
command:
|
||||||
|
name: "otelcol-k8s"
|
||||||
|
|
||||||
|
livenessProbe:
|
||||||
|
initialDelaySeconds: 10
|
||||||
|
periodSeconds: 5
|
||||||
|
timeoutSeconds: 3
|
||||||
|
failureThreshold: 2
|
||||||
|
terminationGracePeriodSeconds: 40
|
||||||
|
httpGet:
|
||||||
|
port: 8989
|
||||||
|
path: /healthz
|
||||||
|
|
||||||
|
readinessProbe:
|
||||||
|
initialDelaySeconds: 10
|
||||||
|
periodSeconds: 5
|
||||||
|
timeoutSeconds: 3
|
||||||
|
successThreshold: 2
|
||||||
|
failureThreshold: 2
|
||||||
|
httpGet:
|
||||||
|
port: 8989
|
||||||
|
path: /healthz
|
||||||
+13
@@ -0,0 +1,13 @@
|
|||||||
|
mode: statefulset
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
|
||||||
|
command:
|
||||||
|
name: "otelcol-k8s"
|
||||||
|
|
||||||
|
replicaCount: 2
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 200M
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
# Examples of chart configuration
|
||||||
|
|
||||||
|
Here is a collection of common configurations for the OpenTelemetry collector. Each folder contains an example `values.yaml` and the resulting configurations that are generated by the opentelemetry-collector helm charts.
|
||||||
|
|
||||||
|
- [Daemonset only](daemonset-only)
|
||||||
|
- [Deployment only](deployment-only)
|
||||||
|
- [Daemonset and deployment](daemonset-and-deployment)
|
||||||
|
- [Log collection, including collector logs](daemonset-collector-logs)
|
||||||
|
- [Add component (hostmetrics)](daemonset-hostmetrics)
|
||||||
|
|
||||||
|
The manifests are rendered using the `helm template` command and the specific example folder's values.yaml.
|
||||||
|
|
||||||
|
Examples are generated by (from root of the repo):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make generate-examples CHARTS=opentelemetry-collector
|
||||||
|
```
|
||||||
+34
@@ -0,0 +1,34 @@
|
|||||||
|
mode: daemonset
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
|
||||||
|
command:
|
||||||
|
name: "otelcol-k8s"
|
||||||
|
|
||||||
|
config:
|
||||||
|
exporters:
|
||||||
|
otlp:
|
||||||
|
endpoint: example-opentelemetry-collector:4317
|
||||||
|
tls:
|
||||||
|
insecure: true
|
||||||
|
service:
|
||||||
|
pipelines:
|
||||||
|
logs:
|
||||||
|
exporters:
|
||||||
|
- otlp
|
||||||
|
- debug
|
||||||
|
metrics:
|
||||||
|
exporters:
|
||||||
|
- otlp
|
||||||
|
- debug
|
||||||
|
traces:
|
||||||
|
exporters:
|
||||||
|
- otlp
|
||||||
|
- debug
|
||||||
|
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 200M
|
||||||
|
|
||||||
+13
@@ -0,0 +1,13 @@
|
|||||||
|
mode: deployment
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: "otel/opentelemetry-collector-k8s"
|
||||||
|
|
||||||
|
command:
|
||||||
|
name: "otelcol-k8s"
|
||||||
|
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 200M
|
||||||
|
|
||||||
+93
@@ -0,0 +1,93 @@
|
|||||||
|
---
|
||||||
|
# Source: opentelemetry-collector/templates/configmap-agent.yaml
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: example-opentelemetry-collector-agent
|
||||||
|
namespace: default
|
||||||
|
labels:
|
||||||
|
helm.sh/chart: opentelemetry-collector-0.93.3
|
||||||
|
app.kubernetes.io/name: opentelemetry-collector
|
||||||
|
app.kubernetes.io/instance: example
|
||||||
|
app.kubernetes.io/version: "0.102.1"
|
||||||
|
app.kubernetes.io/managed-by: Helm
|
||||||
|
|
||||||
|
data:
|
||||||
|
relay: |
|
||||||
|
exporters:
|
||||||
|
debug: {}
|
||||||
|
otlp:
|
||||||
|
endpoint: example-opentelemetry-collector:4317
|
||||||
|
tls:
|
||||||
|
insecure: true
|
||||||
|
extensions:
|
||||||
|
health_check:
|
||||||
|
endpoint: ${env:MY_POD_IP}:13133
|
||||||
|
processors:
|
||||||
|
batch: {}
|
||||||
|
memory_limiter:
|
||||||
|
check_interval: 5s
|
||||||
|
limit_percentage: 80
|
||||||
|
spike_limit_percentage: 25
|
||||||
|
receivers:
|
||||||
|
jaeger:
|
||||||
|
protocols:
|
||||||
|
grpc:
|
||||||
|
endpoint: ${env:MY_POD_IP}:14250
|
||||||
|
thrift_compact:
|
||||||
|
endpoint: ${env:MY_POD_IP}:6831
|
||||||
|
thrift_http:
|
||||||
|
endpoint: ${env:MY_POD_IP}:14268
|
||||||
|
otlp:
|
||||||
|
protocols:
|
||||||
|
grpc:
|
||||||
|
endpoint: ${env:MY_POD_IP}:4317
|
||||||
|
http:
|
||||||
|
endpoint: ${env:MY_POD_IP}:4318
|
||||||
|
prometheus:
|
||||||
|
config:
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: opentelemetry-collector
|
||||||
|
scrape_interval: 10s
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- ${env:MY_POD_IP}:8888
|
||||||
|
zipkin:
|
||||||
|
endpoint: ${env:MY_POD_IP}:9411
|
||||||
|
service:
|
||||||
|
extensions:
|
||||||
|
- health_check
|
||||||
|
pipelines:
|
||||||
|
logs:
|
||||||
|
exporters:
|
||||||
|
- otlp
|
||||||
|
- debug
|
||||||
|
processors:
|
||||||
|
- memory_limiter
|
||||||
|
- batch
|
||||||
|
receivers:
|
||||||
|
- otlp
|
||||||
|
metrics:
|
||||||
|
exporters:
|
||||||
|
- otlp
|
||||||
|
- debug
|
||||||
|
processors:
|
||||||
|
- memory_limiter
|
||||||
|
- batch
|
||||||
|
receivers:
|
||||||
|
- otlp
|
||||||
|
- prometheus
|
||||||
|
traces:
|
||||||
|
exporters:
|
||||||
|
- otlp
|
||||||
|
- debug
|
||||||
|
processors:
|
||||||
|
- memory_limiter
|
||||||
|
- batch
|
||||||
|
receivers:
|
||||||
|
- otlp
|
||||||
|
- jaeger
|
||||||
|
- zipkin
|
||||||
|
telemetry:
|
||||||
|
metrics:
|
||||||
|
address: ${env:MY_POD_IP}:8888
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user