diff --git a/charts/nemo/Chart.lock b/charts/nemo/Chart.lock new file mode 100644 index 0000000..3ba86a0 --- /dev/null +++ b/charts/nemo/Chart.lock @@ -0,0 +1,39 @@ +dependencies: +- name: nemo-guardrails + repository: file://components/guardrails + version: 0.2.0 +- name: nemo-entity-store + repository: file://components/entity-store + version: 0.2.0 +- name: nemo-datastore + repository: file://components/datastore + version: 0.5.1 +- name: nemo-customizer + repository: file://components/customizer + version: 0.11.0-alpha.13 +- name: nemo-evaluator + repository: file://components/evaluator + version: 0.2.4 +- name: nemo-operator + repository: file://components/nemo-operator + version: 0.2.0 +- name: nemo-deployment-management + repository: file://components/deployment-management + version: 0.2.1 +- name: nemo-nim-proxy + repository: file://components/nim-proxy + version: 0.2.0 +- name: nim-llm + repository: https://helm.ngc.nvidia.com/nvidian/nemo-llm + version: 1.7.0-rc1 +- name: k8s-nim-operator + repository: https://helm.ngc.nvidia.com/nvidia/ + version: 1.0.1 +- name: dgxc-admission-controller + repository: https://helm.ngc.nvidia.com/nvidian/nemo-llm + version: 1.642.0 +- name: volcano + repository: https://volcano-sh.github.io/helm-charts + version: 1.9.0 +digest: sha256:a182a437e6ff1efa1a0a6b56cf7533e612fe9a6ee087ad7f813193f63bcf0bd4 +generated: "2025-04-16T22:58:40.429920984Z" diff --git a/charts/nemo/Chart.yaml b/charts/nemo/Chart.yaml new file mode 100644 index 0000000..ce10156 --- /dev/null +++ b/charts/nemo/Chart.yaml @@ -0,0 +1,68 @@ +apiVersion: v2 +dependencies: +- alias: guardrails + condition: guardrails.enabled + name: nemo-guardrails + repository: file://components/guardrails + version: '>=0.1.0-0' +- alias: entity-store + condition: entity-store.enabled + name: nemo-entity-store + repository: file://components/entity-store + version: '>=0.1.0-0' +- alias: data-store + condition: data-store.enabled + name: nemo-datastore + repository: file://components/datastore + version: '>=0.1.0-0' +- alias: customizer + condition: customizer.enabled + name: nemo-customizer + repository: file://components/customizer + version: '>=0.1.0-0' +- alias: evaluator + condition: evaluator.enabled + name: nemo-evaluator + repository: file://components/evaluator + version: '>=0.1.0-0' +- alias: nemo-operator + condition: nemo-operator.enabled + name: nemo-operator + repository: file://components/nemo-operator + version: '>=0.1.0-0' +- alias: deployment-management + condition: deployment-management.enabled + name: nemo-deployment-management + repository: file://components/deployment-management + version: '>=0.1.0-0' +- alias: nim-proxy + condition: nim-proxy.enabled + name: nemo-nim-proxy + repository: file://components/nim-proxy + version: '>=0.1.0-0' +- alias: nim + condition: nim.enabled + name: nim-llm + repository: https://helm.ngc.nvidia.com/nvidian/nemo-llm + version: 1.7.0-rc1 +- alias: nim-operator + condition: nim-operator.enabled + name: k8s-nim-operator + repository: https://helm.ngc.nvidia.com/nvidia/ + version: 1.0.1 +- condition: dgxc-admission-controller.enabled + name: dgxc-admission-controller + repository: https://helm.ngc.nvidia.com/nvidian/nemo-llm + version: 1.642.0 +- condition: volcano.enabled + name: volcano + repository: https://volcano-sh.github.io/helm-charts + version: 1.9.0 +description: NeMo Microservices Helm Chart +home: https://nvidia.com +maintainers: +- name: NVIDIA, Inc. + url: https://www.nvidia.com +name: nemo-microservices-helm-chart +type: application +version: 25.4.0 diff --git a/charts/nemo/README.md b/charts/nemo/README.md new file mode 100644 index 0000000..91830e0 --- /dev/null +++ b/charts/nemo/README.md @@ -0,0 +1,152 @@ +# NeMo Microservices Helm Chart + +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) + +For deployment guide, see [Admin Setup](https://docs.nvidia.com/nemo/microservices/latest/set-up/index.html) in the NeMo Microservices documentation. + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| customizer.customizerConfig.entityStoreURL | string | `"http://nemo-entity-store:8000"` | Specifies the internal K8s DNS record for the NeMo Entity Store service. | +| customizer.customizerConfig.mlflowURL | string | `""` | Specifies the internal K8s DNS record for the MLflow service. | +| customizer.customizerConfig.models | object | `{"meta/llama-3.1-8b-instruct":{"enabled":true}}` | Specifies the model information to configure with the NeMo Customizer microservice. To find a complete list of models that NeMo Customizer supports, see [Model Catalog](https://docs.nvidia.com/nemo/microservices/latest/fine-tune/models/index.html). | +| customizer.customizerConfig.models."meta/llama-3.1-8b-instruct".enabled | bool | `true` | Specifies whether to use the `meta/llama-3.1-8b-instruct` model for a customization job. | +| customizer.customizerConfig.nemoDataStoreURL | string | `"http://nemo-data-store:3000"` | Specifies the internal K8s DNS record for the NeMo Data Store service. | +| customizer.customizerConfig.training.pvc | object | `{"storageClass":""}` | Specifies where to store the customization job information during training. | +| customizer.enabled | bool | `true` | Specifies whether to install the NeMo Customizer microservice. | +| customizer.modelsStorage | object | `{"accessModes":["ReadWriteOnce"],"enabled":true,"size":"20Gi","storageClassName":""}` | Specifies whether to enable the PVC for storing base models. | +| customizer.modelsStorage.accessModes | list | `["ReadWriteOnce"]` | Specifies the access modes for the PVC. | +| customizer.modelsStorage.size | string | `"20Gi"` | Specifies the size of the PVC. | +| customizer.modelsStorage.storageClassName | string | `""` | Specifies the storage class to use for the PVC. | +| customizer.nemoDataStoreTools | object | `{"imagePullSecret":"nvcrimagepullsecret"}` | Specifies the image pull secret for the NeMo Data Store tools. | +| customizer.postgresql.nameOverride | string | `"customizerdb"` | The name override for the PostgreSQL database. | +| customizer.postgresql.serviceAccount.create | bool | `true` | Specifies whether to create a new service account for PostgreSQL. | +| customizer.postgresql.serviceAccount.name | string | `"customizer-postgresql"` | | +| customizer.serviceName | string | `"nemo-customizer"` | | +| customizer.workspaceStorage.enabled | bool | `false` | Specifies whether to enable the PVC for storing a training job's workspace. | +| data-store.enabled | bool | `true` | Specifies whether to install the NeMo Data Store microservice. | +| data-store.objectStore.enabled | bool | `false` | Specifies whether to enable object storage for the NeMo Data Store microservice. | +| data-store.persistence.accessModes | list | `["ReadWriteOnce"]` | The access modes for the persistent volume. | +| data-store.persistence.claimName | string | `"datastore-shared-storage"` | The name of the persistent volume claim. | +| data-store.persistence.enabled | bool | `true` | Specifies whether to enable persistent volume for the NeMo Data Store microservice. | +| data-store.persistence.size | string | `"1Gi"` | The size of the persistent volume. | +| data-store.persistence.storageClass | string | `""` | The storage class to use for the persistent volume. | +| data-store.replicaCount | int | `1` | The number of replicas for the NeMo Data Store microservice deployment. | +| data-store.serviceAccount.create | bool | `true` | Specifies whether to create a service account for the NeMo Data Store microservice. | +| data-store.serviceAccount.name | string | `"gitea"` | The service account name for the NeMo Data Store microservice. | +| data-store.serviceName | string | `"nemo-data-store"` | The service name for the NeMo Data Store microservice. | +| data-store.strategy.type | string | `"Recreate"` | The type of deployment strategy to use for the NeMo Data Store microservice. | +| deployment-management.deployments.entityStoreUrl | string | `"http://nemo-entity-store:8000"` | The URL for the NeMo Entity Store service. | +| deployment-management.deployments.modelSyncPeriod | string | `"30"` | The period in seconds for model synchronization. | +| deployment-management.deployments.nimImagePullSecrets | list | `["nvcrimagepullsecret"]` | The image pull secrets for the NeMo Deployment Management microservice. | +| deployment-management.deployments.nimPeftSource | string | `"http://nemo-entity-store:8000"` | The URL for the NIM PEFT source. | +| deployment-management.enabled | bool | `true` | Specifies whether to install the NeMo Deployment Management microservice. | +| deployment-management.image.pullPolicy | string | `"IfNotPresent"` | The pull policy for the NeMo Deployment Management container image. | +| deployment-management.serviceName | string | `"nemo-deployment-management"` | The service name for the NeMo Deployment Management microservice. | +| dgxc-admission-controller.cloudProviderSpec.dummy | object | `{"type":"dummy"}` | The type of the dummy cloud provider. | +| dgxc-admission-controller.dgxcController.image | string | `"nvcr.io/nvidia/nemo-microservices/dgxc-admission-controller:1.642.0"` | The image for the DGX Cloud Admission Controller. | +| dgxc-admission-controller.enableDGXCRbac | bool | `false` | | +| dgxc-admission-controller.enableIntegrationsSupport | bool | `false` | Specifies whether to enable integrations support. | +| dgxc-admission-controller.enableLabeler | bool | `false` | Specifies whether to enable the labeler. | +| dgxc-admission-controller.enableNetworkPolicies | bool | `false` | Specifies whether to enable network policies. | +| dgxc-admission-controller.enabled | bool | `false` | Specifies whether to install the DGX Cloud Admission Controller. | +| dgxc-admission-controller.imagePullSecret | string | `"nvcrimagepullsecret"` | The image pull secret for the DGX Cloud Admission Controller. | +| dgxc-admission-controller.kubernetesAffinity.enabled | bool | `false` | Specifies whether to enable Kubernetes affinity. | +| entity-store.appConfig.BASE_URL_DATASTORE | string | `"http://nemo-data-store:3000/v1/hf"` | The base URL for the NeMo Data Store microservice. | +| entity-store.appConfig.BASE_URL_NIM | string | `"http://nemo-nim-proxy:8000"` | The base URL for the NIM service used by the NeMo Entity Store microservice. | +| entity-store.enabled | bool | `true` | Specifies whether to install the NeMo Entity Store microservice. | +| entity-store.postgresql.nameOverride | string | `"entity-storedb"` | The name override for the PostgreSQL database. | +| entity-store.postgresql.serviceAccount.create | bool | `true` | Specifies whether to create a new service account for PostgreSQL. | +| entity-store.postgresql.serviceAccount.name | string | `"entity-store-postgresql"` | The service account name for PostgreSQL. | +| entity-store.serviceName | string | `"nemo-entity-store"` | The service name for the NeMo Entity Store microservice. | +| evaluator.argoWorkflows.controller.serviceAccount.name | string | `"argo-workflows-controller"` | The name of the service account for the Argo Workflows controller. | +| evaluator.argoWorkflows.crds.install | bool | `true` | Specifies whether to install the CRDs for Argo Workflows. | +| evaluator.argoWorkflows.enabled | bool | `true` | Specifies whether to enable the default Argo Workflows integration for the NeMo Evaluator microservice. | +| evaluator.argoWorkflows.server.authModes[0] | string | `"server"` | | +| evaluator.argoWorkflows.server.secure | bool | `true` | Specifies whether to secure the connection to the Argo Workflows server. | +| evaluator.argoWorkflows.server.serviceAccount.name | string | `"argo-workflows-server"` | The name of the service account for the Argo Workflows server. | +| evaluator.argoWorkflows.server.servicePort | int | `2746` | The service port for the Argo Workflows server. | +| evaluator.argoWorkflows.serviceName | string | `"argo-workflows-server"` | The Argo Workflows service name for the NeMo Evaluator microservice. | +| evaluator.enabled | bool | `true` | Specifies whether to install the NeMo Evaluator microservice. | +| evaluator.external.dataStore | object | `{"endpoint":"http://nemo-data-store:3000/v1/hf"}` | The external URL of the NeMo Data Store microservice. | +| evaluator.milvus.standalone.persistence.persistentVolumeClaim.storageClass | string | `""` | Specifies the storage class to use for the Milvus PVC. | +| evaluator.postgresql.nameOverride | string | `"evaluatordb"` | The name override for the default PostgreSQL database. | +| evaluator.postgresql.primary.networkPolicy.enabled | bool | `false` | Specifies whether to enable the network policy for the PostgreSQL service. | +| evaluator.postgresql.serviceAccount.name | string | `"evaluator-postgresql"` | The name of the service account for PostgreSQL. | +| evaluator.serviceName | string | `"nemo-evaluator"` | | +| existingImagePullSecret | string | `""` | You can specify an existing Kubernetes image pull secret for pulling images from the container registry. The chart doesn't reference any image pull secret if you leave this empty. | +| existingSecret | string | `""` | You can use an existing Kubernetes secret for pulling images. The chart uses the `ngcAPIKey` value to generate the secret if you leave this empty. | +| global.imagePullSecrets | list | `[{"name":"nvcrimagepullsecret"}]` | The name of the image pull secret to use globally across all services. | +| guardrails.enabled | bool | `true` | Specifies whether to install the NeMo Guardrails microservice. | +| guardrails.env | object | `{"NIM_ENDPOINT_URL":"http://nemo-nim-proxy:8000/v1"}` | The environment variables for the NeMo Guardrails microservice. | +| guardrails.env.NIM_ENDPOINT_URL | string | `"http://nemo-nim-proxy:8000/v1"` | The NIM endpoint URL for the NeMo Guardrails microservice. | +| guardrails.serviceName | string | `"nemo-guardrails"` | The service name for the NeMo Guardrails microservice. | +| imagePullSecrets | list | `[{"name":"nvcrimagepullsecret","password":"YOUR-NGC-API-KEY","registry":"nvcr.io","username":"$$oauthtoken"}]` | List of image pull secrets. Existing secrets override these values if you specify them. Use this only for experimentation when you want to hardcode a secret in your values file. | +| ingress.annotations | object | `{}` | Annotations for the ingress resource. | +| ingress.className | string | `""` | The ingress class to use if your cluster has more than one class. | +| ingress.enabled | bool | `false` | Specifies whether to enable the ingress. | +| ingress.hosts | object | `{"dataStore":{"name":"data-store.test","paths":[{"path":"/","pathType":"Prefix","port":3000,"service":"nemo-data-store"}]},"default":{"name":"","paths":[{"path":"/v1/namespaces","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/projects","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/datasets","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/repos","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/models","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/customization","pathType":"Prefix","port":8000,"service":"nemo-customizer"},{"path":"/v1/evaluation","pathType":"Prefix","port":7331,"service":"nemo-evaluator"},{"path":"/v1/guardrail","pathType":"Prefix","port":7331,"service":"nemo-guardrails"},{"path":"/v1/deployment","pathType":"Prefix","port":8000,"service":"nemo-deployment-management"}]},"nimProxy":{"name":"nim.test","paths":[{"path":"/v1/completions","pathType":"Prefix","port":8000,"service":"nemo-nim-proxy"},{"path":"/v1/chat","pathType":"Prefix","port":8000,"service":"nemo-nim-proxy"},{"path":"/v1/models","pathType":"Prefix","port":8000,"service":"nemo-nim-proxy"}]}}` | A map of hosts and their corresponding paths for the ingress. | +| ingress.hosts.dataStore.name | string | `"data-store.test"` | The host name for the third ingress host for the NeMo Data Store microservice. | +| ingress.hosts.dataStore.paths | list | `[{"path":"/","pathType":"Prefix","port":3000,"service":"nemo-data-store"}]` | The path rules for the third ingress host. | +| ingress.hosts.default.name | string | `""` | The host name for the default ingress host. | +| ingress.hosts.default.paths | list | `[{"path":"/v1/namespaces","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/projects","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/datasets","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/repos","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/models","pathType":"Prefix","port":8000,"service":"nemo-entity-store"},{"path":"/v1/customization","pathType":"Prefix","port":8000,"service":"nemo-customizer"},{"path":"/v1/evaluation","pathType":"Prefix","port":7331,"service":"nemo-evaluator"},{"path":"/v1/guardrail","pathType":"Prefix","port":7331,"service":"nemo-guardrails"},{"path":"/v1/deployment","pathType":"Prefix","port":8000,"service":"nemo-deployment-management"}]` | The path rules for the default ingress host. | +| ingress.hosts.nimProxy.name | string | `"nim.test"` | The host name for the second ingress host for the NIM Proxy microservice. | +| ingress.hosts.nimProxy.paths | list | `[{"path":"/v1/completions","pathType":"Prefix","port":8000,"service":"nemo-nim-proxy"},{"path":"/v1/chat","pathType":"Prefix","port":8000,"service":"nemo-nim-proxy"},{"path":"/v1/models","pathType":"Prefix","port":8000,"service":"nemo-nim-proxy"}]` | The path rules for the second ingress host. | +| ingress.tls | list | `[]` | TLS configurations. | +| nemo-operator.controllerManager.manager.scheduler | string | `"volcano"` | The scheduler to use for the NeMo Operator. Available options: `volcano` and `runai`. | +| nemo-operator.enabled | bool | `true` | Specifies whether to install the NeMo Operator microservice. | +| ngcAPIKey | string | `"YOUR-NGC-API-KEY"` | Your NVIDIA GPU Cloud (NGC) API key authenticates and enables pulling images from the NGC container registry. The existing secret overrides this key if you provide one to the `existingSecret` key. | +| nim-operator.enabled | bool | `true` | Specifies whether to install the NIM Operator. | +| nim-operator.nfd.nodeFeatureRules.deviceID | bool | `false` | Specifies whether to enable device ID feature rules. | +| nim-proxy.enabled | bool | `true` | Specifies whether to install the NIM Proxy microservice. | +| nim-proxy.monitoring.enabled | bool | `false` | Specifies whether to enable monitoring for the NIM Proxy microservice. | +| nim-proxy.serviceName | string | `"nemo-nim-proxy"` | The service name for the NIM Proxy microservice. | +| nim.enabled | bool | `false` | Specifies whether to deploy a NIM for LLM during the Helm installation of the chart. You can deploy a single static NIM by enabling this object and its values. When enabled, the chart deploys `meta/llama-3.1-8b-instruct` as the default NIM. | +| nim.env | list | `[{"name":"NIM_PEFT_SOURCE","value":"http://nemo-entity-store:8000"},{"name":"NIM_PEFT_REFRESH_INTERVAL","value":"30"},{"name":"NIM_MAX_CPU_LORAS","value":"16"},{"name":"NIM_MAX_GPU_LORAS","value":"8"}]` | Environment variables for the NIM service. | +| nim.image.repository | string | `"nvcr.io/nim/meta/llama-3.1-8b-instruct"` | | +| nim.image.tag | string | `"1.8.3"` | | +| nim.imagePullSecrets[0].name | string | `"nvcrimagepullsecret"` | | +| nim.model.name | string | `"meta/llama-3.1-8b-instruct"` | The name of the model to deploy as NIM. | +| nim.model.ngcAPISecret | string | `"ngc-api"` | The NGC API secret for model access. | +| nim.persistence.annotations | object | `{"helm.sh/resource-policy":"keep"}` | Annotations for the PVC. | +| nim.persistence.enabled | bool | `true` | Specifies whether to enable persistence volume claim (PVC) for the NIM service. | +| nim.persistence.storageClass | string | `""` | Specifies the storage class to use for the PVC. | +| nim.resources | object | `{"limits":{"nvidia.com/gpu":1},"requests":{"nvidia.com/gpu":1}}` | Specifies resources for the NIM service. | +| nim.resources.requests | object | `{"nvidia.com/gpu":1}` | Specifies requests for the NIM service. | +| nim.service.labels | object | `{"app.nvidia.com/nim-type":"inference"}` | Labels for the NIM service. | +| nim.statefulSet.enabled | bool | `false` | Specifies whether to enable a stateful set for the NIM service. | +| virtualService | object | A virtual service configuration template. | Specifies whether to enable the virtual service. If you are not using istio and virtualservices, it can be useful to create some virtual services for the NeMo Microservices system. Don't enable unless you use istio. | +| virtualService.additional.data-store | object | `{"entries":{"data-store":{"corsPolicy":{},"match":[{"uri":{"prefix":"/"}}],"route":[{"destination":{"host":"nemo-data-store","port":{"number":3000}}}]}},"gateways":[],"hosts":[]}` | Additional virtual service configurations. | +| virtualService.additional.data-store.entries | object | `{"data-store":{"corsPolicy":{},"match":[{"uri":{"prefix":"/"}}],"route":[{"destination":{"host":"nemo-data-store","port":{"number":3000}}}]}}` | The entries for the virtual NeMo Data Store service. | +| virtualService.additional.data-store.entries.data-store.corsPolicy | object | `{}` | The CORS policy for the virtual NeMo Data Store service. | +| virtualService.additional.data-store.entries.data-store.match | list | `[{"uri":{"prefix":"/"}}]` | The match for the virtual NeMo Data Store service. | +| virtualService.additional.data-store.entries.data-store.route | list | `[{"destination":{"host":"nemo-data-store","port":{"number":3000}}}]` | The route for the virtual NeMo Data Store service. | +| virtualService.additional.data-store.gateways | list | `[]` | The gateways for the virtual NeMo Data Store service. | +| virtualService.additional.data-store.hosts | list | `[]` | The hosts for the virtual NeMo Data Store service. | +| virtualService.additional.nim-proxy.entries.nim-proxy.corsPolicy | object | `{}` | The CORS policy for the virtual NIM Proxy service. | +| virtualService.additional.nim-proxy.entries.nim-proxy.match | list | `[{"uri":{"prefix":"/"}}]` | The match for the virtual NIM Proxy service. | +| virtualService.additional.nim-proxy.entries.nim-proxy.route | list | `[{"destination":{"host":"nemo-nim-proxy","port":{"number":8000}}}]` | The route for the virtual NIM Proxy service. | +| virtualService.additional.nim-proxy.gateways | list | `[]` | The gateways for the virtual NIM Proxy service. | +| virtualService.additional.nim-proxy.hosts | list | `[]` | The hosts for the virtual NIM Proxy service. | +| virtualService.annotations | object | `{}` | Annotations for the virtual service. | +| virtualService.enabled | bool | `false` | Specifies whether to enable the virtual service. | +| virtualService.labels | object | `{}` | Labels for the virtual service. | +| virtualService.main.entries.customizer.corsPolicy | object | `{}` | The CORS policy for the virtual NeMo Customizer service. | +| virtualService.main.entries.customizer.match | list | `[{"uri":{"prefix":"/v1/customization"}}]` | The match for the virtual NeMo Customizer service. | +| virtualService.main.entries.customizer.route | list | `[{"destination":{"host":"nemo-customizer","port":{"number":8000}}}]` | The route for the virtual NeMo Customizer service. | +| virtualService.main.entries.deployment-management.corsPolicy | object | `{}` | The CORS policy for the virtual NeMo Deployment Management service. | +| virtualService.main.entries.deployment-management.match | list | `[{"uri":{"prefix":"/v1/deployment"}}]` | The match for the virtual NeMo Deployment Management service. | +| virtualService.main.entries.deployment-management.route | list | `[{"destination":{"host":"nemo-deployment-management","port":{"number":8000}}}]` | The route for the virtual NeMo Deployment Management service. | +| virtualService.main.entries.entity-store.corsPolicy | object | `{}` | The CORS policy for the virtual NeMo Entity Store service. | +| virtualService.main.entries.entity-store.match | list | `[{"uri":{"prefix":"/v1/namespaces"}},{"uri":{"prefix":"/v1/projects"}},{"uri":{"prefix":"/v1/datasets"}},{"uri":{"prefix":"/v1/repos"}},{"uri":{"prefix":"/v1/models"}}]` | The match for the virtual NeMo Entity Store service. | +| virtualService.main.entries.entity-store.route | list | `[{"destination":{"host":"nemo-entity-store","port":{"number":8000}}}]` | The route for the virtual NeMo Entity Store service. | +| virtualService.main.entries.evaluator.corsPolicy | object | `{}` | The CORS policy for the virtual NeMo Evaluator service. | +| virtualService.main.entries.evaluator.match | list | `[{"uri":{"prefix":"/v1/evaluation"}}]` | The match for the virtual NeMo Evaluator service. | +| virtualService.main.entries.evaluator.route | list | `[{"destination":{"host":"nemo-evaluator","port":{"number":7331}}}]` | The route for the virtual NeMo Evaluator service. | +| virtualService.main.entries.guardrails.corsPolicy | object | `{}` | The CORS policy for the virtual NeMo Guardrails service. | +| virtualService.main.entries.guardrails.match | list | `[{"uri":{"prefix":"/v1/guardrail"}}]` | The match for the virtual NeMo Guardrails service. | +| virtualService.main.entries.guardrails.route | list | `[{"destination":{"host":"nemo-guardrails","port":{"number":7331}}}]` | The route for the virtual NeMo Guardrails service. | +| virtualService.main.gateways | list | `[]` | A list of gateways for the virtual service. | +| virtualService.main.hosts | list | `[]` | A list of hosts for the virtual service. | +| volcano.enabled | bool | `false` | Specifies whether to enable the default Volcano scheduler installation. To learn more, see [Volcano](https://docs.nvidia.com/nemo/microservices/latest/set-up/deploy-as-microservices/customizer.html#volcano). | \ No newline at end of file diff --git a/charts/nemo/charts/dgxc-admission-controller/Chart.yaml b/charts/nemo/charts/dgxc-admission-controller/Chart.yaml new file mode 100644 index 0000000..0cf334c --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: 0.0.1 +description: A Helm chart for Kubernetes +name: dgxc-admission-controller +version: 1.642.0 diff --git a/charts/nemo/charts/dgxc-admission-controller/files/storage.dgxc.nvidia.com_nvstoragelocations.yaml b/charts/nemo/charts/dgxc-admission-controller/files/storage.dgxc.nvidia.com_nvstoragelocations.yaml new file mode 100644 index 0000000..61513ac --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/files/storage.dgxc.nvidia.com_nvstoragelocations.yaml @@ -0,0 +1,220 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: nvstoragelocations.storage.dgxc.nvidia.com +spec: + group: storage.dgxc.nvidia.com + names: + kind: NvStorageLocation + listKind: NvStorageLocationList + plural: nvstoragelocations + singular: nvstoragelocation + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Bucket + jsonPath: .spec.bucket + name: Bucket + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: NvStorageLocation defines storage for DGXC Run:ai accelerated + compute environments. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + description: + type: string + mountPath: + type: string + pvc: + description: '`pvc` represents a PVC to manage.' + properties: + capacity: + description: Requested size of the volume (required for a managed + volume) + format: ByteSize + type: string + mountOptions: + items: + type: string + type: array + objectSpec: + description: '`object` represents a volume that uses object storage.' + properties: + bucket: + type: string + endpointUrl: + type: string + required: + - bucket + - endpointUrl + type: object + parameters: + additionalProperties: + type: string + description: |- + Parameters contains an arbitrary set of key-value pairs + to supply to whatever storage backend we're using + according to our rules for that object storage backend. + Currently the only thing supported is the GCS OIDC ConfigMap + this parameter is "gcs-oidc-configmap" with the value being + the name of the configmap + type: object + required: + - capacity + - objectSpec + type: object + volumeName: + description: volumeName will be used to name the PVC and if needed + the PVs + type: string + required: + - description + - mountPath + type: object + status: + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + pendingCount: + description: The number of PVCs monitored by this NVStorageLocation + that are not in a ready state (e.g. missing, error state). + type: integer + pvcs: + description: |- + A list of PVC statuses. This is a list because we expect to support multiple namespaces, + but it may very well be a list of one object. + items: + properties: + externalReferences: + items: + properties: + clusterName: + type: string + externalID: + type: string + platform: + type: string + required: + - clusterName + - externalID + - platform + type: object + type: array + namespace: + description: The namespace in which the PersistentVolumeClaim + exists. + type: string + pv: + type: string + state: + description: The current state of this volume. + type: string + volumeName: + description: The full name of the PersistentVolumeClaim. + type: string + required: + - namespace + - pv + - state + - volumeName + type: object + type: array + readyCount: + description: The number of PVCs monitored by this NVStorageLocation + that are in a ready state. + type: integer + summary: + description: |- + A short human-readable summary of the status of all managed and monitored PVCs. + Machine-readable data should be sourced from the `Ready` and `NotReady` counts, or + from pvcs[].State for detailed information. + type: string + required: + - pendingCount + - pvcs + - readyCount + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/NOTES.txt b/charts/nemo/charts/dgxc-admission-controller/templates/NOTES.txt new file mode 100644 index 0000000..e2b5c6a --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/NOTES.txt @@ -0,0 +1 @@ +No notes diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/_helpers.tpl b/charts/nemo/charts/dgxc-admission-controller/templates/_helpers.tpl new file mode 100644 index 0000000..5c7d66d --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/_helpers.tpl @@ -0,0 +1,64 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} + +{{- define "dgxcController.name" -}} +{{- default "controller" .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} + +{{- define "dgxcController.fullname" -}} +{{- if .Values.dgxcController.deploymentFullNameOverride -}} +{{- .Values.dgxcController.deploymentFullNameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "dgxcController.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "dgx-application-deploy" -}} +{{ .Release.Name }}-{{ .Release.Revision }} +{{- end -}} + +{{- define "kubernetesAffinityConfig" }} +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ .Values.kubernetesAffinity.antiAffinityKey | default "reason-to-avoid" }} + operator: DoesNotExist + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "nvidia.com/dgx-application-deploy" + operator: In + values: + - {{ include "dgx-application-deploy" . }} + topologyKey: {{ .Values.kubernetesAffinity.topologyKey | default "kubernetes.io/hostname" }} +{{- if .Values.kubernetesAffinity.runOnMasters }} +tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists +{{- end }} +{{- end }} + diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/cm.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/cm.yaml new file mode 100644 index 0000000..a16a535 --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/cm.yaml @@ -0,0 +1,250 @@ +{{- if .Values.enabledComponents.dgxcAdmissionController }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.dgxcController.name}}-config +data: + config.yaml: |- + apiVersion: runai.dgxc.nvidia.com/v1 + kind: DGXCRunaiMutatingWebhook + certs: + namespace: dgxc-admission-controller + service: {{ .Values.serviceName }} + port: 9443 + + organization: {{ .Values.webhooks.organization }} + + mountPath: {{ .Values.dgxcController.certs.mountPath }} + certName: {{ .Values.dgxcController.certs.certName }} + keyName: {{ .Values.dgxcController.certs.keyName }} + + podMutatingConfig: + name: dgxc-create-pod-mwh + kind: {{ .Values.webhooks.podMutatingWebhook.kind }} + fqdn: {{ .Values.webhooks.podMutatingWebhook.fqdn }} + path: {{ .Values.webhooks.podMutatingWebhook.path }} + failurePolicy: {{ .Values.webhooks.podMutatingWebhook.failurePolicy }} + timeoutSeconds: {{ .Values.webhooks.podMutatingWebhook.timeoutSeconds }} + fqdn: {{ .Values.webhooks.podMutatingWebhook.fqdn }} + sideEffects: {{ .Values.webhooks.podMutatingWebhook.sideEffects | default "NoneOnDryRun" }} + admissionReviewVersions: + {{- range $i, $value := .Values.webhooks.podMutatingWebhook.admissionReviewVersions }} + - {{ $value }} + {{- end }} + rules: + {{- range $i, $value := .Values.webhooks.podMutatingWebhook.rules }} + - apiGroups: + {{- range $j, $value2 := $value.apiGroups }} + - {{ $value2 | quote }} + {{- end }} + apiVersions: + {{- range $j, $value2 := $value.apiVersions }} + - {{ $value2 | quote }} + {{- end }} + operations: + {{- range $j, $value2 := $value.operations }} + - {{ $value2 | quote }} + {{- end }} + resources: + {{- range $j, $value2 := $value.resources }} + - {{ $value2 | quote }} + {{- end }} + scope: {{ $value.scope | quote }} + {{- end }} + nsExcludeLabels: + - key: kubernetes.io/metadata.name + values: + - dgxc-admission-controller + - argocd + - nvdiag + objectMatchLabels: + {{- range $i, $value := .Values.webhooks.podMutatingWebhook.objectLabelSelector }} + - key: {{ $value.key}} + values: + {{- range $j, $value2 := $value.values }} + - {{ $value2 | quote }} + {{- end }} + {{- end }} + + podMutatingSpec: + {{- with (first .Values.webhooks.podMutatingWebhook.customSettings) }} + addRdmaEnabled: {{ .addRdmaEnabled }} + rdmaResourcePerGpu: {{ .rdmaResourcePerGpu }} + networkAttachmentDefinition: {{ .networkAttachmentDefinition }} + rdmaResourceName: {{ .rdmaResourceName }} + addScratchEnabled: {{ .scratchEnabled }} + scratchVolName: {{ .scratchVolName | quote }} + scratchMountPath: {{ .scratchMountpath | quote }} + scratchVolSize: {{ .scratchVolSize | quote }} + scratchVolSizePerGpu: {{ .scratchVolSizePerGpu }} + scratchVolEnforceLimit: {{ .scratchVolEnforceLimit }} + {{- if .scratchVolSource }} + scratchVolSource: + {{ .scratchVolSource | toYaml | nindent 8 }} + {{- end}} + scratchOnCpu: {{ .scratchOnCpu | default false }} + addTcpxSidecarEnabled: {{ .addTcpxSidecarEnabled }} + addTcpxoSidecarEnabled: {{ .addTcpxoSidecarEnabled }} + addTcpxNcclVars: {{ .addTcpxNCCLVars }} + addTcpxoNcclVars: {{ .addTcpxoNCCLVars }} + addNcclVars: {{ .addNcclVars }} + csp: {{ .csp}} + addGcpFuseAnnotation: {{ .addGcpFuseAnnotation}} + gcpFuseAnnotationKey: {{ .gcpFuseAnnotationKey | quote }} + gcpFuseWICredentialConfigMapAnnotationKey: {{ .gcpFuseWICredentialConfigMapAnnotationKey | quote }} + gcpFuseStorageClasses: + {{- range .gcpFuseStorageClasses}} + - {{ . | quote }} + {{- end }} + clusterName: {{ .clusterName }} + injectRunaiJob: true + {{- end}} + systemNamespaceNames: + {{- range .Values.clusterScopedKinds }} + {{- if eq .kind "Namespace" }} + {{- range .systemResources}} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + + + pvcMutatingConfig: + name: dgxc-create-pvc-mwh + kind: {{ .Values.webhooks.pvcMutatingWebhook.kind }} + fqdn: {{ .Values.webhooks.pvcMutatingWebhook.fqdn }} + path: {{ .Values.webhooks.pvcMutatingWebhook.path }} + failurePolicy: {{ .Values.webhooks.pvcMutatingWebhook.failurePolicy }} + timeoutSeconds: {{ .Values.webhooks.pvcMutatingWebhook.timeoutSeconds }} + fqdn: {{ .Values.webhooks.pvcMutatingWebhook.fqdn }} + sideEffects: {{ .Values.webhooks.pvcMutatingWebhook.sideEffects | default "NoneOnDryRun" }} + admissionReviewVersions: + {{- range $i, $value := .Values.webhooks.pvcMutatingWebhook.admissionReviewVersions }} + - {{ $value }} + {{- end }} + rules: + {{- range $i, $value := .Values.webhooks.pvcMutatingWebhook.rules }} + - apiGroups: + {{- range $j, $value2 := $value.apiGroups }} + - {{ $value2 | quote }} + {{- end }} + apiVersions: + {{- range $j, $value2 := $value.apiVersions }} + - {{ $value2 | quote }} + {{- end }} + operations: + {{- range $j, $value2 := $value.operations }} + - {{ $value2 | quote }} + {{- end }} + resources: + {{- range $j, $value2 := $value.resources }} + - {{ $value2 | quote }} + {{- end }} + scope: {{ $value.scope | quote }} + {{- end }} + nsMatchLabels: + {{- range $i, $value := .Values.webhooks.pvcMutatingWebhook.namespaceLabelSelector }} + - key: {{ $value.key}} + values: + {{- range $j, $value2 := $value.values }} + - {{ $value2 | quote }} + {{- end }} + {{- end }} + objectMatchLabels: + {{- range $i, $value := .Values.webhooks.pvcMutatingWebhook.objectLabelSelector }} + - key: {{ $value.key}} + values: + {{- range $j, $value2 := $value.values }} + - {{ $value2 | quote }} + {{- end }} + {{- end }} + + pvcMutatingSpec: + rules: + {{- range $i, $value := .Values.webhooks.pvcMutatingWebhook.customSettings }} + - storageClasses: + {{- range $j, $value2 := $value.storageClasses }} + - {{ $value2 | quote }} + {{- end }} + incrementAdjustmentEnabled: {{ $value.incrementCheckEnabled }} + incrementAdjustmentValue: {{ $value.incrementValue | quote }} + unitConversionEnabled: {{ $value.unitConversionEnabled }} + unitConversionFormat: {{ $value.unitConversionFormat }} + {{- if $value.minSizeValue }} + adjustedMinimumSize: {{ $value.minSizeValue | quote }} + {{- end }} + {{- end }} + + pvcValidatingConfig: + name: dgxc-create-pvc-vwh + kind: {{ .Values.webhooks.pvcValidatingWebhook.kind }} + fqdn: {{ .Values.webhooks.pvcValidatingWebhook.fqdn }} + path: {{ .Values.webhooks.pvcValidatingWebhook.path }} + failurePolicy: {{ .Values.webhooks.pvcValidatingWebhook.failurePolicy }} + timeoutSeconds: {{ .Values.webhooks.pvcValidatingWebhook.timeoutSeconds }} + fqdn: {{ .Values.webhooks.pvcValidatingWebhook.fqdn }} + sideEffects: {{ .Values.webhooks.pvcValidatingWebhook.sideEffects | default "NoneOnDryRun" }} + admissionReviewVersions: + {{- range $i, $value := .Values.webhooks.pvcValidatingWebhook.admissionReviewVersions }} + - {{ $value }} + {{- end }} + rules: + {{- range $i, $value := .Values.webhooks.pvcValidatingWebhook.rules }} + - apiGroups: + {{- range $j, $value2 := $value.apiGroups }} + - {{ $value2 | quote }} + {{- end }} + apiVersions: + {{- range $j, $value2 := $value.apiVersions }} + - {{ $value2 | quote }} + {{- end }} + operations: + {{- range $j, $value2 := $value.operations }} + - {{ $value2 | quote }} + {{- end }} + resources: + {{- range $j, $value2 := $value.resources }} + - {{ $value2 | quote }} + {{- end }} + scope: {{ $value.scope | quote }} + {{- end }} + nsMatchLabels: + {{- range $i, $value := .Values.webhooks.pvcValidatingWebhook.namespaceLabelSelector }} + - key: {{ $value.key}} + values: + {{- range $j, $value2 := $value.values }} + - {{ $value2 | quote }} + {{- end }} + {{- end }} + objectMatchLabels: + {{- range $i, $value := .Values.webhooks.pvcValidatingWebhook.objectLabelSelector }} + - key: {{ $value.key}} + values: + {{- range $j, $value2 := $value.values }} + - {{ $value2 | quote }} + {{- end }} + {{- end }} + + pvcValidatingSpec: + rules: + {{- range $i, $value := .Values.webhooks.pvcValidatingWebhook.customSettings }} + - name: {{ $value.name }} + storageClasses: + {{- range $j, $value2 := $value.storageClasses }} + - {{ $value2 | quote }} + {{- end }} + minsizeCheckEnabled: {{ $value.minSizeCheckEnabled }} + minsizeValue: {{ $value.minSizeValue | quote }} + maxsizeCheckEnabled: {{ $value.maxSizeCheckEnabled }} + maxsizeValue: {{ $value.maxSizeValue | quote }} + {{- end }} + + systemNamespaceNames: + {{- range .Values.clusterScopedKinds }} + {{- if eq .kind "Namespace" }} + {{- range .systemResources}} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} +{{ end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/controllercm.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/controllercm.yaml new file mode 100644 index 0000000..2936d81 --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/controllercm.yaml @@ -0,0 +1,161 @@ +{{ if .Values.enabledComponents.dgxcRunaiController }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: dgxc-runai-controller + namespace: {{ .Release.Namespace }} +data: + config.yaml: |- + apiVersion: runai.dgxc.nvidia.com/v1 + kind: Controller +{{- if .Values.prober }} + prober: + intervalInSeconds: {{ .Values.prober.intervalInSeconds }} + timeoutInSeconds: {{ .Values.prober.timeoutInSeconds }} + endpoints: + {{- range .Values.prober.endpoints }} + - name: {{ .name }} + url: {{ .url }} + {{- end }} +{{- end }} +{{- if .Values.sso }} + cluster: {{ .Values.sso.cluster }} + sso: + controlPlaneDomain: {{ .Values.sso.controlPlaneDomain }} + realm: {{ .Values.sso.realm }} + pollIntervalSeconds: {{ .Values.sso.pollIntervalSeconds }} + pollAttempts: {{ .Values.sso.pollAttempts }} +{{- end }} + tenantMetadata: + cluster: {{ .Values.tenantMetadata.cluster }} + platform: {{ .Values.tenantMetadata.platform }} + orgName: {{ .Values.tenantMetadata.orgName }} + tenantName: {{ .Values.tenantMetadata.tenantName }} + tenantType: {{ .Values.tenantMetadata.tenantType }} + startDate: {{ .Values.tenantMetadata.startDate }} + endDate: {{ .Values.tenantMetadata.endDate }} + CSP: {{ .Values.tenantMetadata.CSP }} + allocatedCapacityGPU: {{ .Values.tenantMetadata.allocatedCapacityGPU }} + allocatedCapacityCPU: {{ .Values.tenantMetadata.allocatedCapacityCPU }} + committedCapacityGPU: {{ .Values.tenantMetadata.committedCapacityGPU }} + committedCapacityCPU: {{ .Values.tenantMetadata.committedCapacityCPU }} + CSPProject: {{ .Values.tenantMetadata.CSPProject }} + dataCenterRegion: {{ .Values.tenantMetadata.dataCenterRegion }} + storageType: {{ .Values.tenantMetadata.storageType }} + storageQuota: {{ .Values.tenantMetadata.storageQuota }} +{{- if .Values.initSettings }} + appID: {{ .Values.initSettings.appID }} + tenant: {{ .Values.initSettings.tenant }} + controlPlane: {{ .Values.initSettings.controlPlane }} + initSettings: + supportEmailAddress: {{ .Values.initSettings.supportEmailAddress }} + customerEmailAddress: {{ .Values.initSettings.customerEmailAddress }} + bootstrapEmailAddress: {{ .Values.initSettings.bootstrapEmailAddress }} +{{- end }} + dgxcAdmins: + {{- range .Values.dgxcAdmins }} + - {{ . }} + {{- end }} + tenantSubjects: + {{- range .Values.tenantSubjects.User }} + - {{ . }} + {{- end }} + reservedCPU: {{ .Values.reserved.CPU }} + reservedMemory: {{ .Values.reserved.Memory }} + maxCPUJobs: {{ .Values.maxCPUJobs }} + cloudProviderSpec: + type: {{ .Values.cloudProviderSpec.type }} +{{- if eq .Values.cloudProviderSpec.type "gcp" }} + gcp: + projectId: {{ .Values.cloudProviderSpec.gcp.projectId }} + projectNumber: {{ .Values.cloudProviderSpec.gcp.projectNumber | int64 }} + securityPolicyRuleName: {{ .Values.cloudProviderSpec.gcp.securityPolicyRuleName }} + region: {{ .Values.cloudProviderSpec.gcp.region }} +{{- else if eq .Values.cloudProviderSpec.type "aws" }} + aws: + accountName: {{ .Values.cloudProviderSpec.aws.accountName }} + clusterName: {{ .Values.cloudProviderSpec.aws.clusterName }} + region: {{ .Values.cloudProviderSpec.aws.region }} + securityGroupARN: {{ .Values.cloudProviderSpec.aws.securityGroupARN }} + statefulRuleGroupName: {{ .Values.cloudProviderSpec.aws.statefulRuleGroupName }} +{{- else if eq .Values.cloudProviderSpec.type "oci" }} + oci: + compartmentID: {{ .Values.cloudProviderSpec.oci.compartmentID }} + clusterID: {{ .Values.cloudProviderSpec.oci.clusterID }} + fssAvailabilityDomain: "{{ .Values.cloudProviderSpec.oci.fssAvailabilityDomain }}" + publicSubnetFirewallID: "{{ .Values.cloudProviderSpec.oci.publicSubnetFirewallID }}" + privateSubnetFirewallID: "{{ .Values.cloudProviderSpec.oci.privateSubnetFirewallID }}" + baseFirewallPolicyID: "{{ .Values.cloudProviderSpec.oci.baseFirewallPolicyID }}" +{{- else if eq .Values.cloudProviderSpec.type "azure" }} + azure: + subscriptionId: {{ .Values.cloudProviderSpec.azure.subscriptionId }} + resourceGroupName: {{ .Values.cloudProviderSpec.azure.resourceGroupName }} + ingressNsgName: {{ .Values.cloudProviderSpec.azure.ingressNsgName }} + egressNsgName: {{ .Values.cloudProviderSpec.azure.egressNsgName }} + standardStorageAccountName: {{ .Values.cloudProviderSpec.azure.standardStorageAccountName }} + premiumStorageAccountName: {{ .Values.cloudProviderSpec.azure.premiumStorageAccountName }} + logExporterAnalyticsWorkspaceId: {{ .Values.cloudProviderSpec.azure.logExporterAnalyticsWorkspaceId }} +{{- end }} + logExporterControllerSpec: + enabled: {{ .Values.logExporterControllerSpec.enabled }} + exportFrequency: {{ .Values.logExporterControllerSpec.exportFrequency }} + exportBuffer: {{ .Values.logExporterControllerSpec.exportBuffer }} + fetchTimeout: {{ .Values.logExporterControllerSpec.fetchTimeout }} + systemResources: + {{- range $i, $csk := .Values.clusterScopedKinds }} + {{- $group := $csk.apiGroup | default "core" -}} + {{- range $j, $kind := $csk.resources }} + {{- range $csk.systemResources}} + {{- if and (eq $kind "namespaces") (eq . "dgxc-tenant-cluster-policies") | not }} + - {{ print $group "/" $kind "/" . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + networkControllerSpec: + enabled: {{ .Values.networkControllerSpec.enabled }} + defaultIngressCIDRs: + {{- .Values.bootstrapIngressCIDR | toYaml | nindent 6 }} +{{- if eq .Values.cloudProviderSpec.type "gcp" }} + storageControllerSpec: + enabled: {{ .Values.storageControllerSpecGCP.enabled }} + defaultCustomerPVReclaimPolicy: {{ .Values.storageControllerSpecGCP.defaultCustomerPVReclaimPolicy | default "Retain" }} + defaultStorageTypes: + {{- .Values.storageControllerSpecGCP.defaultStorageTypes | toYaml | nindent 6 }} +{{- else if eq .Values.cloudProviderSpec.type "aws" }} + storageControllerSpec: + enabled: {{ .Values.storageControllerSpecAWS.enabled }} + defaultCustomerPVReclaimPolicy: {{ .Values.storageControllerSpecAWS.defaultCustomerPVReclaimPolicy | default "Retain" }} + defaultStorageTypes: + {{- .Values.storageControllerSpecAWS.defaultStorageTypes | toYaml | nindent 6 }} +{{- else if eq .Values.cloudProviderSpec.type "oci" }} + storageControllerSpec: + enabled: {{ .Values.storageControllerSpecOCI.enabled }} + defaultCustomerPVReclaimPolicy: {{ .Values.storageControllerSpecOCI.defaultCustomerPVReclaimPolicy | default "Retain" }} + defaultStorageTypes: + {{- .Values.storageControllerSpecOCI.defaultStorageTypes | toYaml | nindent 6 }} +{{- else if eq .Values.cloudProviderSpec.type "azure" }} + storageControllerSpec: + enabled: {{ .Values.storageControllerSpecAzure.enabled }} + defaultCustomerPVReclaimPolicy: {{ .Values.storageControllerSpecAzure.defaultCustomerPVReclaimPolicy | default "Retain" }} + defaultStorageTypes: + {{- .Values.storageControllerSpecAzure.defaultStorageTypes | toYaml | nindent 6 }} +{{- end }} + nvStorageControllerSpec: + enabled: {{ .Values.nvStorage.enabled }} + storageClass: {{ .Values.nvStorage.storageClass }} + {{- if .Values.nvStorage.platform }} + platform: + provider: {{ .Values.nvStorage.platform.provider }} + clusterName: {{ .Values.nvStorage.platform.clusterName }} + config: + {{- .Values.nvStorage.platform.config | toYaml | nindent 10 }} + {{- end }} + {{- if .Values.nvStorage.csi }} + csi: + driver: {{ .Values.nvStorage.csi.driver }} + volumeAttributes: + {{- .Values.nvStorage.csi.volumeAttributes | toYaml | nindent 10 }} + {{- end}} + +{{ end}} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/controllercr.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/controllercr.yaml new file mode 100644 index 0000000..9114b01 --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/controllercr.yaml @@ -0,0 +1,28 @@ +{{ if .Values.enabledComponents.dgxcRunaiController }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: dgxc-runai-controller +rules: + - apiGroups: [""] + resources: ["services", "configmaps", "secrets"] + verbs: ["*"] + - apiGroups: ["authentication.gke.io"] + resources: ["clientconfigs"] + verbs: ["*"] + - apiGroups: ["run.ai"] + resources: ["runaiconfigs"] + verbs: ["*"] + - apiGroups: ["runai.dgxc.nvidia.com", "storage.dgxc.nvidia.com", "kyverno.io"] + resources: ["*"] + verbs: ["*"] + - apiGroups: [""] + resources: ["persistentvolumes", "persistentvolumeclaims"] + verbs: ["*"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterrolebindings", "clusterroles", "roles", "rolebindings"] + verbs: ["*"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list", "watch"] +{{ end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/controllercrb.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/controllercrb.yaml new file mode 100644 index 0000000..fe74ee3 --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/controllercrb.yaml @@ -0,0 +1,14 @@ +{{ if .Values.enabledComponents.dgxcRunaiController }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: dgxc-runai-controller +subjects: + - kind: ServiceAccount + name: dgxc-runai-controller + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: dgxc-runai-controller + apiGroup: rbac.authorization.k8s.io +{{ end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/controllerdeployment.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/controllerdeployment.yaml new file mode 100644 index 0000000..aeababf --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/controllerdeployment.yaml @@ -0,0 +1,68 @@ +{{ if .Values.enabledComponents.dgxcRunaiController }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: "dgxc-runai-controller" + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: "dgxc-runai-controller" + helm.sh/chart: {{ include "dgxcController.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: "dgxc-runai-controller" + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app: "dgxc-runai-controller" + app.kubernetes.io/name: "dgxc-runai-controller" + app.kubernetes.io/instance: {{ .Release.Name }} + {{- with .Values.extraControllerPodLabels }} + {{ toYaml . }} + {{- end }} + spec: + serviceAccountName: dgxc-runai-controller + imagePullSecrets: + - name: {{ .Values.imagePullSecret | required ".Values.imagePullSecret" }} + containers: + - name: controller + image: {{ .Values.dgxcController.image | required "dgxcController.image is required." | quote }} + args: + - controller + - --zap-log-level=debug + imagePullPolicy: Always + {{- if eq .Values.cloudProviderSpec.type "oci" }} + env: + - name: OCI_RESOURCE_PRINCIPAL_VERSION + value: {{ .Values.cloudProviderSpec.oci.resourcePrincipalVersion | quote }} + - name: OCI_RESOURCE_PRINCIPAL_REGION + value: {{ .Values.cloudProviderSpec.oci.region }} + {{ end }} + ports: + - containerPort: 8080 + {{- if and (hasKey .Values.nvStorage "platform") (eq .Values.nvStorage.platform.provider "runai" )}} + volumeMounts: + - name: run-ai-secret + mountPath: {{ .Values.nvStorage.platform.config.secret_path | required "config.secret_path is required for runai setup" | quote }} + readOnly: true + {{- end }} + tolerations: + - effect: NoSchedule + key: "{{ .Values.systemTolerationKeyName }}" + operator: Equal + value: "{{ .Values.systemTolerationValueName }}" + - effect: NoExecute + key: "{{ .Values.systemTolerationKeyName }}" + operator: Equal + value: "{{ .Values.systemTolerationValueName }}" + {{- if and (hasKey .Values.nvStorage "platform") (eq .Values.nvStorage.platform.provider "runai" )}} + volumes: + - name: run-ai-secret + secret: + secretName: run-ai-secret + {{- end}} + {{ end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/controllersa.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/controllersa.yaml new file mode 100644 index 0000000..beeb04a --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/controllersa.yaml @@ -0,0 +1,13 @@ +{{ if .Values.enabledComponents.dgxcRunaiController }} +{{- if .Values.controllerServiceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.controllerServiceAccount.name }} + namespace: {{ .Release.Namespace }} + {{- if .Values.controllerServiceAccount.annotations }} + annotations: + {{ toYaml .Values.controllerServiceAccount.annotations }} + {{- end }} +{{ end }} +{{ end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/cr.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/cr.yaml new file mode 100644 index 0000000..c65398c --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/cr.yaml @@ -0,0 +1,13 @@ +{{ if .Values.enabledComponents.dgxcAdmissionController }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.dgxcController.name }} +rules: + - apiGroups: [""] + resources: ["pods", "secrets", "configmaps", "persistentvolumeclaims", "storageclass"] + verbs: ["*"] + - apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] + verbs: ["*"] +{{ end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/crb.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/crb.yaml new file mode 100644 index 0000000..581ca3f --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/crb.yaml @@ -0,0 +1,14 @@ +{{ if .Values.enabledComponents.dgxcAdmissionController }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.dgxcController.name }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.name }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Values.dgxcController.name }} + apiGroup: rbac.authorization.k8s.io +{{ end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/daemonset.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/daemonset.yaml new file mode 100644 index 0000000..4639f4b --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/daemonset.yaml @@ -0,0 +1,71 @@ +{{- if eq .Values.cloudProviderSpec.type "gcp" }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fix-hostname + namespace: kube-system + labels: + nodeGroup: customer-gpu +spec: + selector: + matchLabels: + nodeGroup: customer-gpu + template: + metadata: + labels: + nodeGroup: customer-gpu + spec: + nodeSelector: + nodeGroup: customer-gpu + hostNetwork: true + containers: + - image: gcr.io/google-containers/pause:3.2 + name: pause + initContainers: + - name: fix-hostname + image: ubuntu + env: [ {name: NODE_NAME, valueFrom: { fieldRef: {fieldPath: spec.nodeName } } } ] + command: + - /bin/bash + - -c + - | + set -xeuo pipefail + hostname $(NODE_NAME) + cat << EOF > /host/etc/systemd/network/97-temp.network + [Match] + Name=eth* !eth0 + + [Network] + DHCP=yes + IPv6AcceptRA=yes + + # non-eth0 modification + [DHCPv4] + RoutesToDNS=false + RoutesToNTP=false + + [DHCP] + UseMTU=true + UseDomains=true + + # non-eth0 modification + UseHostname=false + EOF + chroot /host networkctl reload + echo "Success" + + + securityContext: { privileged: true, capabilities: { add: [ NET_ADMIN ] } } + volumeMounts: + - name: host + mountPath: /host + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + volumes: + - name: host + hostPath: + path: / +{{- end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/deployment.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/deployment.yaml new file mode 100644 index 0000000..b164453 --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/deployment.yaml @@ -0,0 +1,97 @@ +{{ if .Values.enabledComponents.dgxcAdmissionController }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "dgxcController.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "dgxcController.name" . }} + helm.sh/chart: {{ include "dgxcController.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + replicas: {{ .Values.dgxcController.replicas | required ".Values.dgxcController.replicas required" }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "dgxcController.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 50% + type: RollingUpdate + template: + metadata: + labels: + app: {{ .Values.dgxcController.name }} + app.kubernetes.io/name: {{ include "dgxcController.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + nvidia.com/dgx-application-deploy: {{ include "dgx-application-deploy" . }} + {{- with .Values.extraPodLabels }} + {{ toYaml . }} + {{- end }} + {{- with .Values.podAnnotations }} + annotations: + {{ toYaml . }} + {{- end }} + spec: + serviceAccountName: {{ .Values.serviceAccount.name }} + imagePullSecrets: + - name: {{ .Values.imagePullSecret | required ".Values.imagePullSecret" }} + initContainers: + - name: init + image: {{ .Values.dgxcController.image | required ".Values.dgxcController.image is required." | quote }} + args: [ "admission-init", "--config", "/etc/{{ .Values.dgxcController.name}}/config.yaml", "--zap-log-level=debug" ] + imagePullPolicy: Always + volumeMounts: + - name: {{ .Values.dgxcController.name}}-config + mountPath: /etc/{{ .Values.dgxcController.name}}/ + readOnly: true + - name: {{ .Values.dgxcController.name}}-certs + mountPath: {{ .Values.dgxcController.certs.mountPath }} + containers: + - name: controller + image: {{ .Values.dgxcController.image | required "dgxcController.image is required." | quote }} + args: ["admission", "--config", "/etc/{{ .Values.dgxcController.name}}/config.yaml", "--zap-log-level=debug"] + imagePullPolicy: Always + ports: [] + volumeMounts: + - name: {{ .Values.dgxcController.name}}-certs + mountPath: {{ .Values.dgxcController.certs.mountPath }} + readOnly: true + - name: {{ .Values.dgxcController.name}}-config + mountPath: /etc/{{ .Values.dgxcController.name}}/ + readOnly: true +{{/* - name: app-share-ns*/}} +{{/* command:*/}} +{{/* - sleep*/}} +{{/* - infinity*/}} +{{/* image: nvcr.io/nvidian/nsvis-corestorage/ubuntu:latest*/}} +{{/* imagePullPolicy: IfNotPresent*/}} +{{/* terminationMessagePath: /dev/termination-log*/}} +{{/* terminationMessagePolicy: File*/}} +{{/* volumeMounts:*/}} +{{/* - name: {{ .Values.dgxcController.name}}-certs*/}} +{{/* mountPath: {{ .Values.dgxcController.certs.mountPath }}*/}} +{{/* readOnly: true*/}} +{{/* - name: {{ .Values.dgxcController.name}}-config*/}} +{{/* mountPath: /etc/{{ .Values.dgxcController.name}}-/*/}} +{{/* readOnly: true*/}} + {{ if .Values.kubernetesAffinity.enabled }} + {{- include "kubernetesAffinityConfig" . | nindent 6 }} + {{ end }} + tolerations: + - effect: NoSchedule + key: "{{ .Values.systemTolerationKeyName }}" + operator: Equal + value: "{{ .Values.systemTolerationValueName }}" + - effect: NoExecute + key: "{{ .Values.systemTolerationKeyName }}" + operator: Equal + value: "{{ .Values.systemTolerationValueName }}" + volumes: + - name: {{ .Values.dgxcController.name}}-certs + emptyDir: {} + - name: {{ .Values.dgxcController.name}}-config + configMap: + name: {{ .Values.dgxcController.name}}-config +{{ end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/exporters/deployment.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/exporters/deployment.yaml new file mode 100644 index 0000000..af0169d --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/exporters/deployment.yaml @@ -0,0 +1,55 @@ +{{ if .Values.enabledComponents.dgxcExporter }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dgxc-exporter + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: dgxc-exporter + helm.sh/chart: {{ include "dgxcController.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: dgxc-exporter + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app: dgxc-exporter + app.kubernetes.io/name: dgxc-exporter + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + serviceAccount: dgxc-exporter + imagePullSecrets: + - name: {{ .Values.imagePullSecret | required ".Values.imagePullSecret" }} + containers: + - name: exporter + image: {{ .Values.dgxcController.image | required "dgxcController.image is required." | quote }} + args: + - exporter + - --port={{ .Values.exporter.metricsPort }} + - --zap-log-level=debug + imagePullPolicy: Always + {{- if eq .Values.cloudProviderSpec.type "oci" }} + env: + - name: OCI_RESOURCE_PRINCIPAL_VERSION + value: {{ .Values.cloudProviderSpec.oci.resourcePrincipalVersion | quote }} + - name: OCI_RESOURCE_PRINCIPAL_REGION + value: {{ .Values.cloudProviderSpec.oci.region }} + {{ end }} + ports: + - containerPort: {{ .Values.exporter.metricsPort }} + protocol: TCP + tolerations: + - effect: NoSchedule + key: "{{ .Values.systemTolerationKeyName }}" + operator: Equal + value: "{{ .Values.systemTolerationValueName }}" + - effect: NoExecute + key: "{{ .Values.systemTolerationKeyName }}" + operator: Equal + value: "{{ .Values.systemTolerationValueName }}" +{{ end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/exporters/rbac.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/exporters/rbac.yaml new file mode 100644 index 0000000..8199dee --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/exporters/rbac.yaml @@ -0,0 +1,32 @@ +{{ if .Values.enabledComponents.dgxcExporter }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: dgxc-exporter + namespace: {{ .Release.Namespace }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: dgxc-exporter +rules: + - apiGroups: [kyverno.io] + resources: [updaterequests] + verbs: [get, list, watch] + - apiGroups: [""] + resources: [configmaps] + verbs: [get, list, watch] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: dgxc-exporter +subjects: + - kind: ServiceAccount + name: dgxc-exporter + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: dgxc-exporter + apiGroup: rbac.authorization.k8s.io +{{ end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/integrations/kyverno.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/integrations/kyverno.yaml new file mode 100644 index 0000000..d7c34db --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/integrations/kyverno.yaml @@ -0,0 +1,416 @@ +{{- if .Values.enableIntegrationsSupport }} +# cluster roles required by kyverno to label cluster scoped resources +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kyverno:dgxc-integrations + labels: + app.kubernetes.io/component: background-controller + app.kubernetes.io/instance: kyverno + app.kubernetes.io/part-of: kyverno +rules: +{{- range .Values.clusterScopedKinds }} +{{- if gt (len .systemResources) 0 }} +- apiGroups: [ {{ .apiGroup | quote }} ] + resources: + {{- range .resources }} + - {{ . | quote }} + {{- end }} + verbs: ['create', 'delete', 'deletecollection', 'get', 'list', 'patch', 'update', 'watch'] +{{- end }} +{{- end }} +- apiGroups: ['rbac.authorization.k8s.io'] + resources: ['clusterroles'] + verbs: ['escalate', 'bind'] +--- +# label system namespaces with dgxc/system-managed: "true" +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: dgxc-integrations-set-tolerations + labels: + dgxc/system-managed: "true" +spec: + background: false # must be done as part of request + validationFailureAction: {{ .Values.validationFailureAction }} + rules: + - name: inject-system-pod-tolerations # inject system node tolerations on system pods + preconditions: # pod doesn't already have system toleration + any: + - key: + - NoExecute + - NoSchedule + operator: AnyNotIn + value: | + {{`{{`}} request.object.spec.tolerations[?(key == '{{ .Values.tolerations.system.key }}' && value == '{{ .Values.tolerations.system.value }}')].effect {{`}}`}} + match: + all: + - resources: # system pods + kinds: [ Pod ] + namespaceSelector: + matchExpressions: [ { key: kubernetes.io/metadata.name , operator: In, values: [ "runai" ] } ] + mutate: + patchesJson6902: |- + - op: add + path: "/spec/tolerations/-" + value: + key: {{ .Values.tolerations.system.key | quote }} + operator: Equal + value: {{ .Values.tolerations.system.value | quote }} + effect: NoSchedule + - op: add + path: "/spec/tolerations/-" + value: + key: {{ .Values.tolerations.system.key | quote }} + operator: Equal + value: {{ .Values.tolerations.system.value | quote }} + effect: NoExecute + - name: inject-customer-pod-tolerations + preconditions: # pod doesn't already have customer toleration + any: + - key: + - NoExecute + - NoSchedule + operator: AnyNotIn + value: | + {{`{{`}} request.object.spec.tolerations[?(key == '{{ .Values.tolerations.customer.key }}' && value == '{{ .Values.tolerations.customer.value }}')].effect {{`}}`}} + match: + all: + - resources: + kinds: [ Pod ] + exclude: + all: + - resources: + namespaceSelector: + matchExpressions: [ { key: dgxc/system-managed , operator: In, values: [ "true" ] } ] + mutate: + patchesJson6902: |- + - op: add + path: "/spec/tolerations/-" + value: + key: {{ .Values.tolerations.customer.key | quote }} + operator: Equal + value: {{ .Values.tolerations.customer.value | quote }} + effect: NoSchedule + - op: add + path: "/spec/tolerations/-" + value: + key: {{ .Values.tolerations.customer.key | quote }} + operator: Equal + value: {{ .Values.tolerations.customer.value | quote }} + effect: NoExecute + - name: inject-customer-ds-tolerations + match: + all: + - resources: + kinds: [ DaemonSet ] + exclude: + all: + - resources: + namespaceSelector: + matchExpressions: [ { key: dgxc/system-managed , operator: In, values: [ "true" ] } ] + preconditions: + any: + - key: + - NoExecute + - NoSchedule + operator: AnyNotIn + value: | + {{`{{`}} request.object.spec.template.spec.tolerations[?(key=='{{ .Values.tolerations.customer.key }}' && value=='{{ .Values.tolerations.customer.value }}')].effect {{`}}`}} + mutate: + patchesJson6902: |- + - op: add + path: "/spec/template/spec/tolerations/-" + value: + key: {{ .Values.tolerations.customer.key | quote }} + operator: Equal + value: {{ .Values.tolerations.customer.value | quote }} + effect: NoSchedule + - op: add + path: "/spec/template/spec/tolerations/-" + value: + key: {{ .Values.tolerations.customer.key | quote }} + operator: Equal + value: {{ .Values.tolerations.customer.value | quote }} + effect: NoExecute + - name: deny-customer-pod-tolerations + match: + all: + - resources: + kinds: [ Pod ] + exclude: + all: + - resources: + namespaceSelector: # resources in system namespaces + matchExpressions: [ { key: dgxc/system-managed , operator: In, values: [ "true" ] } ] + preconditions: + any: + - key: + - NoExecute + - NoSchedule + operator: AnyIn + value: | + {{`{{ request.object.spec.tolerations[?(key==`}}'{{ .Values.tolerations.system.key }}'{{` && value==`}}'{{ .Values.tolerations.system.value }}'{{` || key=='' || key == null)].effect }}`}} + - key: "true" # explicitly bound to system node + operator: Equals + value: | + {{`{{`}} spec.nodename && contains(spec.nodename, 'system') {{`}}`}} + validate: + message: "Customer pods may not have tolerations with an empty key or the key \"{{ .Values.tolerations.system.key }}\" or a nodeName containing 'system' as these allow them to schedule on system pods." + deny: {} + - name: deny-customer-ds-blanket-tolerations + match: + all: + - resources: + kinds: [ DaemonSet ] + exclude: + all: + - resources: + namespaceSelector: # resources in system namespaces + matchExpressions: [ { key: dgxc/system-managed , operator: In, values: [ "true" ] } ] + validate: + message: "Customer daemonsets may not have tolerations with an empty key or empty value." + pattern: + spec: + template: + spec: + =(tolerations): + - key: "?*" + value: "?*" +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: dgxc-integrations-deny-changes-to-system-resources + labels: + dgxc/system-managed: "true" +spec: + background: false + validationFailureAction: {{ .Values.validationFailureAction }} + rules: + # Deny all writes to resources to cluster scoped system resources not made by system accounts + - name: deny-cluster-scoped + match: + any: + # Only below resources are considered dgxc/system-managed + {{- range .Values.clusterScopedKinds }} + {{- if gt (len .systemResources) 0 }} + - resources: + kinds: [ {{ .kind | quote }} ] + names: + {{- range .systemResources }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + exclude: # allow writes by system accounts + any: + - subjects: + # allow writes from service accounts in system namespaces + {{- range .Values.clusterScopedKinds }} + {{- if eq .kind "Namespace" }} + {{- range .systemResources}} + - kind: ServiceAccount + namespace: "{{ . }}" + name: "*" + {{- end }} + {{- end }} + {{- end }} + # allow writes from system users and groups + {{- range $k, $v := .Values.systemSubjects }} + {{- range $v }} + - kind: {{ $k | quote }} + name: {{ . | quote }} + {{- end }} + {{- end }} + # allow writes from system CSP service accounts + {{- range $k, $v := .Values.cspSubjects }} + {{- range $v }} + - kind: {{ $k | quote }} + name: {{ . | quote }} + {{- end }} + {{- end }} + validate: + message: "Changes to system resources are not allowed." + deny: {} + # Deny all writes to resources in system namespaces not made by system accounts + - name: deny-namespace-scoped + match: + any: + - resources: + namespaceSelector: # resources in system namespaces + matchExpressions: [ { key: dgxc/system-managed , operator: In, values: [ "true" ] } ] + kinds: + {{- $kinds := dict }} + {{- range .Values.namespacedKinds }} + {{- if .kind }} + {{- if not (hasKey $kinds .kind) }} + - {{ .kind | quote }} + {{ $_ := set $kinds .kind .kind }} + {{- end }} + {{- end }} + {{- end }} + - 'ServiceAccount/token' + exclude: # allow writes by system accounts + any: + - subjects: + # allow writes from service accounts in system namespaces + {{- range .Values.clusterScopedKinds }} + {{- if eq .kind "Namespace" }} + {{- range .systemResources}} + - kind: ServiceAccount + namespace: "{{ . }}" + name: "*" + {{- end }} + {{- end }} + {{- end }} + # allow writes from system users and groups + {{- range $k, $v := .Values.systemSubjects }} + {{- range $v }} + - kind: {{ $k | quote }} + name: {{ . | quote }} + {{- end }} + {{- end }} + # allow writes from system CSP service accounts + {{- range $k, $v := .Values.cspSubjects }} + {{- range $v }} + - kind: {{ $k | quote }} + name: {{ . | quote }} + {{- end }} + {{- end }} + validate: + message: "Changes to system resources are not allowed." + deny: {} + # Deny customer from applying the dgxc/system-managed label + - name: deny-system-label-on-user-namespaces + match: + any: + {{- range .Values.clusterScopedKinds }} + {{- if gt (len .systemResources) 0 }} + {{- if eq .kind "Namespace" }} + - resources: + kinds: [ {{ .kind | quote }} ] + names: + {{- range .systemResources }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + exclude: # allow writes by system accounts + any: + - subjects: + # allow writes from service accounts in system namespaces + {{- range .Values.clusterScopedKinds }} + {{- if eq .kind "Namespace" }} + {{- range .systemResources}} + - kind: ServiceAccount + namespace: "{{ . }}" + name: "*" + {{- end }} + {{- end }} + {{- end }} + # allow writes from system users and groups + {{- range $k, $v := .Values.systemSubjects }} + {{- range $v }} + - kind: {{ $k | quote }} + name: {{ . | quote }} + {{- end }} + {{- end }} + # allow writes from system CSP service accounts + {{- range $k, $v := .Values.cspSubjects }} + {{- range $v }} + - kind: {{ $k | quote }} + name: {{ . | quote }} + {{- end }} + {{- end }} + validate: + message: "Setting label dgxc/system-managed not allowed" + deny: + conditions: + - key: "{{`{{ request.object.metadata.labels.\"dgxc/system-managed\" || '' }}`}}" + operator: Equals + value: "true" + - name: deny-system-connect + match: + any: + - resources: + kinds: [ 'Pod/exec', 'Pod/portforward', 'Pod/attach', 'Pod/ephemeralcontainers', 'Pod/proxy' ] + namespaceSelector: # resources in system namespaces + matchExpressions: [ { key: dgxc/system-managed , operator: In, values: [ "true" ] } ] + exclude: # allow writes by system accounts + any: + - subjects: + # allow writes from service accounts in system namespaces + {{- range .Values.clusterScopedKinds }} + {{- if eq .kind "Namespace" }} + {{- range .systemResources}} + - kind: ServiceAccount + namespace: "{{ . }}" + name: "*" + {{- end }} + {{- end }} + {{- end }} + # allow writes from system users and groups + {{- range $k, $v := .Values.systemSubjects }} + {{- range $v }} + - kind: {{ $k | quote }} + name: {{ . | quote }} + {{- end }} + {{- end }} + # allow writes from system CSP service accounts + {{- range $k, $v := .Values.cspSubjects }} + {{- range $v }} + - kind: {{ $k | quote }} + name: {{ . | quote }} + {{- end }} + {{- end }} + preconditions: + all: + - key: "{{`{{ request.operation || 'BACKGROUND' }}`}}" + operator: Equals + value: CONNECT + validate: + message: "Changes to system resources are not allowed." + deny: {} +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: dgxc-integrations-generate-rbac + annotations: + argocd.argoproj.io/sync-options: Force=true,Replace=true + labels: + dgxc/system-managed: "true" +spec: + rules: + - name: generate-rbac-for-crd + match: { any: [ resources: { kinds: [ "CustomResourceDefinition" ] } ] } + exclude: + any: + - resources: + names: + {{- range .Values.clusterScopedKinds }} + {{- if eq .kind "CustomResourceDefinition" }} + {{- range .systemResources }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + generate: + synchronize: true + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + name: "dgxc-customer-app-admin-{{ `{{ request.object.spec.names.plural }}` }}" + data: + metadata: + labels: + dgxc/customer-app-admin: "true" + dgxc/system-managed: "true" + dgxc/generated: "true" + rules: + - apiGroups: [ "{{ `{{request.object.spec.group}}` }}" ] + resources: [ "{{ `{{request.object.spec.names.plural}}` }}", "{{ `{{request.object.spec.names.plural}}` }}/finalizers", "{{ `{{request.object.spec.names.plural}}` }}/status", "{{ `{{request.object.spec.names.plural}}` }}/scale" ] + verbs: ['create', 'delete', 'deletecollection', 'get', 'list', 'patch', 'update', 'watch'] +--- +{{- end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/integrations/rbac.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/integrations/rbac.yaml new file mode 100644 index 0000000..ff31038 --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/integrations/rbac.yaml @@ -0,0 +1,37 @@ +{{- if .Values.enableIntegrationsSupport }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: dgxc-customer-app-admin-base + labels: + dgxc/customer-app-admin: "true" + dgxc/system-managed: "true" +rules: +{{- range .Values.namespacedKinds }} +{{- if gt (len .verbs) 0 }} +- apiGroups: [ {{ .apiGroup | quote }} ] + resources: + {{- range .resources }} + - {{ . | quote }} + {{- end }} + verbs: + {{- range .verbs }} + - {{ . | quote }} + {{- end }} +{{- end }} +{{- end }} +{{- range .Values.clusterScopedKinds }} +{{- if gt (len .verbs) 0 }} +- apiGroups: [ {{ .apiGroup | quote }} ] + resources: + {{- range .resources }} + - {{ . | quote }} + {{- end }} + verbs: + {{- range .verbs }} + - {{ . | quote }} + {{- end }} +{{- end }} +{{- end }} +--- +{{- end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/labeler/kyverno.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/labeler/kyverno.yaml new file mode 100644 index 0000000..b91861d --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/labeler/kyverno.yaml @@ -0,0 +1,102 @@ +{{- if .Values.enableLabeler }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kyverno:watch-update-namespaces + labels: + app.kubernetes.io/component: background-controller + app.kubernetes.io/instance: kyverno + app.kubernetes.io/part-of: kyverno +rules: + - apiGroups: + - "" + resources: + - namespaces + verbs: ["get", "watch", "list", "patch", "update"] +--- +# label system namespaces with dgxc/system-managed: "true" +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: dgxc-label-system-resources + labels: + dgxc/system-managed: "true" +spec: + mutateExistingOnPolicyUpdate: true + rules: + - name: label-cluster-scoped-system-resources + mutate: + targets: + - apiVersion: "{{`{{request.object.apiVersion}}`}}" + kind: "{{`{{request.object.kind}}`}}" + name: "{{`{{request.object.metadata.name}}`}}" + patchStrategicMerge: + metadata: + labels: + "dgxc/system-managed": "true" + match: + any: + {{- range .Values.clusterScopedKinds }} + {{- if gt (len .systemResources) 0 }} + {{- if eq .kind "Namespace" }} + - resources: + kinds: [ {{ .kind | quote }} ] + names: + {{- range .systemResources }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: dgxc-copy-node-group-to-node-role +spec: + rules: + - name: cleanup-node-roles + match: + any: + - resources: + kinds: + - Node + context: + - name: nodeGroup + variable: + jmesPath: request.object.metadata.labels.nodeGroup + default: "" + mutate: + foreach: + - list: keys(request.object.metadata.labels)[?starts_with(@, 'node-role.kubernetes.io')] + patchesJson6902: |- + - op: remove + path: "/metadata/labels/{{`{{ replace_all('{{ element }}', '/', '~1') }}`}}" + preconditions: + all: + - key: "{{`{{ element }}`}}" + operator: NotEquals + value: "{{`{{ nodeGroup }}`}}" + - name: copy-node-role + match: + any: + - resources: + kinds: + - Node + context: + - name: nodeGroup + variable: + jmesPath: request.object.metadata.labels.nodeGroup + default: "" + mutate: + patchStrategicMerge: + metadata: + labels: + node-role.kubernetes.io/{{`{{ nodeGroup }}`}}: "" + preconditions: + all: + - key: "{{`{{ nodeGroup }}`}}" + operator: NotEquals + value: "" +--- +{{- end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/monitor.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/monitor.yaml new file mode 100644 index 0000000..9ae6b8c --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/monitor.yaml @@ -0,0 +1,19 @@ +{{- if .Values.enableServiceMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: dgxc-admission-controller + labels: + app: {{ .Values.dgxcController.name }}-metrics +spec: + endpoints: + - interval: 30s + path: /metrics + port: metrics + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + app: {{ .Values.dgxcController.name }}-metrics +{{- end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/nccl/kyverno.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/nccl/kyverno.yaml new file mode 100644 index 0000000..1e05176 --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/nccl/kyverno.yaml @@ -0,0 +1,400 @@ +{{- if and (.Values.cloudProviderSpec) (eq .Values.cloudProviderSpec.type "aws") ((.Values.cloudProviderSpec.aws).efa)}} +# EFA config changes: +# add volume /opt/amazon-efa-ofi to pod, env vars and volumeMount to applicable containers in pod +# Applies to: +# 1) Worker pods resulting from MPIJob or PyTorchJob, and containers only when requested GPUs == gpuAllocatable +# 2) Batch Jobs - these act as launchers for MPIJob and PyTorchJob. Generally no GPU requested. +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + labels: + dgxc/system-managed: "true" + name: dgxc-eks-efa-configs +spec: + background: false + rules: + # attach hostpath volume to Pytorch and MPI workers that are full GPU + - match: + any: + - resources: + kinds: + - Pod + operations: + - CREATE + exclude: + any: + - resources: + annotations: + disable-auto-efa: "true" + mutate: + patchStrategicMerge: + spec: + volumes: + - hostPath: + path: /opt/amazon-efa-ofi + type: Directory + name: amazon-efa + preconditions: + all: + - key: "{{`{{ request.object.spec.containers[].resources.requests.\"nvidia.com/gpu\" || '' }}`}}" + operator: AnyIn + value: [ "{{ .Values.cloudProviderSpec.aws.gpuNode.gpuAllocatable }}" ] + - key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}" + operator: AnyIn + value: [ "PyTorchJob", "MPIJob", "Job", "RunaiJob" ] + - key: "{{`{{ request.object.metadata.ownerReferences[].apiVersion.split(@, '/')[0] }}`}}" + operator: AnyIn + value: [ "batch.volcano.sh", "kubeflow.org", "run.ai" ] + name: efa-pod-volumes-workers + # PyTorch and MPI launchers can be CPU-only Job, still need the hostpath volume + - match: + any: + - resources: + kinds: + - Pod + operations: + - CREATE + exclude: + any: + - resources: + annotations: + disable-auto-efa: "true" + mutate: + patchStrategicMerge: + spec: + volumes: + - hostPath: + path: /opt/amazon-efa-ofi + type: Directory + name: amazon-efa + preconditions: + all: + - key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}" + operator: AnyIn + value: [ "Job" ] + name: efa-pod-volumes-launchers + # Make container changes for EFA - mount and resources + - match: + any: + - resources: + kinds: + - Pod + operations: + - CREATE + exclude: + any: + - resources: + annotations: + disable-auto-efa: "true" + preconditions: + all: + - key: "{{`{{ request.object.spec.containers[].resources.requests.\"nvidia.com/gpu\" || '' }}`}}" + operator: AnyIn + value: [ "{{ .Values.cloudProviderSpec.aws.gpuNode.gpuAllocatable }}" ] + - key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}" + operator: AnyIn + value: [ "PyTorchJob", "MPIJob", "Job", "RunaiJob" ] + - key: "{{`{{ request.object.metadata.ownerReferences[].apiVersion.split(@, '/')[0] }}`}}" + operator: AnyIn + value: [ "batch.volcano.sh", "kubeflow.org", "run.ai" ] + mutate: + foreach: + - list: "request.object.spec.containers" + patchStrategicMerge: + spec: + containers: + - (name): "{{`{{ element.name }}`}}" + volumeMounts: + - mountPath: /opt/amazon-efa-ofi + name: amazon-efa + readOnly: true + resources: + requests: + "vpc.amazonaws.com/efa": "{{ .Values.cloudProviderSpec.aws.gpuNode.efaAllocatable }}" + hugepages-2Mi: "{{`{{ element.resources.requests.\"hugepages-2Mi\" || '`}}{{ .Values.cloudProviderSpec.aws.gpuNode.hugePagesAllocatable }}{{`' }}`}}" + cpu: "{{`{{ element.resources.requests.cpu || '100m' }}`}}" + limits: + "vpc.amazonaws.com/efa": "{{ .Values.cloudProviderSpec.aws.gpuNode.efaAllocatable }}" + hugepages-2Mi: "{{`{{ element.resources.limits.\"hugepages-2Mi\" || '`}}{{ .Values.cloudProviderSpec.aws.gpuNode.hugePagesAllocatable }}{{`' }}`}}" + preconditions: + any: + - key: "{{`{{ element.resources.requests.\"nvidia.com/gpu\" || '' }}`}}" + operator: Equals + value: "{{ .Values.cloudProviderSpec.aws.gpuNode.gpuAllocatable }}" + name: container-efa-volume-mounts-worker + # We treat the launcher differently, doesn't have GPU + - match: + any: + - resources: + kinds: + - Pod + operations: + - CREATE + exclude: + any: + - resources: + annotations: + disable-auto-efa: "true" + preconditions: + all: + - key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}" + operator: AnyIn + value: [ "Job" ] + mutate: + foreach: + - list: "request.object.spec.containers" + patchStrategicMerge: + spec: + containers: + - (name): "{{`{{ element.name }}`}}" + volumeMounts: + - mountPath: /opt/amazon-efa-ofi + name: amazon-efa + readOnly: true + name: container-efa-volume-mounts-launcher + # worker pods need LD_LIBRARY_PATH updated, but we limit this to containers that have gpuAllocatable GPU + - match: + any: + - resources: + kinds: + - Pod + operations: + - CREATE + exclude: + any: + - resources: + annotations: + disable-auto-efa: "true" + preconditions: + all: + - key: "{{`{{ request.object.spec.containers[].resources.requests.\"nvidia.com/gpu\" || '' }}`}}" + operator: AnyIn + value: [ "{{ .Values.cloudProviderSpec.aws.gpuNode.gpuAllocatable }}" ] + - key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}" + operator: AnyIn + value: [ "PyTorchJob", "MPIJob", "Job", "RunaiJob" ] + - key: "{{`{{ request.object.metadata.ownerReferences[].apiVersion.split(@, '/')[0] }}`}}" + operator: AnyIn + value: [ "batch.volcano.sh", "kubeflow.org", "run.ai" ] + mutate: + foreach: + - list: "request.object.spec.containers" + context: + - name: baselibpath + variable: + jmesPath: element.env[?(name=='LD_LIBRARY_PATH')].value | [0] + default: "/usr/lib:/usr/lib64" + - name: extralibpathprefix + variable: + value: "/opt/amazon-efa-ofi/ofi/lib:/opt/amazon-efa-ofi/efa/lib:/opt/amazon-efa-ofi/openmpi/lib:/usr/local/nvidia/lib64" + - name: extralibpathsuffix + variable: + value: "/usr/lib/x86_64-linux-gnu" + patchStrategicMerge: + spec: + containers: + - (name): "{{`{{ element.name }}`}}" + env: + - name: LD_LIBRARY_PATH + value: "{{`{{ extralibpathprefix }}:{{ baselibpath }}:{{ extralibpathsuffix }}`}}" + preconditions: + all: + - key: "{{`{{ element.resources.requests.\"nvidia.com/gpu\" || '' }}`}}" + operator: Equals + value: "{{ .Values.cloudProviderSpec.aws.gpuNode.gpuAllocatable }}" + - key: "{{`{{ element.[env[?(name=='LD_LIBRARY_PATH' && contains(value, 'amazon'))] || `}}`[]`{{` ][] | length(@) }}`}}" + operator: Equals + value: 0 + name: container-worker-library-path + # launcher pods need LD_LIBRARY_PATH updated + - match: + any: + - resources: + kinds: + - Pod + operations: + - CREATE + exclude: + any: + - resources: + annotations: + disable-auto-efa: "true" + preconditions: + all: + - key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}" + operator: AnyIn + value: [ "Job" ] + - key: "{{`{{ request.object.metadata.ownerReferences[].apiVersion.split(@, '/')[0] }}`}}" + operator: AnyIn + value: [ "batch" ] + mutate: + foreach: + - list: "request.object.spec.containers" + context: + - name: baselibpath + variable: + jmesPath: element.env[?(name=='LD_LIBRARY_PATH')].value | [0] + default: "/usr/lib:/usr/lib64" + - name: extralibpathprefix + variable: + value: "/opt/amazon-efa-ofi/ofi/lib:/opt/amazon-efa-ofi/efa/lib:/opt/amazon-efa-ofi/openmpi/lib:/usr/local/nvidia/lib64" + - name: extralibpathsuffix + variable: + value: "/usr/lib/x86_64-linux-gnu" + patchStrategicMerge: + spec: + containers: + - (name): "{{`{{ element.name }}`}}" + env: + - name: LD_LIBRARY_PATH + value: "{{`{{ extralibpathprefix }}:{{ baselibpath }}:{{ extralibpathsuffix }}`}}" + preconditions: + all: + - key: "{{`{{ element.[env[?(name=='LD_LIBRARY_PATH' && contains(value, 'amazon'))] || `}}`[]`{{` ][] | length(@) }}`}}" + operator: Equals + value: 0 + name: container-launcher-library-path + {{- if and ((.Values.cloudProviderSpec.aws).efa) (((.Values.envVars).user).efa)}} + - match: + any: + - resources: + kinds: + - Pod + operations: + - CREATE + exclude: + any: + - resources: + annotations: + disable-auto-efa: "true" + preconditions: + all: + - key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}" + operator: AnyIn + value: [ "PyTorchJob", "MPIJob", "Job", "RunaiJob" ] + - key: "{{`{{ request.object.metadata.ownerReferences[].apiVersion.split(@, '/')[0] }}`}}" + operator: AnyIn + value: [ "batch.volcano.sh", "kubeflow.org", "run.ai" ] + mutate: + foreach: + {{- range $e := .Values.envVars.user.efa }} + - list: "request.object.spec.containers" + patchStrategicMerge: + spec: + containers: + - (name): "{{`{{ element.name }}`}}" + env: + - name: {{ $e.name }} + value: "{{ $e.value }}" + preconditions: + all: + - key: {{ $e.name }} + operator: AnyNotIn + value: "{{`{{ element.env[].name || '' }}`}}" + - key: "{{`{{ element.resources.requests.\"nvidia.com/gpu\" || '' }}`}}" + operator: Equals + value: "{{ $.Values.cloudProviderSpec.aws.gpuNode.gpuAllocatable }}" + {{- end }} + name: extra-gpu-env-vars-workers + - match: + any: + - resources: + kinds: + - Pod + operations: + - CREATE + exclude: + any: + - resources: + annotations: + disable-auto-efa: "true" + preconditions: + all: + - key: "{{`{{ request.object.metadata.ownerReferences[].kind || '' }}`}}" + operator: AnyIn + value: [ "Job" ] + - key: "{{`{{ request.object.metadata.ownerReferences[].apiVersion.split(@, '/')[0] }}`}}" + operator: AnyIn + value: [ "batch" ] + mutate: + foreach: + {{- range $e := .Values.envVars.user.efa }} + - list: "request.object.spec.containers" + patchStrategicMerge: + spec: + containers: + - (name): "{{`{{ element.name }}`}}" + env: + - name: {{ $e.name }} + value: "{{ $e.value }}" + preconditions: + all: + - key: {{ $e.name }} + operator: AnyNotIn + value: "{{`{{ element.env[].name || '' }}`}}" + {{- end }} + name: extra-gpu-env-vars-launchers + {{- end}} +--- +{{- end}} + +{{- if and (.Values.cloudProviderSpec) (eq .Values.cloudProviderSpec.type "azure") (eq .Values.cloudProviderSpec.azure.instanceType "Standard_ND96amsr_A100_v4")}} +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: nccl-topo-configmap-policy +spec: + generateExisting: true + rules: + - name: generate-nccl-topo-configmap + match: + any: + - resources: + kinds: + - Namespace + name: "{{ .Values.ncclTopoConfigMapNamespaces }}" + generate: + apiVersion: v1 + kind: ConfigMap + name: nccl-topo + synchronize: true + namespace: "{{`{{request.object.metadata.name}}`}}" + data: + data: + nccl-topo.xml: | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +--- +{{- end}} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/pdb.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/pdb.yaml new file mode 100644 index 0000000..78b5b66 --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/pdb.yaml @@ -0,0 +1,9 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "dgxcController.fullname" . }} +spec: + minAvailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: {{ include "dgxcController.name" . }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/rbac/rbac.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/rbac/rbac.yaml new file mode 100644 index 0000000..27c587b --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/rbac/rbac.yaml @@ -0,0 +1,29 @@ +{{- if .Values.enableRunAIRbac }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: dgxc-customer-app-admin + labels: + dgxc/system-managed: "true" +aggregationRule: + clusterRoleSelectors: + - matchLabels: + dgxc/customer-app-admin: "true" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: dgxc-customer-app-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: dgxc-customer-app-admin +subjects: +{{- range $k, $v := .Values.tenantSubjects }} +{{- range $v }} +- apiGroup: rbac.authorization.k8s.io + kind: {{ $k | quote }} + name: {{ . | quote }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/runai.dgxc.nvidia.com_runaidgxclogexporters.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/runai.dgxc.nvidia.com_runaidgxclogexporters.yaml new file mode 100644 index 0000000..5b33bbd --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/runai.dgxc.nvidia.com_runaidgxclogexporters.yaml @@ -0,0 +1,152 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: runaidgxclogexporters.runai.dgxc.nvidia.com +spec: + group: runai.dgxc.nvidia.com + names: + kind: RunaiDGXCLogExporter + listKind: RunaiDGXCLogExporterList + plural: runaidgxclogexporters + singular: runaidgxclogexporter + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.destination + name: Destination + type: string + - jsonPath: .status.watermark + name: Watermark + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: RunaiDGXCLogExporter exports audit logs to an object storage + bucket. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + destination: + description: |- + Destination is the cloud storage bucket to write logs to. + + If set, logs of each type specified in LogTypes will be exported to the + bucket periodically. The destination is created and managed by the user. + The Status.WriterPrincipal field will specify the principal that must be + granted write access to the destination. + type: string + logTypes: + default: + - apiserver-audit + description: LogTypes specifies the types of logs to export. + items: + enum: + - apiserver-audit + type: string + minItems: 1 + type: array + required: + - destination + - logTypes + type: object + status: + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + watermark: + description: Watermark is the latest timestamp up until which logs + are exported. + format: date-time + type: string + writerPrincipal: + description: |- + WriterPrincipal is the cloud provider-specific identifier of the log writer. + + The principal must be granted write access to the destination in order for log + writes to succeed. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/runai.dgxc.nvidia.com_runaidgxcnetworkpolicies.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/runai.dgxc.nvidia.com_runaidgxcnetworkpolicies.yaml new file mode 100644 index 0000000..6129d84 --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/runai.dgxc.nvidia.com_runaidgxcnetworkpolicies.yaml @@ -0,0 +1,151 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: runaidgxcnetworkpolicies.runai.dgxc.nvidia.com +spec: + group: runai.dgxc.nvidia.com + names: + kind: RunaiDGXCNetworkPolicy + listKind: RunaiDGXCNetworkPolicyList + plural: runaidgxcnetworkpolicies + singular: runaidgxcnetworkpolicy + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: RunaiDGXCNetworkPolicy defines tenant cluster scoped network + policies. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + clusterAccessPolicy: + description: ClusterAccessPolicy defines how the cluster may be accessed + through the apiserver. + properties: + permittedCIDR: + description: PermittedCIDR defines IP blocks that may access the + cluster through the apiserver. + items: + type: string + type: array + required: + - permittedCIDR + type: object + egressPolicy: + description: EgressPolicy if defined limits the egress NetworkPolicies + that may be defined per-namespace. Applies to all namespaces. + properties: + permittedCIDR: + description: PermittedCIDR defines permitted egress CIDR ranges + items: + properties: + cidr: + description: CIDR defines the CIDR range + type: string + port: + description: Port defines the allowed port + type: string + required: + - cidr + - port + type: object + type: array + required: + - permittedCIDR + type: object + ingressPolicy: + description: IngressPolicy if defined limits the ingress NetworkPolicies + that may be defined per-namespace. Applies to all namespaces. + properties: + permittedCIDR: + description: PermittedCIDR defines permitted ingress CIDR ranges + items: + type: string + type: array + required: + - permittedCIDR + type: object + type: object + status: + properties: + clusterAccessPolicy: + description: ClusterAccessPolicy defines how the cluster may be accessed + through the apiserver. + properties: + permittedCIDR: + description: PermittedCIDR defines IP blocks that may access the + cluster through the apiserver. + items: + type: string + type: array + required: + - permittedCIDR + type: object + egressPolicy: + description: EgressPolicy is the egressPolicy for the last fully reconciled + spec + properties: + permittedCIDR: + description: PermittedCIDR defines permitted egress CIDR ranges + items: + properties: + cidr: + description: CIDR defines the CIDR range + type: string + port: + description: Port defines the allowed port + type: string + required: + - cidr + - port + type: object + type: array + required: + - permittedCIDR + type: object + ingressPolicy: + description: IngressPolicy is the ingressPolicy for the last fully + reconciled spec + properties: + permittedCIDR: + description: PermittedCIDR defines permitted ingress CIDR ranges + items: + type: string + type: array + required: + - permittedCIDR + type: object + reconcileErrors: + description: ReconcileErrors are errors seen during the last reconciliation. + items: + type: string + type: array + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/runai.dgxc.nvidia.com_runaidgxcstorages.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/runai.dgxc.nvidia.com_runaidgxcstorages.yaml new file mode 100644 index 0000000..b16e155 --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/runai.dgxc.nvidia.com_runaidgxcstorages.yaml @@ -0,0 +1,236 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: runaidgxcstorages.runai.dgxc.nvidia.com +spec: + group: runai.dgxc.nvidia.com + names: + kind: RunaiDGXCStorage + listKind: RunaiDGXCStorageList + plural: runaidgxcstorages + singular: runaidgxcstorage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Storage classes + jsonPath: .status.storageClassesDisplayString + name: StorageClasses + type: string + - description: Percent of storage quota used for volumes + jsonPath: .status.storageUsedPct + name: QuotaUsedPct + type: string + - description: Storage quota + jsonPath: .status.storageQuota + name: Quota + type: string + - description: Storage quota used by PVs + jsonPath: .status.storageRequested + name: QuotaUsed + type: string + - description: Storage utilized by PVs + jsonPath: .status.storageConsumed + name: StorageConsumed + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: RunaiDGXCStorage defines storage for DGXC Run:ai accelerated + compute environments. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: RunaiDGXCStorageSpec defines the desired storage state + properties: + instances: + additionalProperties: + properties: + name: + description: Name of the PV + type: string + persistentVolumeReclaimPolicy: + description: PersistentVolumeReclaimPolicy describes a policy + for end-of-life maintenance of persistent volumes. + type: string + type: object + type: object + quota: + type: string + quotaID: + type: string + x-kubernetes-validations: + - message: quotaID is immutable + rule: self == oldSelf + tiers: + items: + type: string + type: array + required: + - quotaID + type: object + status: + properties: + altNames: + description: Alternative names of the storage type + items: + type: string + type: array + conditions: + description: Conditions contains storage conditions + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + description: + description: Short description for the quota + type: string + docs: + description: Documentation for the storage class + type: string + instances: + additionalProperties: + properties: + capacity: + description: Capacity of the PV + type: string + creationTime: + description: When the volume was created + format: date-time + type: string + fileShares: + additionalProperties: + properties: + name: + description: Name of the fileshare + type: string + orphaned: + description: Orphaned is true if the volume is not mounted + in any cluster + type: boolean + values: + additionalProperties: + type: string + description: Usage metrics + type: object + required: + - name + type: object + description: Fileshare used for the volume + type: object + location: + description: Region where the resource is provisioned + type: string + name: + description: Name of the PV + type: string + persistentVolumeReclaimPolicy: + description: PersistentVolumeReclaimPolicy describes a policy + for end-of-life maintenance of persistent volumes. + type: string + state: + description: Current state of the resource + type: string + tier: + description: Storage tier + type: string + type: object + description: PVs consuming the quota + type: object + storageClasses: + description: Storage classes that are bound to this quota + items: + type: string + type: array + storageClassesDisplayString: + description: StorageClassesDisplayString has a display string for + printing the storage classes from kubectl + type: string + storageConsumed: + description: Amount of storage consumed from the requested + type: string + storageQuota: + description: Amount of quota allocated + type: string + storageRequested: + description: Amount of storage requested from the quota + type: string + storageUsedPct: + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/sa.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/sa.yaml new file mode 100644 index 0000000..2694ded --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/sa.yaml @@ -0,0 +1,12 @@ +{{ if .Values.enabledComponents.dgxcAdmissionController }} +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.name }} + {{- if .Values.serviceAccount.annotations }} + annotations: + {{ toYaml .Values.serviceAccount.annotations }} + {{- end }} +{{ end }} +{{ end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/security/kyverno.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/security/kyverno.yaml new file mode 100644 index 0000000..f30f75e --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/security/kyverno.yaml @@ -0,0 +1,308 @@ +{{- if .Values.enableNetworkPolicies }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kyverno:watch-update-services + labels: + app.kubernetes.io/component: background-controller + app.kubernetes.io/instance: kyverno + app.kubernetes.io/part-of: kyverno +rules: + - apiGroups: + - "" + resources: + - services + verbs: ["get", "watch", "list", "patch", "update"] +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: add-runai-cluster-redis-networkpolicy +spec: + generateExisting: true + rules: + - generate: + apiVersion: networking.k8s.io/v1 + data: + metadata: + labels: + dgxc/generated: "true" + spec: + ingress: + - from: + - namespaceSelector: + matchExpressions: + - key: runai/queue + operator: Exists + podSelector: + matchLabels: + run.ai/control-plane-managed: "true" + ports: + - port: 6379 + protocol: TCP + podSelector: + matchLabels: + app: cluster-redis + policyTypes: + - Ingress + kind: NetworkPolicy + name: allow-runai-cluster-redis-ingress + namespace: "{{`{{request.object.metadata.name}}`}}" + synchronize: true + match: + any: + - resources: + kinds: + - Namespace + selector: + matchLabels: + kubernetes.io/metadata.name: "runai" + name: allow-runai-cluster-redis-ingress +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: add-networkpolicy-allow-system-ingress +spec: + generateExisting: true + rules: + {{- if and (.Values.cloudProviderSpec) (eq .Values.cloudProviderSpec.type "oci") }} + - name: allow-system-ingress-via-ciliumnetworkpolicy + match: + any: + - resources: + kinds: + - Namespace + selector: + matchLabels: + dgxc/system-managed: "true" + generate: + apiVersion: cilium.io/v2 + kind: CiliumNetworkPolicy + name: allow-system-ingress + synchronize: true + namespace: "{{`{{request.object.metadata.name}}`}}" + data: + metadata: + labels: + dgxc/generated: "true" + spec: + endpointSelector: {} + ingress: + - fromEntities: + - remote-node + {{- end }} + - name: allow-system-ingress-via-clusterpolicy + match: + any: + - resources: + kinds: + - Namespace + selector: + matchLabels: + dgxc/system-managed: "true" + generate: + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + name: allow-system-ingress + synchronize: true + namespace: "{{`{{request.object.metadata.name}}`}}" + data: + metadata: + labels: + dgxc/generated: "true" + spec: + podSelector: {} + policyTypes: + - Ingress + ingress: + - from: + {{- range .Values.network.systemCIDRs }} + - ipBlock: + cidr: {{ . | quote }} + {{- end }} + - namespaceSelector: + matchExpressions: + - key: dgxc/system-managed + operator: In + values: ["true"] + - name: default-deny-ingress-via-clusterpolicy + match: + any: + - resources: + kinds: + - Namespace + selector: + matchLabels: + dgxc/system-managed: "true" + generate: + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + name: default-deny + namespace: "{{`{{request.object.metadata.name}}`}}" + synchronize: true + data: + metadata: + labels: + dgxc/generated: "true" + spec: + podSelector: {} + policyTypes: + - Ingress +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: manage-loadbalancer-networkpolicy +spec: + generateExisting: true + rules: + - name: allow-internet-access-for-loadbalancer + match: + resources: + kinds: + - Service + preconditions: + any: + - key: "{{`{{request.object.spec.type}}`}}" + operator: Equals + value: "LoadBalancer" + - key: "{{`{{request.object.spec.type}}`}}" + operator: Equals + value: "NodePort" + generate: + kind: NetworkPolicy + name: allow-internet-access + apiVersion: networking.k8s.io/v1 + namespace: "{{`{{request.object.metadata.namespace}}`}}" + synchronize: true + data: + metadata: + labels: + dgxc/generated: "true" + spec: + podSelector: + matchLabels: "{{`{{request.object.spec.selector}}`}}" + policyTypes: + - Ingress + ingress: + - {} +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: deny-metadata-access-policy + annotations: + argocd.argoproj.io/sync-options: Force=true,Replace=true +spec: + generateExisting: true + rules: + - name: deny-metadata-access-rule + match: + any: + - resources: + kinds: + - Namespace + exclude: + any: + {{- range .Values.clusterScopedKinds }} + {{- if gt (len .systemResources) 0 }} + {{- if eq .kind "Namespace" }} + - resources: + kinds: [ {{ .kind | quote }} ] + names: + {{- range .systemResources }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + generate: + kind: NetworkPolicy + name: deny-metadata-access + apiVersion: networking.k8s.io/v1 + namespace: "{{`{{request.object.metadata.name}}`}}" + synchronize: true + data: + metadata: + labels: + dgxc/generated: "true" + spec: + podSelector: {} + policyTypes: + - Egress + egress: + - to: + - ipBlock: + cidr: 0.0.0.0/0 + {{- if not (and + (hasKey .Values "cloudProviderSpec") + (hasKey .Values.cloudProviderSpec "type") + (eq .Values.cloudProviderSpec.type "gcp") + (hasKey .Values.cloudProviderSpec "gcp") + (hasKey .Values.cloudProviderSpec.gcp "workloadIdentityEnabled") + (.Values.cloudProviderSpec.gcp.workloadIdentityEnabled) + ) }} + except: + - 169.254.0.0/16 + {{- end }} + {{- if and (.Values.cloudProviderSpec) (or (eq .Values.cloudProviderSpec.type "gcp") (eq .Values.cloudProviderSpec.type "oci") (eq .Values.cloudProviderSpec.type "azure")) }} + - namespaceSelector: {} + - podSelector: {} + {{- end }} +{{- if and (.Values.cloudProviderSpec) (eq .Values.cloudProviderSpec.type "aws")}} +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: deny-node-traffic +spec: + generateExisting: true + rules: + - name: deny-node-traffic + match: + any: + - resources: + kinds: + - Namespace + exclude: + any: + {{- range .Values.clusterScopedKinds }} + {{- if gt (len .systemResources) 0 }} + {{- if eq .kind "Namespace" }} + - resources: + kinds: [ {{ .kind | quote }} ] + names: + {{- range .systemResources }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + context: + - name: nodesIPs + apiCall: + urlPath: "/api/v1/nodes" + jmesPath: | + map(&join('', [@, '/32']), items[].status[].addresses[?type=='InternalIP'].address | []) + generate: + synchronize: true + kind: NetworkPolicy + name: deny-node-access + apiVersion: networking.k8s.io/v1 + namespace: "{{`{{request.object.metadata.name}}`}}" + data: + metadata: + labels: + dgxc/generated: "true" + spec: + podSelector: {} + policyTypes: + - Egress + egress: + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: "{{`{{ nodesIPs }}`}}" +{{- end }} +{{- end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/security/networkpolicy.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/security/networkpolicy.yaml new file mode 100644 index 0000000..b8eaa5b --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/security/networkpolicy.yaml @@ -0,0 +1,28 @@ +{{- if .Values.enableNetworkPolicies }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-dns + namespace: kube-system +spec: + podSelector: + matchLabels: + k8s-app: "kube-dns" + policyTypes: + - Ingress + ingress: + - from: [] +{{- if and (.Values.cloudProviderSpec) (eq .Values.cloudProviderSpec.type "oci") }} +--- +apiVersion: cilium.io/v2 +kind: CiliumClusterwideNetworkPolicy +metadata: + name: allow-egress-to-all +spec: + endpointSelector: {} + egress: + - toEntities: + - all +{{- end }} +--- +{{- end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/service.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/service.yaml new file mode 100644 index 0000000..fe389cb --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/service.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.serviceName }} + labels: + app: {{ .Values.dgxcController.name }} +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + app: {{ .Values.dgxcController.name }} +--- +apiVersion: v1 +kind: Service +metadata: + name: dgx-controller-metrics + labels: + app: {{ .Values.dgxcController.name }}-metrics +spec: + ports: + - name: metrics + port: 80 + protocol: TCP + targetPort: 8080 + selector: + app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/sre/default.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/sre/default.yaml new file mode 100644 index 0000000..3b0692b --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/sre/default.yaml @@ -0,0 +1,222 @@ +{{- if .Values.enableDGXCRbac }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: sre-cluster-scoped-access +rules: +- apiGroups: [ "" ] + resources: + - nodes + - nodes/proxy + - nodes/status + - componentstatuses + verbs: [ "*" ] +- apiGroups: [ "apiregistration.k8s.io" ] + resources: + - apiservices + - apiservices/status + verbs: [ "*" ] +- apiGroups: [ "authentication.k8s.io" ] + resources: + - selfsubjectreviews + - tokenreviews + verbs: [ "*" ] +- apiGroups: [ "authorization.k8s.io" ] + resources: + - selfsubjectaccessreviews + - selfsubjectrulesreviews + - subjectaccessreviews + verbs: [ "*" ] +- apiGroups: [ "certificates.k8s.io" ] + resources: + - certificatesigningrequests + - certificatesigningrequests/approval + - certificatesigningrequests/status + verbs: [ "*" ] +- apiGroups: [ "networking.k8s.io" ] + resources: + - ingressclasses + verbs: [ "*" ] +- apiGroups: [ "rbac.authorization.k8s.io" ] + resources: + - clusterrolebindings + - clusterroles + verbs: [ "*" ] +- apiGroups: [ "storage.k8s.io" ] + resources: + - csidrivers + - csinodes + - storageclasses + - volumeattachments + - volumeattachments/status + verbs: [ "*" ] +- apiGroups: [ "admissionregistration.k8s.io" ] + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: [ "*" ] +- apiGroups: [ "apiextensions.k8s.io" ] + resources: + - customresourcedefinitions + - customresourcedefinitions/status + verbs: [ "*" ] +- apiGroups: [ "scheduling.k8s.io" ] + resources: + - priorityclasses + verbs: [ "*" ] +- apiGroups: [ "node.k8s.io" ] + resources: + - runtimeclasses + verbs: [ "*" ] +- apiGroups: [ "flowcontrol.apiserver.k8s.io" ] + resources: + - flowschemas + - flowschemas/status + - prioritylevelconfigurations + - prioritylevelconfigurations/status + verbs: [ "*" ] +- apiGroups: [ "auto.gke.io" ] + resources: + - allowlistedworkloads + - allowlistedv2workloads + verbs: [ "*" ] +- apiGroups: [ "cilium.io" ] + resources: + - ciliumexternalworkloads + - ciliumexternalworkloads/status + - ciliumidentities + - ciliumidentities/status + - ciliumnodes + - ciliumnodes/status + verbs: [ "*" ] +- apiGroups: [ "engine.run.ai" ] + resources: + - configs + - configs/status + verbs: [ "*" ] +- apiGroups: [ "external-secrets.io" ] + resources: + - clusterexternalsecrets + - clusterexternalsecrets/status + - clustersecretstores + - clustersecretstores/status + verbs: [ "*" ] +- apiGroups: [ "hub.gke.io" ] + resources: + - memberships + verbs: [ "*" ] +- apiGroups: [ "kyverno.io" ] + resources: + - clusterpolicies + - clusterpolicies/status + verbs: [ "*" ] +- apiGroups: [ "monitoring.googleapis.com" ] + resources: + - clusterpodmonitorings + - clusterpodmonitorings/status + - clusterrules + - clusterrules/status + - globalrules + - globalrules/status + - clusternodemonitorings + - clusternodemonitorings/status + verbs: [ "*" ] +- apiGroups: [ "networking.gke.io" ] + resources: + - gkenetworkparamsets + - gkenetworkparamsets/status + - networks + - networks/status + verbs: [ "*" ] +- apiGroups: [ "nfd.k8s-sigs.io" ] + resources: + - nodefeaturerules + verbs: [ "*" ] +- apiGroups: [ "reports.kyverno.io" ] + resources: + - clusterephemeralreports + verbs: [ "*" ] +- apiGroups: [ "skyhook.nvidia.com" ] + resources: + - skyhooks + - skyhooks/status + verbs: [ "*" ] +- apiGroups: [ "snapshot.storage.k8s.io" ] + resources: + - volumesnapshotcontents + - volumesnapshotcontents/status + - volumesnapshotclasses + verbs: [ "*" ] +- apiGroups: [ "warden.gke.io" ] + resources: + - audits + - audits/status + verbs: [ "*" ] +- apiGroups: [ "wgpolicyk8s.io" ] + resources: + - clusterpolicyreports + verbs: [ "*" ] +- apiGroups: [ "metrics.k8s.io" ] + resources: + - nodes + verbs: [ "*" ] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: sre-cluster-scoped-access +subjects: + - kind: Group + name: {{ .Values.sre.group.default }} + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: sre-cluster-scoped-access + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: dgxc-sre-generate-namespace-rolebindings + annotations: + argocd.argoproj.io/sync-options: Force=true,Replace=true + labels: + dgxc/system-managed: "true" +spec: + generateExisting: true + rules: + - name: generate-sre-namespace-rolebinding + match: + any: + - resources: + kinds: + - Namespace + names: + {{- range .Values.clusterScopedKinds }} + {{- if eq .kind "Namespace" }} + {{- range .systemResources }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + generate: + synchronize: true + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + name: "sre-ns-{{ `{{ request.object.metadata.name }}` }}-binding" + namespace: "{{ `{{ request.object.metadata.name }}` }}" + data: + metadata: + labels: + dgxc/customer-app-admin: "true" + dgxc/system-managed: "true" + dgxc/generated: "true" + subjects: + - kind: Group + name: {{ .Values.sre.group.default }} + apiGroup: rbac.authorization.k8s.io + roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/sre/privileged.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/sre/privileged.yaml new file mode 100644 index 0000000..16059e3 --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/sre/privileged.yaml @@ -0,0 +1,14 @@ +{{- if .Values.enableDGXCRbac }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: sre-full-access-binding +subjects: + - kind: Group + name: {{ .Values.sre.group.privileged }} + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/storage.dgxc.nvidia.com_nvstoragelocations.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/storage.dgxc.nvidia.com_nvstoragelocations.yaml new file mode 100644 index 0000000..29f4107 --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/storage.dgxc.nvidia.com_nvstoragelocations.yaml @@ -0,0 +1,3 @@ +{{- if .Values.nvStorage.enabled }} +{{ $.Files.Get "files/storage.dgxc.nvidia.com_nvstoragelocations.yaml" }} +{{- end }} \ No newline at end of file diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/tenantclusterpolicies/clusterrole.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/tenantclusterpolicies/clusterrole.yaml new file mode 100644 index 0000000..4f8eeeb --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/tenantclusterpolicies/clusterrole.yaml @@ -0,0 +1,26 @@ +{{- if .Values.enableDGXCAPIBootstrap }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: dgxc-customer-network-policy + labels: + dgxc/customer-app-admin: "true" + dgxc/system-managed: "true" +rules: +- apiGroups: [ "runai.dgxc.nvidia.com" ] + resources: + - runaidgxcnetworkpolicies + verbs: ["get", "list", "watch", "update", "patch"] +- apiGroups: [ "runai.dgxc.nvidia.com" ] + resources: + - runaidgxcstorages + verbs: ["get", "list", "watch", "update", "patch"] +- apiGroups: [ "runai.dgxc.nvidia.com" ] + resources: + - runaidgxclogexporters + verbs: ["get", "list", "watch", "update", "patch"] +- apiGroups: [ "storage.dgxc.nvidia.com" ] + resources: + - nvstoragelocations + verbs: ["*"] +{{- end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/templates/tenantclusterpolicies/namespace.yaml b/charts/nemo/charts/dgxc-admission-controller/templates/tenantclusterpolicies/namespace.yaml new file mode 100644 index 0000000..3676272 --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/templates/tenantclusterpolicies/namespace.yaml @@ -0,0 +1,6 @@ +{{- if .Values.enableDGXCAPIBootstrap }} +apiVersion: v1 +kind: Namespace +metadata: + name: dgxc-tenant-cluster-policies +{{- end }} diff --git a/charts/nemo/charts/dgxc-admission-controller/values-aws-custom.yaml b/charts/nemo/charts/dgxc-admission-controller/values-aws-custom.yaml new file mode 100644 index 0000000..06d49d1 --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/values-aws-custom.yaml @@ -0,0 +1,66 @@ +--- +kubernetesAffinity: + enabled: false + +systemTolerationKeyName: dedicated +systemTolerationValueName: "system-workload" + +tolerations: + customer: + key: "dedicated" + value: "user-workload" + system: + key: "dedicated" + value: "system-workload" + +cloudProviderSpec: + type: aws + aws: + accountName: nv-dgxc-non-prod-runai-dev + clusterName: t11111111-dgxc-k8s-runai-aws-us-east-1-dev + efa: true + region: us-east-1 + securityGroupARN: arn:aws:ec2:us-east-1:111111111111:security-group/sg-11111111111111111 + statefulRuleGroupName: dgxc-runai-dev-fw-nacls + gpuNode: + gpuAllocatable: "8" + hugePagesAllocatable: 42242Mi + efaAllocatable: "32" + +envVars: + user: + efa: + - name: FI_EFA_USE_DEVICE_RDMA + value: "1" + - name: OPAL_PREFIX + value: "/opt/amazon-efa-ofi/openmpi" + - name: NVIDIA_GDRCOPY + value: "enabled" + +storageControllerSpecAWS: + enabled: false + defaultStorageTypes: + - apiVersion: runai.dgxc.nvidia.com/v1beta1 + kind: RunaiDGXCStorage + metadata: + name: amazon-fsx-for-lustre + namespace: dgxc-tenant-cluster-policies + spec: + quotaID: L-C8640C82 + status: + altNames: [ "fsx" ] + storageClasses: + - lustre-sc + - dgxc-enterprise-file + - dgxc-standard-file + storageClassesDisplayString: "lustre-sc" + docs: https://aws.amazon.com/fsx/when-to-choose-fsx/ + +nvStorage: + # NVStorageLocation CRD management + enabled: false + storageClass: "dgxc-standard-object" + csi: + driver: "s3.csi.aws.com" + volumeAttributes: + authenticationSource: pod \ No newline at end of file diff --git a/charts/nemo/charts/dgxc-admission-controller/values-aws.yaml b/charts/nemo/charts/dgxc-admission-controller/values-aws.yaml new file mode 100644 index 0000000..67c933a --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/values-aws.yaml @@ -0,0 +1,1059 @@ + +#################################################################### +##### The portion below this marker is automatically generated ##### +##### DO NOT EDIT BY HAND ##### +#################################################################### + +clusterScopedKinds: + - apiGroup: extensions + resources: + - ingresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + systemResources: [] + - apiGroup: "" + resources: + - persistentvolumeclaims/finalizers + verbs: + - create + - update + - delete + - get + systemResources: [] + - apiGroup: argoproj.io + resources: + - eventsources + - sensors + verbs: + - "*" + systemResources: [] + - apiGroup: route.openshift.io + resources: + - "*" + verbs: + - "*" + systemResources: [] + - apiGroup: "" + resources: + - configmaps/status + verbs: + - "*" + systemResources: [] + - apiGroup: nvidia.com + kind: ClusterPolicy + resources: + - clusterpolicies + - clusterpolicies/status + verbs: + - "*" + systemResources: + - cluster-policy + - apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + resources: + - clusterroles + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + # - bind + # - escalate + systemResources: + - admin + - argocd-application-controller + - argocd-notifications-controller + - argocd-server + - assets-sync + - assets-sync-cluster + - aws-node + - aws-secrets-store-csi-driver-secrets-store-csi-driver-provider-cluster-role + - cert-manager-cainjector + - cert-manager-cluster-view + - cert-manager-controller-approve:cert-manager-io + - cert-manager-controller-certificates + - cert-manager-controller-certificatesigningrequests + - cert-manager-controller-challenges + - cert-manager-controller-clusterissuers + - cert-manager-controller-ingress-shim + - cert-manager-controller-issuers + - cert-manager-controller-orders + - cert-manager-edit + - cert-manager-view + - cert-manager-webhook:subjectaccessreviews + - cluster-admin + - cluster-sync + - cluster-sync-project + - dgxc-admission-controller + - dgxc-customer-app-admin + - dgxc-customer-app-admin-base + - dgxc-customer-network-policy + - dgxc-exporter + - dgxc-logging-alloy + - dgxc-runai-controller + - ebs-csi-node-role + - ebs-external-attacher-role + - ebs-external-provisioner-role + - ebs-external-resizer-role + - ebs-external-snapshotter-role + - edit + - eks:addon-manager + - eks:az-poller + - eks:certificate-controller-approver + - eks:certificate-controller-manager + - eks:certificate-controller-signer + - eks:cloud-controller-manager + - eks:cloud-provider-extraction-migration + - eks:cluster-event-watcher + - eks:coredns-autoscaler + - eks:extension-metrics-apiserver + - eks:fargate-manager + - eks:fargate-scheduler + - eks:k8s-metrics + - eks:network-policy-controller + - eks:network-webhooks + - eks:node-bootstrapper + - eks:node-manager + - eks:nodewatcher + - eks:pod-identity-mutating-webhook + - eks:service-operations + - eks:tagging-controller + - fsx-csi-external-provisioner-role + - fsx-csi-node-role + - fsx-external-resizer-role + - gpu-operator + - gpu-operator-node-feature-discovery + - gpu-operator-node-feature-discovery-gc + - ingress-nginx + - knative-serving-activator-cluster + - knative-serving-addressable-resolver + - knative-serving-admin + - knative-serving-aggregated-addressable-resolver + - knative-serving-core + - knative-serving-namespaced-admin + - knative-serving-namespaced-edit + - knative-serving-namespaced-view + - knative-serving-podspecable-binding + - kratos-fluent-bit-read + - kubeflow-mpijobs-admin + - kubeflow-mpijobs-edit + - kubeflow-mpijobs-view + - kyverno-cleanup-jobs + - kyverno:admission-controller + - kyverno:admission-controller:additional + - kyverno:admission-controller:core + - kyverno:background-controller + - kyverno:background-controller:additional + - kyverno:background-controller:core + - kyverno:cleanup-controller + - kyverno:cleanup-controller:core + - kyverno:dgxc-integrations + - kyverno:rbac:admin:policies + - kyverno:rbac:admin:policyreports + - kyverno:rbac:admin:reports + - kyverno:rbac:admin:updaterequests + - kyverno:rbac:view:policies + - kyverno:rbac:view:policyreports + - kyverno:rbac:view:reports + - kyverno:rbac:view:updaterequests + - kyverno:reports-controller + - kyverno:reports-controller:core + - kyverno:watch-update-namespaces + - kyverno:watch-update-services + - metrics-exporter + - mpi-operator + - net-kourier + - node-problem-detector-cpu + - node-problem-detector-gpu + - node-scale-adjuster-ro + - nodepool-controller + - nvidia-device-plugin + - nvidia-device-plugin-mps-control-daemon + - nvidia-driver + - nvidia-gpu-feature-discovery + - nvidia-mig-manager + - nvidia-operator-validator + - nvsentinel + - pod-group-assigner + - pod-group-controller + - pod-group-controller-ro + - pod-grouper + - prometheus-kube-state-metrics + - prometheus-operator + - prometheus-prometheus + - queue-controller + - researcher-service-ro + - researcher-service-rw + - runai-accessrule-controller + - runai-accessrule-controller-project + - runai-admission-controller-project + - runai-admission-controller-ro + - runai-agent + - runai-binder-cluster-wide + - runai-binder-project + - runai-cluster-api + - runai-cluster-api-project + - runai-compute-resource-administrator + - runai-compute-resource-administrator-ns + - runai-compute-resource-administrator-runai + - runai-credentials-administrator + - runai-credentials-administrator-ns + - runai-credentials-administrator-runai + - runai-data-source-administrator + - runai-data-source-administrator-ns + - runai-data-source-administrator-runai + - runai-data-source-administrator-runai-cs + - runai-data-volume-administrator + - runai-data-volume-administrator-ns + - runai-data-volume-administrator-runai + - runai-department-administrator + - runai-department-administrator-ns + - runai-department-administrator-runai + - runai-department-viewer + - runai-department-viewer-ns + - runai-department-viewer-runai + - runai-device-plugin + - runai-editor + - runai-editor-ns + - runai-editor-runai + - runai-engine-operator + - runai-environment-administrator + - runai-environment-administrator-ns + - runai-environment-administrator-runai + - runai-external-workload-integrator + - runai-external-workload-integrator-project + - runai-inference-workload-controller + - runai-inference-workload-controller-project + - runai-init-ca + - runai-job-controller + - runai-job-controller-project + - runai-l1-researcher + - runai-l1-researcher-ns + - runai-l1-researcher-runai + - runai-l2-researcher + - runai-l2-researcher-ns + - runai-l2-researcher-runai + - runai-mig-parted + - runai-ml-engineer + - runai-ml-engineer-ns + - runai-ml-engineer-runai + - runai-operator + - runai-project-controller + - runai-project-controller-cluster-pvc + - runai-project-controller-cluster-pvc-per-project + - runai-project-controller-cluster-secret + - runai-project-controller-cluster-secret-per-project + - runai-project-controller-limit-range + - runai-project-controller-limit-range-per-project + - runai-project-controller-ns-creation + - runai-project-controller-resources-patcher + - runai-project-controller-rolebinding-creation + - runai-project-controller-servicemonitor-creation + - runai-prometheus + - runai-research-manager + - runai-research-manager-ns + - runai-research-manager-runai + - runai-resource-reservation-rw + - runai-scheduler-ro + - runai-scheduler-rw + - runai-shared-objects-controller + - runai-shared-objects-controller-project + - runai-system-administrator + - runai-system-administrator-ns + - runai-system-administrator-runai + - runai-system-administrator-runai-cs + - runai-template-administrator + - runai-template-administrator-ns + - runai-template-administrator-runai + - runai-viewer + - runai-viewer-ns + - runai-viewer-runai + - runai-workload-exporter + - s3-csi-driver-cluster-role + - secretproviderclasses-admin-role + - secretproviderclasses-role + - secretproviderclasses-viewer-role + - secretproviderclasspodstatuses-viewer-role + - shoreline-agent + - skyhook-operator-manager-role + - skyhook-operator-metrics-reader + - skyhook-operator-proxy-role + - sre-cluster-scoped-access + - status-updater + - status-updater-ro + - system:aggregate-to-admin + - system:aggregate-to-edit + - system:aggregate-to-view + - system:auth-delegator + - system:basic-user + - system:certificates.k8s.io:certificatesigningrequests:nodeclient + - system:certificates.k8s.io:certificatesigningrequests:selfnodeclient + - system:certificates.k8s.io:kube-apiserver-client-approver + - system:certificates.k8s.io:kube-apiserver-client-kubelet-approver + - system:certificates.k8s.io:kubelet-serving-approver + - system:certificates.k8s.io:legacy-unknown-approver + - system:controller:attachdetach-controller + - system:controller:certificate-controller + - system:controller:clusterrole-aggregation-controller + - system:controller:cronjob-controller + - system:controller:daemon-set-controller + - system:controller:deployment-controller + - system:controller:disruption-controller + - system:controller:endpoint-controller + - system:controller:endpointslice-controller + - system:controller:endpointslicemirroring-controller + - system:controller:ephemeral-volume-controller + - system:controller:expand-controller + - system:controller:generic-garbage-collector + - system:controller:horizontal-pod-autoscaler + - system:controller:job-controller + - system:controller:legacy-service-account-token-cleaner + - system:controller:namespace-controller + - system:controller:node-controller + - system:controller:persistent-volume-binder + - system:controller:pod-garbage-collector + - system:controller:pv-protection-controller + - system:controller:pvc-protection-controller + - system:controller:replicaset-controller + - system:controller:replication-controller + - system:controller:resourcequota-controller + - system:controller:root-ca-cert-publisher + - system:controller:route-controller + - system:controller:service-account-controller + - system:controller:service-controller + - system:controller:statefulset-controller + - system:controller:ttl-after-finished-controller + - system:controller:ttl-controller + - system:controller:validatingadmissionpolicy-status-controller + - system:coredns + - system:discovery + - system:heapster + - system:kube-aggregator + - system:kube-controller-manager + - system:kube-dns + - system:kube-scheduler + - system:kubelet-api-admin + - system:monitoring + - system:node + - system:node-bootstrapper + - system:node-problem-detector + - system:node-proxier + - system:persistent-volume-provisioner + - system:public-info-viewer + - system:service-account-issuer-discovery + - system:volume-scheduler + - training-operator + - vace-role + - view + - vpc-resource-controller-role + - workload-controller-ro + - workload-controller-rw + - workload-overseer + - apiGroup: rbac.authorization.k8s.io + kind: ClusterRoleBinding + resources: + - clusterrolebindings + verbs: + - "*" + systemResources: + - argocd-application-controller + - argocd-notifications-controller + - argocd-server + - assets-sync + - aws-node + - aws-secrets-store-csi-driver-secrets-store-csi-driver-provider-cluster-role-binding + - cert-manager-cainjector + - cert-manager-controller-approve:cert-manager-io + - cert-manager-controller-certificates + - cert-manager-controller-certificatesigningrequests + - cert-manager-controller-challenges + - cert-manager-controller-clusterissuers + - cert-manager-controller-ingress-shim + - cert-manager-controller-issuers + - cert-manager-controller-orders + - cert-manager-webhook:subjectaccessreviews + - cluster-admin + - cluster-sync + - dgxc-admission-controller + - dgxc-customer-app-admin + # - dgxc-customer-app-admin-access + - dgxc-exporter + - dgxc-logging-alloy + - dgxc-runai-controller + - ebs-csi-attacher-binding + - ebs-csi-node-getter-binding + - ebs-csi-provisioner-binding + - ebs-csi-resizer-binding + - ebs-csi-snapshotter-binding + - eks:addon-cluster-admin + - eks:addon-manager + - eks:az-poller + - eks:certificate-controller + - eks:certificate-controller-approver + - eks:certificate-controller-manager + - eks:certificate-controller-signer + - eks:cloud-controller-manager + - eks:cloud-provider-extraction-migration + - eks:cluster-event-watcher + - eks:coredns-autoscaler + - eks:extension-metrics-apiserver + - eks:extension-metrics-apiserver-auth-delegator + - eks:fargate-manager + - eks:fargate-scheduler + - eks:k8s-metrics + - eks:kube-proxy + - eks:kube-proxy-fargate + - eks:kube-proxy-windows + - eks:network-policy-controller + - eks:network-webhooks + - eks:node-bootstrapper + - eks:node-manager + - eks:nodewatcher + - eks:pod-identity-mutating-webhook + - eks:service-operations + - eks:tagging-controller + - fsx-csi-external-provisioner-binding + - fsx-csi-node-getter-binding + - fsx-csi-resizer-binding + - gpu-operator + - gpu-operator-node-feature-discovery + - gpu-operator-node-feature-discovery-gc + - ingress-nginx + - knative-serving-activator-cluster + - knative-serving-controller-addressable-resolver + - knative-serving-controller-admin + - kratos-fluent-bit-read + - kyverno-cleanup-jobs + - kyverno:admission-controller + - kyverno:background-controller + - kyverno:cleanup-controller + - kyverno:reports-controller + - metrics-exporter + - mountpoint-s3-csi-node-binding + - mpi-operator + - net-kourier + - node-problem-detector-cpu + - node-problem-detector-gpu + - node-scale-adjuster-ro + - nodepool-controller + - nvidia-device-plugin + - nvidia-device-plugin-mps-control-daemon + - nvidia-driver + - nvidia-gpu-feature-discovery + - nvidia-mig-manager + - nvidia-operator-validator + - nvsentinel + - nvsentinel-sa-view-binding + - nvsentinel-shoreline + - pod-group-assigner + - pod-group-controller-ro + - pod-grouper + - prometheus-kube-state-metrics + - prometheus-operator + - prometheus-prometheus + - queue-controller + - researcher-service-ro + - runai-accessrule-controller + - runai-admission-controller-ro + - runai-agent + - runai-binder-cluster-wide + - runai-cluster-api + - runai-compute-resource-administrator + - runai-credentials-administrator + - runai-data-source-administrator + - runai-data-volume-administrator + - runai-department-administrator + - runai-department-viewer + - runai-device-plugin + - runai-editor + - runai-engine-operator + - runai-environment-administrator + - runai-external-workload-integrator + - runai-inference-workload-controller + - runai-init-ca + - runai-job-controller + - runai-l1-researcher + - runai-l2-researcher + - runai-mig-parted + - runai-ml-engineer + - runai-operator + - runai-project-controller + - runai-project-controller-cluster-pvc + - runai-project-controller-cluster-secret + - runai-project-controller-limit-range + - runai-project-controller-ns-creation + - runai-project-controller-rolebinding-creation + - runai-project-controller-servicemonitor-creation + - runai-prometheus + - runai-research-manager + - runai-scheduler-ro + - runai-shared-objects-controller + - runai-system-administrator + - runai-template-administrator + - runai-viewer + - runai-workload-exporter + - secretproviderclasses-rolebinding + - skyhook-operator-manager-rolebinding + - skyhook-operator-proxy-rolebinding + - sre-cluster-scoped-access + - sre-full-access-binding + - status-updater-ro + - system:basic-user + - system:controller:attachdetach-controller + - system:controller:certificate-controller + - system:controller:clusterrole-aggregation-controller + - system:controller:cronjob-controller + - system:controller:daemon-set-controller + - system:controller:deployment-controller + - system:controller:disruption-controller + - system:controller:endpoint-controller + - system:controller:endpointslice-controller + - system:controller:endpointslicemirroring-controller + - system:controller:ephemeral-volume-controller + - system:controller:expand-controller + - system:controller:generic-garbage-collector + - system:controller:horizontal-pod-autoscaler + - system:controller:job-controller + - system:controller:legacy-service-account-token-cleaner + - system:controller:namespace-controller + - system:controller:node-controller + - system:controller:persistent-volume-binder + - system:controller:pod-garbage-collector + - system:controller:pv-protection-controller + - system:controller:pvc-protection-controller + - system:controller:replicaset-controller + - system:controller:replication-controller + - system:controller:resourcequota-controller + - system:controller:root-ca-cert-publisher + - system:controller:route-controller + - system:controller:service-account-controller + - system:controller:service-controller + - system:controller:statefulset-controller + - system:controller:ttl-after-finished-controller + - system:controller:ttl-controller + - system:controller:validatingadmissionpolicy-status-controller + - system:coredns + - system:discovery + - system:kube-controller-manager + - system:kube-dns + - system:kube-scheduler + - system:monitoring + - system:node + - system:node-proxier + - system:public-info-viewer + - system:service-account-issuer-discovery + - system:volume-scheduler + - training-operator + - vace-rolebinding + - vpc-resource-controller-rolebinding + - workload-controller-ro + - workload-overseer + - apiGroup: "" + kind: ComponentStatus + resources: + - componentstatuses + verbs: + - "*" + systemResources: + - etcd-0 + - scheduler + - controller-manager + - apiGroup: apiextensions.k8s.io + kind: CustomResourceDefinition + resources: + - customresourcedefinitions + - customresourcedefinitions/status + verbs: + - "*" + systemResources: + - accessrules.run.ai + - admissionreports.kyverno.io + - alertmanagerconfigs.monitoring.coreos.com + - alertmanagers.monitoring.coreos.com + - applications.argoproj.io + - applicationsets.argoproj.io + - appprojects.argoproj.io + - backgroundscanreports.kyverno.io + - bindrequests.scheduling.run.ai + - certificaterequests.cert-manager.io + - certificates.cert-manager.io + - certificates.networking.internal.knative.dev + - challenges.acme.cert-manager.io + - cleanuppolicies.kyverno.io + - clusteradmissionreports.kyverno.io + - clusterbackgroundscanreports.kyverno.io + - clustercleanuppolicies.kyverno.io + - clusterdomainclaims.networking.internal.knative.dev + - clusterissuers.cert-manager.io + - clusterpolicies.kyverno.io + - clusterpolicies.nvidia.com + - clusterpolicyreports.wgpolicyk8s.io + - cninodes.vpcresources.k8s.aws + - configs.engine.run.ai + - configurations.serving.knative.dev + - datavolumes.run.ai + - departments.scheduling.run.ai + - distributedpolicies.run.ai + - distributedworkloads.run.ai + - domainmappings.serving.knative.dev + - eniconfigs.crd.k8s.amazonaws.com + - externalworkloads.run.ai + - images.caching.internal.knative.dev + - inferencepolicies.run.ai + - inferenceworkloads.run.ai + - ingresses.networking.internal.knative.dev + - interactivepolicies.run.ai + - interactiveworkloads.run.ai + - issuers.cert-manager.io + - metrics.autoscaling.internal.knative.dev + - mpijobs.kubeflow.org + - mxjobs.kubeflow.org + - nodefeaturegroups.nfd.k8s-sigs.io + - nodefeaturerules.nfd.k8s-sigs.io + - nodefeatures.nfd.k8s-sigs.io + - nodepools.run.ai + - nvidiadrivers.nvidia.com + - orders.acme.cert-manager.io + - paddlejobs.kubeflow.org + - podautoscalers.autoscaling.internal.knative.dev + - podgroups.scheduling.run.ai + - podlogs.monitoring.grafana.com + - podmonitors.monitoring.coreos.com + - policies.kyverno.io + - policyendpoints.networking.k8s.aws + - policyexceptions.kyverno.io + - policyreports.wgpolicyk8s.io + - probes.monitoring.coreos.com + - projects.run.ai + - prometheusagents.monitoring.coreos.com + - prometheuses.monitoring.coreos.com + - prometheusrules.monitoring.coreos.com + - pytorchjobs.kubeflow.org + - queues.scheduling.run.ai + - revisions.serving.knative.dev + - routes.serving.knative.dev + - runaiconfigs.run.ai + - runaidgxclogexporters.runai.dgxc.nvidia.com + - runaidgxcnetworkpolicies.runai.dgxc.nvidia.com + - runaidgxcstorages.runai.dgxc.nvidia.com + - runaijobs.run.ai + - schedulingshards.engine.run.ai + - scrapeconfigs.monitoring.coreos.com + - secretproviderclasses.secrets-store.csi.x-k8s.io + - secretproviderclasspodstatuses.secrets-store.csi.x-k8s.io + - securitygrouppolicies.vpcresources.k8s.aws + - serverlessservices.networking.internal.knative.dev + - servicemonitors.monitoring.coreos.com + - services.serving.knative.dev + - skyhooks.skyhook.nvidia.com + - tfjobs.kubeflow.org + - thanosrulers.monitoring.coreos.com + - trainingpolicies.run.ai + - trainingworkloads.run.ai + - updaterequests.kyverno.io + - xgboostjobs.kubeflow.org + - apiGroup: crd.k8s.amazonaws.com + kind: ENIConfig + resources: + - eniconfigs + verbs: + - "*" + systemResources: [] + - apiGroup: networking.k8s.io + kind: IngressClass + resources: + - ingressclasses + verbs: + # - create + # - delete + # - deletecollection + - get + - list + # - patch + # - update + - watch + systemResources: + - nginx + - apiGroup: "" + kind: Namespace + resources: + - namespaces + - namespaces/finalize + - namespaces/status + verbs: + - "*" + systemResources: + - argocd + - cert-manager + - default + - dgxc-admission-controller + - dgxc-cluster-validator + - dgxc-logging + - dgxc-tenant-cluster-policies + - gpu-operator + - ingress-nginx + - knative-serving + - kourier-system + - kube-node-lease + - kube-public + - kube-system + - kubeflow + - kyverno + - monitoring + - mpi-operator + - node-problem-detector + - nvdiag + - nvsentinel + - observability + - prometheus + - runai + - skyhook + - apiGroup: "" + kind: PersistentVolume + resources: + - persistentvolumes + - persistentvolumes/status + verbs: + # - create + # - delete + # - deletecollection + - get + - list + # - patch + # - update + - watch + systemResources: [] + - apiGroup: scheduling.k8s.io + kind: PriorityClass + resources: + - priorityclasses + verbs: + - "*" + systemResources: + - build + - inference + - interactive-preemptible + - runai-critical + - runai-engine-critical + - system-cluster-critical + - system-node-critical + - train + - train-critical + - train-high + - apiGroup: authorization.k8s.io + kind: SelfSubjectAccessReview + resources: + - selfsubjectaccessreviews + verbs: + - "*" + systemResources: [] + - apiGroup: authentication.k8s.io + kind: SelfSubjectReview + resources: + - selfsubjectreviews + verbs: + - "*" + systemResources: [] + - apiGroup: authorization.k8s.io + kind: SelfSubjectRulesReview + resources: + - selfsubjectrulesreviews + verbs: + - "*" + systemResources: [] + - apiGroup: authorization.k8s.io + kind: SubjectAccessReview + resources: + - subjectaccessreviews + verbs: + - "*" + systemResources: [] + - apiGroup: authentication.k8s.io + kind: TokenReview + resources: + - tokenreviews + verbs: + - "*" + systemResources: [] + +namespacedKinds: + - apiGroup: policy + resources: + - pods/eviction + verbs: + - create + - apiGroup: "" + kind: Binding + resources: + - bindings + verbs: + - "*" + - apiGroup: "" + kind: ConfigMap + resources: + - configmaps + verbs: + - "*" + - apiGroup: apps + kind: ControllerRevision + resources: + - controllerrevisions + verbs: + - "*" + - apiGroup: batch + kind: CronJob + resources: + - cronjobs + - cronjobs/status + verbs: + - "*" + - apiGroup: apps + kind: DaemonSet + resources: + - daemonsets + - daemonsets/status + verbs: + - "*" + - apiGroup: apps + kind: Deployment + resources: + - deployments + - deployments/scale + - deployments/status + verbs: + - "*" + - apiGroup: discovery.k8s.io + kind: EndpointSlice + resources: + - endpointslices + verbs: + - "*" + - apiGroup: "" + kind: Endpoints + resources: + - endpoints + verbs: + - "*" + - apiGroup: "" + kind: Event + resources: + - events + verbs: + - "*" + - apiGroup: events.k8s.io + kind: Event + resources: + - events + verbs: + - "*" + - apiGroup: autoscaling + kind: HorizontalPodAutoscaler + resources: + - horizontalpodautoscalers + - horizontalpodautoscalers/status + verbs: + - "*" + - apiGroup: networking.k8s.io + kind: Ingress + resources: + - ingresses + - ingresses/status + verbs: + - "*" + - apiGroup: batch + kind: Job + resources: + - jobs + - jobs/status + verbs: + - "*" + - apiGroup: coordination.k8s.io + kind: Lease + resources: + - leases + verbs: + - "*" + - apiGroup: "" + kind: LimitRange + resources: + - limitranges + verbs: + - "*" + - apiGroup: authorization.k8s.io + kind: LocalSubjectAccessReview + resources: + - localsubjectaccessreviews + verbs: + - "*" + - apiGroup: networking.k8s.io + kind: NetworkPolicy + resources: + - networkpolicies + verbs: + - "*" + - apiGroup: "" + kind: PersistentVolumeClaim + resources: + - persistentvolumeclaims + - persistentvolumeclaims/status + verbs: + - "*" + - apiGroup: "" + kind: Pod + resources: + - pods + - pods/attach + - pods/binding + - pods/ephemeralcontainers + - pods/eviction + - pods/exec + - pods/log + - pods/portforward + - pods/proxy + - pods/status + verbs: + - "*" + - apiGroup: policy + kind: PodDisruptionBudget + resources: + - poddisruptionbudgets + - poddisruptionbudgets/status + verbs: + - "*" + - apiGroup: "" + kind: PodTemplate + resources: + - podtemplates + verbs: + - "*" + - apiGroup: apps + kind: ReplicaSet + resources: + - replicasets + - replicasets/scale + - replicasets/status + verbs: + - "*" + - apiGroup: "" + kind: ReplicationController + resources: + - replicationcontrollers + - replicationcontrollers/scale + - replicationcontrollers/status + verbs: + - "*" + - apiGroup: "" + kind: ResourceQuota + resources: + - resourcequotas + - resourcequotas/status + verbs: + - "*" + - apiGroup: rbac.authorization.k8s.io + kind: Role + resources: + - roles + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + # - bind + # - escalate + - apiGroup: rbac.authorization.k8s.io + kind: RoleBinding + resources: + - rolebindings + verbs: + - "*" + - apiGroup: "" + kind: Secret + resources: + - secrets + verbs: + - "*" + - apiGroup: "" + kind: Service + resources: + - services + - services/proxy + - services/status + verbs: + - "*" + - apiGroup: "" + kind: ServiceAccount + resources: + - serviceaccounts + - serviceaccounts/token + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + # - impersonate + - apiGroup: apps + kind: StatefulSet + resources: + - statefulsets + - statefulsets/scale + - statefulsets/status + verbs: + - "*" + +systemSubjects: + User: + - eks:addon-manager + - eks:az-poller + - eks:certificate-controller + - eks:cloud-controller-manager + - eks:cluster-event-watcher + - eks:coredns-autoscaler + - eks:extension-metrics-apiserver + - eks:fargate-manager + - eks:fargate-scheduler + - eks:k8s-metrics + - eks:network-policy-controller + - eks:network-webhooks + - eks:node-manager + - eks:nodewatcher + - eks:pod-identity-mutating-webhook + - eks:vpc-resource-controller + - kyverno@test.com + - ladithyav@nvidia.com + - pwittrock@nvidia.com + - system:kube-controller-manager + - system:kube-proxy + - system:kube-scheduler + Group: + - access-dgx-cloud-runai-non-prod-aws-admin + - access-dgx-cloud-runai-non-prod-aws-engineer + - access-dgx-cloud-runai-prod-aws-admin + - access-dgx-cloud-runai-prod-aws-engineer + - eks:kube-proxy-windows + - eks:service-operations + - system:bootstrappers + - system:masters + - system:monitoring + - system:node-proxier + - system:nodes + - system:serviceaccounts + - system:unauthenticated + diff --git a/charts/nemo/charts/dgxc-admission-controller/values-azure-custom.yaml b/charts/nemo/charts/dgxc-admission-controller/values-azure-custom.yaml new file mode 100644 index 0000000..01b0645 --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/values-azure-custom.yaml @@ -0,0 +1,78 @@ +kubernetesAffinity: + enabled: false + +systemTolerationKeyName: dedicated +systemTolerationValueName: "system-workload" + +tolerations: + customer: + key: "dedicated" + value: "user-workload" + system: + key: "dedicated" + value: "system-workload" + +cloudProviderSpec: + type: azure + azure: + subscriptionId: 99e004ca-5389-45c5-9e7b-402c0a46550c + subscriptionName: sub-dgxc-runai-non-prod-test + resourceGroupName: rg-runai-scus-dev-test + logExporterAnalyticsWorkspaceId: workspace-id + ingressNsgName: ingress-name + egressNsgName: egress-name + standardStorageAccountName: standardtz990000 + premiumStorageAccountName: premiumtz990000 + instanceType: Standard_ND96amsr_A100_v4 + +storageControllerSpecAzure: + enabled: false + defaultStorageTypes: + - apiVersion: runai.dgxc.nvidia.com/v1beta1 + kind: RunaiDGXCStorage + metadata: + name: dgxc-azurefile + namespace: dgxc-tenant-cluster-policies + spec: + quotaID: AzureFileStandard + status: + altNames: [ "dgxc-azurefile" ] + storageClasses: + - dgxc-azurefile + storageClassesDisplayString: "dgxc-azurefile" + docs: https://learn.microsoft.com/en-us/azure/storage/files/storage-files-planning + - apiVersion: runai.dgxc.nvidia.com/v1beta1 + kind: RunaiDGXCStorage + metadata: + name: dgxc-azurefile-premium-pool-1 + namespace: dgxc-tenant-cluster-policies + spec: + quotaID: AzureFilePremiumPool1 + status: + altNames: [ "dgxc-azurefile-premium-pool-1" ] + storageClasses: + - dgxc-azurefile-premium-pool-1 + storageClassesDisplayString: "dgxc-azurefile-premium-pool-1" + - apiVersion: runai.dgxc.nvidia.com/v1beta1 + kind: RunaiDGXCStorage + metadata: + name: dgxc-azurefile-premium-pool-2 + namespace: dgxc-tenant-cluster-policies + spec: + quotaID: AzureFilePremiumPool2 + status: + altNames: [ "dgxc-azurefile-premium-pool-2" ] + storageClasses: + - dgxc-azurefile-premium-pool-2 + storageClassesDisplayString: "dgxc-azurefile-premium-pool-2" + +podMutatingWebhook: + customSettings: + - name: modify-pod-rule + addRdmaEnabled: true + rdmaResourcePerGpu: 1 + rdmaResourceName: nvidia.com/mlnxnics + addNcclVars: true + csp: "azure" + +ncclTopoConfigMapNamespaces: runai-* diff --git a/charts/nemo/charts/dgxc-admission-controller/values-azure.yaml b/charts/nemo/charts/dgxc-admission-controller/values-azure.yaml new file mode 100644 index 0000000..9b4f08c --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/values-azure.yaml @@ -0,0 +1,1094 @@ + +#################################################################### +##### The portion below this marker is automatically generated ##### +##### DO NOT EDIT BY HAND ##### +#################################################################### + +clusterScopedKinds: + - apiGroup: "" + resources: + - configmaps/status + verbs: + - "*" + systemResources: [] + - apiGroup: "" + resources: + - persistentvolumeclaims/finalizers + verbs: + - create + - update + - delete + - get + systemResources: [] + - apiGroup: extensions + resources: + - ingresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + systemResources: [] + - apiGroup: argoproj.io + resources: + - eventsources + - sensors + verbs: + - "*" + systemResources: [] + - apiGroup: route.openshift.io + resources: + - "*" + verbs: + - "*" + systemResources: [] + - apiGroup: nvidia.com + kind: ClusterPolicy + resources: + - clusterpolicies + - clusterpolicies/status + verbs: + - "*" + systemResources: + - cluster-policy + - apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + resources: + - clusterroles + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + # - bind + # - escalate + systemResources: + - acn-multitenancy-editor + - admin + - aks-secretproviderclasses-admin-role + - aks-secretproviderclasses-role + - aks-secretproviderclasses-viewer-role + - aks-secretprovidersyncing-role + - aks-secretprovidertokenrequest-role + - aks-service + - ama-logs-reader + - appmonitoringconfig-user + - argocd-application-controller + - argocd-notifications-controller + - argocd-server + - assets-sync + - assets-sync-cluster + - azure-wi-webhook-manager-role + - cert-manager-cainjector + - cert-manager-cluster-view + - cert-manager-controller-approve:cert-manager-io + - cert-manager-controller-certificates + - cert-manager-controller-certificatesigningrequests + - cert-manager-controller-challenges + - cert-manager-controller-clusterissuers + - cert-manager-controller-ingress-shim + - cert-manager-controller-issuers + - cert-manager-controller-orders + - cert-manager-edit + - cert-manager-view + - cert-manager-webhook:subjectaccessreviews + - cilium + - cilium-operator + - cloud-node-manager + - cluster-admin + - cluster-sync + - cluster-sync-project + - container-health-log-reader + - csi-azuredisk-node-role + - csi-azurefile-node-secret-role + - dgxc-admission-controller + - dgxc-customer-app-admin + - dgxc-customer-app-admin-base + - dgxc-customer-network-policy + - dgxc-exporter + - dgxc-logging-alloy + - dgxc-runai-controller + - edit + - gpu-operator + - gpu-operator-node-feature-discovery + - gpu-operator-node-feature-discovery-gc + - ingress-nginx + - knative-serving-activator-cluster + - knative-serving-addressable-resolver + - knative-serving-admin + - knative-serving-aggregated-addressable-resolver + - knative-serving-core + - knative-serving-namespaced-admin + - knative-serving-namespaced-edit + - knative-serving-namespaced-view + - knative-serving-podspecable-binding + - kratos-fluent-bit-read + - kube-oidc-proxy + - kubeflow-mpijobs-admin + - kubeflow-mpijobs-edit + - kubeflow-mpijobs-view + - kyverno-cleanup-jobs + - kyverno:admission-controller + - kyverno:admission-controller:additional + - kyverno:admission-controller:core + - kyverno:background-controller + - kyverno:background-controller:additional + - kyverno:background-controller:core + - kyverno:cleanup-controller + - kyverno:cleanup-controller:core + - kyverno:dgxc-integrations + - kyverno:rbac:admin:policies + - kyverno:rbac:admin:policyreports + - kyverno:rbac:admin:reports + - kyverno:rbac:admin:updaterequests + - kyverno:rbac:view:policies + - kyverno:rbac:view:policyreports + - kyverno:rbac:view:reports + - kyverno:rbac:view:updaterequests + - kyverno:reports-controller + - kyverno:reports-controller:core + - kyverno:watch-update-namespaces + - kyverno:watch-update-services + - metrics-exporter + - mpi-operator + - net-kourier + - network-operator + - node-scale-adjuster-ro + - nodepool-controller + - nv-ipam-controller + - nv-ipam-node + - nvidia-device-plugin + - nvidia-device-plugin-mps-control-daemon + - nvidia-driver + - nvidia-gpu-feature-discovery + - nvidia-mig-manager + - nvidia-operator-validator + - ofed-driver + - pod-group-assigner + - pod-group-controller + - pod-group-controller-ro + - pod-grouper + - pod-reader-all-namespaces + - prometheus-kube-state-metrics + - prometheus-operator + - prometheus-prometheus + - queue-controller + - researcher-service-ro + - researcher-service-rw + - runai-accessrule-controller + - runai-accessrule-controller-project + - runai-admission-controller-project + - runai-admission-controller-ro + - runai-agent + - runai-agent-project + - runai-binder-cluster-wide + - runai-binder-project + - runai-cluster-api + - runai-cluster-api-project + - runai-compute-resource-administrator + - runai-compute-resource-administrator-ns + - runai-compute-resource-administrator-runai + - runai-credentials-administrator + - runai-credentials-administrator-ns + - runai-credentials-administrator-runai + - runai-data-source-administrator + - runai-data-source-administrator-ns + - runai-data-source-administrator-runai + - runai-data-source-administrator-runai-cs + - runai-data-volume-administrator + - runai-data-volume-administrator-ns + - runai-data-volume-administrator-runai + - runai-department-administrator + - runai-department-administrator-ns + - runai-department-administrator-runai + - runai-department-viewer + - runai-department-viewer-ns + - runai-department-viewer-runai + - runai-device-plugin + - runai-editor + - runai-editor-ns + - runai-editor-runai + - runai-engine-operator + - runai-environment-administrator + - runai-environment-administrator-ns + - runai-environment-administrator-runai + - runai-external-workload-integrator + - runai-external-workload-integrator-project + - runai-inference-workload-controller + - runai-inference-workload-controller-project + - runai-init-ca + - runai-job-controller + - runai-job-controller-project + - runai-l1-researcher + - runai-l1-researcher-ns + - runai-l1-researcher-runai + - runai-l2-researcher + - runai-l2-researcher-ns + - runai-l2-researcher-runai + - runai-ml-engineer + - runai-ml-engineer-ns + - runai-ml-engineer-runai + - runai-node-exporter-role + - runai-operator + - runai-project-controller + - runai-project-controller-cluster-configmap-per-project + - runai-project-controller-cluster-pvc + - runai-project-controller-cluster-pvc-per-project + - runai-project-controller-cluster-secret + - runai-project-controller-cluster-secret-per-project + - runai-project-controller-limit-range + - runai-project-controller-limit-range-per-project + - runai-project-controller-ns-creation + - runai-project-controller-resources-patcher + - runai-project-controller-rolebinding-creation + - runai-project-controller-servicemonitor-creation + - runai-prometheus + - runai-research-manager + - runai-research-manager-ns + - runai-research-manager-runai + - runai-resource-reservation-rw + - runai-scheduler-ro + - runai-scheduler-rw + - runai-shared-objects-controller + - runai-shared-objects-controller-project + - runai-system-administrator + - runai-system-administrator-ns + - runai-system-administrator-runai + - runai-system-administrator-runai-cs + - runai-template-administrator + - runai-template-administrator-ns + - runai-template-administrator-runai + - runai-viewer + - runai-viewer-ns + - runai-viewer-runai + - runai-workload-exporter + - secretproviderrotation-role + - skyhook-operator-manager-role + - skyhook-operator-metrics-reader + - skyhook-operator-proxy-role + - sre-cluster-scoped-access + - status-updater + - status-updater-ro + - system:aggregate-to-admin + - system:aggregate-to-edit + - system:aggregate-to-view + - system:auth-delegator + - system:azure-cloud-provider + - system:azure-cloud-provider-secret-getter + - system:basic-user + - system:certificates.k8s.io:certificatesigningrequests:nodeclient + - system:certificates.k8s.io:certificatesigningrequests:selfnodeclient + - system:certificates.k8s.io:kube-apiserver-client-approver + - system:certificates.k8s.io:kube-apiserver-client-kubelet-approver + - system:certificates.k8s.io:kubelet-serving-approver + - system:certificates.k8s.io:legacy-unknown-approver + - system:controller:attachdetach-controller + - system:controller:certificate-controller + - system:controller:clusterrole-aggregation-controller + - system:controller:cronjob-controller + - system:controller:daemon-set-controller + - system:controller:deployment-controller + - system:controller:disruption-controller + - system:controller:endpoint-controller + - system:controller:endpointslice-controller + - system:controller:endpointslicemirroring-controller + - system:controller:ephemeral-volume-controller + - system:controller:expand-controller + - system:controller:generic-garbage-collector + - system:controller:horizontal-pod-autoscaler + - system:controller:job-controller + - system:controller:legacy-service-account-token-cleaner + - system:controller:namespace-controller + - system:controller:node-controller + - system:controller:persistent-volume-binder + - system:controller:pod-garbage-collector + - system:controller:pv-protection-controller + - system:controller:pvc-protection-controller + - system:controller:replicaset-controller + - system:controller:replication-controller + - system:controller:resourcequota-controller + - system:controller:root-ca-cert-publisher + - system:controller:route-controller + - system:controller:service-account-controller + - system:controller:service-controller + - system:controller:statefulset-controller + - system:controller:ttl-after-finished-controller + - system:controller:ttl-controller + - system:coredns + - system:coredns-autoscaler + - system:discovery + - system:heapster + - system:kube-aggregator + - system:kube-controller-manager + - system:kube-dns + - system:kube-scheduler + - system:kubelet-api-admin + - system:metrics-server + - system:monitoring + - system:node + - system:node-bootstrapper + - system:node-problem-detector + - system:node-proxier + - system:persistent-volume-provisioner + - system:persistent-volume-secret-operator + - system:prometheus + - system:public-info-viewer + - system:service-account-issuer-discovery + - system:volume-scheduler + - training-operator + - vace-role + - view + - workload-controller-ro + - workload-controller-rw + - workload-overseer + - apiGroup: rbac.authorization.k8s.io + kind: ClusterRoleBinding + resources: + - clusterrolebindings + verbs: + - "*" + systemResources: + - acn-multitenancy-editor-binding + - aks-cluster-admin-binding + - aks-cluster-admin-binding-aad + - aks-secretproviderclasses-rolebinding + - aks-secretprovidersyncing-rolebinding + - aks-secretprovidertokenrequest-rolebinding + - aks-service-rolebinding + - amalogsclusterrolebinding + - appmonitoringconfig-user-global + - argocd-application-controller + - argocd-notifications-controller + - argocd-server + - assets-sync + - auto-approve-csrs-for-group + - auto-approve-renewals-for-nodes + - azure-wi-webhook-manager-rolebinding + - billing-admin-binding + - cert-manager-cainjector + - cert-manager-controller-approve:cert-manager-io + - cert-manager-controller-certificates + - cert-manager-controller-certificatesigningrequests + - cert-manager-controller-challenges + - cert-manager-controller-clusterissuers + - cert-manager-controller-ingress-shim + - cert-manager-controller-issuers + - cert-manager-controller-orders + - cert-manager-webhook:subjectaccessreviews + - cilium + - cilium-operator + - cloud-node-manager + - cluster-admin + - cluster-sync + - container-health-read-logs-global + - create-csrs-for-bootstrapping + - csi-azuredisk-node-binding + - csi-azurefile-node-secret-binding + - dgxc-admission-controller + - dgxc-customer-app-admin + # - dgxc-customer-app-admin-access + - dgxc-exporter + - dgxc-logging-alloy + - dgxc-runai-controller + - extension-operator + - gpu-operator + - gpu-operator-node-feature-discovery + - gpu-operator-node-feature-discovery-gc + - ingress-nginx + - knative-serving-activator-cluster + - knative-serving-controller-addressable-resolver + - knative-serving-controller-admin + - kratos-fluent-bit-read + - kube-oidc-proxy + - kyverno-cleanup-jobs + - kyverno:admission-controller + - kyverno:background-controller + - kyverno:cleanup-controller + - kyverno:reports-controller + - metrics-exporter + - metrics-server:system:auth-delegator + - mpi-operator + - net-kourier + - network-operator + - node-scale-adjuster-ro + - nodepool-controller + - nv-ipam-controller + - nv-ipam-node + - nvidia-device-plugin + - nvidia-device-plugin-mps-control-daemon + - nvidia-driver + - nvidia-gpu-feature-discovery + - nvidia-mig-manager + - nvidia-operator-validator + - ofed-driver + - pod-group-assigner + - pod-group-controller-ro + - pod-grouper + - pod-reader-all-namespaces-binding + - prometheus-kube-state-metrics + - prometheus-operator + - prometheus-prometheus + - queue-controller + - researcher-service-ro + - runai-accessrule-controller + - runai-admission-controller-ro + - runai-agent + - runai-binder-cluster-wide + - runai-cluster-api + - runai-compute-resource-administrator + - runai-credentials-administrator + - runai-data-source-administrator + - runai-data-volume-administrator + - runai-department-administrator + - runai-department-viewer + - runai-device-plugin + - runai-editor + - runai-engine-operator + - runai-environment-administrator + - runai-external-workload-integrator + - runai-inference-workload-controller + - runai-init-ca + - runai-job-controller + - runai-l1-researcher + - runai-l2-researcher + - runai-ml-engineer + - runai-node-exporter-role + - runai-operator + - runai-project-controller + - runai-project-controller-cluster-pvc + - runai-project-controller-cluster-secret + - runai-project-controller-limit-range + - runai-project-controller-ns-creation + - runai-project-controller-rolebinding-creation + - runai-project-controller-servicemonitor-creation + - runai-prometheus + - runai-research-manager + - runai-scheduler-ro + - runai-shared-objects-controller + - runai-system-administrator + - runai-template-administrator + - runai-viewer + - runai-workload-exporter + - secretproviderrotation-rolebinding + - skyhook-operator-manager-rolebinding + - skyhook-operator-proxy-rolebinding + - sre-cluster-scoped-access + - sre-full-access-binding + - status-updater-ro + - system:aks-client-node-proxier + - system:azure-cloud-provider + - system:azure-cloud-provider-secret-getter + - system:basic-user + - system:controller:attachdetach-controller + - system:controller:certificate-controller + - system:controller:clusterrole-aggregation-controller + - system:controller:cronjob-controller + - system:controller:daemon-set-controller + - system:controller:deployment-controller + - system:controller:disruption-controller + - system:controller:endpoint-controller + - system:controller:endpointslice-controller + - system:controller:endpointslicemirroring-controller + - system:controller:ephemeral-volume-controller + - system:controller:expand-controller + - system:controller:generic-garbage-collector + - system:controller:horizontal-pod-autoscaler + - system:controller:job-controller + - system:controller:legacy-service-account-token-cleaner + - system:controller:namespace-controller + - system:controller:node-controller + - system:controller:persistent-volume-binder + - system:controller:pod-garbage-collector + - system:controller:pv-protection-controller + - system:controller:pvc-protection-controller + - system:controller:replicaset-controller + - system:controller:replication-controller + - system:controller:resourcequota-controller + - system:controller:root-ca-cert-publisher + - system:controller:route-controller + - system:controller:service-account-controller + - system:controller:service-controller + - system:controller:statefulset-controller + - system:controller:ttl-after-finished-controller + - system:controller:ttl-controller + - system:coredns + - system:coredns-autoscaler + - system:discovery + - system:kube-controller-manager + - system:kube-dns + - system:kube-scheduler + - system:metrics-server + - system:monitoring + - system:node + - system:node-proxier + - system:persistent-volume-binding + - system:prometheus + - system:public-info-viewer + - system:service-account-issuer-discovery + - system:volume-scheduler + - training-operator + - vace-rolebinding + - workload-controller-ro + - workload-overseer + - apiGroup: "" + kind: ComponentStatus + resources: + - componentstatuses + verbs: + - "*" + systemResources: + - controller-manager + - scheduler + - etcd-1 + - etcd-0 + - apiGroup: apiextensions.k8s.io + kind: CustomResourceDefinition + resources: + - customresourcedefinitions + - customresourcedefinitions/status + verbs: + - "*" + systemResources: + - accessrules.run.ai + - admissionreports.kyverno.io + - alertmanagerconfigs.monitoring.coreos.com + - alertmanagers.monitoring.coreos.com + - applications.argoproj.io + - applicationsets.argoproj.io + - appprojects.argoproj.io + - backgroundscanreports.kyverno.io + - billingerrors.clusterconfig.azure.com + - billingsettingses.clusterconfig.azure.com + - billingstorages.clusterconfig.azure.com + - billingusages.clusterconfig.azure.com + - bindrequests.scheduling.run.ai + - blockdeviceclaims.openebs.io + - blockdevices.openebs.io + - capacityprovisionerconfigs.containerstorage.azure.com + - certificaterequests.cert-manager.io + - certificates.cert-manager.io + - certificates.networking.internal.knative.dev + - challenges.acme.cert-manager.io + - cidrpools.nv-ipam.nvidia.com + - ciliumcidrgroups.cilium.io + - ciliumclusterwidenetworkpolicies.cilium.io + - ciliumendpoints.cilium.io + - ciliumexternalworkloads.cilium.io + - ciliumidentities.cilium.io + - ciliuml2announcementpolicies.cilium.io + - ciliumloadbalancerippools.cilium.io + - ciliumnetworkpolicies.cilium.io + - ciliumnodeconfigs.cilium.io + - ciliumnodes.cilium.io + - ciliumpodippools.cilium.io + - cleanuppolicies.kyverno.io + - clusteradmissionreports.kyverno.io + - clusterbackgroundscanreports.kyverno.io + - clustercleanuppolicies.kyverno.io + - clusterdomainclaims.networking.internal.knative.dev + - clusterissuers.cert-manager.io + - clusterpolicies.kyverno.io + - clusterpolicies.nvidia.com + - clusterpolicyreports.wgpolicyk8s.io + - configs.engine.run.ai + - configsyncstatuses.clusterconfig.azure.com + - configurations.serving.knative.dev + - datavolumes.run.ai + - departments.scheduling.run.ai + - diskpools.containerstorage.azure.com + - diskpools.openebs.io + - distributedpolicies.run.ai + - distributedworkloads.run.ai + - domainmappings.serving.knative.dev + - etcdclusters.etcd.database.coreos.com + - etcdrecoveries.containerstorage.azure.com + - extensionconfigs.clusterconfig.azure.com + - externalworkloads.run.ai + - hostdevicenetworks.mellanox.com + - images.caching.internal.knative.dev + - inferencepolicies.run.ai + - inferenceworkloads.run.ai + - ingresses.networking.internal.knative.dev + - interactivepolicies.run.ai + - interactiveworkloads.run.ai + - ipoibnetworks.mellanox.com + - ippools.nv-ipam.nvidia.com + - issuers.cert-manager.io + - jaegers.jaegertracing.io + - macvlannetworks.mellanox.com + - metrics.autoscaling.internal.knative.dev + - mpijobs.kubeflow.org + - mxjobs.kubeflow.org + - network-attachment-definitions.k8s.cni.cncf.io + - nicclusterpolicies.mellanox.com + - nodefeaturegroups.nfd.k8s-sigs.io + - nodefeaturerules.nfd.k8s-sigs.io + - nodefeatures.nfd.k8s-sigs.io + - nodenetworkconfigs.acn.azure.com + - nodepools.run.ai + - nvidiadrivers.nvidia.com + - orders.acme.cert-manager.io + - paddlejobs.kubeflow.org + - podautoscalers.autoscaling.internal.knative.dev + - podgroups.scheduling.run.ai + - podlogs.monitoring.grafana.com + - podmonitors.monitoring.coreos.com + - policies.kyverno.io + - policyexceptions.kyverno.io + - policyreports.wgpolicyk8s.io + - probes.monitoring.coreos.com + - projects.run.ai + - prometheusagents.monitoring.coreos.com + - prometheuses.monitoring.coreos.com + - prometheusrules.monitoring.coreos.com + - pytorchjobs.kubeflow.org + - queues.scheduling.run.ai + - revisions.serving.knative.dev + - routes.serving.knative.dev + - runaiconfigs.run.ai + - runaidgxclogexporters.runai.dgxc.nvidia.com + - runaidgxcnetworkpolicies.runai.dgxc.nvidia.com + - runaidgxcstorages.runai.dgxc.nvidia.com + - runaijobs.run.ai + - schedulingshards.engine.run.ai + - scrapeconfigs.monitoring.coreos.com + - secretproviderclasses.secrets-store.csi.x-k8s.io + - secretproviderclasspodstatuses.secrets-store.csi.x-k8s.io + - serverlessservices.networking.internal.knative.dev + - servicemonitors.monitoring.coreos.com + - services.serving.knative.dev + - skyhooks.skyhook.nvidia.com + - storagepools.containerstorage.azure.com + - tfjobs.kubeflow.org + - thanosrulers.monitoring.coreos.com + - trainingpolicies.run.ai + - trainingworkloads.run.ai + - updaterequests.kyverno.io + - volumesnapshotclasses.snapshot.storage.k8s.io + - volumesnapshotcontents.snapshot.storage.k8s.io + - volumesnapshots.snapshot.storage.k8s.io + - xgboostjobs.kubeflow.org + - apiGroup: mellanox.com + kind: HostDeviceNetwork + resources: + - hostdevicenetworks + - hostdevicenetworks/status + verbs: + - "*" + systemResources: [] + - apiGroup: mellanox.com + kind: IPoIBNetwork + resources: + - ipoibnetworks + - ipoibnetworks/status + verbs: + - "*" + systemResources: [] + - apiGroup: networking.k8s.io + kind: IngressClass + resources: + - ingressclasses + verbs: + # - create + # - delete + # - deletecollection + - get + - list + # - patch + # - update + - watch + systemResources: + - nginx + - apiGroup: mellanox.com + kind: MacvlanNetwork + resources: + - macvlannetworks + - macvlannetworks/status + verbs: + - "*" + systemResources: [] + - apiGroup: "" + kind: Namespace + resources: + - namespaces + - namespaces/finalize + - namespaces/status + verbs: + - "*" + systemResources: + - acstor + - argocd + - azure-extensions-usage-system + - cert-manager + - default + - dgxc-admission-controller + - dgxc-logging + - dgxc-tenant-cluster-policies + - gpu-operator + - ingress-nginx + - knative-serving + - kourier-system + - kube-node-lease + - kube-oidc-proxy + - kube-public + - kube-system + - kubeflow + - kyverno + - mpi-operator + - network-operator + - ngcc-csi-lvm + - nvdiag + - nvsentinel + - observability + - prometheus + - runai + - skyhook + - apiGroup: mellanox.com + kind: NicClusterPolicy + resources: + - nicclusterpolicies + - nicclusterpolicies/status + verbs: + - "*" + systemResources: + - nic-cluster-policy + - apiGroup: "" + kind: PersistentVolume + resources: + - persistentvolumes + - persistentvolumes/status + verbs: + # - create + # - delete + # - deletecollection + - get + - list + # - patch + # - update + - watch + systemResources: + - pvc-09658801-ecb4-4df6-a922-a744e01aa739 + - pvc-243e7721-54d1-4c2c-b0f3-6304e5568571 + - pvc-5910d71c-6c60-408b-ac17-a2a6929579ed + - pvc-d0bef359-dc21-4a5e-8ae8-c9a54509e5ae + - pvc-d4435c08-216a-4281-859c-26de19b98fc9 + - apiGroup: scheduling.k8s.io + kind: PriorityClass + resources: + - priorityclasses + verbs: + - "*" + systemResources: + - build + - inference + - interactive-preemptible + - runai-critical + - runai-engine-critical + - system-cluster-critical + - system-node-critical + - train + - train-critical + - train-high + - apiGroup: authorization.k8s.io + kind: SelfSubjectAccessReview + resources: + - selfsubjectaccessreviews + verbs: + - "*" + systemResources: [] + - apiGroup: authentication.k8s.io + kind: SelfSubjectReview + resources: + - selfsubjectreviews + verbs: + - "*" + systemResources: [] + - apiGroup: authorization.k8s.io + kind: SelfSubjectRulesReview + resources: + - selfsubjectrulesreviews + verbs: + - "*" + systemResources: [] + - apiGroup: authorization.k8s.io + kind: SubjectAccessReview + resources: + - subjectaccessreviews + verbs: + - "*" + systemResources: [] + - apiGroup: authentication.k8s.io + kind: TokenReview + resources: + - tokenreviews + verbs: + - "*" + systemResources: [] + +namespacedKinds: + - apiGroup: policy + resources: + - pods/eviction + verbs: + - create + - apiGroup: "" + kind: Binding + resources: + - bindings + verbs: + - "*" + - apiGroup: "" + kind: ConfigMap + resources: + - configmaps + verbs: + - "*" + - apiGroup: apps + kind: ControllerRevision + resources: + - controllerrevisions + verbs: + - "*" + - apiGroup: batch + kind: CronJob + resources: + - cronjobs + - cronjobs/status + verbs: + - "*" + - apiGroup: apps + kind: DaemonSet + resources: + - daemonsets + - daemonsets/status + verbs: + - "*" + - apiGroup: apps + kind: Deployment + resources: + - deployments + - deployments/scale + - deployments/status + verbs: + - "*" + - apiGroup: discovery.k8s.io + kind: EndpointSlice + resources: + - endpointslices + verbs: + - "*" + - apiGroup: "" + kind: Endpoints + resources: + - endpoints + verbs: + - "*" + - apiGroup: "" + kind: Event + resources: + - events + verbs: + - "*" + - apiGroup: events.k8s.io + kind: Event + resources: + - events + verbs: + - "*" + - apiGroup: autoscaling + kind: HorizontalPodAutoscaler + resources: + - horizontalpodautoscalers + - horizontalpodautoscalers/status + verbs: + - "*" + - apiGroup: networking.k8s.io + kind: Ingress + resources: + - ingresses + - ingresses/status + verbs: + - "*" + - apiGroup: batch + kind: Job + resources: + - jobs + - jobs/status + verbs: + - "*" + - apiGroup: coordination.k8s.io + kind: Lease + resources: + - leases + verbs: + - "*" + - apiGroup: "" + kind: LimitRange + resources: + - limitranges + verbs: + - "*" + - apiGroup: authorization.k8s.io + kind: LocalSubjectAccessReview + resources: + - localsubjectaccessreviews + verbs: + - "*" + - apiGroup: networking.k8s.io + kind: NetworkPolicy + resources: + - networkpolicies + verbs: + - "*" + - apiGroup: "" + kind: PersistentVolumeClaim + resources: + - persistentvolumeclaims + - persistentvolumeclaims/status + verbs: + - "*" + - apiGroup: "" + kind: Pod + resources: + - pods + - pods/attach + - pods/binding + - pods/ephemeralcontainers + - pods/eviction + - pods/exec + - pods/log + - pods/portforward + - pods/proxy + - pods/status + verbs: + - "*" + - apiGroup: policy + kind: PodDisruptionBudget + resources: + - poddisruptionbudgets + - poddisruptionbudgets/status + verbs: + - "*" + - apiGroup: "" + kind: PodTemplate + resources: + - podtemplates + verbs: + - "*" + - apiGroup: apps + kind: ReplicaSet + resources: + - replicasets + - replicasets/scale + - replicasets/status + verbs: + - "*" + - apiGroup: "" + kind: ReplicationController + resources: + - replicationcontrollers + - replicationcontrollers/scale + - replicationcontrollers/status + verbs: + - "*" + - apiGroup: "" + kind: ResourceQuota + resources: + - resourcequotas + - resourcequotas/status + verbs: + - "*" + - apiGroup: rbac.authorization.k8s.io + kind: Role + resources: + - roles + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + # - bind + # - escalate + - apiGroup: rbac.authorization.k8s.io + kind: RoleBinding + resources: + - rolebindings + verbs: + - "*" + - apiGroup: "" + kind: Secret + resources: + - secrets + verbs: + - "*" + - apiGroup: "" + kind: Service + resources: + - services + - services/proxy + - services/status + verbs: + - "*" + - apiGroup: "" + kind: ServiceAccount + resources: + - serviceaccounts + - serviceaccounts/token + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + # - impersonate + - apiGroup: apps + kind: StatefulSet + resources: + - statefulsets + - statefulsets/scale + - statefulsets/status + verbs: + - "*" + +systemSubjects: + User: + - aks-support + - clusterAdmin + - clusterMonitoringUser + - clusterUser + - hcpService + - konnectivity-prometheus-client + - kyverno@test.com + - ladithyav@nvidia.com + - pwittrock@nvidia.com + - system:kube-controller-manager + - system:kube-proxy + - system:kube-scheduler + Group: + - 07d60667-263d-401a-8b36-119b74fbb6b1 + - 93bb39a4-e70f-489f-8d9e-b08bbc8f557 + - 93bb39a4-e70f-489f-8d9e-b08bbc8f557b + - aece7cfa-d4e2-48f4-a812-3053722e561b + - ed8eca09-efb9-4909-9e73-295d09445297 + - system:bootstrappers + - system:masters + - system:monitoring + - system:nodes + - system:serviceaccounts + - system:serviceaccounts:network-operator + - system:unauthenticated + diff --git a/charts/nemo/charts/dgxc-admission-controller/values-gcp-custom.yaml b/charts/nemo/charts/dgxc-admission-controller/values-gcp-custom.yaml new file mode 100644 index 0000000..276ac1e --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/values-gcp-custom.yaml @@ -0,0 +1,107 @@ +--- +kubernetesAffinity: + enabled: true + # we will make an effort to not take up valuable GPU resources + # so try and run on a CPU node that is not reserved for the Kubernetes Master. + + # matchExpressions: + #- key: "node-role.kubernetes.io/fss-workload" + # operator: Exists + # matchExpressions: + #- key: "kubernetes.io/role" + # operator: In + # values: ["fss-workload"] + # + # The below configurationa replaces the match expressions. + # The default setting is the handle legacy clusters without the above + # matching updated label. + antiAffinityKey: cloud.google.com/gke-gpu + antiAffinityValue: true + # The below is used for pod anti-affinity to ensure pods are running on + # different nodes + topologyKey: kubernetes.io/hostname + # Run on Masters should remain false unless there is a very specific need + # (ie. if you have been given a minimal dev cluster, with _only_ masters) + runOnMasters: false +systemTolerationKeyName: components.gke.io/gke-managed-components +systemTolerationValueName: "true" + +tolerations: + customer: + key: "dedicated" + value: "user-workload" + system: + key: "components.gke.io/gke-managed-components" + value: "true" + +cloudProviderSpec: + type: gcp + gcp: + projectId: proj-dgxc-runai-np-dev-mega + projectNumber: 574718269555 + securityPolicyRuleName: dgxc-runai-us-east5-ngc-00546849-system-lb-backend-policy + region: us-east5 + # GCP Workload Identity requires access to the metadata server + # Setting this flag, disables the kyverno security policy preventing + # Pods for accessing the metadata server. + workloadIdentityEnabled: false + +storageControllerSpecGCP: + enabled: false + defaultStorageTypes: + - apiVersion: runai.dgxc.nvidia.com/v1beta1 + kind: RunaiDGXCStorage + metadata: + name: "filestore-enterprise" + namespace: dgxc-tenant-cluster-policies + spec: + quotaID: "EnterpriseStorageGibPerRegion" + status: + altNames: [ "enterprise" ] + storageClasses: + - enterprise-multishare-rwx + - enterprise-rwx + storageClassesDisplayString: "enterprise-multishare-rwx,enterprise-rwx" + docs: https://cloud.google.com/filestore/docs/service-tiers + - apiVersion: runai.dgxc.nvidia.com/v1beta1 + kind: RunaiDGXCStorage + metadata: + name: "filestore-premium" + namespace: dgxc-tenant-cluster-policies + spec: + quotaID: "PremiumStorageGbPerRegion" + status: + altNames: [ "premium" ] + storageClasses: + - premium-rwo + - premium-rwx + storageClassesDisplayString: "premium-rwo,premium-rwx" + docs: https://cloud.google.com/filestore/docs/service-tiers + - apiVersion: runai.dgxc.nvidia.com/v1beta1 + kind: RunaiDGXCStorage + metadata: + name: "filestore-standard" + namespace: dgxc-tenant-cluster-policies + spec: + quotaID: "StandardStorageGbPerRegion" + status: + altNames: [ "filestore-standard" ] + storageClasses: + - standard + - standard-rwo + - standard-rwx + storageClassesDisplayString: "standard,standard-rwo,standard-rwx" + docs: https://cloud.google.com/filestore/docs/service-tiers + - apiVersion: runai.dgxc.nvidia.com/v1beta1 + kind: RunaiDGXCStorage + metadata: + name: "filestore-high-scale" + namespace: dgxc-tenant-cluster-policies + spec: + quotaID: "HighScaleSSDStorageGibPerRegion" + status: + altNames: [ "zonal" ] + storageClasses: + - zonal-rwx + storageClassesDisplayString: "zonal-rwx" + docs: https://cloud.google.com/filestore/docs/service-tiers diff --git a/charts/nemo/charts/dgxc-admission-controller/values-gcp.yaml b/charts/nemo/charts/dgxc-admission-controller/values-gcp.yaml new file mode 100644 index 0000000..03270f6 --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/values-gcp.yaml @@ -0,0 +1,1183 @@ + +#################################################################### +##### The portion below this marker is automatically generated ##### +##### DO NOT EDIT BY HAND ##### +#################################################################### + +clusterScopedKinds: + - apiGroup: argoproj.io + resources: + - eventsources + - sensors + verbs: + - "*" + systemResources: [] + - apiGroup: extensions + resources: + - ingresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + systemResources: [] + - apiGroup: "" + resources: + - persistentvolumeclaims/finalizers + verbs: + - create + - update + - delete + - get + systemResources: [] + - apiGroup: "" + resources: + - configmaps/status + verbs: + - "*" + systemResources: [] + - apiGroup: route.openshift.io + resources: + - "*" + verbs: + - "*" + systemResources: [] + - apiGroup: nvidia.com + kind: ClusterPolicy + resources: + - clusterpolicies + - clusterpolicies/status + verbs: + - "*" + systemResources: + - cluster-policy + - apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + resources: + - clusterroles + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + # - bind + # - escalate + systemResources: + - admin + - anet-operator-cluster-role + - antrea-agent + - antrea-controller + - antrea-cpha + - argocd-application-controller + - argocd-notifications-controller + - argocd-server + - assets-sync + - assets-sync-cluster + - ca-cr-actor + - ca-pr-beta-actor + - cert-manager-cainjector + - cert-manager-cluster-view + - cert-manager-controller-approve:cert-manager-io + - cert-manager-controller-certificates + - cert-manager-controller-certificatesigningrequests + - cert-manager-controller-challenges + - cert-manager-controller-clusterissuers + - cert-manager-controller-ingress-shim + - cert-manager-controller-issuers + - cert-manager-controller-orders + - cert-manager-edit + - cert-manager-view + - cert-manager-webhook:subjectaccessreviews + - cilium + - cilium-win + - cloud-provider + - cluster-admin + - cluster-autoscaler + - cluster-sync + - cluster-sync-project + - dgxc-admission-controller + - dgxc-customer-app-admin + - dgxc-customer-app-admin-base + - dgxc-customer-network-policy + - dgxc-exporter + - dgxc-runai-controller + - edit + - external-metrics-reader + - filestorecsi-controller-cluster-role + - filestorecsi-node-driver-cluster-role + - filestorecsi-provisioner-role + - filestorecsi-resizer-role + - filestorecsi-snapshotter-role + - fluentbit-gke-pod-label-reader + - gce:beta:kubelet-certificate-bootstrap + - gce:beta:kubelet-certificate-rotation + - gce:cloud-provider + - gce:gke-metadata-server-reader + - gcs-fuse-csi-publisher-role + - gke-oidc-envoy-impersonate + - gke-oidc-operator-role + - gke-spiffe-issuer + - gke-spiffe-user + - gmp-system:collector + - gmp-system:operator + - gpu-operator + - gpu-operator-node-feature-discovery + - gpu-operator-node-feature-discovery-gc + - gpu-operator-upgrade-crd-hook-role + - ingress-nginx + - knative-serving-activator-cluster + - knative-serving-addressable-resolver + - knative-serving-admin + - knative-serving-aggregated-addressable-resolver + - knative-serving-core + - knative-serving-namespaced-admin + - knative-serving-namespaced-edit + - knative-serving-namespaced-view + - knative-serving-podspecable-binding + - konnectivity-agent-cpha + - kratos-fluent-bit-read + - kubeflow-mpijobs-admin + - kubeflow-mpijobs-edit + - kubeflow-mpijobs-view + - kubelet-api-admin + - kyverno-cleanup-jobs + - kyverno:admission-controller + - kyverno:admission-controller:additional + - kyverno:admission-controller:core + - kyverno:background-controller + - kyverno:background-controller:additional + - kyverno:background-controller:core + - kyverno:cleanup-controller + - kyverno:cleanup-controller:core + - kyverno:dgxc-integrations + - kyverno:rbac:admin:policies + - kyverno:rbac:admin:policyreports + - kyverno:rbac:admin:reports + - kyverno:rbac:admin:updaterequests + - kyverno:rbac:view:policies + - kyverno:rbac:view:policyreports + - kyverno:rbac:view:reports + - kyverno:rbac:view:updaterequests + - kyverno:reports-controller + - kyverno:reports-controller:core + - kyverno:watch-update-namespaces + - kyverno:watch-update-services + - maintenance-handler + - metrics-exporter + - mpi-operator + - net-kourier + - netd + - node-problem-detector-cpu + - node-problem-detector-gpu + - node-scale-adjuster-ro + - nodepool-controller + - nvidia-device-plugin + - nvidia-device-plugin-mps-control-daemon + - nvidia-gpu-feature-discovery + - nvidia-mig-manager + - nvidia-operator-validator + - pdcsi-attacher-role + - pdcsi-provisioner-role + - pdcsi-resizer-role + - pdcsi-snapshotter-role + - pod-group-assigner + - pod-group-controller + - pod-group-controller-ro + - pod-grouper + - pods-list-osmo-agent + - prometheus-kube-state-metrics + - prometheus-operator + - prometheus-prometheus + - queue-controller + - read-updateinfo + - researcher-service-ro + - researcher-service-rw + - runai-accessrule-controller + - runai-accessrule-controller-project + - runai-admission-controller-project + - runai-admission-controller-ro + - runai-agent + - runai-binder-cluster-wide + - runai-binder-project + - runai-cluster-api + - runai-cluster-api-project + - runai-compute-resource-administrator + - runai-compute-resource-administrator-ns + - runai-compute-resource-administrator-runai + - runai-credentials-administrator + - runai-credentials-administrator-ns + - runai-credentials-administrator-runai + - runai-data-source-administrator + - runai-data-source-administrator-ns + - runai-data-source-administrator-runai + - runai-data-source-administrator-runai-cs + - runai-data-volume-administrator + - runai-data-volume-administrator-ns + - runai-data-volume-administrator-runai + - runai-department-administrator + - runai-department-administrator-ns + - runai-department-administrator-runai + - runai-department-viewer + - runai-department-viewer-ns + - runai-department-viewer-runai + - runai-device-plugin + - runai-editor + - runai-editor-ns + - runai-editor-runai + - runai-engine-operator + - runai-environment-administrator + - runai-environment-administrator-ns + - runai-environment-administrator-runai + - runai-external-workload-integrator + - runai-external-workload-integrator-project + - runai-inference-workload-controller + - runai-inference-workload-controller-project + - runai-init-ca + - runai-job-controller + - runai-job-controller-project + - runai-l1-researcher + - runai-l1-researcher-ns + - runai-l1-researcher-runai + - runai-l2-researcher + - runai-l2-researcher-ns + - runai-l2-researcher-runai + - runai-mig-parted + - runai-ml-engineer + - runai-ml-engineer-ns + - runai-ml-engineer-runai + - runai-operator + - runai-project-controller + - runai-project-controller-cluster-pvc + - runai-project-controller-cluster-pvc-per-project + - runai-project-controller-cluster-secret + - runai-project-controller-cluster-secret-per-project + - runai-project-controller-limit-range + - runai-project-controller-limit-range-per-project + - runai-project-controller-ns-creation + - runai-project-controller-resources-patcher + - runai-project-controller-rolebinding-creation + - runai-project-controller-servicemonitor-creation + - runai-prometheus + - runai-research-manager + - runai-research-manager-ns + - runai-research-manager-runai + - runai-resource-reservation-rw + - runai-scheduler-ro + - runai-scheduler-rw + - runai-shared-objects-controller + - runai-shared-objects-controller-project + - runai-system-administrator + - runai-system-administrator-ns + - runai-system-administrator-runai + - runai-system-administrator-runai-cs + - runai-template-administrator + - runai-template-administrator-ns + - runai-template-administrator-runai + - runai-viewer + - runai-viewer-ns + - runai-viewer-runai + - runai-workload-exporter + - secretproviderclasses-role-gke + - secrets-store-csi-driver-provider-gke-role + - skyhook-operator-manager-role + - skyhook-operator-metrics-reader + - skyhook-operator-proxy-role + - snapshot-controller-runner + - sre-cluster-scoped-access + - status-updater + - status-updater-ro + - system:aggregate-to-admin + - system:aggregate-to-edit + - system:aggregate-to-view + - system:anet-manager-cluster-role + - system:auth-delegator + - system:basic-user + - system:certificates.k8s.io:certificatesigningrequests:nodeclient + - system:certificates.k8s.io:certificatesigningrequests:selfnodeclient + - system:certificates.k8s.io:kube-apiserver-client-approver + - system:certificates.k8s.io:kube-apiserver-client-kubelet-approver + - system:certificates.k8s.io:kubelet-serving-approver + - system:certificates.k8s.io:legacy-unknown-approver + - system:cloud-controller-manager + - system:clustermetrics + - system:controller:attachdetach-controller + - system:controller:certificate-controller + - system:controller:cloud-node-controller + - system:controller:clusterrole-aggregation-controller + - system:controller:cronjob-controller + - system:controller:daemon-set-controller + - system:controller:deployment-controller + - system:controller:disruption-controller + - system:controller:endpoint-controller + - system:controller:endpointslice-controller + - system:controller:endpointslicemirroring-controller + - system:controller:ephemeral-volume-controller + - system:controller:expand-controller + - system:controller:generic-garbage-collector + - system:controller:glbc + - system:controller:horizontal-pod-autoscaler + - system:controller:job-controller + - system:controller:legacy-service-account-token-cleaner + - system:controller:namespace-controller + - system:controller:node-controller + - system:controller:persistent-volume-binder + - system:controller:pod-garbage-collector + - system:controller:pv-protection-controller + - system:controller:pvc-protection-controller + - system:controller:replicaset-controller + - system:controller:replication-controller + - system:controller:resourcequota-controller + - system:controller:root-ca-cert-publisher + - system:controller:route-controller + - system:controller:service-account-controller + - system:controller:service-controller + - system:controller:statefulset-controller + - system:controller:ttl-after-finished-controller + - system:controller:ttl-controller + - system:controller:validatingadmissionpolicy-status-controller + - system:discovery + - system:gcp-controller-manager + - system:gke-common-webhooks + - system:gke-controller + - system:gke-hpa-actor + - system:gke-hpa-service-reader + - system:gke-uas-collection-reader + - system:gke-uas-metrics-reader + - system:glbc-status + - system:heapster + - system:kube-aggregator + - system:kube-controller-manager + - system:kube-dns + - system:kube-dns-autoscaler + - system:kube-scheduler + - system:kubelet-api-admin + - system:kubestore-collector + - system:maintenance-controller-cluster-role + - system:managed-certificate-controller + - system:master-monitoring-role + - system:metrics-server + - system:metrics-server-nanny + - system:monitoring + - system:multinet-endpointslice-controller + - system:network-controller-manager + - system:node + - system:node-bootstrapper + - system:node-problem-detector + - system:node-proxier + - system:persistent-volume-provisioner + - system:public-info-viewer + - system:resource-tracker + - system:service-account-issuer-discovery + - system:service-steering + - system:slo-monitor + - system:volume-scheduler + - training-operator + - vace-role + - view + - workload-controller-ro + - workload-controller-rw + - workload-overseer + - apiGroup: rbac.authorization.k8s.io + kind: ClusterRoleBinding + resources: + - clusterrolebindings + verbs: + - "*" + systemResources: + - anet-operator-role-binding + - antrea-agent + - antrea-controller + - antrea-cpha + - argocd-application-controller + - argocd-notifications-controller + - argocd-server + - assets-sync + - ca-cr + - ca-pr-beta + - cert-manager-cainjector + - cert-manager-controller-approve:cert-manager-io + - cert-manager-controller-certificates + - cert-manager-controller-certificatesigningrequests + - cert-manager-controller-challenges + - cert-manager-controller-clusterissuers + - cert-manager-controller-ingress-shim + - cert-manager-controller-issuers + - cert-manager-controller-orders + - cert-manager-webhook:subjectaccessreviews + - cilium + - cilium-win + - cluster-admin + - cluster-autoscaler + - cluster-autoscaler-updateinfo + - cluster-sync + - dgxc-admission-controller + - dgxc-customer-app-admin + # - dgxc-customer-app-admin-access + - dgxc-exporter + - dgxc-runai-controller + - event-exporter-rb + - filestorecsi-controller-cluster-role-binding + - filestorecsi-controller-provisioner-binding + - filestorecsi-controller-resizer-binding + - filestorecsi-node-driver-cluster-role-binding + - filestorecsi-snapshotter-binding + - fluentbit-gke-pod-label-reader + - gce:beta:kubelet-certificate-bootstrap + - gce:beta:kubelet-certificate-rotation + - gce:cloud-provider + - gce:gke-metadata-server-reader + - gce:podsecuritypolicy:gcsfusecsi-node-sa + - gke-oidc-envoy-impersonate + - gke-oidc-operator-binding + - gke-spiffe-controller-is-a-gke-spiffe-issuer + - gmp-system:collector + - gmp-system:operator + - gpu-operator + - gpu-operator-node-feature-discovery + - gpu-operator-node-feature-discovery-gc + - gpu-operator-upgrade-crd-hook-binding + - ingress-nginx + - knative-serving-activator-cluster + - knative-serving-controller-addressable-resolver + - knative-serving-controller-admin + - konnectivity-agent-cpha + - kratos-fluent-bit-read + - kube-apiserver-kubelet-api-admin + - kubelet-bootstrap + - kubelet-bootstrap-certificate-bootstrap + - kubelet-bootstrap-node-bootstrapper + - kubelet-cluster-admin + - kubelet-user-npd-binding + - kyverno-cleanup-jobs + - kyverno:admission-controller + - kyverno:background-controller + - kyverno:cleanup-controller + - kyverno:reports-controller + - maintenance-controller + - maintenance-handler + - master-monitoring-role-binding + - metrics-exporter + - metrics-server-nanny:system:auth-delegator + - metrics-server:system:auth-delegator + - mpi-operator + - net-kourier + - netd + - node-problem-detector-cpu + - node-problem-detector-gpu + - node-scale-adjuster-ro + - nodepool-controller + - nodes-are-gke-spiffe-users + - npd-binding + - nvidia-device-plugin + - nvidia-device-plugin-mps-control-daemon + - nvidia-gpu-feature-discovery + - nvidia-mig-manager + - nvidia-operator-validator + - pdcsi-controller-attacher-binding + - pdcsi-controller-provisioner-binding + - pdcsi-controller-resizer-binding + - pdcsi-snapshotter-binding + - pod-group-assigner + - pod-group-controller-ro + - pod-grouper + - pods-list-osmo-agent + - prometheus-kube-state-metrics + - prometheus-operator + - prometheus-prometheus + - queue-controller + - researcher-service-ro + - runai-accessrule-controller + - runai-admission-controller-ro + - runai-agent + - runai-binder-cluster-wide + - runai-cluster-api + - runai-compute-resource-administrator + - runai-credentials-administrator + - runai-data-source-administrator + - runai-data-volume-administrator + - runai-department-administrator + - runai-department-viewer + - runai-device-plugin + - runai-editor + - runai-engine-operator + - runai-environment-administrator + - runai-external-workload-integrator + - runai-inference-workload-controller + - runai-init-ca + - runai-job-controller + - runai-l1-researcher + - runai-l2-researcher + - runai-mig-parted + - runai-ml-engineer + - runai-operator + - runai-project-controller + - runai-project-controller-cluster-pvc + - runai-project-controller-cluster-secret + - runai-project-controller-limit-range + - runai-project-controller-ns-creation + - runai-project-controller-rolebinding-creation + - runai-project-controller-servicemonitor-creation + - runai-prometheus + - runai-research-manager + - runai-scheduler-ro + - runai-shared-objects-controller + - runai-system-administrator + - runai-template-administrator + - runai-viewer + - runai-workload-exporter + - secretproviderclasses-rolebinding-gke + - secrets-store-csi-driver-provider-gke-rolebinding + - skyhook-operator-manager-rolebinding + - skyhook-operator-proxy-rolebinding + - snapshot-controller-role + - sre-cluster-scoped-access + - sre-full-access-binding + - status-updater-ro + - system:anet-manager-cluster-role-binding + - system:basic-user + - system:cloud-controller-manager + - system:clustermetrics + - system:controller:attachdetach-controller + - system:controller:certificate-controller + - system:controller:clusterrole-aggregation-controller + - system:controller:cronjob-controller + - system:controller:daemon-set-controller + - system:controller:deployment-controller + - system:controller:disruption-controller + - system:controller:endpoint-controller + - system:controller:endpointslice-controller + - system:controller:endpointslicemirroring-controller + - system:controller:ephemeral-volume-controller + - system:controller:expand-controller + - system:controller:generic-garbage-collector + - system:controller:horizontal-pod-autoscaler + - system:controller:job-controller + - system:controller:legacy-service-account-token-cleaner + - system:controller:namespace-controller + - system:controller:node-controller + - system:controller:persistent-volume-binder + - system:controller:pod-garbage-collector + - system:controller:pv-protection-controller + - system:controller:pvc-protection-controller + - system:controller:replicaset-controller + - system:controller:replication-controller + - system:controller:resourcequota-controller + - system:controller:root-ca-cert-publisher + - system:controller:route-controller + - system:controller:service-account-controller + - system:controller:service-controller + - system:controller:statefulset-controller + - system:controller:ttl-after-finished-controller + - system:controller:ttl-controller + - system:controller:validatingadmissionpolicy-status-controller + - system:discovery + - system:gcp-controller-manager + - system:gke-common-webhooks + - system:gke-controller + - system:gke-hpa-actor + - system:gke-hpa-service-reader + - system:gke-uas-collection-reader + - system:gke-uas-hpa-controller + - system:gke-uas-metrics-reader + - system:glbc-status + - system:konnectivity-server + - system:kube-controller-manager + - system:kube-dns + - system:kube-dns-autoscaler + - system:kube-proxy + - system:kube-scheduler + - system:kubestore-collector + - system:managed-certificate-controller + - system:metrics-server + - system:metrics-server-nanny + - system:monitoring + - system:multinet-endpointslice-controller + - system:network-controller-manager + - system:node + - system:node-proxier + - system:public-info-viewer + - system:resource-tracker + - system:service-account-issuer-discovery + - system:service-steering + - system:slo-monitor + - system:volume-scheduler + - training-operator + - uas-hpa-external-metrics-reader + - vace-rolebinding + - workload-controller-ro + - workload-overseer + - apiGroup: "" + kind: ComponentStatus + resources: + - componentstatuses + verbs: + - "*" + systemResources: + - scheduler + - controller-manager + - etcd-0 + - etcd-1 + - apiGroup: cloud.google.com + kind: ComputeClass + resources: + - computeclasses + - computeclasses/status + verbs: + - "*" + systemResources: [] + - apiGroup: apiextensions.k8s.io + kind: CustomResourceDefinition + resources: + - customresourcedefinitions + - customresourcedefinitions/status + verbs: + - "*" + systemResources: + - accessrules.run.ai + - admissionreports.kyverno.io + - alertmanagerconfigs.monitoring.coreos.com + - alertmanagers.monitoring.coreos.com + # - allowlistedv2workloads.auto.gke.io GKE has a webhook that denies changes + # - allowlistedworkloads.auto.gke.io GKE has a webhook that denies changes + - applications.argoproj.io + - applicationsets.argoproj.io + - appprojects.argoproj.io + # - audits.warden.gke.io GKE has a webhook that denies changes + - backendconfigs.cloud.google.com + - backgroundscanreports.kyverno.io + - bindrequests.scheduling.run.ai + - capacityrequests.internal.autoscaling.gke.io + - certificaterequests.cert-manager.io + - certificates.cert-manager.io + - certificates.networking.internal.knative.dev + - challenges.acme.cert-manager.io + - ciliumendpoints.cilium.io + - ciliumendpointslices.cilium.io + - ciliumexternalworkloads.cilium.io + - ciliumidentities.cilium.io + - ciliumlocalredirectpolicies.cilium.io + - ciliumnodes.cilium.io + - cleanuppolicies.kyverno.io + - clientconfigs.authentication.gke.io + - clusteradmissionreports.kyverno.io + - clusterbackgroundscanreports.kyverno.io + - clustercleanuppolicies.kyverno.io + - clusterdomainclaims.networking.internal.knative.dev + - clusterissuers.cert-manager.io + - clusternodemonitorings.monitoring.googleapis.com + - clusterpodmonitorings.monitoring.googleapis.com + - clusterpolicies.kyverno.io + - clusterpolicies.nvidia.com + - clusterpolicyreports.wgpolicyk8s.io + - clusterrules.monitoring.googleapis.com + - compositecontrollerdefinitions.extensions.dgxc.nvidia.com + - computeclasses.cloud.google.com + - configs.engine.run.ai + - configurations.serving.knative.dev + - crontests.test.dgxc.nvidia.com + - datavolumes.run.ai + - departments.scheduling.run.ai + - distributedpolicies.run.ai + - distributedworkloads.run.ai + - domainmappings.serving.knative.dev + - externalworkloads.run.ai + - frontendconfigs.networking.gke.io + - gcpresourceallowlists.node.gke.io + - gitlabresources.gitlab.dgxc.nvidia.com + - gkenetworkparamsets.networking.gke.io + - globalrules.monitoring.googleapis.com + - images.caching.internal.knative.dev + - inferencepolicies.run.ai + - inferenceworkloads.run.ai + - ingresses.networking.internal.knative.dev + - interactivepolicies.run.ai + - interactiveworkloads.run.ai + - issuers.cert-manager.io + - managedcertificates.networking.gke.io + - memberships.hub.gke.io + - metrics.autoscaling.internal.knative.dev + - mpijobs.kubeflow.org + - mxjobs.kubeflow.org + - networkinterfaces.networking.gke.io + - networkloggings.networking.gke.io + - networks.networking.gke.io + - nodefeaturegroups.nfd.k8s-sigs.io + - nodefeaturerules.nfd.k8s-sigs.io + - nodefeatures.nfd.k8s-sigs.io + - nodepools.run.ai + - nvidiadrivers.nvidia.com + - operatorconfigs.monitoring.googleapis.com + - orders.acme.cert-manager.io + - paddlejobs.kubeflow.org + - podautoscalers.autoscaling.internal.knative.dev + - podgroups.scheduling.run.ai + - podlogs.monitoring.grafana.com + - podmonitorings.monitoring.googleapis.com + - podmonitors.monitoring.coreos.com + - policies.kyverno.io + - policyexceptions.kyverno.io + - policyreports.wgpolicyk8s.io + - probes.monitoring.coreos.com + - projects.run.ai + - prometheusagents.monitoring.coreos.com + - prometheuses.monitoring.coreos.com + - prometheusrules.monitoring.coreos.com + - provisioningrequests.autoscaling.x-k8s.io + - pytorchjobs.kubeflow.org + - queues.scheduling.run.ai + - redirectservices.networking.gke.io + - remotenodes.networking.gke.io + - revisions.serving.knative.dev + - routes.serving.knative.dev + - rules.monitoring.googleapis.com + - runaiaces.runai.dgxc.nvidia.com + - runaiconfigs.run.ai + - runaidgxclogexporters.runai.dgxc.nvidia.com + - runaidgxcnetworkpolicies.runai.dgxc.nvidia.com + - runaidgxcstorages.runai.dgxc.nvidia.com + - runaijobs.run.ai + - runaiplatforms.runai.dgxc.nvidia.com + - runairuntimes.runai.dgxc.nvidia.com + - schedulingshards.engine.run.ai + - scrapeconfigs.monitoring.coreos.com + - secretproviderclasses.secrets-store.csi.x-k8s.io + - secretproviderclasspodstatuses.secrets-store.csi.x-k8s.io + - serverlessservices.networking.internal.knative.dev + - serviceattachments.networking.gke.io + - servicefunctionchains.networking.gke.io + - servicemonitors.monitoring.coreos.com + - servicenetworkendpointgroups.networking.gke.io + - services.serving.knative.dev + - skyhooks.skyhook.nvidia.com + - sriovnetworknodestates.networking.gke.io + - sriovoperatorconfigs.networking.gke.io + - tfjobs.kubeflow.org + - thanosrulers.monitoring.coreos.com + - trafficselectors.networking.gke.io + - trainingpolicies.run.ai + - trainingworkloads.run.ai + - trustconfigs.security.cloud.google.com + - updateinfos.nodemanagement.gke.io + - updaterequests.kyverno.io + - volumesnapshotclasses.snapshot.storage.k8s.io + - volumesnapshotcontents.snapshot.storage.k8s.io + - volumesnapshots.snapshot.storage.k8s.io + - workloadallowlists.auto.gke.io + - workloadcertificateconfigs.security.cloud.google.com + - xgboostjobs.kubeflow.org + - apiGroup: networking.k8s.io + kind: IngressClass + resources: + - ingressclasses + verbs: + # - create + # - delete + # - deletecollection + - get + - list + # - patch + # - update + - watch + systemResources: + - nginx + - apiGroup: "" + kind: Namespace + resources: + - namespaces + - namespaces/finalize + - namespaces/status + verbs: + - "*" + systemResources: + - anthos-identity-service + - argocd + - cert-manager + - default + - dgxc-admission-controller + - dgxc-cluster-validator + - dgxc-tenant-cluster-policies + - gke-managed-filestorecsi + - gke-managed-system + - gmp-public + - gmp-system + - gpu-operator + - ingress-nginx + - knative-serving + - kourier-system + - kube-node-lease + - kube-public + - kube-system + - kubeflow + - kyverno + - lalit + - monitoring + - mpi-operator + - node-problem-detector + - nvdiag + - nvsentinel + - observability + - osmo + - prometheus + - runai + - skyhook + - test-ns + - apiGroup: "" + kind: PersistentVolume + resources: + - persistentvolumes + - persistentvolumes/status + verbs: + # - create + # - delete + # - deletecollection + - get + - list + # - patch + # - update + - watch + systemResources: [] + - apiGroup: scheduling.k8s.io + kind: PriorityClass + resources: + - priorityclasses + verbs: + - "*" + systemResources: + - build + - gmp-critical + - inference + - interactive-preemptible + - runai-critical + - runai-engine-critical + - system-cluster-critical + - system-node-critical + - train + - train-critical + - train-high + - apiGroup: authorization.k8s.io + kind: SelfSubjectAccessReview + resources: + - selfsubjectaccessreviews + verbs: + - "*" + systemResources: [] + - apiGroup: authentication.k8s.io + kind: SelfSubjectReview + resources: + - selfsubjectreviews + verbs: + - "*" + systemResources: [] + - apiGroup: authorization.k8s.io + kind: SelfSubjectRulesReview + resources: + - selfsubjectrulesreviews + verbs: + - "*" + systemResources: [] + - apiGroup: authorization.k8s.io + kind: SubjectAccessReview + resources: + - subjectaccessreviews + verbs: + - "*" + systemResources: [] + - apiGroup: authentication.k8s.io + kind: TokenReview + resources: + - tokenreviews + verbs: + - "*" + systemResources: [] + +namespacedKinds: + - apiGroup: policy + resources: + - pods/eviction + verbs: + - create + - apiGroup: "" + kind: Binding + resources: + - bindings + verbs: + - "*" + - apiGroup: "" + kind: ConfigMap + resources: + - configmaps + verbs: + - "*" + - apiGroup: apps + kind: ControllerRevision + resources: + - controllerrevisions + verbs: + - "*" + - apiGroup: batch + kind: CronJob + resources: + - cronjobs + - cronjobs/status + verbs: + - "*" + - apiGroup: apps + kind: DaemonSet + resources: + - daemonsets + - daemonsets/status + verbs: + - "*" + - apiGroup: apps + kind: Deployment + resources: + - deployments + - deployments/scale + - deployments/status + verbs: + - "*" + - apiGroup: discovery.k8s.io + kind: EndpointSlice + resources: + - endpointslices + verbs: + - "*" + - apiGroup: "" + kind: Endpoints + resources: + - endpoints + verbs: + - "*" + - apiGroup: events.k8s.io + kind: Event + resources: + - events + verbs: + - "*" + - apiGroup: "" + kind: Event + resources: + - events + verbs: + - "*" + - apiGroup: autoscaling + kind: HorizontalPodAutoscaler + resources: + - horizontalpodautoscalers + - horizontalpodautoscalers/status + verbs: + - "*" + - apiGroup: networking.k8s.io + kind: Ingress + resources: + - ingresses + - ingresses/status + verbs: + - "*" + - apiGroup: batch + kind: Job + resources: + - jobs + - jobs/status + verbs: + - "*" + - apiGroup: coordination.k8s.io + kind: Lease + resources: + - leases + verbs: + - "*" + - apiGroup: "" + kind: LimitRange + resources: + - limitranges + verbs: + - "*" + - apiGroup: authorization.k8s.io + kind: LocalSubjectAccessReview + resources: + - localsubjectaccessreviews + verbs: + - "*" + - apiGroup: networking.k8s.io + kind: NetworkPolicy + resources: + - networkpolicies + verbs: + - "*" + - apiGroup: "" + kind: PersistentVolumeClaim + resources: + - persistentvolumeclaims + - persistentvolumeclaims/status + verbs: + - "*" + - apiGroup: "" + kind: Pod + resources: + - pods + - pods/attach + - pods/binding + - pods/ephemeralcontainers + - pods/eviction + - pods/exec + - pods/log + - pods/portforward + - pods/proxy + - pods/status + verbs: + - "*" + - apiGroup: policy + kind: PodDisruptionBudget + resources: + - poddisruptionbudgets + - poddisruptionbudgets/status + verbs: + - "*" + - apiGroup: "" + kind: PodTemplate + resources: + - podtemplates + verbs: + - "*" + - apiGroup: apps + kind: ReplicaSet + resources: + - replicasets + - replicasets/scale + - replicasets/status + verbs: + - "*" + - apiGroup: "" + kind: ReplicationController + resources: + - replicationcontrollers + - replicationcontrollers/scale + - replicationcontrollers/status + verbs: + - "*" + - apiGroup: "" + kind: ResourceQuota + resources: + - resourcequotas + - resourcequotas/status + verbs: + - "*" + - apiGroup: rbac.authorization.k8s.io + kind: Role + resources: + - roles + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + # - bind + # - escalate + - apiGroup: rbac.authorization.k8s.io + kind: RoleBinding + resources: + - rolebindings + verbs: + - "*" + - apiGroup: "" + kind: Secret + resources: + - secrets + verbs: + - "*" + - apiGroup: "" + kind: Service + resources: + - services + - services/proxy + - services/status + verbs: + - "*" + - apiGroup: "" + kind: ServiceAccount + resources: + - serviceaccounts + - serviceaccounts/token + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + # - impersonate + - apiGroup: apps + kind: StatefulSet + resources: + - statefulsets + - statefulsets/scale + - statefulsets/status + verbs: + - "*" + +systemSubjects: + User: + - cluster-autoscaler + - kube-apiserver + - kubelet + - kubelet-bootstrap + - kyverno@test.com + - ladithyav@nvidia.com + - pwittrock@nvidia.com + - system:anet-controller-manager + - system:anet-operator + - system:cloud-controller-manager + - system:cluster-autoscaler + - system:clustermetrics + - system:controller:glbc + - system:filestorecsi-controller + - system:gcp-controller-manager + - system:gke-common-webhooks + - system:gke-spiffe-controller + - system:konnectivity-server + - system:kube-controller-manager + - system:kube-proxy + - system:kube-scheduler + - system:kubestore-collector + - system:l7-lb-controller + - system:maintenance-controller + - system:managed-certificate-controller + - system:master-prom-to-sd-monitor + - system:metrics-server-nanny + - system:network-controller-manager + - system:node-problem-detector + - system:pdcsi-controller + - system:resource-tracker + - system:service-steering + - system:slo-monitor + - system:snapshot-controller + - system:vpa-recommender + Group: + - access-f-dgx-cloud-runai-non-prod-gcp-admin@nvidia.com + - access-f-dgx-cloud-runai-non-prod-gcp-engineer@nvidia.com + - access-f-dgx-cloud-runai-prod-gcp-admin@nvidia.com + - access-f-dgx-cloud-runai-prod-gcp-engineer@nvidia.com + - hcuf-f-dgx-cloud-runai-non-prod-gcp-admin@nvidia.com + - hcuf-f-dgx-cloud-runai-non-prod-gcp-engineer@nvidia.com + - hcuf-f-dgx-cloud-runai-prod-gcp-admin@nvidia.com + - hcuf-f-dgx-cloud-runai-prod-gcp-engineer@nvidia.com + - system:masters + - system:monitoring + - system:nodes + - system:serviceaccounts + - system:unauthenticated + diff --git a/charts/nemo/charts/dgxc-admission-controller/values-oci-custom.yaml b/charts/nemo/charts/dgxc-admission-controller/values-oci-custom.yaml new file mode 100644 index 0000000..480ccca --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/values-oci-custom.yaml @@ -0,0 +1,61 @@ +--- +kubernetesAffinity: + enabled: false + +systemTolerationKeyName: CriticalAddonsOnly +systemTolerationValueName: "true" + +tolerations: + customer: + key: "dedicated" + value: "user-workload" + system: + key: "CriticalAddonsOnly" + value: "true" + +cloudProviderSpec: + type: oci + oci: + compartmentID: ocid1.compartment.oc1..aaaaaaaadt4vviaydftj2dknihhot2dhpmb4cvjn6lr57idmp4q2tbnhj6kq + clusterID: ocid1.cluster.oc1.iad.aaaaaaaalrxcu6og5prnj4hm2kcp7qvkyu26spioeic34i5ytc5bnhgiz67q + fssAvailabilityDomain: VSwX:US-ASHBURN-AD-1 + publicSubnetFirewallID: ocid1.networkfirewall.oc1.iad.amaaaaaaaqwjupqaedtz64tuorazbidw36a647cyluuqs2gf4kwev4ayf3yq + privateSubnetFirewallID: ocid1.networkfirewall.oc1.iad.amaaaaaaaqwjupqadoaciovfm5lpledoh4vbnx3iuytj2gwhzu74hlpfcfha + baseFirewallPolicyID: ocid1.fpid.oc1.iad.aaaaaaaalrxcu6og5prnj4hm2kcp7qvkyu26spioeic34i5ytc5bnhgiz67q + resourcePrincipalVersion: "2.2" + region: us-ashburn-1 + +storageControllerSpecOCI: + enabled: true + defaultStorageTypes: + - apiVersion: runai.dgxc.nvidia.com/v1beta1 + kind: RunaiDGXCStorage + metadata: + name: fss + namespace: dgxc-tenant-cluster-policies + spec: + quotaID: unlimited + status: + altNames: [ "fss" ] + storageClasses: + - fss + storageClassesDisplayString: "fss" + +webhooks: + registeredWebhooks: + - name: dgxc-create-pvc-vwh + enabled: false + - name: dgxc-create-pvc-mwh + enabled: false + - name: dgxc-create-pod-mwh + enabled: true + + podMutatingWebhook: + customSettings: + - name: modify-pod-rule + addRdmaEnabled: true + rdmaResourcePerGpu: 2 + networkAttachmentDefinition: network-operator/sriov-net + rdmaResourceName: nvidia.com/mlnxnics + addNcclVars: true + csp: "oci" diff --git a/charts/nemo/charts/dgxc-admission-controller/values-oci.yaml b/charts/nemo/charts/dgxc-admission-controller/values-oci.yaml new file mode 100644 index 0000000..c0d5a0e --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/values-oci.yaml @@ -0,0 +1,1048 @@ + +#################################################################### +##### The portion below this marker is automatically generated ##### +##### DO NOT EDIT BY HAND ##### +#################################################################### + +clusterScopedKinds: + - apiGroup: "" + resources: + - persistentvolumeclaims/finalizers + verbs: + - create + - update + - delete + - get + systemResources: [] + - apiGroup: argoproj.io + resources: + - eventsources + - sensors + verbs: + - "*" + systemResources: [] + - apiGroup: extensions + resources: + - ingresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + systemResources: [] + - apiGroup: route.openshift.io + resources: + - "*" + verbs: + - "*" + systemResources: [] + - apiGroup: "" + resources: + - configmaps/status + verbs: + - "*" + systemResources: [] + - apiGroup: nvidia.com + kind: ClusterPolicy + resources: + - clusterpolicies + - clusterpolicies/status + verbs: + - "*" + systemResources: + - cluster-policy + - apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + resources: + - clusterroles + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + # - bind + # - escalate + systemResources: + - admin + - approve-node-client-csr + - approve-node-client-renewal-csr + - approve-node-server-renewal-csr + - argocd-application-controller + - argocd-notifications-controller + - argocd-server + - assets-sync + - assets-sync-cluster + - cert-manager-cainjector + - cert-manager-cluster-view + - cert-manager-controller-approve:cert-manager-io + - cert-manager-controller-certificates + - cert-manager-controller-certificatesigningrequests + - cert-manager-controller-challenges + - cert-manager-controller-clusterissuers + - cert-manager-controller-ingress-shim + - cert-manager-controller-issuers + - cert-manager-controller-orders + - cert-manager-edit + - cert-manager-view + - cert-manager-webhook:subjectaccessreviews + - cilium + - cilium-operator + - cluster-admin + - cluster-sync + - cluster-sync-project + - csi-oci-driver-registrar-role + - dgxc-admission-controller + - dgxc-customer-app-admin + - dgxc-customer-app-admin-base + - dgxc-customer-network-policy + - dgxc-exporter + - dgxc-logging-alloy + - dgxc-runai-controller + - edit + - gpu-operator + - gpu-operator-node-feature-discovery + - gpu-operator-node-feature-discovery-gc + - ingress-nginx + - knative-serving-activator-cluster + - knative-serving-addressable-resolver + - knative-serving-admin + - knative-serving-aggregated-addressable-resolver + - knative-serving-core + - knative-serving-namespaced-admin + - knative-serving-namespaced-edit + - knative-serving-namespaced-view + - knative-serving-podspecable-binding + - kratos-fluent-bit-read + - kube-dns-autoscaler + - kubeadm:get-nodes + - kubeflow-mpijobs-admin + - kubeflow-mpijobs-edit + - kubeflow-mpijobs-view + - kyverno-cleanup-jobs + - kyverno:admission-controller + - kyverno:admission-controller:additional + - kyverno:admission-controller:core + - kyverno:background-controller + - kyverno:background-controller:additional + - kyverno:background-controller:core + - kyverno:cleanup-controller + - kyverno:cleanup-controller:core + - kyverno:dgxc-integrations + - kyverno:rbac:admin:policies + - kyverno:rbac:admin:policyreports + - kyverno:rbac:admin:reports + - kyverno:rbac:admin:updaterequests + - kyverno:rbac:view:policies + - kyverno:rbac:view:policyreports + - kyverno:rbac:view:reports + - kyverno:rbac:view:updaterequests + - kyverno:reports-controller + - kyverno:reports-controller:core + - kyverno:watch-update-namespaces + - kyverno:watch-update-services + - metrics-exporter + - mpi-operator + - multus + - net-kourier + - network-operator + - node-problem-detector-cpu + - node-problem-detector-gpu + - node-scale-adjuster-ro + - node-termination-handler + - nodepool-controller + - nv-ipam-controller + - nv-ipam-node + - nvidia-device-plugin + - nvidia-device-plugin-mps-control-daemon + - nvidia-driver + - nvidia-gpu-feature-discovery + - nvidia-mig-manager + - nvidia-operator-validator + - nvsentinel + - oci-secrets-store-csi-driver-provider-cluster-role + - oke-nvidia-device-plugin + - pod-group-assigner + - pod-group-controller + - pod-group-controller-ro + - pod-grouper + - prometheus-kube-state-metrics + - prometheus-operator + - prometheus-prometheus + - queue-controller + - researcher-service-ro + - researcher-service-rw + - runai-accessrule-controller + - runai-accessrule-controller-project + - runai-admission-controller-project + - runai-admission-controller-ro + - runai-agent + - runai-binder-cluster-wide + - runai-binder-project + - runai-cluster-api + - runai-cluster-api-project + - runai-compute-resource-administrator + - runai-compute-resource-administrator-ns + - runai-compute-resource-administrator-runai + - runai-credentials-administrator + - runai-credentials-administrator-ns + - runai-credentials-administrator-runai + - runai-data-source-administrator + - runai-data-source-administrator-ns + - runai-data-source-administrator-runai + - runai-data-source-administrator-runai-cs + - runai-data-volume-administrator + - runai-data-volume-administrator-ns + - runai-data-volume-administrator-runai + - runai-department-administrator + - runai-department-administrator-ns + - runai-department-administrator-runai + - runai-department-viewer + - runai-department-viewer-ns + - runai-department-viewer-runai + - runai-device-plugin + - runai-editor + - runai-editor-ns + - runai-editor-runai + - runai-engine-operator + - runai-environment-administrator + - runai-environment-administrator-ns + - runai-environment-administrator-runai + - runai-external-workload-integrator + - runai-external-workload-integrator-project + - runai-inference-workload-controller + - runai-inference-workload-controller-project + - runai-init-ca + - runai-job-controller + - runai-job-controller-project + - runai-l1-researcher + - runai-l1-researcher-ns + - runai-l1-researcher-runai + - runai-l2-researcher + - runai-l2-researcher-ns + - runai-l2-researcher-runai + - runai-mig-parted + - runai-ml-engineer + - runai-ml-engineer-ns + - runai-ml-engineer-runai + - runai-operator + - runai-project-controller + - runai-project-controller-cluster-pvc + - runai-project-controller-cluster-pvc-per-project + - runai-project-controller-cluster-secret + - runai-project-controller-cluster-secret-per-project + - runai-project-controller-limit-range + - runai-project-controller-limit-range-per-project + - runai-project-controller-ns-creation + - runai-project-controller-resources-patcher + - runai-project-controller-rolebinding-creation + - runai-project-controller-servicemonitor-creation + - runai-prometheus + - runai-research-manager + - runai-research-manager-ns + - runai-research-manager-runai + - runai-resource-reservation-rw + - runai-scheduler-ro + - runai-scheduler-rw + - runai-shared-objects-controller + - runai-shared-objects-controller-project + - runai-system-administrator + - runai-system-administrator-ns + - runai-system-administrator-runai + - runai-system-administrator-runai-cs + - runai-template-administrator + - runai-template-administrator-ns + - runai-template-administrator-runai + - runai-viewer + - runai-viewer-ns + - runai-viewer-runai + - runai-workload-exporter + - secretproviderclasses-admin-role + - secretproviderclasses-role + - secretproviderclasses-viewer-role + - secretproviderclasspodstatuses-viewer-role + - skyhook-operator-manager-role + - skyhook-operator-metrics-reader + - skyhook-operator-proxy-role + - sre-cluster-scoped-access + - status-updater + - status-updater-ro + - system:aggregate-to-admin + - system:aggregate-to-edit + - system:aggregate-to-view + - system:auth-delegator + - system:basic-user + - system:certificates.k8s.io:certificatesigningrequests:nodeclient + - system:certificates.k8s.io:certificatesigningrequests:selfnodeclient + - system:certificates.k8s.io:kube-apiserver-client-approver + - system:certificates.k8s.io:kube-apiserver-client-kubelet-approver + - system:certificates.k8s.io:kubelet-serving-approver + - system:certificates.k8s.io:legacy-unknown-approver + - system:controller:attachdetach-controller + - system:controller:certificate-controller + - system:controller:clusterrole-aggregation-controller + - system:controller:cronjob-controller + - system:controller:daemon-set-controller + - system:controller:deployment-controller + - system:controller:disruption-controller + - system:controller:endpoint-controller + - system:controller:endpointslice-controller + - system:controller:endpointslicemirroring-controller + - system:controller:ephemeral-volume-controller + - system:controller:expand-controller + - system:controller:generic-garbage-collector + - system:controller:horizontal-pod-autoscaler + - system:controller:job-controller + - system:controller:legacy-service-account-token-cleaner + - system:controller:namespace-controller + - system:controller:node-controller + - system:controller:persistent-volume-binder + - system:controller:pod-garbage-collector + - system:controller:pv-protection-controller + - system:controller:pvc-protection-controller + - system:controller:replicaset-controller + - system:controller:replication-controller + - system:controller:resourcequota-controller + - system:controller:root-ca-cert-publisher + - system:controller:route-controller + - system:controller:service-account-controller + - system:controller:service-controller + - system:controller:statefulset-controller + - system:controller:ttl-after-finished-controller + - system:controller:ttl-controller + - system:controller:validatingadmissionpolicy-status-controller + - system:coredns + - system:discovery + - system:heapster + - system:kube-aggregator + - system:kube-controller-manager + - system:kube-dns + - system:kube-scheduler + - system:kubelet-api-admin + - system:monitoring + - system:node + - system:node-bootstrapper + - system:node-problem-detector + - system:node-proxier + - system:persistent-volume-provisioner + - system:proxymux + - system:public-info-viewer + - system:service-account-issuer-discovery + - system:volume-scheduler + - training-operator + - vace-role + - view + - workload-controller-ro + - workload-controller-rw + - workload-overseer + - apiGroup: rbac.authorization.k8s.io + kind: ClusterRoleBinding + resources: + - clusterrolebindings + verbs: + - "*" + systemResources: + - argocd-application-controller + - argocd-notifications-controller + - argocd-server + - assets-sync + - cert-manager-cainjector + - cert-manager-controller-approve:cert-manager-io + - cert-manager-controller-certificates + - cert-manager-controller-certificatesigningrequests + - cert-manager-controller-challenges + - cert-manager-controller-clusterissuers + - cert-manager-controller-ingress-shim + - cert-manager-controller-issuers + - cert-manager-controller-orders + - cert-manager-webhook:subjectaccessreviews + - cilium + - cilium-operator + - cluster-admin + - cluster-sync + - csi-oci-driver-registrar-binding + - dgxc-admission-controller + - dgxc-customer-app-admin + # - dgxc-customer-app-admin-access + - dgxc-exporter + - dgxc-logging-alloy + - dgxc-runai-controller + - gpu-operator + - gpu-operator-node-feature-discovery + - gpu-operator-node-feature-discovery-gc + - ingress-nginx + - knative-serving-activator-cluster + - knative-serving-controller-addressable-resolver + - knative-serving-controller-admin + - kratos-fluent-bit-read + - kube-dns-autoscaler + - kubeadm:get-nodes + - kubeadm:kubelet-bootstrap + - kubeadm:node-autoapprove-bootstrap + - kubeadm:node-autoapprove-certificate-rotation + - kubeadm:node-proxier + - kubelet-bootstrap + - kyverno-cleanup-jobs + - kyverno:admission-controller + - kyverno:background-controller + - kyverno:cleanup-controller + - kyverno:reports-controller + - metrics-exporter + - mpi-operator + - multus + - net-kourier + - network-operator + - node-bootstrapper-auto-approve-csrs + - node-bootstrapper-client-cert-renewal + - node-problem-detector-cpu + - node-problem-detector-gpu + - node-scale-adjuster-ro + - node-termination-handler + - nodepool-controller + - nv-ipam-controller + - nv-ipam-node + - nvidia-device-plugin + - nvidia-device-plugin-mps-control-daemon + - nvidia-driver + - nvidia-gpu-feature-discovery + - nvidia-mig-manager + - nvidia-operator-validator + - nvsentinel + - nvsentinel-shoreline + - oci-secrets-store-csi-driver-provider-cluster-rolebinding + - oke-nvidia-device-plugin + - pod-group-assigner + - pod-group-controller-ro + - pod-grouper + - prometheus-kube-state-metrics + - prometheus-operator + - prometheus-prometheus + - queue-controller + - researcher-service-ro + - runai-accessrule-controller + - runai-admission-controller-ro + - runai-agent + - runai-binder-cluster-wide + - runai-cluster-api + - runai-compute-resource-administrator + - runai-credentials-administrator + - runai-data-source-administrator + - runai-data-volume-administrator + - runai-department-administrator + - runai-department-viewer + - runai-device-plugin + - runai-editor + - runai-engine-operator + - runai-environment-administrator + - runai-external-workload-integrator + - runai-inference-workload-controller + - runai-init-ca + - runai-job-controller + - runai-l1-researcher + - runai-l2-researcher + - runai-mig-parted + - runai-ml-engineer + - runai-operator + - runai-project-controller + - runai-project-controller-cluster-pvc + - runai-project-controller-cluster-secret + - runai-project-controller-limit-range + - runai-project-controller-ns-creation + - runai-project-controller-rolebinding-creation + - runai-project-controller-servicemonitor-creation + - runai-prometheus + - runai-research-manager + - runai-scheduler-ro + - runai-shared-objects-controller + - runai-system-administrator + - runai-template-administrator + - runai-viewer + - runai-workload-exporter + - secretproviderclasses-rolebinding + - skyhook-operator-manager-rolebinding + - skyhook-operator-proxy-rolebinding + - sre-cluster-scoped-access + - sre-full-access-binding + - status-updater-ro + - system:basic-user + - system:controller:attachdetach-controller + - system:controller:certificate-controller + - system:controller:clusterrole-aggregation-controller + - system:controller:cronjob-controller + - system:controller:daemon-set-controller + - system:controller:deployment-controller + - system:controller:disruption-controller + - system:controller:endpoint-controller + - system:controller:endpointslice-controller + - system:controller:endpointslicemirroring-controller + - system:controller:ephemeral-volume-controller + - system:controller:expand-controller + - system:controller:generic-garbage-collector + - system:controller:horizontal-pod-autoscaler + - system:controller:job-controller + - system:controller:legacy-service-account-token-cleaner + - system:controller:namespace-controller + - system:controller:node-controller + - system:controller:persistent-volume-binder + - system:controller:pod-garbage-collector + - system:controller:pv-protection-controller + - system:controller:pvc-protection-controller + - system:controller:replicaset-controller + - system:controller:replication-controller + - system:controller:resourcequota-controller + - system:controller:root-ca-cert-publisher + - system:controller:route-controller + - system:controller:service-account-controller + - system:controller:service-controller + - system:controller:statefulset-controller + - system:controller:ttl-after-finished-controller + - system:controller:ttl-controller + - system:controller:validatingadmissionpolicy-status-controller + - system:coredns + - system:discovery + - system:kube-controller-manager + - system:kube-dns + - system:kube-nodes + - system:kube-scheduler + - system:monitoring + - system:node + - system:node-proxier + - system:proxymux + - system:public-info-viewer + - system:service-account-issuer-discovery + - system:volume-scheduler + - training-operator + - vace-rolebinding + - workload-controller-ro + - workload-overseer + - apiGroup: "" + kind: ComponentStatus + resources: + - componentstatuses + verbs: + - "*" + systemResources: + - scheduler + - controller-manager + - etcd-0 + - apiGroup: apiextensions.k8s.io + kind: CustomResourceDefinition + resources: + - customresourcedefinitions + - customresourcedefinitions/status + verbs: + - "*" + systemResources: + - accessrules.run.ai + - admissionreports.kyverno.io + - alertmanagerconfigs.monitoring.coreos.com + - alertmanagers.monitoring.coreos.com + - applications.argoproj.io + - applicationsets.argoproj.io + - appprojects.argoproj.io + - backgroundscanreports.kyverno.io + - bindrequests.scheduling.run.ai + - certificaterequests.cert-manager.io + - certificates.cert-manager.io + - certificates.networking.internal.knative.dev + - challenges.acme.cert-manager.io + - cidrpools.nv-ipam.nvidia.com + - ciliumcidrgroups.cilium.io + - ciliumclusterwidenetworkpolicies.cilium.io + - ciliumendpoints.cilium.io + - ciliumexternalworkloads.cilium.io + - ciliumidentities.cilium.io + - ciliuml2announcementpolicies.cilium.io + - ciliumloadbalancerippools.cilium.io + - ciliumnetworkpolicies.cilium.io + - ciliumnodeconfigs.cilium.io + - ciliumnodes.cilium.io + - ciliumpodippools.cilium.io + - cleanuppolicies.kyverno.io + - clusteradmissionreports.kyverno.io + - clusterbackgroundscanreports.kyverno.io + - clustercleanuppolicies.kyverno.io + - clusterdomainclaims.networking.internal.knative.dev + - clusterissuers.cert-manager.io + - clusterpolicies.kyverno.io + - clusterpolicies.nvidia.com + - clusterpolicyreports.wgpolicyk8s.io + - configs.engine.run.ai + - configurations.serving.knative.dev + - datavolumes.run.ai + - departments.scheduling.run.ai + - distributedpolicies.run.ai + - distributedworkloads.run.ai + - domainmappings.serving.knative.dev + - externalworkloads.run.ai + - hostdevicenetworks.mellanox.com + - images.caching.internal.knative.dev + - inferencepolicies.run.ai + - inferenceworkloads.run.ai + - ingresses.networking.internal.knative.dev + - interactivepolicies.run.ai + - interactiveworkloads.run.ai + - ipoibnetworks.mellanox.com + - ippools.nv-ipam.nvidia.com + - issuers.cert-manager.io + - macvlannetworks.mellanox.com + - metrics.autoscaling.internal.knative.dev + - mpijobs.kubeflow.org + - mxjobs.kubeflow.org + - network-attachment-definitions.k8s.cni.cncf.io + - nicclusterpolicies.mellanox.com + - nodefeaturegroups.nfd.k8s-sigs.io + - nodefeaturerules.nfd.k8s-sigs.io + - nodefeatures.nfd.k8s-sigs.io + - nodepools.run.ai + - nvidiadrivers.nvidia.com + - orders.acme.cert-manager.io + - paddlejobs.kubeflow.org + - podautoscalers.autoscaling.internal.knative.dev + - podgroups.scheduling.run.ai + - podlogs.monitoring.grafana.com + - podmonitors.monitoring.coreos.com + - policies.kyverno.io + - policyexceptions.kyverno.io + - policyreports.wgpolicyk8s.io + - probes.monitoring.coreos.com + - projects.run.ai + - prometheusagents.monitoring.coreos.com + - prometheuses.monitoring.coreos.com + - prometheusrules.monitoring.coreos.com + - pytorchjobs.kubeflow.org + - queues.scheduling.run.ai + - revisions.serving.knative.dev + - routes.serving.knative.dev + - runaiconfigs.run.ai + - runaidgxclogexporters.runai.dgxc.nvidia.com + - runaidgxcnetworkpolicies.runai.dgxc.nvidia.com + - runaidgxcstorages.runai.dgxc.nvidia.com + - runaijobs.run.ai + - schedulingshards.engine.run.ai + - scrapeconfigs.monitoring.coreos.com + - secretproviderclasses.secrets-store.csi.x-k8s.io + - secretproviderclasspodstatuses.secrets-store.csi.x-k8s.io + - serverlessservices.networking.internal.knative.dev + - servicemonitors.monitoring.coreos.com + - services.serving.knative.dev + - skyhooks.skyhook.nvidia.com + - tfjobs.kubeflow.org + - thanosrulers.monitoring.coreos.com + - trainingpolicies.run.ai + - trainingworkloads.run.ai + - updaterequests.kyverno.io + - xgboostjobs.kubeflow.org + - apiGroup: mellanox.com + kind: HostDeviceNetwork + resources: + - hostdevicenetworks + - hostdevicenetworks/status + verbs: + - "*" + systemResources: [] + - apiGroup: mellanox.com + kind: IPoIBNetwork + resources: + - ipoibnetworks + - ipoibnetworks/status + verbs: + - "*" + systemResources: [] + - apiGroup: networking.k8s.io + kind: IngressClass + resources: + - ingressclasses + verbs: + # - create + # - delete + # - deletecollection + - get + - list + # - patch + # - update + - watch + systemResources: + - nginx + - apiGroup: mellanox.com + kind: MacvlanNetwork + resources: + - macvlannetworks + - macvlannetworks/status + verbs: + - "*" + systemResources: [] + - apiGroup: "" + kind: Namespace + resources: + - namespaces + - namespaces/finalize + - namespaces/status + verbs: + - "*" + systemResources: + - argocd + - cert-manager + - default + - dgxc-admission-controller + - dgxc-logging + - dgxc-tenant-cluster-policies + - gpu-operator + - ingress-nginx + - knative-serving + - kourier-system + - kube-node-lease + - kube-public + - kube-system + - kubeflow + - kyverno + - monitoring + - mpi-operator + - network-operator + - node-problem-detector + - nvdiag + - nvsentinel + - observability + - prometheus + - runai + - skyhook + - apiGroup: mellanox.com + kind: NicClusterPolicy + resources: + - nicclusterpolicies + - nicclusterpolicies/status + verbs: + - "*" + systemResources: + - nic-cluster-policy + - apiGroup: "" + kind: PersistentVolume + resources: + - persistentvolumes + - persistentvolumes/status + verbs: + # - create + # - delete + # - deletecollection + - get + - list + # - patch + # - update + - watch + systemResources: + - csi-021ed734-f8f3-4987-b429-e779055c1d8b + - csi-7f4d2119-f378-4519-b79b-631a37063d52 + - apiGroup: scheduling.k8s.io + kind: PriorityClass + resources: + - priorityclasses + verbs: + - "*" + systemResources: + - build + - inference + - interactive-preemptible + - runai-critical + - runai-engine-critical + - system-cluster-critical + - system-node-critical + - train + - train-critical + - train-high + - apiGroup: authorization.k8s.io + kind: SelfSubjectAccessReview + resources: + - selfsubjectaccessreviews + verbs: + - "*" + systemResources: [] + - apiGroup: authentication.k8s.io + kind: SelfSubjectReview + resources: + - selfsubjectreviews + verbs: + - "*" + systemResources: [] + - apiGroup: authorization.k8s.io + kind: SelfSubjectRulesReview + resources: + - selfsubjectrulesreviews + verbs: + - "*" + systemResources: [] + - apiGroup: authorization.k8s.io + kind: SubjectAccessReview + resources: + - subjectaccessreviews + verbs: + - "*" + systemResources: [] + - apiGroup: authentication.k8s.io + kind: TokenReview + resources: + - tokenreviews + verbs: + - "*" + systemResources: [] + +namespacedKinds: + - apiGroup: policy + resources: + - pods/eviction + verbs: + - create + - apiGroup: "" + kind: Binding + resources: + - bindings + verbs: + - "*" + - apiGroup: "" + kind: ConfigMap + resources: + - configmaps + verbs: + - "*" + - apiGroup: apps + kind: ControllerRevision + resources: + - controllerrevisions + verbs: + - "*" + - apiGroup: batch + kind: CronJob + resources: + - cronjobs + - cronjobs/status + verbs: + - "*" + - apiGroup: apps + kind: DaemonSet + resources: + - daemonsets + - daemonsets/status + verbs: + - "*" + - apiGroup: apps + kind: Deployment + resources: + - deployments + - deployments/scale + - deployments/status + verbs: + - "*" + - apiGroup: discovery.k8s.io + kind: EndpointSlice + resources: + - endpointslices + verbs: + - "*" + - apiGroup: "" + kind: Endpoints + resources: + - endpoints + verbs: + - "*" + - apiGroup: "" + kind: Event + resources: + - events + verbs: + - "*" + - apiGroup: events.k8s.io + kind: Event + resources: + - events + verbs: + - "*" + - apiGroup: autoscaling + kind: HorizontalPodAutoscaler + resources: + - horizontalpodautoscalers + - horizontalpodautoscalers/status + verbs: + - "*" + - apiGroup: networking.k8s.io + kind: Ingress + resources: + - ingresses + - ingresses/status + verbs: + - "*" + - apiGroup: batch + kind: Job + resources: + - jobs + - jobs/status + verbs: + - "*" + - apiGroup: coordination.k8s.io + kind: Lease + resources: + - leases + verbs: + - "*" + - apiGroup: "" + kind: LimitRange + resources: + - limitranges + verbs: + - "*" + - apiGroup: authorization.k8s.io + kind: LocalSubjectAccessReview + resources: + - localsubjectaccessreviews + verbs: + - "*" + - apiGroup: networking.k8s.io + kind: NetworkPolicy + resources: + - networkpolicies + verbs: + - "*" + - apiGroup: "" + kind: PersistentVolumeClaim + resources: + - persistentvolumeclaims + - persistentvolumeclaims/status + verbs: + - "*" + - apiGroup: "" + kind: Pod + resources: + - pods + - pods/attach + - pods/binding + - pods/ephemeralcontainers + - pods/eviction + - pods/exec + - pods/log + - pods/portforward + - pods/proxy + - pods/status + verbs: + - "*" + - apiGroup: policy + kind: PodDisruptionBudget + resources: + - poddisruptionbudgets + - poddisruptionbudgets/status + verbs: + - "*" + - apiGroup: "" + kind: PodTemplate + resources: + - podtemplates + verbs: + - "*" + - apiGroup: apps + kind: ReplicaSet + resources: + - replicasets + - replicasets/scale + - replicasets/status + verbs: + - "*" + - apiGroup: "" + kind: ReplicationController + resources: + - replicationcontrollers + - replicationcontrollers/scale + - replicationcontrollers/status + verbs: + - "*" + - apiGroup: "" + kind: ResourceQuota + resources: + - resourcequotas + - resourcequotas/status + verbs: + - "*" + - apiGroup: rbac.authorization.k8s.io + kind: Role + resources: + - roles + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + # - bind + # - escalate + - apiGroup: rbac.authorization.k8s.io + kind: RoleBinding + resources: + - rolebindings + verbs: + - "*" + - apiGroup: "" + kind: Secret + resources: + - secrets + verbs: + - "*" + - apiGroup: "" + kind: Service + resources: + - services + - services/proxy + - services/status + verbs: + - "*" + - apiGroup: "" + kind: ServiceAccount + resources: + - serviceaccounts + - serviceaccounts/token + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + # - impersonate + - apiGroup: apps + kind: StatefulSet + resources: + - statefulsets + - statefulsets/scale + - statefulsets/status + verbs: + - "*" + +systemSubjects: + User: + - kubelet-bootstrap + - kyverno@test.com + - ladithyav@nvidia.com + - pwittrock@nvidia.com + - system:kube-controller-manager + - system:kube-proxy + - system:kube-scheduler + Group: + - ocid1.group.oc1..aaaaaaaaxe752um3jcucgpytvu3msgueh5bdnrcsmalp6ztj5mjt26lrxegq + - ocid1.group.oc1..aaaaaaaajapqxojgiat5c3sabi2utodp3nffgq7bue6qm7c2bosxomdvscuq + - ocid1.group.oc1..aaaaaaaawythahrsnnmblptsvfaeasv5pf65rdjhdbimkwu2mwquh2arly7a + - ocid1.group.oc1..aaaaaaaaxn2dnkms7yddi257jwepzsv6weypbg4w2eksrajbvvoyh7lk2hza + - system:bootstrappers:kubeadm:default-node-token + - system:masters + - system:monitoring + - system:node-bootstrapper + - system:nodes + - system:serviceaccounts + - system:serviceaccounts:network-operator + - system:unauthenticated + diff --git a/charts/nemo/charts/dgxc-admission-controller/values.yaml b/charts/nemo/charts/dgxc-admission-controller/values.yaml new file mode 100644 index 0000000..a6d3229 --- /dev/null +++ b/charts/nemo/charts/dgxc-admission-controller/values.yaml @@ -0,0 +1,282 @@ +--- +imagePullSecret: "nvidia-ngcuser-pull-secret" + +enabledComponents: + dgxcAdmissionController: true + dgxcRunaiController: true + dgxcExporter: true + +# The storage controller is the thing we're running +dgxcController: + name: dgxc-admission-controller + replicas: 3 + image: "nvcr.io/nv-ngc-devops/dgxc-admission-controller:latest" + # shared between the init container and the controller container + certs: + mountPath: "/tmp/k8s-webhook-server/serving-certs" + certName: tls.crt + keyName: tls.key + +nvStorage: + # NVStorageLocation CRD management + enabled: false + storageClass: "" + csi: {} + +serviceName: dgx-controller-svc +serviceAccount: + create: true + name: dgxc-admission-controller-sa + annotations: {} + +controllerServiceAccount: + create: true + name: dgxc-runai-controller + annotations: {} + +webhooks: + organization: dgxc.nvidia.com + registeredWebhooks: + - name: dgxc-create-pvc-vwh + type: PvcValidatingWebhook + cmName: dgxc-create-pvc-vwh-config + enabled: true + - name: dgxc-create-pvc-mwh + type: PvcMutatingWebhook + cmName: dgxc-create-pvc-mwh-config + enabled: true + - name: dgxc-create-pod-mwh + type: PodMutatingWebhook + cmName: dgxc-create-pod-mwh-config + enabled: true + + pvcValidatingWebhook: + cmName: dgxc-create-pvc-vwh-config + type: PvcValidatingWebhook + kind: ValidatingWebhookConfiguration + fqdn: dgxc-create-pvc-vwh.dgxc.nvidia.com + path: "/pvccreate/v1/validation" + failurePolicy: Fail + timeoutSeconds: 10 + sideEffects: NoneOnDryRun + admissionReviewVersions: + - v1 + rules: + - apiGroups: + - "" + apiVersions: + - "v1" + operations: + - CREATE + resources: + - persistentvolumeclaims + scope: "*" + namespaceLabelSelector: + - key: runai/namespace-version + values: + - "v1" + - "v2" + objectLabelSelector: + customSettings: + - name: filestore-pvc-premium-rule + storageClasses: + - zonal-rwx + minSizeCheckEnabled: false + minSizeValue: 10Ti + maxSizeCheckEnabled: false + maxSizeValue: 100Ti + - name: filestore-pvc-standard-rule + storageClasses: + - standard-rwx + minSizeCheckEnabled: false + minSizeValue: 1Ti + maxSizeCheckEnabled: false + maxSizeValue: 10Ti + pvcMutatingWebhook: + cmName: dgxc-create-pvc-mwh-config + type: PvcMutatingWebhook + kind: MutatingWebhookConfiguration + fqdn: dgxc-create-pvc-mwh.dgxc.nvidia.com + path: "/pvccreate/v1/mutation" + failurePolicy: Fail + timeoutSeconds: 10 + sideEffects: NoneOnDryRun + admissionReviewVersions: + - v1 + rules: + - apiGroups: + - "" + apiVersions: + - "v1" + operations: + - CREATE + resources: + - persistentvolumeclaims + scope: "*" + namespaceLabelSelector: + - key: runai/namespace-version + values: + - "v1" + - "v2" + objectLabelSelector: + + podMutatingWebhook: + cmName: dgxc-create-pod-mwh-config + kind: MutatingWebhookConfiguration + type: PodMutatingWebhook + fqdn: dgxc-create-pod-mwh.dgxc.mvp.nvidia.com + path: "/podcreate/v1/mutation" + failurePolicy: Fail + timeoutSeconds: 10 + sideEffects: NoneOnDryRun + admissionReviewVersions: + - v1 + rules: + - apiGroups: + - "" + apiVersions: + - "v1" + operations: + - CREATE + resources: + - pods + scope: "*" + namespaceLabelSelector: + objectLabelSelector: + customSettings: + - name: modify-pod-rule + scratchEnabled: true + scratchVolName: "scratch" + scratchMountpath: "/scratch" + scratchVolSize: "200Gi" + scratchVolSizePerGpu: true + scratchVolEnforceLimit: true + addTcpxSidecarEnabled: true + addTcpxoSidecarEnabled: false + clusterName: "test" + addTcpxNCCLVars: true + addTcpxoNCCLVars: false + addRdmaEnabled: false + addRoceOciNcclVars: false + addGcpFuseAnnotation: true + gcpFuseAnnotationKey: "gke-gcsfuse/volumes" + gcpFuseWICredentialConfigMapAnnotationKey: "gke-gcsfuse/workload-identity-credential-configmap" # This is the annotation key for the ConfigMap that contains the GCP Workload Identity credentials. + gcpFuseStorageClasses: + - dgxc-standard-object +prober: + intervalInSeconds: 60 + timeoutInSeconds: 5 + endpoints: + - name: dev-cluster + url: https://dgxc-runai-us-east5-ngc-11918940.dgxc.nonprod-nvkong.com + +sso: + controlPlaneDomain: run.ai + cluster: "TODO" + realm: dgxc + pollIntervalSeconds: 10 + pollAttempts: 180 # 30 minutes + +initSettings: + enabled: false + appID: dgxc + tenant: dgxc + controlPlane: run.ai + supportEmailAddress: support@nvidia.com + customerEmailAddress: customer@nvidia.com + bootstrapEmailAddress: bootstrap@nvidia.com + +crdRBACName: "{{request.object.spec.names.plural}}" + +validationFailureAction: Enforce + +tenantSubjects: + User: + - ebalduf@test.nvidia.com + - kelhussein@test.nvidia.com + - ladithyav@test.nvidia.com + - pwittrock@test.nvidia.com + +tolerations: + customer: + key: "dedicated" + value: "user-workload" + system: + key: "components.gke.io/gke-managed-components" + value: "true" + +bootstrapIngressCIDR: [ "216.228.125.128/30", "12.97.180.32/27", "216.228.127.128/30", "202.164.25.0/27", "120.31.162.32/28", "203.18.50.0/27", "103.74.125.160/28", "216.228.112.0/26" ] + +dgxcAdmins: + - Run:ai + - svctdcsqa1@nvidia.com + +reserved: + CPU: 2 + Memory: 5Gi + +maxCPUJobs: 10 + +logExporterControllerSpec: + enabled: false + exportFrequency: 5m + exportBuffer: 15m + fetchTimeout: 5m + +networkControllerSpec: + enabled: false + +storageControllerSpecGCP: + enabled: false + +storageControllerSpecAWS: + enabled: false + +storageControllerSpecOCI: + enabled: false + +storageControllerSpecAzure: + enabled: false + +exporter: + metricsPort: 8080 + +sre: + group: + default: "non-existing-engineer@nvidia.com" + privileged: "non-existing-admin@nvidia.com" + +tenantMetadata: + cluster: TODO + platform: runai + orgName: nvidian + tenantName: test-mega + tenantType: Internal + startDate: '8/02/24' + endDate: '01/01/99' + CSP: gcp + allocatedCapacityGPU: 2 + allocatedCapacityCPU: 2 + committedCapacityGPU: 2 + committedCapacityCPU: 2 + CSPProject: proj-dgxc-runai-np-test-mega + dataCenterRegion: us-central1 + storageType: GCPFilestore + storageQuota: 1 + +cspSubjects: + User: + - "107878228028626601324" + +network: + systemCIDRs: + - "10.0.3.0/24" + - "10.0.4.0/24" + +enableIntegrationsSupport: true +enableLabeler: true +enableRunAIRbac: true +enableNetworkPolicies: true +enableDGXCRbac: true +enableDGXCAPIBootstrap: true +enableServiceMonitor: true diff --git a/charts/nemo/charts/k8s-nim-operator/.helmignore b/charts/nemo/charts/k8s-nim-operator/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/nemo/charts/k8s-nim-operator/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/nemo/charts/k8s-nim-operator/Chart.yaml b/charts/nemo/charts/k8s-nim-operator/Chart.yaml new file mode 100644 index 0000000..ba8b4e5 --- /dev/null +++ b/charts/nemo/charts/k8s-nim-operator/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +appVersion: 1.0.1 +description: NVIDIA NIM Operator creates/configures/manages NVIDIA NIM microservices + in Kubernetes +home: https://docs.nvidia.com/nim-operator/latest/index.html +icon: https://assets.nvidiagrid.net/ngc/logos/NIMOperator.png +keywords: +- gpu +- nim +- nemo +- operator +- ai +name: k8s-nim-operator +sources: +- https://github.com/NVIDIA/k8s-nim-operator +type: application +version: 1.0.1 diff --git a/charts/nemo/charts/k8s-nim-operator/crds/apps.nvidia.com_nimcaches.yaml b/charts/nemo/charts/k8s-nim-operator/crds/apps.nvidia.com_nimcaches.yaml new file mode 100644 index 0000000..e1f5c15 --- /dev/null +++ b/charts/nemo/charts/k8s-nim-operator/crds/apps.nvidia.com_nimcaches.yaml @@ -0,0 +1,500 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.2 + name: nimcaches.apps.nvidia.com +spec: + group: apps.nvidia.com + names: + kind: NIMCache + listKind: NIMCacheList + plural: nimcaches + singular: nimcache + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .status.pvc + name: PVC + type: string + - format: date-time + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: NIMCache is the Schema for the nimcaches API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NIMCacheSpec defines the desired state of NIMCache + properties: + certConfig: + description: |- + CertConfig is the name of the ConfigMap containing the custom certificates. + for secure communication. + properties: + mountPath: + description: MountPath is the path where the certificates should + be mounted in the container. + type: string + name: + description: Name of the ConfigMap containing the certificate + data. + type: string + required: + - mountPath + - name + type: object + env: + description: Env are the additional custom environment variabes for + the caching job + items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot + be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + groupID: + description: GroupID is the group ID for the caching job + format: int64 + type: integer + nodeSelector: + additionalProperties: + type: string + description: NodeSelector is the node selector labels to schedule + the caching job. + type: object + resources: + description: Resources defines the minimum resources required for + the caching job to run(cpu, memory, gpu). + properties: + cpu: + anyOf: + - type: integer + - type: string + description: CPU indicates the minimum number of CPUs to use while + caching NIM + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + description: |- + Memory indicates the minimum amount of memory to use while caching NIM + Valid values are numbers followed by one of the suffixes Ki, Mi, Gi, or Ti (e.g. "4Gi", "4096Mi"). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + runtimeClassName: + description: RuntimeClassName is the runtimeclass for the caching + job + type: string + source: + description: Source is the NIM model source to cache + properties: + dataStore: + description: NGCSource represents models stored in NVIDIA DataStore + service + properties: + authSecret: + description: The name of an existing auth secret containing + the AUTH_TOKEN" + type: string + checkpointName: + type: string + datasetName: + type: string + endpoint: + description: The endpoint for datastore + type: string + modelName: + description: Name of either model/checkpoint or dataset to + download + type: string + modelPuller: + description: ModelPuller is the container image that can pull + the model + type: string + pullSecret: + description: PullSecret for the model puller image + type: string + required: + - authSecret + - endpoint + - modelPuller + type: object + ngc: + description: NGCSource represents models stored in NGC + properties: + authSecret: + description: The name of an existing pull secret containing + the NGC_API_KEY + type: string + model: + description: Model spec for caching + properties: + buildable: + description: Buildable indicates generic model profiles + that can be optimized with an NVIDIA engine for any + GPUs + type: boolean + engine: + description: Engine is the backend engine (tensort_llm, + vllm) + type: string + gpus: + description: GPU is the spec for matching GPUs for caching + optimized models + items: + description: GPUSpec is the spec required to cache models + for selected gpu type + properties: + ids: + description: IDs are the device-ids for a specific + GPU SKU + items: + type: string + type: array + product: + description: Product is the GPU product string (h100, + a100, l40s) + type: string + type: object + type: array + lora: + description: Lora indicates a finetuned model with LoRa + adapters + type: boolean + precision: + description: Precision is the precision for model quantization + type: string + profiles: + description: Profiles are the specific model profiles + to cache. When these are provided, rest of the model + parameters for profile selection are ignored + items: + type: string + type: array + qosProfile: + description: QoSProfile is the supported QoS profile types + for the models (throughput, latency) + type: string + tensorParallelism: + description: TensorParallelism is the minimum GPUs required + for the model computations + type: string + type: object + modelPuller: + description: ModelPuller is the container image that can pull + the model + type: string + x-kubernetes-validations: + - message: modelPuller is an immutable field. Please create + a new NIMCache resource instead when you want to change + this container. + rule: self == oldSelf + pullSecret: + description: PullSecret to pull the model puller image + type: string + required: + - authSecret + - modelPuller + type: object + type: object + storage: + description: Storage is the target storage for caching NIM model + properties: + hostPath: + description: HostPath is the host path volume for caching NIM + type: string + pvc: + description: PersistentVolumeClaim is the pvc volume used for + caching NIM + properties: + create: + description: Create indicates to create a new PVC + type: boolean + name: + description: Name is the name of the PVC + type: string + size: + description: Size of the NIM cache in Gi, used during PVC + creation + type: string + storageClass: + description: StorageClass to be used for PVC creation. Leave + it as empty if the PVC is already created. + type: string + subPath: + type: string + volumeAccessMode: + description: VolumeAccessMode is the volume access mode of + the PVC + type: string + type: object + type: object + tolerations: + description: Tolerations for running the job to cache the NIM model + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + userID: + description: UserID is the user ID for the caching job + format: int64 + type: integer + required: + - source + - storage + type: object + status: + description: NIMCacheStatus defines the observed state of NIMCache + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + profiles: + items: + description: NIMProfile defines the profiles that were cached + properties: + config: + additionalProperties: + type: string + type: object + model: + type: string + name: + type: string + release: + type: string + type: object + type: array + pvc: + type: string + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/nemo/charts/k8s-nim-operator/crds/apps.nvidia.com_nimpipelines.yaml b/charts/nemo/charts/k8s-nim-operator/crds/apps.nvidia.com_nimpipelines.yaml new file mode 100644 index 0000000..0422497 --- /dev/null +++ b/charts/nemo/charts/k8s-nim-operator/crds/apps.nvidia.com_nimpipelines.yaml @@ -0,0 +1,2309 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.2 + name: nimpipelines.apps.nvidia.com +spec: + group: apps.nvidia.com + names: + kind: NIMPipeline + listKind: NIMPipelineList + plural: nimpipelines + singular: nimpipeline + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - format: date-time + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: NIMPipeline is the Schema for the nimpipelines API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NIMPipelineSpec defines the desired state of NIMPipeline + properties: + services: + description: NIMService configures attributes to deploy a NIM service + as part of the pipeline + items: + description: NIMServicePipelineSpec defines the desired state of + NIMService as part of the NIMPipeline + properties: + dependencies: + items: + description: ServiceDependency defines service dependencies + properties: + envName: + description: EnvName is the dependent service endpoint + environment variable name + type: string + envValue: + description: EnvValue is the dependent service endpoint + environment variable value + type: string + name: + description: Name is the dependent service name + type: string + port: + description: Port is the dependent service port + format: int32 + type: integer + required: + - name + - port + type: object + type: array + enabled: + type: boolean + name: + type: string + spec: + description: NIMServiceSpec defines the desired state of NIMService + properties: + annotations: + additionalProperties: + type: string + type: object + args: + items: + type: string + type: array + authSecret: + description: The name of an existing pull secret containing + the NGC_API_KEY + type: string + command: + items: + type: string + type: array + env: + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the + pod's namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + expose: + description: Expose defines attributes to expose the service + properties: + ingress: + description: Ingress defines attributes to enable ingress + for the service + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + description: ingress, or virtualService - not both + type: boolean + spec: + description: IngressSpec describes the Ingress the + user wishes to exist. + properties: + defaultBackend: + description: |- + defaultBackend is the backend that should handle requests that don't + match any rule. If Rules are not specified, DefaultBackend must be specified. + If DefaultBackend is not set, the handling of requests that do not match any + of the rules will be up to the Ingress controller. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + ingressClassName: + description: |- + ingressClassName is the name of an IngressClass cluster resource. Ingress + controller implementations use this field to know whether they should be + serving this Ingress resource, by a transitive connection + (controller -> IngressClass -> Ingress resource). Although the + `kubernetes.io/ingress.class` annotation (simple constant name) was never + formally defined, it was widely supported by Ingress controllers to create + a direct binding between Ingress controller and Ingress resources. Newly + created Ingress resources should prefer using the field. However, even + though the annotation is officially deprecated, for backwards compatibility + reasons, ingress controllers should still honor that annotation if present. + type: string + rules: + description: |- + rules is a list of host rules used to configure the Ingress. If unspecified, + or no rule matches, all traffic is sent to the default backend. + items: + description: |- + IngressRule represents the rules mapping the paths under a specified host to + the related backend services. Incoming requests are first evaluated for a host + match, then routed to the backend associated with the matching IngressRuleValue. + properties: + host: + description: "host is the fully qualified + domain name of a network host, as defined + by RFC 3986.\nNote the following deviations + from the \"host\" part of the\nURI as + defined in RFC 3986:\n1. IPs are not + allowed. Currently an IngressRuleValue + can only apply to\n the IP in the + Spec of the parent Ingress.\n2. The + `:` delimiter is not respected because + ports are not allowed.\n\t Currently + the port of an Ingress is implicitly + :80 for http and\n\t :443 for https.\nBoth + these may change in the future.\nIncoming + requests are matched against the host + before the\nIngressRuleValue. If the + host is unspecified, the Ingress routes + all\ntraffic based on the specified + IngressRuleValue.\n\nhost can be \"precise\" + which is a domain name without the terminating + dot of\na network host (e.g. \"foo.bar.com\") + or \"wildcard\", which is a domain name\nprefixed + with a single wildcard label (e.g. \"*.foo.com\").\nThe + wildcard character '*' must appear by + itself as the first DNS label and\nmatches + only a single label. You cannot have + a wildcard label by itself (e.g. Host + == \"*\").\nRequests will be matched + against the Host field in the following + way:\n1. If host is precise, the request + matches this rule if the http host header + is equal to Host.\n2. If host is a wildcard, + then the request matches this rule if + the http host header\nis to equal to + the suffix (removing the first label) + of the wildcard rule." + type: string + http: + description: |- + HTTPIngressRuleValue is a list of http selectors pointing to backends. + In the example: http:///? -> backend where + where parts of the url correspond to RFC 3986, this resource will be used + to match against everything after the last '/' and before the first '?' + or '#'. + properties: + paths: + description: paths is a collection + of paths that map requests to backends. + items: + description: |- + HTTPIngressPath associates a path with a backend. Incoming urls matching the + path are forwarded to the backend. + properties: + backend: + description: |- + backend defines the referenced service endpoint to which the traffic + will be forwarded to. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is + the type of resource + being referenced + type: string + name: + description: Name is + the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + path: + description: |- + path is matched against the path of an incoming request. Currently it can + contain characters disallowed from the conventional "path" part of a URL + as defined by RFC 3986. Paths must begin with a '/' and must be present + when using PathType with value "Exact" or "Prefix". + type: string + pathType: + description: |- + pathType determines the interpretation of the path matching. PathType can + be one of the following values: + * Exact: Matches the URL path exactly. + * Prefix: Matches based on a URL path prefix split by '/'. Matching is + done on a path element by element basis. A path element refers is the + list of labels in the path split by the '/' separator. A request is a + match for path p if every p is an element-wise prefix of p of the + request path. Note that if the last element of the path is a substring + of the last element in request path, it is not a match (e.g. /foo/bar + matches /foo/bar/baz, but does not match /foo/barbaz). + * ImplementationSpecific: Interpretation of the Path matching is up to + the IngressClass. Implementations can treat this as a separate PathType + or treat it identically to Prefix or Exact path types. + Implementations are required to support all path types. + type: string + required: + - backend + - pathType + type: object + type: array + x-kubernetes-list-type: atomic + required: + - paths + type: object + type: object + type: array + x-kubernetes-list-type: atomic + tls: + description: |- + tls represents the TLS configuration. Currently the Ingress only supports a + single TLS port, 443. If multiple members of this list specify different hosts, + they will be multiplexed on the same port according to the hostname specified + through the SNI TLS extension, if the ingress controller fulfilling the + ingress supports SNI. + items: + description: IngressTLS describes the transport + layer security associated with an ingress. + properties: + hosts: + description: |- + hosts is a list of hosts included in the TLS certificate. The values in + this list must match the name/s used in the tlsSecret. Defaults to the + wildcard host setting for the loadbalancer controller fulfilling this + Ingress, if left unspecified. + items: + type: string + type: array + x-kubernetes-list-type: atomic + secretName: + description: |- + secretName is the name of the secret used to terminate TLS traffic on + port 443. Field is left optional to allow TLS routing based on SNI + hostname alone. If the SNI host in a listener conflicts with the "Host" + header field used by an IngressRule, the SNI host is used for termination + and value of the "Host" header is used for routing. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + service: + description: Service defines attributes to create a + service + properties: + annotations: + additionalProperties: + type: string + type: object + name: + description: override the default service name + type: string + port: + default: 8000 + format: int32 + type: integer + type: + description: Service Type string describes ingress + methods for a service + type: string + required: + - port + type: object + type: object + groupID: + format: int64 + type: integer + image: + description: Image defines image attributes + properties: + pullPolicy: + type: string + pullSecrets: + items: + type: string + type: array + repository: + type: string + tag: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + livenessProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + metrics: + description: Metrics defines attributes to setup metrics + collection + properties: + enabled: + type: boolean + serviceMonitor: + description: for use with the Prometheus Operator and + the primary service object + properties: + additionalLabels: + additionalProperties: + type: string + type: object + annotations: + additionalProperties: + type: string + type: object + interval: + description: |- + Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. + Supported units: y, w, d, h, m, s, ms + Examples: `30s`, `1m`, `1h20m15s`, `15d` + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + scrapeTimeout: + description: |- + Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. + Supported units: y, w, d, h, m, s, ms + Examples: `30s`, `1m`, `1h20m15s`, `15d` + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + podAffinity: + description: Pod affinity is a group of inter pod affinity + scheduling rules. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + readinessProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + replicas: + default: 1 + minimum: 1 + type: integer + resources: + description: ResourceRequirements describes the compute + resource requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in + PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + runtimeClassName: + type: string + scale: + description: Autoscaling defines attributes to automatically + scale the service based on metrics + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + type: boolean + hpa: + description: HorizontalPodAutoscalerSpec defines the + parameters required to setup HPA + properties: + behavior: + description: |- + HorizontalPodAutoscalerBehavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single + policy which must hold true for a specified + past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify + the scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single + policy which must hold true for a specified + past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify + the scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + type: integer + metrics: + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + This is an alpha feature and can be enabled by the HPAContainerMetrics feature flag. + properties: + container: + description: container is the name of + the container in the pods of the scaling + target + type: string + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target + value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether + the metric type is Utilization, + Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value + of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target + metric by name and selector + properties: + name: + description: name is the name of the + given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target + value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether + the metric type is Utilization, + Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value + of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies + the descriptions of a object,such as + kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API + version of the referent + type: string + kind: + description: 'kind is the kind of + the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of + the referent; More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target + metric by name and selector + properties: + name: + description: name is the name of the + given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target + value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether + the metric type is Utilization, + Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value + of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target + metric by name and selector + properties: + name: + description: name is the name of the + given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target + value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether + the metric type is Utilization, + Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value + of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target + value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether + the metric type is Utilization, + Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value + of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: |- + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. + Note: "ContainerResource" type is available on when the feature-gate + HPAContainerMetrics is enabled + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + type: integer + required: + - maxReplicas + type: object + type: object + startupProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + storage: + description: Storage is the target storage for caching NIM + model if NIMCache is not provided + properties: + hostPath: + description: HostPath is the host path volume for caching + NIM + type: string + nimCache: + description: NIMCacheVolSpec defines the spec to use + NIMCache volume + properties: + name: + type: string + profile: + type: string + type: object + pvc: + description: PersistentVolumeClaim is the pvc volume + used for caching NIM + properties: + create: + description: Create indicates to create a new PVC + type: boolean + name: + description: Name is the name of the PVC + type: string + size: + description: Size of the NIM cache in Gi, used during + PVC creation + type: string + storageClass: + description: StorageClass to be used for PVC creation. + Leave it as empty if the PVC is already created. + type: string + subPath: + type: string + volumeAccessMode: + description: VolumeAccessMode is the volume access + mode of the PVC + type: string + type: object + readOnly: + description: ReadOnly mode indicates if the volume should + be mounted as read-only + type: boolean + type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + userID: + format: int64 + type: integer + required: + - authSecret + type: object + type: object + type: array + type: object + status: + description: NIMPipelineStatus defines the observed state of NIMPipeline + properties: + conditions: + description: |- + INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + Important: Run "make" to regenerate code after modifying this file + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + state: + description: State indicates the overall state of the pipeline + type: string + states: + additionalProperties: + type: string + description: States indicate state of individual services in the pipeline + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/nemo/charts/k8s-nim-operator/crds/apps.nvidia.com_nimservices.yaml b/charts/nemo/charts/k8s-nim-operator/crds/apps.nvidia.com_nimservices.yaml new file mode 100644 index 0000000..419ef0f --- /dev/null +++ b/charts/nemo/charts/k8s-nim-operator/crds/apps.nvidia.com_nimservices.yaml @@ -0,0 +1,2217 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.2 + name: nimservices.apps.nvidia.com +spec: + group: apps.nvidia.com + names: + kind: NIMService + listKind: NIMServiceList + plural: nimservices + singular: nimservice + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - format: date-time + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: NIMService is the Schema for the nimservices API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NIMServiceSpec defines the desired state of NIMService + properties: + annotations: + additionalProperties: + type: string + type: object + args: + items: + type: string + type: array + authSecret: + description: The name of an existing pull secret containing the NGC_API_KEY + type: string + command: + items: + type: string + type: array + env: + items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot + be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + expose: + description: Expose defines attributes to expose the service + properties: + ingress: + description: Ingress defines attributes to enable ingress for + the service + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + description: ingress, or virtualService - not both + type: boolean + spec: + description: IngressSpec describes the Ingress the user wishes + to exist. + properties: + defaultBackend: + description: |- + defaultBackend is the backend that should handle requests that don't + match any rule. If Rules are not specified, DefaultBackend must be specified. + If DefaultBackend is not set, the handling of requests that do not match any + of the rules will be up to the Ingress controller. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + ingressClassName: + description: |- + ingressClassName is the name of an IngressClass cluster resource. Ingress + controller implementations use this field to know whether they should be + serving this Ingress resource, by a transitive connection + (controller -> IngressClass -> Ingress resource). Although the + `kubernetes.io/ingress.class` annotation (simple constant name) was never + formally defined, it was widely supported by Ingress controllers to create + a direct binding between Ingress controller and Ingress resources. Newly + created Ingress resources should prefer using the field. However, even + though the annotation is officially deprecated, for backwards compatibility + reasons, ingress controllers should still honor that annotation if present. + type: string + rules: + description: |- + rules is a list of host rules used to configure the Ingress. If unspecified, + or no rule matches, all traffic is sent to the default backend. + items: + description: |- + IngressRule represents the rules mapping the paths under a specified host to + the related backend services. Incoming requests are first evaluated for a host + match, then routed to the backend associated with the matching IngressRuleValue. + properties: + host: + description: "host is the fully qualified domain + name of a network host, as defined by RFC 3986.\nNote + the following deviations from the \"host\" part + of the\nURI as defined in RFC 3986:\n1. IPs are + not allowed. Currently an IngressRuleValue can + only apply to\n the IP in the Spec of the parent + Ingress.\n2. The `:` delimiter is not respected + because ports are not allowed.\n\t Currently + the port of an Ingress is implicitly :80 for http + and\n\t :443 for https.\nBoth these may change + in the future.\nIncoming requests are matched + against the host before the\nIngressRuleValue. + If the host is unspecified, the Ingress routes + all\ntraffic based on the specified IngressRuleValue.\n\nhost + can be \"precise\" which is a domain name without + the terminating dot of\na network host (e.g. \"foo.bar.com\") + or \"wildcard\", which is a domain name\nprefixed + with a single wildcard label (e.g. \"*.foo.com\").\nThe + wildcard character '*' must appear by itself as + the first DNS label and\nmatches only a single + label. You cannot have a wildcard label by itself + (e.g. Host == \"*\").\nRequests will be matched + against the Host field in the following way:\n1. + If host is precise, the request matches this rule + if the http host header is equal to Host.\n2. + If host is a wildcard, then the request matches + this rule if the http host header\nis to equal + to the suffix (removing the first label) of the + wildcard rule." + type: string + http: + description: |- + HTTPIngressRuleValue is a list of http selectors pointing to backends. + In the example: http:///? -> backend where + where parts of the url correspond to RFC 3986, this resource will be used + to match against everything after the last '/' and before the first '?' + or '#'. + properties: + paths: + description: paths is a collection of paths + that map requests to backends. + items: + description: |- + HTTPIngressPath associates a path with a backend. Incoming urls matching the + path are forwarded to the backend. + properties: + backend: + description: |- + backend defines the referenced service endpoint to which the traffic + will be forwarded to. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + path: + description: |- + path is matched against the path of an incoming request. Currently it can + contain characters disallowed from the conventional "path" part of a URL + as defined by RFC 3986. Paths must begin with a '/' and must be present + when using PathType with value "Exact" or "Prefix". + type: string + pathType: + description: |- + pathType determines the interpretation of the path matching. PathType can + be one of the following values: + * Exact: Matches the URL path exactly. + * Prefix: Matches based on a URL path prefix split by '/'. Matching is + done on a path element by element basis. A path element refers is the + list of labels in the path split by the '/' separator. A request is a + match for path p if every p is an element-wise prefix of p of the + request path. Note that if the last element of the path is a substring + of the last element in request path, it is not a match (e.g. /foo/bar + matches /foo/bar/baz, but does not match /foo/barbaz). + * ImplementationSpecific: Interpretation of the Path matching is up to + the IngressClass. Implementations can treat this as a separate PathType + or treat it identically to Prefix or Exact path types. + Implementations are required to support all path types. + type: string + required: + - backend + - pathType + type: object + type: array + x-kubernetes-list-type: atomic + required: + - paths + type: object + type: object + type: array + x-kubernetes-list-type: atomic + tls: + description: |- + tls represents the TLS configuration. Currently the Ingress only supports a + single TLS port, 443. If multiple members of this list specify different hosts, + they will be multiplexed on the same port according to the hostname specified + through the SNI TLS extension, if the ingress controller fulfilling the + ingress supports SNI. + items: + description: IngressTLS describes the transport layer + security associated with an ingress. + properties: + hosts: + description: |- + hosts is a list of hosts included in the TLS certificate. The values in + this list must match the name/s used in the tlsSecret. Defaults to the + wildcard host setting for the loadbalancer controller fulfilling this + Ingress, if left unspecified. + items: + type: string + type: array + x-kubernetes-list-type: atomic + secretName: + description: |- + secretName is the name of the secret used to terminate TLS traffic on + port 443. Field is left optional to allow TLS routing based on SNI + hostname alone. If the SNI host in a listener conflicts with the "Host" + header field used by an IngressRule, the SNI host is used for termination + and value of the "Host" header is used for routing. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + service: + description: Service defines attributes to create a service + properties: + annotations: + additionalProperties: + type: string + type: object + name: + description: override the default service name + type: string + port: + default: 8000 + format: int32 + type: integer + type: + description: Service Type string describes ingress methods + for a service + type: string + required: + - port + type: object + type: object + groupID: + format: int64 + type: integer + image: + description: Image defines image attributes + properties: + pullPolicy: + type: string + pullSecrets: + items: + type: string + type: array + repository: + type: string + tag: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + livenessProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP + port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + metrics: + description: Metrics defines attributes to setup metrics collection + properties: + enabled: + type: boolean + serviceMonitor: + description: for use with the Prometheus Operator and the primary + service object + properties: + additionalLabels: + additionalProperties: + type: string + type: object + annotations: + additionalProperties: + type: string + type: object + interval: + description: |- + Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. + Supported units: y, w, d, h, m, s, ms + Examples: `30s`, `1m`, `1h20m15s`, `15d` + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + scrapeTimeout: + description: |- + Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. + Supported units: y, w, d, h, m, s, ms + Examples: `30s`, `1m`, `1h20m15s`, `15d` + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + podAffinity: + description: Pod affinity is a group of inter pod affinity scheduling + rules. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with + the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + readinessProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP + port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + replicas: + default: 1 + minimum: 1 + type: integer + resources: + description: ResourceRequirements describes the compute resource requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + runtimeClassName: + type: string + scale: + description: Autoscaling defines attributes to automatically scale + the service based on metrics + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + type: boolean + hpa: + description: HorizontalPodAutoscalerSpec defines the parameters + required to setup HPA + properties: + behavior: + description: |- + HorizontalPodAutoscalerBehavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling + policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling + policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + type: integer + metrics: + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + This is an alpha feature and can be enabled by the HPAContainerMetrics feature flag. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource in + question. + type: string + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version of + the referent + type: string + kind: + description: 'kind is the kind of the referent; + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource in + question. + type: string + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: |- + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. + Note: "ContainerResource" type is available on when the feature-gate + HPAContainerMetrics is enabled + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + type: integer + required: + - maxReplicas + type: object + type: object + startupProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP + port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + storage: + description: Storage is the target storage for caching NIM model if + NIMCache is not provided + properties: + hostPath: + description: HostPath is the host path volume for caching NIM + type: string + nimCache: + description: NIMCacheVolSpec defines the spec to use NIMCache + volume + properties: + name: + type: string + profile: + type: string + type: object + pvc: + description: PersistentVolumeClaim is the pvc volume used for + caching NIM + properties: + create: + description: Create indicates to create a new PVC + type: boolean + name: + description: Name is the name of the PVC + type: string + size: + description: Size of the NIM cache in Gi, used during PVC + creation + type: string + storageClass: + description: StorageClass to be used for PVC creation. Leave + it as empty if the PVC is already created. + type: string + subPath: + type: string + volumeAccessMode: + description: VolumeAccessMode is the volume access mode of + the PVC + type: string + type: object + readOnly: + description: ReadOnly mode indicates if the volume should be mounted + as read-only + type: boolean + type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + userID: + format: int64 + type: integer + required: + - authSecret + type: object + status: + description: NIMServiceStatus defines the observed state of NIMService + properties: + availableReplicas: + format: int32 + type: integer + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/nemo/charts/k8s-nim-operator/templates/_helpers.tpl b/charts/nemo/charts/k8s-nim-operator/templates/_helpers.tpl new file mode 100644 index 0000000..08b4baf --- /dev/null +++ b/charts/nemo/charts/k8s-nim-operator/templates/_helpers.tpl @@ -0,0 +1,58 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "k8s-nim-operator.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "k8s-nim-operator.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "k8s-nim-operator.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "k8s-nim-operator.labels" -}} +helm.sh/chart: {{ include "k8s-nim-operator.chart" . }} +{{ include "k8s-nim-operator.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "k8s-nim-operator.selectorLabels" -}} +app.kubernetes.io/name: {{ include "k8s-nim-operator.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Full image name with tag +*/}} +{{- define "k8s-nim-operator.fullimage" -}} +{{- .Values.operator.image.repository -}}:{{- .Values.operator.image.tag | default (printf "v%s" .Chart.AppVersion) -}} +{{- end }} diff --git a/charts/nemo/charts/k8s-nim-operator/templates/deployment.yaml b/charts/nemo/charts/k8s-nim-operator/templates/deployment.yaml new file mode 100644 index 0000000..393cb68 --- /dev/null +++ b/charts/nemo/charts/k8s-nim-operator/templates/deployment.yaml @@ -0,0 +1,78 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "k8s-nim-operator.fullname" . }} + labels: + app.kubernetes.io/component: manager + app.kubernetes.io/created-by: k8s-nim-operator + app.kubernetes.io/part-of: k8s-nim-operator + control-plane: controller-manager + {{- include "k8s-nim-operator.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.operator.replicas }} + selector: + matchLabels: + control-plane: controller-manager + {{- include "k8s-nim-operator.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + control-plane: controller-manager + {{- include "k8s-nim-operator.selectorLabels" . | nindent 8 }} + annotations: + kubectl.kubernetes.io/default-container: manager + spec: + containers: + - args: {{- toYaml .Values.operator.args | nindent 8 }} + command: + - /manager + image: {{ include "k8s-nim-operator.fullimage" . }} + imagePullPolicy: {{ .Values.operator.image.pullPolicy }} + env: + - name: WATCH_NAMESPACE + value: "" + - name: OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: manager + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: {{- toYaml .Values.operator.resources | nindent 10 }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + imagePullSecrets: + {{- range .Values.operator.image.pullSecrets }} + - name: {{ . }} + {{- end }} + securityContext: + seccompProfile: + type: RuntimeDefault + runAsNonRoot: true + serviceAccountName: k8s-nim-operator + terminationGracePeriodSeconds: 10 + {{- with .Values.operator.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/nemo/charts/k8s-nim-operator/templates/leader-election-rbac.yaml b/charts/nemo/charts/k8s-nim-operator/templates/leader-election-rbac.yaml new file mode 100644 index 0000000..3b29c3f --- /dev/null +++ b/charts/nemo/charts/k8s-nim-operator/templates/leader-election-rbac.yaml @@ -0,0 +1,59 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: k8s-nim-operator-leader-election-role + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: k8s-nim-operator + app.kubernetes.io/part-of: k8s-nim-operator + {{- include "k8s-nim-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: k8s-nim-operator-leader-election-rolebinding + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: k8s-nim-operator + app.kubernetes.io/part-of: k8s-nim-operator + {{- include "k8s-nim-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: k8s-nim-operator-leader-election-role +subjects: +- kind: ServiceAccount + name: k8s-nim-operator + namespace: '{{ .Release.Namespace }}' diff --git a/charts/nemo/charts/k8s-nim-operator/templates/manager-rbac.yaml b/charts/nemo/charts/k8s-nim-operator/templates/manager-rbac.yaml new file mode 100644 index 0000000..38d9222 --- /dev/null +++ b/charts/nemo/charts/k8s-nim-operator/templates/manager-rbac.yaml @@ -0,0 +1,315 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: k8s-nim-operator-role + labels: + {{- include "k8s-nim-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - configmaps + - persistentvolumeclaims + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - update + - patch +- apiGroups: + - "" + resources: + - endpoints + - pods + - pods/log + - pods/eviction + - serviceaccounts + - services + - services/finalizers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - get + - list + - watch +- apiGroups: + - apps + resources: + - deployments + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps.nvidia.com + resources: + - nimcaches + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps.nvidia.com + resources: + - nimcaches/finalizers + verbs: + - update +- apiGroups: + - apps.nvidia.com + resources: + - nimcaches/status + verbs: + - get + - patch + - update +- apiGroups: + - apps.nvidia.com + resources: + - nimpipelines + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps.nvidia.com + resources: + - nimpipelines/finalizers + verbs: + - update +- apiGroups: + - apps.nvidia.com + resources: + - nimpipelines/status + verbs: + - get + - patch + - update +- apiGroups: + - apps.nvidia.com + resources: + - nimservices + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps.nvidia.com + resources: + - nimservices/finalizers + verbs: + - update +- apiGroups: + - apps.nvidia.com + resources: + - nimservices/status + verbs: + - get + - patch + - update +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - config.openshift.io + resources: + - clusterversions + - proxies + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - create + - delete + - get + - list +- apiGroups: + - monitoring.coreos.com + resources: + - prometheusrules + - servicemonitors + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + - roles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - create + - get + - list + - patch + - update + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - create + - get + - list + - patch + - update + - watch + - delete +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - scheduling.k8s.io + resources: + - priorityclasses + verbs: + - create + - get + - list + - watch +- apiGroups: + - security.openshift.io + resourceNames: + - nonroot + resources: + - securitycontextconstraints + verbs: + - use +- apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: k8s-nim-operator-rolebinding + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: k8s-nim-operator + app.kubernetes.io/part-of: k8s-nim-operator + {{- include "k8s-nim-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: k8s-nim-operator-role +subjects: +- kind: ServiceAccount + name: k8s-nim-operator + namespace: {{ .Release.Namespace }} diff --git a/charts/nemo/charts/k8s-nim-operator/templates/metrics-reader-rbac.yaml b/charts/nemo/charts/k8s-nim-operator/templates/metrics-reader-rbac.yaml new file mode 100644 index 0000000..fe274e7 --- /dev/null +++ b/charts/nemo/charts/k8s-nim-operator/templates/metrics-reader-rbac.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: k8s-nim-operator-metrics-reader + labels: + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: k8s-nim-operator + app.kubernetes.io/part-of: k8s-nim-operator + {{- include "k8s-nim-operator.labels" . | nindent 4 }} +rules: +- nonResourceURLs: + - /metrics + verbs: + - get diff --git a/charts/nemo/charts/k8s-nim-operator/templates/metrics-service.yaml b/charts/nemo/charts/k8s-nim-operator/templates/metrics-service.yaml new file mode 100644 index 0000000..9327aa5 --- /dev/null +++ b/charts/nemo/charts/k8s-nim-operator/templates/metrics-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: k8s-nim-operator-metrics-service + labels: + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: k8s-nim-operator + app.kubernetes.io/part-of: k8s-nim-operator + control-plane: controller-manager + {{- include "k8s-nim-operator.labels" . | nindent 4 }} +spec: + type: {{ .Values.metricsService.type }} + selector: + control-plane: controller-manager + {{- include "k8s-nim-operator.selectorLabels" . | nindent 4 }} + ports: + {{- .Values.metricsService.ports | toYaml | nindent 2 -}} diff --git a/charts/nemo/charts/k8s-nim-operator/templates/node-feature-rule.yaml b/charts/nemo/charts/k8s-nim-operator/templates/node-feature-rule.yaml new file mode 100644 index 0000000..166ff13 --- /dev/null +++ b/charts/nemo/charts/k8s-nim-operator/templates/node-feature-rule.yaml @@ -0,0 +1,17 @@ +{{- if .Values.nfd.nodeFeatureRules.deviceID }} +kind: NodeFeatureRule +apiVersion: nfd.k8s-sigs.io/v1alpha1 +metadata: + name: nfd-pci-device-rule +spec: + rules: + - name: "NVIDIA PCIe Device Label" + labelsTemplate: | + {{`{{ range .pci.device }}pci-{{ .vendor }}-{{ .device }}.present=true`}} + {{`{{ end }}`}} + matchFeatures: + - feature: pci.device + matchExpressions: + vendor: {op: In, value: ["10de"]} + class: {op: In, value: ["0300", "0302"]} +{{- end }} diff --git a/charts/nemo/charts/k8s-nim-operator/templates/serviceaccount.yaml b/charts/nemo/charts/k8s-nim-operator/templates/serviceaccount.yaml new file mode 100644 index 0000000..b2b1b27 --- /dev/null +++ b/charts/nemo/charts/k8s-nim-operator/templates/serviceaccount.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: k8s-nim-operator + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: k8s-nim-operator + app.kubernetes.io/part-of: k8s-nim-operator + {{- include "k8s-nim-operator.labels" . | nindent 4 }} + annotations: + {{- toYaml .Values.operator.annotations | nindent 4 }} diff --git a/charts/nemo/charts/k8s-nim-operator/templates/upgrade_crd.yaml b/charts/nemo/charts/k8s-nim-operator/templates/upgrade_crd.yaml new file mode 100644 index 0000000..fb07cd4 --- /dev/null +++ b/charts/nemo/charts/k8s-nim-operator/templates/upgrade_crd.yaml @@ -0,0 +1,100 @@ +{{- if .Values.operator.upgradeCRD }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nim-operator-upgrade-crd-hook-sa + annotations: + helm.sh/hook: pre-upgrade + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation + helm.sh/hook-weight: "0" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nim-operator-upgrade-crd-hook-role + annotations: + helm.sh/hook: pre-upgrade + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation + helm.sh/hook-weight: "0" +rules: + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch + - patch + - update +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nim-operator-upgrade-crd-hook-binding + annotations: + helm.sh/hook: pre-upgrade + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation + helm.sh/hook-weight: "0" +subjects: + - kind: ServiceAccount + name: nim-operator-upgrade-crd-hook-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: nim-operator-upgrade-crd-hook-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: nim-operator-upgrade-crd + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation + labels: + {{- include "k8s-nim-operator.labels" . | nindent 4 }} + app.kubernetes.io/component: "nim-operator" +spec: + template: + metadata: + name: nim-operator-upgrade-crd + labels: + {{- include "k8s-nim-operator.labels" . | nindent 8 }} + app.kubernetes.io/component: "nim-operator" + spec: + serviceAccountName: nim-operator-upgrade-crd-hook-sa + {{- if .Values.operator.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.operator.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.operator.tolerations }} + securityContext: + seccompProfile: + type: RuntimeDefault + runAsNonRoot: true + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: upgrade-crd + image: {{ include "k8s-nim-operator.fullimage" . }} + imagePullPolicy: {{ .Values.operator.image.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + command: ["kubectl"] + args: + - "apply" + - "-f" + - "/opt/nim-operator/crds/*.yaml" + restartPolicy: OnFailure +{{- end }} diff --git a/charts/nemo/charts/k8s-nim-operator/values.yaml b/charts/nemo/charts/k8s-nim-operator/values.yaml new file mode 100644 index 0000000..3213d67 --- /dev/null +++ b/charts/nemo/charts/k8s-nim-operator/values.yaml @@ -0,0 +1,46 @@ +operator: + replicas: 1 + upgradeCRD: true + image: + repository: nvcr.io/nvidia/cloud-native/k8s-nim-operator + tag: v1.0.1 + pullSecrets: [] + pullPolicy: Always + args: + - --health-probe-bind-address=:8081 + - --metrics-bind-address=:8080 + - --leader-elect + resources: + limits: + cpu: "1" + memory: 256Mi + requests: + cpu: 500m + memory: 128Mi + nodeSelector: {} + annotations: {} + tolerations: + - key: "node-role.kubernetes.io/control-plane" + operator: "Equal" + value: "" + effect: "NoSchedule" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/control-plane" + operator: In + values: [""] + +metricsService: + ports: + - name: metrics + port: 8080 + protocol: TCP + type: ClusterIP + +nfd: + nodeFeatureRules: + deviceID: true diff --git a/charts/nemo/charts/nemo-customizer/.helmignore b/charts/nemo/charts/nemo-customizer/.helmignore new file mode 100644 index 0000000..66e23c6 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/.helmignore @@ -0,0 +1,27 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ + +# Our example-config.yaml is just an example, users should provide their own values. +example-config.yaml +ephemeral.values.yaml diff --git a/charts/nemo/charts/nemo-customizer/Chart.lock b/charts/nemo/charts/nemo-customizer/Chart.lock new file mode 100644 index 0000000..73d843b --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/Chart.lock @@ -0,0 +1,12 @@ +dependencies: +- name: postgresql + repository: oci://dockerhub.nvidia.com/bitnamicharts + version: 13.3.1 +- name: opentelemetry-collector + repository: https://open-telemetry.github.io/opentelemetry-helm-charts + version: 0.93.3 +- name: nemo-common + repository: file://../../../library/ + version: 0.1.0 +digest: sha256:a30bc670003a830025a3cc57257082923e852c35a108629d30768e9aa2e69c5e +generated: "2025-04-16T22:06:20.120607586Z" diff --git a/charts/nemo/charts/nemo-customizer/Chart.yaml b/charts/nemo/charts/nemo-customizer/Chart.yaml new file mode 100644 index 0000000..9e31fbe --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +appVersion: "25.04" +dependencies: +- condition: postgresql.enabled + name: postgresql + repository: oci://dockerhub.nvidia.com/bitnamicharts + version: 13.3.1 +- condition: opentelemetry-collector.enabled + name: opentelemetry-collector + repository: https://open-telemetry.github.io/opentelemetry-helm-charts + version: 0.93.3 +- name: nemo-common + repository: file://../../../library/ + version: '>=0.1.0-0' +description: NeMo Customizer Microservice Helm Chart +name: nemo-customizer +type: application +version: 25.4.0 diff --git a/charts/nemo/charts/nemo-customizer/README.md b/charts/nemo/charts/nemo-customizer/README.md new file mode 100644 index 0000000..64a3c6f --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/README.md @@ -0,0 +1,188 @@ +# NeMo Customizer Microservice Helm Chart + +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) + +For deployment guide, see [Admin Setup](https://docs.nvidia.com/nemo/microservices/latest/set-up/index.html) in the NeMo Microservices documentation. + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| apiImage | object | This object has the following default values for the NeMo Customizer API only image. | Customizer API only image configuration. | +| apiImage.imagePullPolicy | string | `"IfNotPresent"` | Image pull policy for the NeMo Customizer API image. | +| apiImage.registry | string | `"nvcr.io"` | Registry for the NeMo Customizer API image. | +| apiImage.repository | string | `"nvidia/nemo-microservices/customizer-api"` | Repository for the NeMo Customizer API image. | +| customizerConfig | object | This object has default values for the following fields. | Configuration for the NeMo Customizer microservice. | +| customizerConfig.entityStoreURL | string | `"http://nemo-gateway.nemo-gateway.svc.cluster.local:8000"` | URL for the NeMo Entity Store microservice. | +| customizerConfig.mlflowURL | string | `"http://mlflow-tracking.mlflow-system.svc.cluster.local:80"` | URL for the MLflow tracking server. | +| customizerConfig.models | object | This object has default values for the supported models. | Models configuration. | +| customizerConfig.models."meta/llama-3.1-70b-instruct" | object | This object has the following default values for the Llama 3.1 70B Instruct model. | Llama 3.1 70B Instruct model configuration. | +| customizerConfig.models."meta/llama-3.1-70b-instruct".enabled | bool | `false` | Whether to enable the model. | +| customizerConfig.models."meta/llama-3.1-70b-instruct".max_seq_length | int | `4096` | Maximum sequence length for input tokens. | +| customizerConfig.models."meta/llama-3.1-70b-instruct".micro_batch_size | int | `1` | Micro batch size for training. | +| customizerConfig.models."meta/llama-3.1-70b-instruct".model_path | string | `"llama-3_1-70b-instruct_0_0_1"` | Path where model files are stored. | +| customizerConfig.models."meta/llama-3.1-70b-instruct".model_uri | string | `"ngc://nvidia/nemo/llama-3_1-70b-instruct-nemo:2.0"` | NGC model URI for Llama 3.1 70B Instruct model. | +| customizerConfig.models."meta/llama-3.1-70b-instruct".num_parameters | int | `70000000000` | Number of model parameters. | +| customizerConfig.models."meta/llama-3.1-70b-instruct".precision | string | `"bf16-mixed"` | Model precision format. | +| customizerConfig.models."meta/llama-3.1-70b-instruct".prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. | +| customizerConfig.models."meta/llama-3.1-70b-instruct".training_options | list | `[{"finetuning_type":"lora","num_gpus":4,"num_nodes":1,"tensor_parallel_size":4,"training_type":"sft"}]` | Training options for different fine-tuning methods. | +| customizerConfig.models."meta/llama-3.1-8b-instruct" | object | This object has the following default values for the Llama 3.1 8B Instruct model. | Llama 3.1 8B Instruct model configuration. | +| customizerConfig.models."meta/llama-3.1-8b-instruct".enabled | bool | `false` | Whether to enable the model. | +| customizerConfig.models."meta/llama-3.1-8b-instruct".max_seq_length | int | `4096` | Maximum sequence length for input tokens. | +| customizerConfig.models."meta/llama-3.1-8b-instruct".micro_batch_size | int | `1` | Micro batch size for training. | +| customizerConfig.models."meta/llama-3.1-8b-instruct".model_path | string | `"llama-3_1-8b-instruct_0_0_1"` | Path where model files are stored. | +| customizerConfig.models."meta/llama-3.1-8b-instruct".model_uri | string | `"ngc://nvidia/nemo/llama-3_1-8b-instruct-nemo:2.0"` | NGC model URI for Llama 3.1 8B Instruct model. | +| customizerConfig.models."meta/llama-3.1-8b-instruct".num_parameters | int | `8000000000` | Number of model parameters. | +| customizerConfig.models."meta/llama-3.1-8b-instruct".precision | string | `"bf16-mixed"` | Model precision format. | +| customizerConfig.models."meta/llama-3.1-8b-instruct".prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. | +| customizerConfig.models."meta/llama-3.1-8b-instruct".training_options | list | `[{"finetuning_type":"lora","num_gpus":1,"training_type":"sft"},{"finetuning_type":"all_weights","num_gpus":8,"num_nodes":1,"tensor_parallel_size":4,"training_type":"sft"}]` | Training options for different fine-tuning methods. | +| customizerConfig.models."meta/llama-3.2-1b" | object | This object has the following default values for the Llama 3.2 1B model. | Llama 3.2 1B model configuration. | +| customizerConfig.models."meta/llama-3.2-1b".enabled | bool | `false` | Whether to enable the model. | +| customizerConfig.models."meta/llama-3.2-1b".max_seq_length | int | `4096` | Maximum sequence length for input tokens. | +| customizerConfig.models."meta/llama-3.2-1b".micro_batch_size | int | `1` | Micro batch size for training. | +| customizerConfig.models."meta/llama-3.2-1b".model_path | string | `"llama32_1b"` | Path where model files are stored. | +| customizerConfig.models."meta/llama-3.2-1b".model_uri | string | `"ngc://nvidia/nemo/llama-3_2-1b:2.0"` | NGC model URI for Llama 3.2 1B model. | +| customizerConfig.models."meta/llama-3.2-1b".num_parameters | int | `1000000000` | Number of model parameters. | +| customizerConfig.models."meta/llama-3.2-1b".precision | string | `"bf16-mixed"` | Model precision format. | +| customizerConfig.models."meta/llama-3.2-1b".prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. | +| customizerConfig.models."meta/llama-3.2-1b".training_options | list | `[{"finetuning_type":"lora","num_gpus":1,"num_nodes":1,"tensor_parallel_size":1,"training_type":"sft"},{"finetuning_type":"all_weights","num_gpus":1,"num_nodes":1,"tensor_parallel_size":1,"training_type":"sft"}]` | Training options for different fine-tuning methods. | +| customizerConfig.models."meta/llama-3.2-1b-instruct" | object | This object has the following default values for the Llama 3.2 1B Instruct model. | Llama 3.2 1B Instruct model configuration. | +| customizerConfig.models."meta/llama-3.2-1b-instruct".enabled | bool | `false` | Whether to enable the model. | +| customizerConfig.models."meta/llama-3.2-1b-instruct".max_seq_length | int | `4096` | Maximum sequence length for input tokens. | +| customizerConfig.models."meta/llama-3.2-1b-instruct".micro_batch_size | int | `1` | Micro batch size for training. | +| customizerConfig.models."meta/llama-3.2-1b-instruct".model_path | string | `"llama32_1b-instruct"` | Path where model files are stored. | +| customizerConfig.models."meta/llama-3.2-1b-instruct".model_uri | string | `"ngc://nvidia/nemo/llama-3_2-1b-instruct:2.0"` | NGC model URI for Llama 3.2 1B Instruct model. | +| customizerConfig.models."meta/llama-3.2-1b-instruct".num_parameters | int | `1000000000` | Number of model parameters. | +| customizerConfig.models."meta/llama-3.2-1b-instruct".precision | string | `"bf16-mixed"` | Model precision format. | +| customizerConfig.models."meta/llama-3.2-1b-instruct".prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. | +| customizerConfig.models."meta/llama-3.2-1b-instruct".training_options | list | `[{"finetuning_type":"lora","num_gpus":1,"num_nodes":1,"tensor_parallel_size":1,"training_type":"sft"},{"finetuning_type":"all_weights","num_gpus":1,"num_nodes":1,"tensor_parallel_size":1,"training_type":"sft"}]` | Training options for different fine-tuning methods. | +| customizerConfig.models."meta/llama-3.2-3b-instruct" | object | This object has the following default values for the Llama 3.2 3B Instruct model. | Llama 3.2 3B Instruct model configuration. | +| customizerConfig.models."meta/llama-3.2-3b-instruct".enabled | bool | `false` | Whether to enable the model. | +| customizerConfig.models."meta/llama-3.2-3b-instruct".max_seq_length | int | `4096` | Maximum sequence length for input tokens. | +| customizerConfig.models."meta/llama-3.2-3b-instruct".micro_batch_size | int | `1` | Micro batch size for training. | +| customizerConfig.models."meta/llama-3.2-3b-instruct".model_path | string | `"llama32_3b-instruct"` | Path where model files are stored. | +| customizerConfig.models."meta/llama-3.2-3b-instruct".model_uri | string | `"ngc://nvidia/nemo/llama-3_2-3b-instruct:2.0"` | NGC model URI. | +| customizerConfig.models."meta/llama-3.2-3b-instruct".num_parameters | int | `3000000000` | Number of model parameters. | +| customizerConfig.models."meta/llama-3.2-3b-instruct".precision | string | `"bf16-mixed"` | Model precision format. | +| customizerConfig.models."meta/llama-3.2-3b-instruct".prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. | +| customizerConfig.models."meta/llama-3.2-3b-instruct".training_options | list | `[{"finetuning_type":"lora","num_gpus":1,"num_nodes":1,"tensor_parallel_size":1,"training_type":"sft"}]` | Training options for different fine-tuning methods. | +| customizerConfig.models."meta/llama-3.3-70b-instruct" | object | This object has the following default values for the Llama 3.3 70B Instruct model. | Llama 3.3 70B Instruct model configuration. | +| customizerConfig.models."meta/llama-3.3-70b-instruct".enabled | bool | `false` | Whether to enable the model. | +| customizerConfig.models."meta/llama-3.3-70b-instruct".max_seq_length | int | `4096` | Maximum sequence length for input tokens. | +| customizerConfig.models."meta/llama-3.3-70b-instruct".micro_batch_size | int | `1` | Micro batch size for training. | +| customizerConfig.models."meta/llama-3.3-70b-instruct".model_path | string | `"llama-3_3-70b-instruct_0_0_1"` | Path where model files are stored. | +| customizerConfig.models."meta/llama-3.3-70b-instruct".model_uri | string | `"ngc://nvidia/nemo/llama-3_3-70b-instruct:2.0"` | NGC model URI for Llama 3.3 70B Instruct model. | +| customizerConfig.models."meta/llama-3.3-70b-instruct".num_parameters | int | `70000000000` | Number of model parameters. | +| customizerConfig.models."meta/llama-3.3-70b-instruct".precision | string | `"bf16-mixed"` | Model precision format. | +| customizerConfig.models."meta/llama-3.3-70b-instruct".prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. | +| customizerConfig.models."meta/llama-3.3-70b-instruct".training_options | list | `[{"finetuning_type":"lora","num_gpus":4,"num_nodes":1,"tensor_parallel_size":4,"training_type":"sft"}]` | Training options for different fine-tuning methods. | +| customizerConfig.models.meta/llama3-70b-instruct | object | This object has the following default values for the Llama 3 70B Instruct model. | Llama 3 70B Instruct model configuration. | +| customizerConfig.models.meta/llama3-70b-instruct.enabled | bool | `false` | Whether to enable the model. | +| customizerConfig.models.meta/llama3-70b-instruct.max_seq_length | int | `4096` | Maximum sequence length for input tokens. | +| customizerConfig.models.meta/llama3-70b-instruct.micro_batch_size | int | `1` | Micro batch size for training. | +| customizerConfig.models.meta/llama3-70b-instruct.model_path | string | `"llama-3-70b-bf16"` | Path where model files are stored. | +| customizerConfig.models.meta/llama3-70b-instruct.model_uri | string | `"ngc://nvidia/nemo/llama-3-70b-instruct-nemo:2.0"` | NGC model URI for Llama 3 70B Instruct model. | +| customizerConfig.models.meta/llama3-70b-instruct.num_parameters | int | `70000000000` | Number of model parameters. | +| customizerConfig.models.meta/llama3-70b-instruct.precision | string | `"bf16-mixed"` | Model precision format. | +| customizerConfig.models.meta/llama3-70b-instruct.prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. | +| customizerConfig.models.meta/llama3-70b-instruct.training_options | list | `[{"finetuning_type":"lora","num_gpus":4,"num_nodes":1,"tensor_parallel_size":4,"training_type":"sft"}]` | Training options for different fine-tuning methods. | +| customizerConfig.models.microsoft/phi-4 | object | This object has the following default values for the Phi-4. | Phi-4 model configuration. | +| customizerConfig.models.microsoft/phi-4.enabled | bool | `false` | Whether to enable the model. | +| customizerConfig.models.microsoft/phi-4.max_seq_length | int | `4096` | Maximum sequence length for input tokens. | +| customizerConfig.models.microsoft/phi-4.micro_batch_size | int | `1` | Micro batch size for training. | +| customizerConfig.models.microsoft/phi-4.model_path | string | `"phi-4"` | Path where model files are stored. | +| customizerConfig.models.microsoft/phi-4.model_uri | string | `"ngc://nvidia/nemo/phi-4:1.0"` | NGC model URI for Phi-4 model. | +| customizerConfig.models.microsoft/phi-4.num_parameters | int | `14659507200` | Number of model parameters. | +| customizerConfig.models.microsoft/phi-4.precision | string | `"bf16"` | Model precision format. | +| customizerConfig.models.microsoft/phi-4.prompt_template | string | `"{prompt} {completion}"` | Template for formatting prompts. | +| customizerConfig.models.microsoft/phi-4.training_options | list | `[{"finetuning_type":"lora","num_gpus":1,"num_nodes":1,"training_type":"sft"},{"finetuning_type":"all_weights","num_gpus":4,"num_nodes":2,"tensor_parallel_size":8,"training_type":"sft"}]` | Training options for different fine-tuning methods. | +| customizerConfig.nemoDataStoreURL | string | `"http://nds-datastore-http.nds-v2.svc.cluster.local:3000"` | URL for the NeMo Data Store microservice. | +| customizerConfig.openTelemetry | object | This object has the following default values for the OpenTelemetry settings. | OpenTelemetry settings. | +| customizerConfig.openTelemetry.enabled | bool | `true` | Whether to enable OpenTelemetry. | +| customizerConfig.openTelemetry.exporterOtlpEndpoint | string | `""` | Endpoint to access a custom OTLP collector listening on port 4317. Example: "http://$(HOST_IP):4317". | +| customizerConfig.openTelemetry.logsExporter | string | `"otlp"` | Sets the logs exporter type (otlp, console, none). | +| customizerConfig.openTelemetry.metricsExporter | string | `"otlp"` | Sets the metrics exporter type (otlp, console, none). | +| customizerConfig.openTelemetry.tracesExporter | string | `"otlp"` | Sets the traces exporter type (otlp, console, none). | +| customizerConfig.tolerations | list | `[]` | Tolerations on the customization job pods. | +| customizerConfig.training | object | This object has the following default values for the training configuration. | Training configuration for customization jobs. | +| customizerConfig.training.container_defaults | object | `{"env":null,"imagePullPolicy":"IfNotPresent"}` | Default container configuration for training jobs. | +| customizerConfig.training.container_defaults.env | string | `nil` | Environment variables for the training container. Cannot override env variables reserved by NeMo Customizer. | +| customizerConfig.training.pvc.size | string | `"5Gi"` | Size of the training job PVC. | +| customizerConfig.training.pvc.storageClass | string | `"local-nfs"` | Storage class for the training job PVC. | +| customizerConfig.training.pvc.volumeAccessMode | string | `"ReadWriteMany"` | Volume access mode for the training job PVC. | +| customizerConfig.training.queue | string | `"default"` | Queue name used by the underlying scheduler of NemoTrainingJob. Maps to "resourceGroup" in NemoTrainingJob. | +| customizerConfig.training.workspace_dir | string | `"/pvc/workspace"` | Directory path for training workspace. | +| customizerConfig.trainingNetworking | list | `[{"name":"NCCL_IB_SL","value":0},{"name":"NCCL_IB_TC","value":41},{"name":"NCCL_IB_QPS_PER_CONNECTION","value":4},{"name":"UCX_TLS","value":"TCP"},{"name":"UCX_NET_DEVICES","value":"eth0"},{"name":"HCOLL_ENABLE_MCAST_ALL","value":0},{"name":"NCCL_IB_GID_INDEX","value":3}]` | Network configuration for training jobs on Oracle Kubernetes Engine (OKE) on Oracle Cloud Infrastructure (OCI). | +| customizerConfig.wandb | object | `{"entity":null,"project":"nvidia-nemo-customizer"}` | Weights and Biases (WandB) Python SDK intialization configuration for logging and monitoring training jobs in WandB. | +| customizerConfig.wandb.entity | string | `nil` | Reference: https://docs.wandb.ai/ref/python/init/ | +| env | object | `{}` | A map of environment variables to inject into the NeMo Customizer app container. Example: `{HOST_IP: valueFrom: fieldRef: fieldPath: status.hostIP OTEL_EXPORTER_OTLP_ENDPOINT: "http://$(HOST_IP):4317"}` | +| externalDatabase | object | This object has the following default values for the external PostgreSQL configuration. | External PostgreSQL configuration. | +| externalDatabase.database | string | `"finetuning"` | Name of the database to use. | +| externalDatabase.existingSecret | string | `""` | Name of an existing secret resource containing the database credentials. | +| externalDatabase.existingSecretPasswordKey | string | `""` | Name of an existing secret key containing the database credentials. | +| externalDatabase.host | string | `"localhost"` | External database host address. | +| externalDatabase.password | string | `"nemo"` | Password for the non-root username for the NeMo Customizer microservice. | +| externalDatabase.port | int | `5432` | External database port number. | +| externalDatabase.user | string | `"nemo"` | Non-root username for the NeMo Customizer microservice. | +| fullnameOverride | string | `""` | String to fully override the chart and release name on resulting objects when deployed. | +| image | object | This object has the following default values for the NeMo Customizer microservice image. | NeMo Customizer image that supports training and standalone mode. | +| image.imagePullPolicy | string | `"IfNotPresent"` | Image pull policy for the NeMo Customizer image. | +| image.registry | string | `"nvcr.io"` | Registry for the NeMo Customizer image. | +| image.repository | string | `"nvidia/nemo-microservices/customizer"` | Repository for the NeMo Customizer image. | +| imagePullSecrets | list | `[{"name":"nvcrimagepullsecret"}]` | Image pull secrets configuration. | +| ingress | object | This object has the following default values for the Ingress configuration. | Ingress configuration. | +| ingress.annotations | object | `{}` | Additional annotations for the Ingress resource. | +| ingress.className | string | `""` | Ingress class name. | +| ingress.enabled | bool | `false` | Whether to enable the ingress resource. | +| ingress.hostname | string | `""` | Hostname for the ingress resource. | +| ingress.hosts | list | [] | A list of maps, each containing the keys `host` and `paths` for the ingress resource. You must specify a list for configuring ingress for the microservice. | +| ingress.tls | list | `[]` | TLS configuration for the ingress resource. | +| logging | object | `{"logHealthEndpoints":false,"logLevel":"INFO"}` | Logging configuration. | +| logging.logHealthEndpoints | bool | `false` | Enable logging for health endpoints. | +| logging.logLevel | string | `"INFO"` | Log level for the application. | +| modelDownloader | object | This object has the following default values for the model downloader. | Download models to PVC model cache configuration. | +| modelDownloader.pollIntervalSeconds | int | `15` | Interval in seconds to poll for model download status. | +| modelDownloader.securityContext | object | `{"fsGroup":1000,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}` | Security context for the model downloader. | +| modelDownloader.ttlSecondsAfterFinished | int | `7200` | Time to live in seconds after the job finishes. | +| modelsStorage | object | `{"accessModes":["ReadWriteMany"],"enabled":true,"size":"1Ti","storageClassName":""}` | Configure the PVC for models mount, where we store the parent/base models. | +| modelsStorage.accessModes | list | `["ReadWriteMany"]` | Access modes for the persistent volume. | +| modelsStorage.enabled | bool | `true` | Enable persistent volume for model storage. | +| modelsStorage.size | string | `"1Ti"` | Size of the persistent volume. | +| modelsStorage.storageClassName | string | `""` | Storage class name for the models PVC. Empty string uses the default storage class. | +| nameOverride | string | `""` | String to override chart name on resulting objects when deployed. | +| nemoDataStoreTools | object | This object has the following default values for the NeMo Data Store tools image. | Tools configuration for downloading and uploading entities to NeMo Data Store. | +| nemoDataStoreTools.imagePullSecret | string | `"nvcrimagepullsecret"` | Image pull secret for the NeMo Data Store tools image. | +| nemoDataStoreTools.registry | string | `"nvcr.io"` | Registry for the NeMo Data Store tools image. | +| nemoDataStoreTools.repository | string | `"nvidia/nemo-microservices/nds-v2-huggingface-cli"` | Repository for the NeMo Data Store tools image. | +| nemoDataStoreTools.tag | string | `""` | Tag for the NeMo Data Store tools image. | +| ngcAPISecret | string | `"ngc-api"` | Secret used for auto hydrating the model cache from NGC for enabled models. | +| ngcAPISecretKey | string | `"NGC_API_KEY"` | Key in the NGC API secret containing the API key. | +| opentelemetry-collector | object | This object has the following default values for the Open Telemetry Collector configuration. | Open Telemetry Collector configuration. | +| opentelemetry-collector.config | object | `{"exporters":{"debug":{"verbosity":"detailed"}},"extensions":{"health_check":{},"zpages":{"endpoint":"0.0.0.0:55679"}},"processors":{"batch":{}},"receivers":{"otlp":{"protocols":{"grpc":{},"http":{"cors":{"allowed_origins":["*"]}}}}},"service":{"extensions":["zpages","health_check"],"pipelines":{"logs":{"exporters":["debug"],"processors":["batch"],"receivers":["otlp"]},"metrics":{"exporters":["debug"],"processors":["batch"],"receivers":["otlp"]},"traces":{"exporters":["debug"],"processors":["batch"],"receivers":["otlp"]}}}}` | Base collector configuration for Open Telemetry Collector. | +| opentelemetry-collector.enabled | bool | `true` | Switch to enable or disable Open Telemetry Collector. | +| opentelemetry-collector.image.repository | string | `"otel/opentelemetry-collector-k8s"` | Repository for Open Telemetry Collector image. | +| opentelemetry-collector.image.tag | string | `"0.102.1"` | Overrides the image tag whose default is the chart appVersion. | +| opentelemetry-collector.mode | string | `"deployment"` | Deployment mode for Open Telemetry Collector. Valid values are "daemonset", "deployment", and "statefulset". | +| postgresql | object | This object has the following default values for the PostgreSQL configuration. | PostgreSQL configuration for the NeMo Customizer microservice. | +| postgresql.architecture | string | `"standalone"` | PostgreSQL architecture (`standalone` or `replication`). | +| postgresql.auth.database | string | `"finetuning"` | Name for a custom database to create. | +| postgresql.auth.enablePostgresUser | bool | `true` | Whether to assign a password to the "postgres" admin user. Otherwise, remote access will be blocked for this user. | +| postgresql.auth.existingSecret | string | `""` | Name of existing secret to use for PostgreSQL credentials. | +| postgresql.auth.password | string | `"nemo"` | Password for the custom user to create. | +| postgresql.auth.username | string | `"nemo"` | Name for a custom user to create. | +| postgresql.enabled | bool | `true` | Whether to enable or disable the PostgreSQL helm chart. | +| replicaCount | int | `1` | Number of replicas to deploy. | +| service | object | `{"internalPort":9009,"port":8000,"type":"ClusterIP"}` | Service configuration. | +| service.internalPort | int | `9009` | Internal port for the service. | +| service.port | int | `8000` | External port for the service. | +| service.type | string | `"ClusterIP"` | Type of Kubernetes service to create. | +| serviceAccount | object | `{"annotations":{},"automountServiceAccountToken":true,"create":true,"name":""}` | Service account configuration. | +| serviceAccount.annotations | object | `{}` | Annotations to add to the service account. | +| serviceAccount.automountServiceAccountToken | bool | `true` | Automatically mount a ServiceAccount's API credentials. | +| serviceAccount.create | bool | `true` | Specifies whether a service account should be created. | +| serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated. | +| useRunAIExecutor | bool | `false` | Enable or disable RunAI executor. | +| wandb | object | This object has the following default values for the WandB configuration. | WandB configuration. | +| wandb.existingSecret | string | `""` | Name of an existing Kubernetes secret resource for the WandB encryption secret. | +| wandb.existingSecretKey | string | `""` | Name of the key in the existing WandB secret containing the secret value. The secret value must be exactly 32 alphanumeric characters: ^[a-zA-Z0-9]{32}$ | +| wandb.secretValue | string | `"ec60d96b639764ccf9859bc10d4363d1"` | WandB secret value. Must contain exactly 32 alphanumeric characters. Creates a new Kubernetes secret named "wandb-secret" with key-value pair "encryption_key=". Ignored if wandb.existingSecret is set. | \ No newline at end of file diff --git a/charts/nemo/charts/nemo-customizer/charts/nemo-common/.helmignore b/charts/nemo/charts/nemo-customizer/charts/nemo-common/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/nemo-common/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/nemo/charts/nemo-customizer/charts/nemo-common/Chart.yaml b/charts/nemo/charts/nemo-customizer/charts/nemo-common/Chart.yaml new file mode 100644 index 0000000..f88c163 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/nemo-common/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +appVersion: latest +description: Library chart for common elements across NeMo Microservices helm charts +keywords: +- library +- common +kubeVersion: '>=v1.23.0-0' +name: nemo-common +type: library +version: 0.1.0 diff --git a/charts/nemo/charts/nemo-customizer/charts/nemo-common/README.md b/charts/nemo/charts/nemo-customizer/charts/nemo-common/README.md new file mode 100644 index 0000000..199d185 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/nemo-common/README.md @@ -0,0 +1,5 @@ +# NeMo Common Library Chart + +This is a library helm chart of templates that are useful for consistency across helm charts and because defined templates exist in the global scope. + +All charts should use it's templates if they have relevant objects defined. When importing the chart dependency, use the version ">=0.1.0-0" to ensure all versions are captured and it stays up to date. diff --git a/charts/nemo/charts/nemo-customizer/charts/nemo-common/templates/_env.tpl b/charts/nemo/charts/nemo-customizer/charts/nemo-common/templates/_env.tpl new file mode 100644 index 0000000..2ecba37 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/nemo-common/templates/_env.tpl @@ -0,0 +1,121 @@ +{{/* +nemo-common.env generates an env var array out of a dict or array of supported values +*/}} +{{- define "nemo-common.env" -}} +{{- if and .Values.env (kindIs "slice" .Values.env) -}} +{{- toYaml .Values.env -}} +{{- else if and .Values.env (kindIs "map" .Values.env) -}} +{{- range $k, $v := .Values.env }} +- name: {{ $k }} + {{- if kindIs "map" $v }} + valueFrom: + {{ toYaml $v.valueFrom | nindent 4 | trim }} + {{- else }} + value: {{ $v | quote }} + {{- end }} +{{- end }} +{{- else if .Values.envVars }} +{{- range $k, $v := .Values.envVars }} +- name: {{ $k }} + value: {{ $v | quote }} +{{- end }} +{{- end -}} +{{- end -}} + +{{/* +nemo-common.postgresql.uri provides a template for constructing a postgresql database +connection string as an environment variable value +*/}} +{{- define "nemo-common.postgresql.uri" -}} +{{- if and .Values.externalDatabase.uriSecret .Values.externalDatabase.uriSecret.name .Values.externalDatabase.uriSecret.key }} +valueFrom: + secretKeyRef: + name: {{ .Values.externalDatabase.uriSecret.name }} + key: {{ .Values.externalDatabase.uriSecret.key }} +{{- else }} +value: {{ printf "postgresql://%s:$(POSTGRES_DB_PASSWORD)@%s:%s/%s" (include "nemo-common.postgresql.user" .) (include "nemo-common.postgresql.host" .) (include "nemo-common.postgresql.port" .) (include "nemo-common.postgresql.name" .) | quote }} +{{- end }} +{{- end -}} + +{{/* +PostgreSQL Hostname +*/}} +{{- define "nemo-common.postgresql.host" -}} +{{- if .Values.postgresql.enabled -}} +{{ template "postgresql.v1.primary.fullname" .Subcharts.postgresql }} +{{- else -}} +{{ .Values.externalDatabase.host }} +{{- end -}} +{{- end -}} + +{{/* +nemo-common.database.port chooses between externalDatabase and the bitnami postgresql port values +*/}} +{{- define "nemo-common.postgresql.port" -}} +{{- if .Values.postgresql.enabled -}} +{{- printf "%d" (.Values.postgresql.primary.service.ports.postgresql | int) -}} +{{- else -}} +{{- printf "%d" (.Values.externalDatabase.port | int) -}} +{{- end -}} +{{- end -}} + +{{/* +nemo-common.database.user chooses between externalDatabase and the bitnami postgresql user values +*/}} +{{- define "nemo-common.postgresql.user" -}} +{{- if .Values.postgresql.enabled -}} +{{- print .Values.postgresql.auth.username -}} +{{- else -}} +{{- print .Values.externalDatabase.user -}} +{{- end -}} +{{- end -}} + +{{/* +nemo-common.database.name chooses between externalDatabase and the bitnami postgresql db name values +*/}} +{{- define "nemo-common.postgresql.name" -}} +{{- if .Values.postgresql.enabled -}} +{{- print .Values.postgresql.auth.database -}} +{{- else -}} +{{- print .Values.externalDatabase.database -}} +{{- end -}} +{{- end -}} + +{{/* +nemo-common.database.secret-name chooses between externalDatabase and the bitnami postgresql existing secret values +*/}} +{{- define "nemo-common.postgresql.secret-name" -}} +{{- if .Values.postgresql.enabled -}} +{{- if .Values.postgresql.auth.existingSecret -}} +{{- print .Values.postgresql.auth.existingSecret -}} +{{- else -}} +{{- template "postgresql.v1.primary.fullname" .Subcharts.postgresql -}} +{{- end -}} +{{- else if .Values.externalDatabase.existingSecret -}} +{{- print .Values.externalDatabase.existingSecret -}} +{{- end -}} +{{- end -}} + +{{/* +nemo-common.database.password-key chooses between externalDatabase and the bitnami postgresql existing secret key values +*/}} +{{- define "nemo-common.postgresql.password-key" -}} +{{- if or .Values.postgresql.enabled (not .Values.externalDatabase.existingSecret) -}} +{{- print "password" -}} +{{- else -}} +{{- print .Values.externalDatabase.existingSecretPasswordKey -}} +{{- end -}} +{{- end -}} + +{{/* +nemo-common.database.password generates a POSTGRES_DB_PASSWORD environment value if a full URI isn't used +*/}} +{{- define "nemo-common.postgresql.password" -}} +{{- if not (and .Values.externalDatabase.uriSecret .Values.externalDatabase.uriSecret.name .Values.externalDatabase.uriSecret.key) }} +- name: POSTGRES_DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "nemo-common.postgresql.secret-name" .}} + key: {{ include "nemo-common.postgresql.password-key" .}} +{{- end }} +{{- end -}} diff --git a/charts/nemo/charts/nemo-customizer/charts/nemo-common/templates/_image.tpl b/charts/nemo/charts/nemo-customizer/charts/nemo-common/templates/_image.tpl new file mode 100644 index 0000000..6f70171 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/nemo-common/templates/_image.tpl @@ -0,0 +1,27 @@ +{{/* +Image Definition Parsing +Favor not using a separate registry because it is confusing, but support it. +*/}} +{{- define "nemo-common.mainimage" -}} +{{- if .Values.image.registry -}} +{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} +{{- else -}} +{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} +{{- end }} +{{- end }} + +{{/* +Image Pull Secrets +Use globals if they are there, but define sensibly. +*/}} +{{- define "nemo-common.imagepullsecrets" -}} +{{- if and .Values.global .Values.global.imagePullSecrets }} +{{- range .Values.global.imagePullSecrets }} +- {{ . | toYaml }} +{{- end }} +{{- else }} +{{- range .Values.imagePullSecrets }} +- {{ . | toYaml }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/nemo/charts/nemo-customizer/charts/nemo-common/templates/_service.tpl b/charts/nemo/charts/nemo-customizer/charts/nemo-common/templates/_service.tpl new file mode 100644 index 0000000..eedd107 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/nemo-common/templates/_service.tpl @@ -0,0 +1,21 @@ +{{/* +Global level service name definition. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +This will allow top level overrides of service names when using the platform helm chart +*/}} +{{- define "nemo-common.servicename" -}} +{{- if .Values.serviceName }} +{{- .Values.serviceName | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/.helmignore b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/.helmignore new file mode 100644 index 0000000..7e03f4b --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/.helmignore @@ -0,0 +1,27 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ + +# Ignore unittest +tests/ +*/__snapshot__/* diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/CONTRIBUTING.md b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/CONTRIBUTING.md new file mode 100644 index 0000000..aa0bf26 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/CONTRIBUTING.md @@ -0,0 +1,12 @@ +# Collector Chart Contributing Guide + +All changes to the chart require a bump to the version in `chart.yaml`. See the [Contributing Guide](https://github.com/open-telemetry/opentelemetry-helm-charts/blob/main/CONTRIBUTING.md#versioning) for our versioning requirements. + +Once the chart version is bumped, the examples must be regenerated. You can regenerate examples by running `make generate-examples CHARTS=opentelemetry-collector`. + +## Bumping Default Collector Version + +1. Increase the minor version of the chart by one and set the patch version to zero. +2. Update the chart's `appVersion` to match the new collector version. This version will be used as the image tag by default. +3. Review the corresponding release notes in [Collector Core](https://github.com/open-telemetry/opentelemetry-collector/releases), [Collector Contrib](https://github.com/open-telemetry/opentelemetry-collector-contrib/releases), and [Collector Releases](https://github.com/open-telemetry/opentelemetry-collector-releases/releases). If any changes affect the helm charts, adjust the helm chart accordingly. +4. Run `make generate-examples CHARTS=opentelemetry-collector`. diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/Chart.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/Chart.yaml new file mode 100644 index 0000000..e82d90b --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/Chart.yaml @@ -0,0 +1,14 @@ +apiVersion: v2 +appVersion: 0.102.1 +description: OpenTelemetry Collector Helm chart for Kubernetes +home: https://opentelemetry.io/ +icon: https://opentelemetry.io/img/logos/opentelemetry-logo-nav.png +maintainers: +- name: dmitryax +- name: TylerHelmuth +name: opentelemetry-collector +sources: +- https://github.com/open-telemetry/opentelemetry-collector +- https://github.com/open-telemetry/opentelemetry-collector-contrib +type: application +version: 0.93.3 diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/README.md b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/README.md new file mode 100644 index 0000000..a085a88 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/README.md @@ -0,0 +1,251 @@ +# OpenTelemetry Collector Helm Chart + +The helm chart installs [OpenTelemetry Collector](https://github.com/open-telemetry/opentelemetry-collector) +in kubernetes cluster. + +## Prerequisites + +- Kubernetes 1.24+ +- Helm 3.9+ + +## Installing the Chart + +Add OpenTelemetry Helm repository: + +```console +helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts +``` + +To install the chart with the release name my-opentelemetry-collector, run the following command: + +```console +helm install my-opentelemetry-collector open-telemetry/opentelemetry-collector --set mode= --set image.repository="otel/opentelemetry-collector-k8s" --set command.name="otelcol-k8s" +``` + +Where the `mode` value needs to be set to one of `daemonset`, `deployment` or `statefulset`. + +For an in-depth walk through getting started in Kubernetes using this helm chart, see [OpenTelemetry Kubernetes Getting Started](https://opentelemetry.io/docs/kubernetes/getting-started/). + +## Upgrading + +See [UPGRADING.md](UPGRADING.md). + +## Security Considerations + +OpenTelemetry Collector recommends to bind receivers' servers to addresses that limit connections to authorized users. +For this reason, by default the chart binds all the Collector's endpoints to the pod's IP. + +More info is available in the [Security Best Practices docummentation](https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks) + +Some care must be taken when using `hostNetwork: true`, as then OpenTelemetry Collector will listen on all the addresses in the host network namespace. + +## Configuration + +### Default configuration + +By default this chart will deploy an OpenTelemetry Collector with three pipelines (logs, metrics and traces) +and debug exporter enabled by default. The collector can be installed either as daemonset (agent), deployment or stateful set. + +*Example*: Install collector as a deployment. + +```yaml +mode: deployment +``` + +By default collector has the following receivers enabled: + +- **metrics**: OTLP and prometheus. Prometheus is configured only for scraping collector's own metrics. +- **traces**: OTLP, zipkin and jaeger (thrift and grpc). +- **logs**: OTLP (to enable container logs, see [Configuration for Kubernetes container logs](#configuration-for-kubernetes-container-logs)). + +### Basic Top Level Configuration + +The Collector's configuration is set via the `config` section. Default components can be removed with `null`. Remember that lists in helm are not merged, so if you want to modify any default list you must specify all items, including any default items you want to keep. + +*Example*: Disable metrics and logs pipelines and non-otlp receivers: + +```yaml +config: + receivers: + jaeger: null + prometheus: null + zipkin: null + service: + pipelines: + traces: + receivers: + - otlp + metrics: null + logs: null +``` + +The chart also provides several presets, detailed below, to help configure important Kubernetes components. For more details on each component, see [Kubernetes Collector Components](https://opentelemetry.io/docs/kubernetes/collector/components/). + +### Configuration for Kubernetes Container Logs + +The collector can be used to collect logs sent to standard output by Kubernetes containers. +This feature is disabled by default. It has the following requirements: + +- It needs agent collector to be deployed. +- It requires the [Filelog receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#filelog-receiver) to be included in the collector, such as [k8s](https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s) version of the collector image. + +To enable this feature, set the `presets.logsCollection.enabled` property to `true`. +Here is an example `values.yaml`: + +```yaml +mode: daemonset + +presets: + logsCollection: + enabled: true + includeCollectorLogs: true +``` + +The way this feature works is it adds a `filelog` receiver on the `logs` pipeline. This receiver is preconfigured +to read the files where Kubernetes container runtime writes all containers' console output to. + +#### :warning: Warning: Risk of looping the exported logs back into the receiver, causing "log explosion" + +The container logs pipeline uses the `debug` exporter by default. +Paired with the default `filelog` receiver that receives all containers' console output, +it is easy to accidentally feed the exported logs back into the receiver. + +Also note that using the `--verbosity=detailed` option for the `debug` exporter causes it to output +multiple lines per single received log, which when looped, would amplify the logs exponentially. + +To prevent the looping, the default configuration of the receiver excludes logs from the collector's containers. + +If you want to include the collector's logs, make sure to replace the `debug` exporter +with an exporter that does not send logs to collector's standard output. + +Here's an example `values.yaml` file that replaces the default `debug` exporter on the `logs` pipeline +with an `otlphttp` exporter that sends the container logs to `https://example.com:55681` endpoint. +It also clears the `filelog` receiver's `exclude` property, for collector logs to be included in the pipeline. + +```yaml +mode: daemonset + +presets: + logsCollection: + enabled: true + includeCollectorLogs: true + +config: + exporters: + otlphttp: + endpoint: https://example.com:55681 + service: + pipelines: + logs: + exporters: + - otlphttp +``` + +### Configuration for Kubernetes Attributes Processor + +The collector can be configured to add Kubernetes metadata, such as pod name and namespace name, as resource attributes to incoming logs, metrics and traces. + +This feature is disabled by default. It has the following requirements: + +- It requires the [Kubernetes Attributes processor](https://opentelemetry.io/docs/kubernetes/collector/components/#kubernetes-attributes-processor) to be included in the collector, such as [k8s](https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s) version of the collector image. + +To enable this feature, set the `presets.kubernetesAttributes.enabled` property to `true`. +Here is an example `values.yaml`: + +```yaml +mode: daemonset +presets: + kubernetesAttributes: + enabled: true + # You can also configure the preset to add all of the associated pod's labels and annotations to you telemetry. + # The label/annotation name will become the resource attribute's key. + extractAllPodLabels: true + extractAllPodAnnotations: true +``` + +### Configuration for Retrieving Kubelet Metrics + +The collector can be configured to collect node, pod, and container metrics from the API server on a kubelet. + +This feature is disabled by default. It has the following requirements: + +- It requires the [Kubeletstats receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#kubeletstats-receiver) to be included in the collector, such as [k8s](https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s) version of the collector image. + +To enable this feature, set the `presets.kubeletMetrics.enabled` property to `true`. +Here is an example `values.yaml`: + +```yaml +mode: daemonset +presets: + kubeletMetrics: + enabled: true +``` + +### Configuration for Kubernetes Cluster Metrics + +The collector can be configured to collects cluster-level metrics from the Kubernetes API server. A single instance of this receiver can be used to monitor a cluster. + +This feature is disabled by default. It has the following requirements: + +- It requires the [Kubernetes Cluster receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#kubernetes-cluster-receiver) to be included in the collector, such as [k8s](https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s) version of the collector image. +- It requires statefulset or deployment mode with a single replica. + +To enable this feature, set the `presets.clusterMetrics.enabled` property to `true`. + +Here is an example `values.yaml`: + +```yaml +mode: deployment +replicaCount: 1 +presets: + clusterMetrics: + enabled: true +``` + +### Configuration for Retrieving Kubernetes Events + +The collector can be configured to collect Kubernetes events. + +This feature is disabled by default. It has the following requirements: + +- It requires [Kubernetes Objects receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#kubernetes-objects-receiver) to be included in the collector, such as [k8s](https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s) version of the collector image. + +To enable this feature, set the `presets.kubernetesEvents.enabled` property to `true`. +Here is an example `values.yaml`: + +```yaml +mode: deployment +replicaCount: 1 +presets: + kubernetesEvents: + enabled: true +``` + +### Configuration for Host Metrics + +The collector can be configured to collect host metrics for Kubernetes nodes. + +This feature is disabled by default. It has the following requirements: + +- It requires [Host Metrics receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#host-metrics-receiver) to be included in the collector, such as [k8s](https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s) version of the collector image. + +To enable this feature, set the `presets.hostMetrics.enabled` property to `true`. +Here is an example `values.yaml`: + +```yaml +mode: daemonset +presets: + hostMetrics: + enabled: true +``` + +## CRDs + +At this time, Prometheus CRDs are supported but other CRDs are not. + +### Other configuration options + +The [values.yaml](./values.yaml) file contains information about all other configuration +options for this chart. + +For more examples see [Examples](examples). diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/UPGRADING.md b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/UPGRADING.md new file mode 100644 index 0000000..92ac2be --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/UPGRADING.md @@ -0,0 +1,424 @@ +# Upgrade guidelines + +These upgrade guidelines only contain instructions for version upgrades which require manual modifications on the user's side. +If the version you want to upgrade to is not listed here, then there is nothing to do for you. +Just upgrade and enjoy. + +## Feature Flags + +### useGOMEMLIMIT + +A new flag, `useGOMEMLIMIT` has been added that allows specifying whether or not the chart should use the `GOMEMLIMIT` environment variable or the Memory Ballast Extension. +When enabled, the chart will remove the Memory Ballast Extension from the collector's configuration AND will setup a `GOMEMLIMIT` environment variable that is set to 80% +of the configured `resources.limits.memory`. If no `resources.limits.memory` are set when `useGOMEMLIMIT` is enabled then a `GOMEMLIMIT` environment variable WILL NOT be +created but the Memory Ballast Extension will still be removed. + +Depending on the progress made in [Issue 891](https://github.com/open-telemetry/opentelemetry-helm-charts/issues/891), +the use of `GOMEMLIMIT` may completely replace the Memory Ballast Extension in the future. + +## 0.88.0 to 0.89.0 + +> [!WARNING] +> Critical content demanding immediate user attention due to potential risks. + +As part of working towards using the [OpenTelemetry Collector Kubernetes Distro](https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s) by default, the chart now requires users to explicitly set an image repository. If you are already explicitly setting an image repository this breaking change does not affect you. + +If you are using a OpenTelemetry Community distribution of the Collector we recommend you use `otel/opentelemetry-collector-k8s`, but carefully review the [components included in this distribution](https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/distributions/otelcol-k8s/manifest.yaml) to make sure it includes all the components you use in your configuration. In the future this distribution will become the default image used for the chart. + +You can use the OpenTelemetry Collector Kubernetes Distro by adding these lines to your values.yaml: + +```yaml +image: + repository: "otel/opentelemetry-collector-k8s" +``` + +If you want to stick with using the Contrib distribution, add these lines to your values.yaml: + +```yaml +image: + repository: "otel/opentelemetry-collector-contrib" +``` + +For more details see [#1135](https://github.com/open-telemetry/opentelemetry-helm-charts/issues/1135). + +## 0.84.0 to 0.85.0 + +The `loggingexporter` has been removed from the default configuration. Use the `debugexporter` instead. + +## 0.78.2 to 0.78.3 + +[Update Health Check Extension's endpoints to use Pod IP Instead of 0.0.0.0](https://github.com/open-telemetry/opentelemetry-helm-charts/pull/1012) + +The [Collector's security guidelines were updated](https://github.com/open-telemetry/opentelemetry-collector/pull/6959) to include containerized environments when discussing safeguards against denial of service attacks. +To be in compliance with the Collector's security best practices the chart has been updated to use the Collector's pod IP in place of `0.0.0.0`. + +The chart will continue to allow complete configuration of the Collector via the `config` field in the values.yaml. If pod IP does not suite your needs you can use `config` to set something different. + +See [Security Best Practices docummentation](https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks) for more details. + +## 0.75.1 to 0.76.0 + +Enable the `useGOMEMLIMIT` feature flag by default. This means by default the chart now does not use the Memory Ballast Extension and any custom configuraiton applied to the Memory Ballast Extension is ignored. + +**If you're still interested in using the Memory Ballast Extension set this back to false.** + +## 0.69.3 to 0.70.0 + +The following deprecated fields have been removed. Please use the new values: + +- `extraConfigMapMounts` -> `extraVolumes` +- `extraHostPathMounts` -> `extraVolumes` +- `secretMounts` -> `extraVolumes` +- `containerLogs` -> `presets.logsCollection` + +## 0.69.0 to 0.69.1 & 0.69.2 + +The `loggingexporter` was replaced with the `debugexporter`. This ended up being an accidental breaking change for any user that depended on the default logging exporter config when explicitly listing the logging exporter in an exporter list. + +When using versions `0.69.1` or `0.69.2` you should explicitly list the debugging exporter instead of the logging exporter. You other option is to skip these version and use `0.69.3` or newer, which includes the logging exporter configuration. + +**The logging exporter will be removed in a future version.** We highly recommend switching to the debug exporter. + +## 0.67 to 0.68 + +The `preset.kubernetesEvents` preset now excludes `DELETED` watch types so that an log is not ingested when Kubernetes deletes an event. +The intention behind this change is to cleanup the data ingested by the preset as the `DELETED` updated for a Kubernetes Events is +uninteresting. If you want to keep ingesting `DELETED` updates for Kubernetes Events you will need to configure the `k8sobjectsreceiver` manually. + +## 0.62 to 0.63 + +The `kubernetesAttributes` preset now respects order of processors in logs, metrics and traces pipelines. +This implicitly might break your pipelines if you relied on having the `k8sAttributes` processor rendered as the first processor but also explicitly listed it in the signal's pipeline somewhere else. + +## 0.55.2 to 0.56 + +The `tpl` function has been added to references of pod labels and ingress hosts. This adds the ability to add some reusability in +charts values through referencing global values. If you are currently using any `{{ }}` syntax in pod labels or ingress hosts it will now be rendered. To escape existing instances of {{ }}, use {{` `}}. + +```yaml +global: + region: us-east-1 + environment: stage + +# Tests `tpl` function reference used in pod labels and +# ingress.hosts[*] +podLabels: + environment: "{{ .Values.global.environment }}" + +ingress: + enabled: true + hosts: + - host: "otlp-collector-{{ .Values.global.region }}-{{ .Values.global.environment }}-example.dev" + paths: + - path: / + pathType: Prefix + port: 4318 +``` + +Note that only global Helm values can be referenced as the Helm Chart schema currently does not allow `additionalValues`. + +## 0.55.0 to 0.55.1 + +As of v0.55.1 Collector chart use `${env:ENV}` style syntax when getting environment variables and that $`{env:ENV}` syntax is not supported before collector 0.71. If you upgrade collector chart to v0.55.1, you need to make sure your collector version is after than 0.71 (default is v0.76.1). + +## 0.53.1 to 0.54.0 + +As of v0.54.0 Collector chart, the default resource limits are removed. If you want to keep old values you can use the following configuration: + +``` +resources: + limits: + # CPU units are in fractions of 1000; memory in powers of 2 + cpu: 250m + memory: 512Mi +``` + +See [the 644 issue](https://github.com/open-telemetry/opentelemetry-helm-charts/issues/644) for more information. + +## 0.46.0 to 0.47.0 + +[Update Collector Endpoints to use Pod IP Instead of 0.0.0.0](https://github.com/open-telemetry/opentelemetry-helm-charts/pull/603) + +The [Collector's security guidelines were updated](https://github.com/open-telemetry/opentelemetry-collector/pull/6959) to include containerized environments when discussing safeguards against denial of service attacks. +To be in compliance with the Collector's security best practices the chart has been updated to use the Collector's pod IP in place of `0.0.0.0`. + +The chart will continue to allow complete configuration of the Collector via the `config` field in the values.yaml. If pod IP does not suite your needs you can use `config` to set something different. + +See [Security Best Practices docummentation](https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks) for more details. + +The new default of binding to the pod IP, rather than `0.0.0.0`, will cause `kubectl port-forward` to fail. If port-forwarding is desired, the following `value.yaml` snippet will allow the Collector bind to `127.0.0.1` inside the pod, in addition to the pod's IP: + +```yaml +config: + receivers: + jaeger/local: + protocols: + grpc: + endpoint: 127.0.0.1:14250 + thrift_compact: + endpoint: 127.0.0.1:6831 + thrift_http: + endpoint: 127.0.0.1:14268 + otlp/local: + protocols: + grpc: + endpoint: 127.0.0.1:4317 + http: + endpoint: 127.0.0.1:4318 + zipkin/local: + endpoint: 127.0.0.1:9411 + service: + pipelines: + traces: + receivers: + - otlp + - otlp/local + - jaeger + - jaeger/local + - zipkin + - zipkin/local +``` + +## 0.40.7 to 0.41.0 + +[Require Kubernetes version 1.23 or later](https://github.com/open-telemetry/opentelemetry-helm-charts/pull/541) + +If you enable use of a _HorizontalPodAutoscaler_ for the collector when running in the "deployment" mode by way of `.Values.autoscaling.enabled`, the manifest now uses the "autoscaling/v2" API group version, which [is available only as recently as Kubernetes version 1.23](https://kubernetes.io/blog/2021/12/07/kubernetes-1-23-release-announcement/#horizontalpodautoscaler-v2-graduates-to-ga). As [all previous versions of this API group are deprecated and removed as of Kubernetes version 1.26](https://kubernetes.io/docs/reference/using-api/deprecation-guide/#horizontalpodautoscaler-v126), we don't offer support for Kubernetes versions older than 1.23. + +## 0.34.0 to 0.34.0 + +[config supports templating](TBD) + +The chart now supports templating in `.Values.config`. If you are currently using any `{{ }}` syntax in `.Values.yaml` it will now be rendered. To escape existing instances of `{{ }}`, use ``` {{` `}} ```. For example, `{{ REDACTED_EMAIL }}` becomes ``` {{` {{ REDACTED_EMAIL }} `}} ```. + +## 0.28.0 to 0.29.0 + +[Reduce requested resources](https://github.com/open-telemetry/opentelemetry-helm-charts/pull/273) + +Resource `limits` have been reduced. Upgrades/installs of chart 0.29.0 will now use fewer resources. In order to set the resources back to what they were, you will need to override the `resources` section in the `values.yaml`. + +*Example*: + +```yaml +resources: + limits: + cpu: 1 + memory: 2Gi +``` + +## 0.23.1 to 0.24.0 + +[Remove containerLogs in favor of presets.logsCollection]() + +The ability to enable logs collection from the collector has been moved from `containerLogs.enabled` to `presets.logsCollection.enabled`. If you are currently using `containerLogs.enabled`, you should instead use the preset: + +```yaml +presets: + logsCollection: + enabled: true +``` + +If you are using `containerLogs.enabled` and also enabling collection of the collector logs you can use `includeCollectorLogs` + +```yaml +presets: + logsCollection: + enabled: true + includeCollectorLogs: true +``` + +You no longer need to update `config.service.pipelines.logs` to include the filelog receiver yourself as the preset will automatically update the logs pipeline to include the filelog receiver. + +The filelog's preset configuration can modified by `config.receivers`, but preset configuration cannot be removed. If you need to remove any filelog receiver configuration generated by the preset you should not use the preset. Instead, configure the filelog receiver manually in `config.receivers` and set any other necessary fields in the values.yaml to modify k8s as needed. + +See the [daemonset-collector-logs example](https://github.com/open-telemetry/opentelemetry-helm-charts/tree/main/charts/opentelemetry-collector/examples/daemonset-collector-logs) to see an example of the preset in action. + +## 0.18.0 to 0.19.0 + +[Remove agentCollector and standaloneCollector settings](https://github.com/open-telemetry/opentelemetry-helm-charts/pull/216) + +The `agentCollector` and `standaloneCollector` config sections have been removed. Upgrades/installs of chart 0.19.0 will fail if `agentCollector` or `standaloneCollector` are in the values.yaml. See the [Migrate to mode](#migrate-to-mode) steps for instructions on how to replace `agentCollector` and `standaloneCollector` with `mode`. + +## 0.13.0 to 0.14.0 + +[Remove two-deployment mode](https://github.com/open-telemetry/opentelemetry-helm-charts/pull/159) + +The ability to install both the agent and standalone collectors simultaneous with the chart has been removed. Installs/upgrades where both `.Values.agentCollector.enabled` and `.Values.standloneCollector.enables` are true will fail. `agentCollector` and `standloneCollector` have also be deprecated, but backward compatibility has been maintained. + +### To run both a deployment and daemonset + +Install a deployment version of the collector. This is done by setting `.Values.mode` to `deployment` + +```yaml +mode: deployment +``` + +Next, install an daemonset version of the collector that is configured to send traffic to the previously installed deployment. This is done by setting `.Values.mode` to `daemonset` and updating `.Values.config` so that data is exported to the deployment. + +```yaml +mode: daemonset + +config: + exporters: + otlp: + endpoint: example-opentelemetry-collector:4317 + tls: + insecure: true + service: + pipelines: + logs: + exporters: + - otlp + - logging + metrics: + exporters: + - otlp + - logging + traces: + exporters: + - otlp + - logging +``` + +See the [daemonset-and-deployment](examples/daemonset-and-deployment) example to see the rendered config. + +### Migrate to `mode`: + +The `agentCollector` and `standaloneCollector` sections in values.yaml have been deprecated. Instead there is a new field, `mode`, that determines if the collector is being installed as a daemonset or deployment. + +```yaml +# Valid values are "daemonset" and "deployment". +# If set, agentCollector and standaloneCollector are ignored. +mode: +``` + +The following fields have also been added to the root-level to replace the depracated `agentCollector` and `standaloneCollector` settings. + +```yaml +containerLogs: + enabled: false + +resources: + limits: + cpu: 1 + memory: 2Gi + +podAnnotations: {} + +podLabels: {} + +# Host networking requested for this pod. Use the host's network namespace. +hostNetwork: false + +# only used with deployment mode +replicaCount: 1 + +annotations: {} +``` + +When using `mode`, these settings should be used instead of their counterparts in `agentCollector` and `standaloneCollector`. + +Set `mode` to `daemonset` if `agentCollector` was being used. Move all `agentCollector` settings to the corresponding root-level setting. If `agentCollector.configOverride` was being used, merge the settings with `.Values.config`. + +Example agentCollector values.yaml: + +```yaml +agentCollector: + resources: + limits: + cpu: 3 + memory: 6Gi + configOverride: + receivers: + hostmetrics: + scrapers: + cpu: + disk: + filesystem: + service: + pipelines: + metrics: + receivers: [otlp, prometheus, hostmetrics] +``` + +Example mode values.yaml: + +```yaml +mode: daemonset + +resources: + limits: + cpu: 3 + memory: 6Gi + +config: + receivers: + hostmetrics: + scrapers: + cpu: + disk: + filesystem: + service: + pipelines: + metrics: + receivers: [otlp, prometheus, hostmetrics] +``` + +Set `mode` to `deployment` if `standaloneCollector` was being used. Move all `standaloneCollector` settings to the corresponding root-level setting. If `standaloneCollector.configOverride` was being used, merge the settings with `.Values.config`. + +Example standaloneCollector values.yaml: + +```yaml +standaloneCollector: + enabled: true + replicaCount: 2 + configOverride: + receivers: + podman_stats: + endpoint: unix://run/podman/podman.sock + timeout: 10s + collection_interval: 10s + service: + pipelines: + metrics: + receivers: [otlp, prometheus, podman_stats] +``` + +Example mode values.yaml: + +```yaml +mode: deployment + +replicaCount: 2 + +config: + receivers: + receivers: + podman_stats: + endpoint: unix://run/podman/podman.sock + timeout: 10s + collection_interval: 10s + service: + pipelines: + metrics: + receivers: [otlp, prometheus, podman_stats] +``` + +Default configuration in `.Values.config` can now be removed with `null`. When changing a pipeline, you must explicitly list all the components that are in the pipeline, including any default components. + +*Example*: Disable metrics and logging pipelines and non-otlp receivers: + +```yaml +config: + receivers: + jaeger: null + prometheus: null + zipkin: null + service: + pipelines: + traces: + receivers: + - otlp + metrics: null + logs: null +``` diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/GOMEMLIMIT-values.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/GOMEMLIMIT-values.yaml new file mode 100644 index 0000000..cc0570f --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/GOMEMLIMIT-values.yaml @@ -0,0 +1,9 @@ +mode: deployment + +image: + repository: "otel/opentelemetry-collector-k8s" + +command: + name: "otelcol-k8s" + +useGOMEMLIMIT: true diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/clusterrole-values.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/clusterrole-values.yaml new file mode 100644 index 0000000..01acd44 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/clusterrole-values.yaml @@ -0,0 +1,27 @@ +mode: daemonset + +image: + repository: "otel/opentelemetry-collector-k8s" + +command: + name: "otelcol-k8s" + +clusterRole: + create: true + name: "testing-clusterrole" + rules: + - apiGroups: + - '' + resources: + - 'pods' + - 'nodes' + verbs: + - 'get' + - 'list' + - 'watch' + clusterRoleBinding: + name: "testing-clusterrolebinding" +resources: + limits: + cpu: 100m + memory: 200M diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/config-override-values.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/config-override-values.yaml new file mode 100644 index 0000000..577cf4d --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/config-override-values.yaml @@ -0,0 +1,26 @@ +mode: daemonset + +image: + repository: "otel/opentelemetry-collector-k8s" + +command: + name: "otelcol-k8s" + +config: + receivers: + jaeger: null + otlp: null + zipkin: null + hostmetrics: + scrapers: + cpu: + disk: + filesystem: + service: + pipelines: + metrics: + receivers: + - prometheus + - hostmetrics + traces: null + logs: null diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/daemonset-values.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/daemonset-values.yaml new file mode 100644 index 0000000..0f1d619 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/daemonset-values.yaml @@ -0,0 +1,12 @@ +mode: daemonset + +image: + repository: "otel/opentelemetry-collector-k8s" + +command: + name: "otelcol-k8s" + +resources: + limits: + cpu: 100m + memory: 200M diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/deployment-values.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/deployment-values.yaml new file mode 100644 index 0000000..447a3f1 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/deployment-values.yaml @@ -0,0 +1,29 @@ +global: + test: templated-value + +mode: deployment + +image: + repository: "otel/opentelemetry-collector-k8s" + +command: + name: "otelcol-k8s" + +resources: + limits: + cpu: 100m + memory: 200M + +# Tests `tpl` function reference used in pod labels and +# ingress.hosts[*] +podLabels: + testLabel: "{{ .Values.global.test }}" + +ingress: + enabled: true + hosts: + - host: "{{ .Values.global.test }}" + paths: + - path: / + pathType: Prefix + port: 4318 diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/disabling-protocols-values.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/disabling-protocols-values.yaml new file mode 100644 index 0000000..b7a67bd --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/disabling-protocols-values.yaml @@ -0,0 +1,21 @@ +mode: deployment + +image: + repository: "otel/opentelemetry-collector-k8s" + +command: + name: "otelcol-k8s" + +ports: + jaeger-compact: + enabled: false + jaeger-thrift: + enabled: false + jaeger-grpc: + enabled: false + zipkin: + enabled: false +resources: + limits: + cpu: 100m + memory: 200M diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/hpa-deployment-values.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/hpa-deployment-values.yaml new file mode 100644 index 0000000..3fe3853 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/hpa-deployment-values.yaml @@ -0,0 +1,15 @@ +mode: deployment + +image: + repository: "otel/opentelemetry-collector-k8s" + +command: + name: "otelcol-k8s" + +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 10 + behavior: {} + targetCPUUtilizationPercentage: 80 + targetMemoryUtilizationPercentage: 80 diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/hpa-statefulset-values.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/hpa-statefulset-values.yaml new file mode 100644 index 0000000..cbf3c60 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/hpa-statefulset-values.yaml @@ -0,0 +1,15 @@ +mode: statefulset + +image: + repository: "otel/opentelemetry-collector-k8s" + +command: + name: "otelcol-k8s" + +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 10 + behavior: {} + targetCPUUtilizationPercentage: 80 + targetMemoryUtilizationPercentage: 80 diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/multiple-ingress-values.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/multiple-ingress-values.yaml new file mode 100644 index 0000000..4b9df75 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/multiple-ingress-values.yaml @@ -0,0 +1,49 @@ +mode: deployment + +image: + repository: "otel/opentelemetry-collector-k8s" + +command: + name: "otelcol-k8s" + +resources: + limits: + cpu: 100m + memory: 200M + +ingress: + enabled: true + + ingressClassName: nginx + annotations: + test.io/collector: default + hosts: + - host: defaultcollector.example.com + paths: + - path: / + pathType: Prefix + port: 4318 + + additionalIngresses: + - name: additional-basic + hosts: + - host: additional-basic.example.com + paths: + - path: / + pathType: Prefix + port: 4318 + + - name: additional-advanced + ingressClassName: nginx + annotations: + test.io/ingress: additional-advanced + hosts: + - host: additional-advanced.example.com + paths: + - path: / + pathType: Exact + port: 4318 + tls: + - secretName: somesecret + hosts: + - additional-advanced.example.com diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/networkpolicy-override-values.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/networkpolicy-override-values.yaml new file mode 100644 index 0000000..d08d368 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/networkpolicy-override-values.yaml @@ -0,0 +1,37 @@ +mode: daemonset + +image: + repository: "otel/opentelemetry-collector-k8s" + +command: + name: "otelcol-k8s" + +resources: + limits: + cpu: 100m + memory: 200M + +networkPolicy: + enabled: true + + allowIngressFrom: + - namespaceSelector: {} + - ipBlock: + cidr: 127.0.0.1/32 + + extraIngressRules: + - ports: + - port: metrics + protocol: TCP + from: + - ipBlock: + cidr: 127.0.0.1/32 + + egressRules: + - to: + - podSelector: + matchLabels: + app: jaeger + ports: + - port: 4317 + protocol: TCP diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/networkpolicy-values.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/networkpolicy-values.yaml new file mode 100644 index 0000000..7e80cb4 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/networkpolicy-values.yaml @@ -0,0 +1,15 @@ +mode: deployment + +image: + repository: "otel/opentelemetry-collector-k8s" + +command: + name: "otelcol-k8s" + +resources: + limits: + cpu: 100m + memory: 200M + +networkPolicy: + enabled: true diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/preset-clustermetrics-values.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/preset-clustermetrics-values.yaml new file mode 100644 index 0000000..8c21aeb --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/preset-clustermetrics-values.yaml @@ -0,0 +1,16 @@ +mode: deployment + +image: + repository: "otel/opentelemetry-collector-k8s" + +command: + name: "otelcol-k8s" + +presets: + clusterMetrics: + enabled: true + +resources: + limits: + cpu: 100m + memory: 200M diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/preset-hostmetrics-values.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/preset-hostmetrics-values.yaml new file mode 100644 index 0000000..4889c01 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/preset-hostmetrics-values.yaml @@ -0,0 +1,16 @@ +mode: daemonset + +image: + repository: "otel/opentelemetry-collector-k8s" + +command: + name: "otelcol-k8s" + +presets: + hostMetrics: + enabled: true + +resources: + limits: + cpu: 100m + memory: 200M diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/preset-k8sevents-values.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/preset-k8sevents-values.yaml new file mode 100644 index 0000000..6578291 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/preset-k8sevents-values.yaml @@ -0,0 +1,16 @@ +mode: deployment + +image: + repository: "otel/opentelemetry-collector-k8s" + +command: + name: "otelcol-k8s" + +presets: + kubernetesEvents: + enabled: true + +resources: + limits: + cpu: 100m + memory: 200M diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/preset-kubeletmetrics-values.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/preset-kubeletmetrics-values.yaml new file mode 100644 index 0000000..83dd9ca --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/preset-kubeletmetrics-values.yaml @@ -0,0 +1,16 @@ +mode: daemonset + +image: + repository: "otel/opentelemetry-collector-k8s" + +command: + name: "otelcol-k8s" + +presets: + kubeletMetrics: + enabled: true + +resources: + limits: + cpu: 100m + memory: 200M diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/preset-kubernetesattributes-values.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/preset-kubernetesattributes-values.yaml new file mode 100644 index 0000000..ef32ac9 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/preset-kubernetesattributes-values.yaml @@ -0,0 +1,16 @@ +mode: daemonset + +image: + repository: "otel/opentelemetry-collector-k8s" + +command: + name: "otelcol-k8s" + +presets: + kubernetesAttributes: + enabled: true + +resources: + limits: + cpu: 100m + memory: 200M diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/preset-logscollection-values.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/preset-logscollection-values.yaml new file mode 100644 index 0000000..1dd9a8e --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/preset-logscollection-values.yaml @@ -0,0 +1,17 @@ +mode: daemonset + +image: + repository: "otel/opentelemetry-collector-k8s" + +command: + name: "otelcol-k8s" + +presets: + logsCollection: + enabled: true + includeCollectorLogs: true + +resources: + limits: + cpu: 100m + memory: 200M diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/probes-values.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/probes-values.yaml new file mode 100644 index 0000000..831f9ef --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/probes-values.yaml @@ -0,0 +1,27 @@ +mode: daemonset + +image: + repository: "otel/opentelemetry-collector-k8s" + +command: + name: "otelcol-k8s" + +livenessProbe: + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 2 + terminationGracePeriodSeconds: 40 + httpGet: + port: 8989 + path: /healthz + +readinessProbe: + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + successThreshold: 2 + failureThreshold: 2 + httpGet: + port: 8989 + path: /healthz diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/statefulset-values.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/statefulset-values.yaml new file mode 100644 index 0000000..76de30c --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/ci/statefulset-values.yaml @@ -0,0 +1,13 @@ +mode: statefulset + +image: + repository: "otel/opentelemetry-collector-k8s" + +command: + name: "otelcol-k8s" + +replicaCount: 2 +resources: + limits: + cpu: 100m + memory: 200M diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/README.md b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/README.md new file mode 100644 index 0000000..8f2ac4c --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/README.md @@ -0,0 +1,17 @@ +# Examples of chart configuration + +Here is a collection of common configurations for the OpenTelemetry collector. Each folder contains an example `values.yaml` and the resulting configurations that are generated by the opentelemetry-collector helm charts. + +- [Daemonset only](daemonset-only) +- [Deployment only](deployment-only) +- [Daemonset and deployment](daemonset-and-deployment) +- [Log collection, including collector logs](daemonset-collector-logs) +- [Add component (hostmetrics)](daemonset-hostmetrics) + +The manifests are rendered using the `helm template` command and the specific example folder's values.yaml. + +Examples are generated by (from root of the repo): + +```sh +make generate-examples CHARTS=opentelemetry-collector +``` diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/daemonset-values.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/daemonset-values.yaml new file mode 100644 index 0000000..5b21b7b --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/daemonset-values.yaml @@ -0,0 +1,34 @@ +mode: daemonset + +image: + repository: "otel/opentelemetry-collector-k8s" + +command: + name: "otelcol-k8s" + +config: + exporters: + otlp: + endpoint: example-opentelemetry-collector:4317 + tls: + insecure: true + service: + pipelines: + logs: + exporters: + - otlp + - debug + metrics: + exporters: + - otlp + - debug + traces: + exporters: + - otlp + - debug + +resources: + limits: + cpu: 100m + memory: 200M + diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/deployment-values.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/deployment-values.yaml new file mode 100644 index 0000000..1c80d15 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/deployment-values.yaml @@ -0,0 +1,13 @@ +mode: deployment + +image: + repository: "otel/opentelemetry-collector-k8s" + +command: + name: "otelcol-k8s" + +resources: + limits: + cpu: 100m + memory: 200M + diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/rendered/configmap-agent.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/rendered/configmap-agent.yaml new file mode 100644 index 0000000..cf354e8 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/rendered/configmap-agent.yaml @@ -0,0 +1,93 @@ +--- +# Source: opentelemetry-collector/templates/configmap-agent.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: example-opentelemetry-collector-agent + namespace: default + labels: + helm.sh/chart: opentelemetry-collector-0.93.3 + app.kubernetes.io/name: opentelemetry-collector + app.kubernetes.io/instance: example + app.kubernetes.io/version: "0.102.1" + app.kubernetes.io/managed-by: Helm + +data: + relay: | + exporters: + debug: {} + otlp: + endpoint: example-opentelemetry-collector:4317 + tls: + insecure: true + extensions: + health_check: + endpoint: ${env:MY_POD_IP}:13133 + processors: + batch: {} + memory_limiter: + check_interval: 5s + limit_percentage: 80 + spike_limit_percentage: 25 + receivers: + jaeger: + protocols: + grpc: + endpoint: ${env:MY_POD_IP}:14250 + thrift_compact: + endpoint: ${env:MY_POD_IP}:6831 + thrift_http: + endpoint: ${env:MY_POD_IP}:14268 + otlp: + protocols: + grpc: + endpoint: ${env:MY_POD_IP}:4317 + http: + endpoint: ${env:MY_POD_IP}:4318 + prometheus: + config: + scrape_configs: + - job_name: opentelemetry-collector + scrape_interval: 10s + static_configs: + - targets: + - ${env:MY_POD_IP}:8888 + zipkin: + endpoint: ${env:MY_POD_IP}:9411 + service: + extensions: + - health_check + pipelines: + logs: + exporters: + - otlp + - debug + processors: + - memory_limiter + - batch + receivers: + - otlp + metrics: + exporters: + - otlp + - debug + processors: + - memory_limiter + - batch + receivers: + - otlp + - prometheus + traces: + exporters: + - otlp + - debug + processors: + - memory_limiter + - batch + receivers: + - otlp + - jaeger + - zipkin + telemetry: + metrics: + address: ${env:MY_POD_IP}:8888 diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/rendered/configmap.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/rendered/configmap.yaml new file mode 100644 index 0000000..81705f7 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/rendered/configmap.yaml @@ -0,0 +1,86 @@ +--- +# Source: opentelemetry-collector/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: example-opentelemetry-collector + namespace: default + labels: + helm.sh/chart: opentelemetry-collector-0.93.3 + app.kubernetes.io/name: opentelemetry-collector + app.kubernetes.io/instance: example + app.kubernetes.io/version: "0.102.1" + app.kubernetes.io/managed-by: Helm + +data: + relay: | + exporters: + debug: {} + extensions: + health_check: + endpoint: ${env:MY_POD_IP}:13133 + processors: + batch: {} + memory_limiter: + check_interval: 5s + limit_percentage: 80 + spike_limit_percentage: 25 + receivers: + jaeger: + protocols: + grpc: + endpoint: ${env:MY_POD_IP}:14250 + thrift_compact: + endpoint: ${env:MY_POD_IP}:6831 + thrift_http: + endpoint: ${env:MY_POD_IP}:14268 + otlp: + protocols: + grpc: + endpoint: ${env:MY_POD_IP}:4317 + http: + endpoint: ${env:MY_POD_IP}:4318 + prometheus: + config: + scrape_configs: + - job_name: opentelemetry-collector + scrape_interval: 10s + static_configs: + - targets: + - ${env:MY_POD_IP}:8888 + zipkin: + endpoint: ${env:MY_POD_IP}:9411 + service: + extensions: + - health_check + pipelines: + logs: + exporters: + - debug + processors: + - memory_limiter + - batch + receivers: + - otlp + metrics: + exporters: + - debug + processors: + - memory_limiter + - batch + receivers: + - otlp + - prometheus + traces: + exporters: + - debug + processors: + - memory_limiter + - batch + receivers: + - otlp + - jaeger + - zipkin + telemetry: + metrics: + address: ${env:MY_POD_IP}:8888 diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/rendered/daemonset.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/rendered/daemonset.yaml new file mode 100644 index 0000000..04856b7 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/rendered/daemonset.yaml @@ -0,0 +1,104 @@ +--- +# Source: opentelemetry-collector/templates/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: example-opentelemetry-collector-agent + namespace: default + labels: + helm.sh/chart: opentelemetry-collector-0.93.3 + app.kubernetes.io/name: opentelemetry-collector + app.kubernetes.io/instance: example + app.kubernetes.io/version: "0.102.1" + app.kubernetes.io/managed-by: Helm + +spec: + selector: + matchLabels: + app.kubernetes.io/name: opentelemetry-collector + app.kubernetes.io/instance: example + component: agent-collector + updateStrategy: + type: RollingUpdate + template: + metadata: + annotations: + checksum/config: f1c8c67d823932ab369e0e3284ebfe7763446d7af037aed7c7c7a3a3ab5d7e96 + + labels: + app.kubernetes.io/name: opentelemetry-collector + app.kubernetes.io/instance: example + component: agent-collector + + spec: + + serviceAccountName: example-opentelemetry-collector + securityContext: + {} + containers: + - name: opentelemetry-collector + command: + - /otelcol-k8s + args: + - --config=/conf/relay.yaml + securityContext: + {} + image: "otel/opentelemetry-collector-k8s:0.102.1" + imagePullPolicy: IfNotPresent + ports: + + - name: jaeger-compact + containerPort: 6831 + protocol: UDP + hostPort: 6831 + - name: jaeger-grpc + containerPort: 14250 + protocol: TCP + hostPort: 14250 + - name: jaeger-thrift + containerPort: 14268 + protocol: TCP + hostPort: 14268 + - name: otlp + containerPort: 4317 + protocol: TCP + hostPort: 4317 + - name: otlp-http + containerPort: 4318 + protocol: TCP + hostPort: 4318 + - name: zipkin + containerPort: 9411 + protocol: TCP + hostPort: 9411 + env: + - name: MY_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: GOMEMLIMIT + value: "152MiB" + livenessProbe: + httpGet: + path: / + port: 13133 + readinessProbe: + httpGet: + path: / + port: 13133 + resources: + limits: + cpu: 100m + memory: 200M + volumeMounts: + - mountPath: /conf + name: opentelemetry-collector-configmap + volumes: + - name: opentelemetry-collector-configmap + configMap: + name: example-opentelemetry-collector-agent + items: + - key: relay + path: relay.yaml + hostNetwork: false diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/rendered/deployment.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/rendered/deployment.yaml new file mode 100644 index 0000000..043eee9 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/rendered/deployment.yaml @@ -0,0 +1,100 @@ +--- +# Source: opentelemetry-collector/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: example-opentelemetry-collector + namespace: default + labels: + helm.sh/chart: opentelemetry-collector-0.93.3 + app.kubernetes.io/name: opentelemetry-collector + app.kubernetes.io/instance: example + app.kubernetes.io/version: "0.102.1" + app.kubernetes.io/managed-by: Helm + +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/name: opentelemetry-collector + app.kubernetes.io/instance: example + component: standalone-collector + strategy: + type: RollingUpdate + template: + metadata: + annotations: + checksum/config: 753d31810a0a1ab3a69b9f1f78bb692c23fd5b10e0cc3c4029fb5bf9fc9080e7 + + labels: + app.kubernetes.io/name: opentelemetry-collector + app.kubernetes.io/instance: example + component: standalone-collector + + spec: + + serviceAccountName: example-opentelemetry-collector + securityContext: + {} + containers: + - name: opentelemetry-collector + command: + - /otelcol-k8s + args: + - --config=/conf/relay.yaml + securityContext: + {} + image: "otel/opentelemetry-collector-k8s:0.102.1" + imagePullPolicy: IfNotPresent + ports: + + - name: jaeger-compact + containerPort: 6831 + protocol: UDP + - name: jaeger-grpc + containerPort: 14250 + protocol: TCP + - name: jaeger-thrift + containerPort: 14268 + protocol: TCP + - name: otlp + containerPort: 4317 + protocol: TCP + - name: otlp-http + containerPort: 4318 + protocol: TCP + - name: zipkin + containerPort: 9411 + protocol: TCP + env: + - name: MY_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: GOMEMLIMIT + value: "152MiB" + livenessProbe: + httpGet: + path: / + port: 13133 + readinessProbe: + httpGet: + path: / + port: 13133 + resources: + limits: + cpu: 100m + memory: 200M + volumeMounts: + - mountPath: /conf + name: opentelemetry-collector-configmap + volumes: + - name: opentelemetry-collector-configmap + configMap: + name: example-opentelemetry-collector + items: + - key: relay + path: relay.yaml + hostNetwork: false diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/rendered/service.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/rendered/service.yaml new file mode 100644 index 0000000..47135ee --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/rendered/service.yaml @@ -0,0 +1,49 @@ +--- +# Source: opentelemetry-collector/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: example-opentelemetry-collector + namespace: default + labels: + helm.sh/chart: opentelemetry-collector-0.93.3 + app.kubernetes.io/name: opentelemetry-collector + app.kubernetes.io/instance: example + app.kubernetes.io/version: "0.102.1" + app.kubernetes.io/managed-by: Helm + + component: standalone-collector +spec: + type: ClusterIP + ports: + + - name: jaeger-compact + port: 6831 + targetPort: 6831 + protocol: UDP + - name: jaeger-grpc + port: 14250 + targetPort: 14250 + protocol: TCP + - name: jaeger-thrift + port: 14268 + targetPort: 14268 + protocol: TCP + - name: otlp + port: 4317 + targetPort: 4317 + protocol: TCP + appProtocol: grpc + - name: otlp-http + port: 4318 + targetPort: 4318 + protocol: TCP + - name: zipkin + port: 9411 + targetPort: 9411 + protocol: TCP + selector: + app.kubernetes.io/name: opentelemetry-collector + app.kubernetes.io/instance: example + component: standalone-collector + internalTrafficPolicy: Cluster diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/rendered/serviceaccount.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/rendered/serviceaccount.yaml new file mode 100644 index 0000000..54eaff2 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-and-deployment/rendered/serviceaccount.yaml @@ -0,0 +1,13 @@ +--- +# Source: opentelemetry-collector/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: example-opentelemetry-collector + namespace: default + labels: + helm.sh/chart: opentelemetry-collector-0.93.3 + app.kubernetes.io/name: opentelemetry-collector + app.kubernetes.io/instance: example + app.kubernetes.io/version: "0.102.1" + app.kubernetes.io/managed-by: Helm diff --git a/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-collector-logs/rendered/configmap-agent.yaml b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-collector-logs/rendered/configmap-agent.yaml new file mode 100644 index 0000000..4542672 --- /dev/null +++ b/charts/nemo/charts/nemo-customizer/charts/opentelemetry-collector/examples/daemonset-collector-logs/rendered/configmap-agent.yaml @@ -0,0 +1,166 @@ +--- +# Source: opentelemetry-collector/templates/configmap-agent.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: example-opentelemetry-collector-agent + namespace: default + labels: + helm.sh/chart: opentelemetry-collector-0.93.3 + app.kubernetes.io/name: opentelemetry-collector + app.kubernetes.io/instance: example + app.kubernetes.io/version: "0.102.1" + app.kubernetes.io/managed-by: Helm + +data: + relay: | + exporters: + debug: {} + extensions: + health_check: + endpoint: ${env:MY_POD_IP}:13133 + processors: + batch: {} + memory_limiter: + check_interval: 5s + limit_percentage: 80 + spike_limit_percentage: 25 + receivers: + filelog: + exclude: [] + include: + - /var/log/pods/*/*/*.log + include_file_name: false + include_file_path: true + operators: + - id: get-format + routes: + - expr: body matches "^\\{" + output: parser-docker + - expr: body matches "^[^ Z]+ " + output: parser-crio + - expr: body matches "^[^ Z]+Z" + output: parser-containerd + type: router + - id: parser-crio + regex: ^(?P