You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
282 lines
6.6 KiB
282 lines
6.6 KiB
---
|
|
imagePullSecret: "nvidia-ngcuser-pull-secret"
|
|
|
|
enabledComponents:
|
|
dgxcAdmissionController: true
|
|
dgxcRunaiController: true
|
|
dgxcExporter: true
|
|
|
|
# The storage controller is the thing we're running
|
|
dgxcController:
|
|
name: dgxc-admission-controller
|
|
replicas: 3
|
|
image: "nvcr.io/nv-ngc-devops/dgxc-admission-controller:latest"
|
|
# shared between the init container and the controller container
|
|
certs:
|
|
mountPath: "/tmp/k8s-webhook-server/serving-certs"
|
|
certName: tls.crt
|
|
keyName: tls.key
|
|
|
|
nvStorage:
|
|
# NVStorageLocation CRD management
|
|
enabled: false
|
|
storageClass: ""
|
|
csi: {}
|
|
|
|
serviceName: dgx-controller-svc
|
|
serviceAccount:
|
|
create: true
|
|
name: dgxc-admission-controller-sa
|
|
annotations: {}
|
|
|
|
controllerServiceAccount:
|
|
create: true
|
|
name: dgxc-runai-controller
|
|
annotations: {}
|
|
|
|
webhooks:
|
|
organization: dgxc.nvidia.com
|
|
registeredWebhooks:
|
|
- name: dgxc-create-pvc-vwh
|
|
type: PvcValidatingWebhook
|
|
cmName: dgxc-create-pvc-vwh-config
|
|
enabled: true
|
|
- name: dgxc-create-pvc-mwh
|
|
type: PvcMutatingWebhook
|
|
cmName: dgxc-create-pvc-mwh-config
|
|
enabled: true
|
|
- name: dgxc-create-pod-mwh
|
|
type: PodMutatingWebhook
|
|
cmName: dgxc-create-pod-mwh-config
|
|
enabled: true
|
|
|
|
pvcValidatingWebhook:
|
|
cmName: dgxc-create-pvc-vwh-config
|
|
type: PvcValidatingWebhook
|
|
kind: ValidatingWebhookConfiguration
|
|
fqdn: dgxc-create-pvc-vwh.dgxc.nvidia.com
|
|
path: "/pvccreate/v1/validation"
|
|
failurePolicy: Fail
|
|
timeoutSeconds: 10
|
|
sideEffects: NoneOnDryRun
|
|
admissionReviewVersions:
|
|
- v1
|
|
rules:
|
|
- apiGroups:
|
|
- ""
|
|
apiVersions:
|
|
- "v1"
|
|
operations:
|
|
- CREATE
|
|
resources:
|
|
- persistentvolumeclaims
|
|
scope: "*"
|
|
namespaceLabelSelector:
|
|
- key: runai/namespace-version
|
|
values:
|
|
- "v1"
|
|
- "v2"
|
|
objectLabelSelector:
|
|
customSettings:
|
|
- name: filestore-pvc-premium-rule
|
|
storageClasses:
|
|
- zonal-rwx
|
|
minSizeCheckEnabled: false
|
|
minSizeValue: 10Ti
|
|
maxSizeCheckEnabled: false
|
|
maxSizeValue: 100Ti
|
|
- name: filestore-pvc-standard-rule
|
|
storageClasses:
|
|
- standard-rwx
|
|
minSizeCheckEnabled: false
|
|
minSizeValue: 1Ti
|
|
maxSizeCheckEnabled: false
|
|
maxSizeValue: 10Ti
|
|
pvcMutatingWebhook:
|
|
cmName: dgxc-create-pvc-mwh-config
|
|
type: PvcMutatingWebhook
|
|
kind: MutatingWebhookConfiguration
|
|
fqdn: dgxc-create-pvc-mwh.dgxc.nvidia.com
|
|
path: "/pvccreate/v1/mutation"
|
|
failurePolicy: Fail
|
|
timeoutSeconds: 10
|
|
sideEffects: NoneOnDryRun
|
|
admissionReviewVersions:
|
|
- v1
|
|
rules:
|
|
- apiGroups:
|
|
- ""
|
|
apiVersions:
|
|
- "v1"
|
|
operations:
|
|
- CREATE
|
|
resources:
|
|
- persistentvolumeclaims
|
|
scope: "*"
|
|
namespaceLabelSelector:
|
|
- key: runai/namespace-version
|
|
values:
|
|
- "v1"
|
|
- "v2"
|
|
objectLabelSelector:
|
|
|
|
podMutatingWebhook:
|
|
cmName: dgxc-create-pod-mwh-config
|
|
kind: MutatingWebhookConfiguration
|
|
type: PodMutatingWebhook
|
|
fqdn: dgxc-create-pod-mwh.dgxc.mvp.nvidia.com
|
|
path: "/podcreate/v1/mutation"
|
|
failurePolicy: Fail
|
|
timeoutSeconds: 10
|
|
sideEffects: NoneOnDryRun
|
|
admissionReviewVersions:
|
|
- v1
|
|
rules:
|
|
- apiGroups:
|
|
- ""
|
|
apiVersions:
|
|
- "v1"
|
|
operations:
|
|
- CREATE
|
|
resources:
|
|
- pods
|
|
scope: "*"
|
|
namespaceLabelSelector:
|
|
objectLabelSelector:
|
|
customSettings:
|
|
- name: modify-pod-rule
|
|
scratchEnabled: true
|
|
scratchVolName: "scratch"
|
|
scratchMountpath: "/scratch"
|
|
scratchVolSize: "200Gi"
|
|
scratchVolSizePerGpu: true
|
|
scratchVolEnforceLimit: true
|
|
addTcpxSidecarEnabled: true
|
|
addTcpxoSidecarEnabled: false
|
|
clusterName: "test"
|
|
addTcpxNCCLVars: true
|
|
addTcpxoNCCLVars: false
|
|
addRdmaEnabled: false
|
|
addRoceOciNcclVars: false
|
|
addGcpFuseAnnotation: true
|
|
gcpFuseAnnotationKey: "gke-gcsfuse/volumes"
|
|
gcpFuseWICredentialConfigMapAnnotationKey: "gke-gcsfuse/workload-identity-credential-configmap" # This is the annotation key for the ConfigMap that contains the GCP Workload Identity credentials.
|
|
gcpFuseStorageClasses:
|
|
- dgxc-standard-object
|
|
prober:
|
|
intervalInSeconds: 60
|
|
timeoutInSeconds: 5
|
|
endpoints:
|
|
- name: dev-cluster
|
|
url: https://dgxc-runai-us-east5-ngc-11918940.dgxc.nonprod-nvkong.com
|
|
|
|
sso:
|
|
controlPlaneDomain: run.ai
|
|
cluster: "TODO"
|
|
realm: dgxc
|
|
pollIntervalSeconds: 10
|
|
pollAttempts: 180 # 30 minutes
|
|
|
|
initSettings:
|
|
enabled: false
|
|
appID: dgxc
|
|
tenant: dgxc
|
|
controlPlane: run.ai
|
|
supportEmailAddress: support@nvidia.com
|
|
customerEmailAddress: customer@nvidia.com
|
|
bootstrapEmailAddress: bootstrap@nvidia.com
|
|
|
|
crdRBACName: "{{request.object.spec.names.plural}}"
|
|
|
|
validationFailureAction: Enforce
|
|
|
|
tenantSubjects:
|
|
User:
|
|
- ebalduf@test.nvidia.com
|
|
- kelhussein@test.nvidia.com
|
|
- ladithyav@test.nvidia.com
|
|
- pwittrock@test.nvidia.com
|
|
|
|
tolerations:
|
|
customer:
|
|
key: "dedicated"
|
|
value: "user-workload"
|
|
system:
|
|
key: "components.gke.io/gke-managed-components"
|
|
value: "true"
|
|
|
|
bootstrapIngressCIDR: [ "216.228.125.128/30", "12.97.180.32/27", "216.228.127.128/30", "202.164.25.0/27", "120.31.162.32/28", "203.18.50.0/27", "103.74.125.160/28", "216.228.112.0/26" ]
|
|
|
|
dgxcAdmins:
|
|
- Run:ai
|
|
- svctdcsqa1@nvidia.com
|
|
|
|
reserved:
|
|
CPU: 2
|
|
Memory: 5Gi
|
|
|
|
maxCPUJobs: 10
|
|
|
|
logExporterControllerSpec:
|
|
enabled: false
|
|
exportFrequency: 5m
|
|
exportBuffer: 15m
|
|
fetchTimeout: 5m
|
|
|
|
networkControllerSpec:
|
|
enabled: false
|
|
|
|
storageControllerSpecGCP:
|
|
enabled: false
|
|
|
|
storageControllerSpecAWS:
|
|
enabled: false
|
|
|
|
storageControllerSpecOCI:
|
|
enabled: false
|
|
|
|
storageControllerSpecAzure:
|
|
enabled: false
|
|
|
|
exporter:
|
|
metricsPort: 8080
|
|
|
|
sre:
|
|
group:
|
|
default: "non-existing-engineer@nvidia.com"
|
|
privileged: "non-existing-admin@nvidia.com"
|
|
|
|
tenantMetadata:
|
|
cluster: TODO
|
|
platform: runai
|
|
orgName: nvidian
|
|
tenantName: test-mega
|
|
tenantType: Internal
|
|
startDate: '8/02/24'
|
|
endDate: '01/01/99'
|
|
CSP: gcp
|
|
allocatedCapacityGPU: 2
|
|
allocatedCapacityCPU: 2
|
|
committedCapacityGPU: 2
|
|
committedCapacityCPU: 2
|
|
CSPProject: proj-dgxc-runai-np-test-mega
|
|
dataCenterRegion: us-central1
|
|
storageType: GCPFilestore
|
|
storageQuota: 1
|
|
|
|
cspSubjects:
|
|
User:
|
|
- "107878228028626601324"
|
|
|
|
network:
|
|
systemCIDRs:
|
|
- "10.0.3.0/24"
|
|
- "10.0.4.0/24"
|
|
|
|
enableIntegrationsSupport: true
|
|
enableLabeler: true
|
|
enableRunAIRbac: true
|
|
enableNetworkPolicies: true
|
|
enableDGXCRbac: true
|
|
enableDGXCAPIBootstrap: true
|
|
enableServiceMonitor: true
|
|
|