update kubeflow dip-catalog

This commit is contained in:
ChanghoWoo
2025-01-13 02:31:27 +00:00
parent 1dc1181a03
commit 5451f16d72
1959 changed files with 602337 additions and 0 deletions
@@ -0,0 +1,69 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: katib-controller
namespace: kubeflow
labels:
katib.kubeflow.org/component: controller
spec:
replicas: 1
selector:
matchLabels:
katib.kubeflow.org/component: controller
template:
metadata:
labels:
katib.kubeflow.org/component: controller
katib.kubeflow.org/metrics-collector-injection: disabled
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "8080"
sidecar.istio.io/inject: "false"
spec:
serviceAccountName: katib-controller
containers:
- name: katib-controller
image: docker.io/kubeflowkatib/katib-controller
command: ["./katib-controller"]
args:
- --katib-config=/katib-config.yaml
ports:
- containerPort: 8443
name: webhook
protocol: TCP
- containerPort: 8080
name: metrics
protocol: TCP
- containerPort: 18080
name: healthz
protocol: TCP
readinessProbe:
httpGet:
path: /readyz
port: healthz
livenessProbe:
httpGet:
path: /healthz
port: healthz
env:
- name: KATIB_CORE_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
volumeMounts:
- mountPath: /tmp/cert
name: cert
readOnly: true
- mountPath: /katib-config.yaml
name: katib-config
subPath: katib-config.yaml
readOnly: true
volumes:
- name: cert
secret:
defaultMode: 420
secretName: katib-webhook-cert
- name: katib-config
configMap:
name: katib-config
@@ -0,0 +1,9 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- controller.yaml
- rbac.yaml
- service.yaml
- trial-templates.yaml
@@ -0,0 +1,148 @@
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: katib-controller
rules:
- apiGroups:
- ""
resources:
- services
verbs:
- "get"
- "list"
- "watch"
- "create"
- "delete"
- apiGroups:
- ""
resources:
- events
verbs:
- "create"
- "patch"
- "update"
- apiGroups:
- ""
resources:
- serviceaccounts
- persistentvolumes
- persistentvolumeclaims
verbs:
- "get"
- "list"
- "watch"
- "create"
- apiGroups:
- ""
resources:
- namespaces
- configmaps
verbs:
- "get"
- "list"
- "watch"
- apiGroups:
- ""
resources:
- pods
- pods/status
verbs:
- "get"
- apiGroups:
- ""
resources:
- secrets
verbs:
- "get"
- "list"
- "watch"
- "patch"
- apiGroups:
- apps
resources:
- deployments
verbs:
- "get"
- "list"
- "watch"
- "create"
- "delete"
- apiGroups:
- rbac.authorization.k8s.io
resources:
- roles
- rolebindings
verbs:
- "get"
- "create"
- "list"
- "watch"
- apiGroups:
- batch
resources:
- jobs
- cronjobs
verbs:
- "get"
- "list"
- "watch"
- "create"
- "delete"
- apiGroups:
- kubeflow.org
resources:
- tfjobs
- pytorchjobs
- mpijobs
- xgboostjobs
- mxjobs
verbs:
- "get"
- "list"
- "watch"
- "create"
- "delete"
- apiGroups:
- kubeflow.org
resources:
- experiments
- experiments/status
- experiments/finalizers
- trials
- trials/status
- trials/finalizers
- suggestions
- suggestions/status
- suggestions/finalizers
verbs:
- "*"
- apiGroups:
- admissionregistration.k8s.io
resources:
- validatingwebhookconfigurations
- mutatingwebhookconfigurations
verbs:
- "get"
- "watch"
- "list"
- "patch"
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: katib-controller
namespace: kubeflow
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: katib-controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: katib-controller
subjects:
- kind: ServiceAccount
name: katib-controller
namespace: kubeflow
@@ -0,0 +1,26 @@
---
apiVersion: v1
kind: Service
metadata:
name: katib-controller
namespace: kubeflow
labels:
katib.kubeflow.org/component: controller
annotations:
prometheus.io/port: "8080"
prometheus.io/scheme: http
prometheus.io/scrape: "true"
spec:
ports:
- port: 443
protocol: TCP
targetPort: 8443
name: webhook
- name: metrics
port: 8080
targetPort: 8080
- name: healthz
port: 18080
targetPort: 18080
selector:
katib.kubeflow.org/component: controller
@@ -0,0 +1,77 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: trial-templates
namespace: kubeflow
labels:
katib.kubeflow.org/component: trial-templates
data:
defaultTrialTemplate.yaml: |-
apiVersion: batch/v1
kind: Job
spec:
template:
spec:
containers:
- name: training-container
image: docker.io/kubeflowkatib/pytorch-mnist-cpu:v0.17.0
command:
- "python3"
- "/opt/pytorch-mnist/mnist.py"
- "--epochs=1"
- "--batch-size=16"
- "--lr=${trialParameters.learningRate}"
- "--momentum=${trialParameters.momentum}"
restartPolicy: Never
# For ConfigMap templates double quotes must set in commands to correct parse JSON parameters in Trial Template (e.g nn_config, architecture)
enasCPUTemplate: |-
apiVersion: batch/v1
kind: Job
spec:
template:
spec:
containers:
- name: training-container
image: docker.io/kubeflowkatib/enas-cnn-cifar10-cpu:v0.17.0
command:
- python3
- -u
- RunTrial.py
- --num_epochs=1
- "--architecture=\"${trialParameters.neuralNetworkArchitecture}\""
- "--nn_config=\"${trialParameters.neuralNetworkConfig}\""
restartPolicy: Never
pytorchJobTemplate: |-
apiVersion: kubeflow.org/v1
kind: PyTorchJob
spec:
pytorchReplicaSpecs:
Master:
replicas: 1
restartPolicy: OnFailure
template:
spec:
containers:
- name: pytorch
image: docker.io/kubeflowkatib/pytorch-mnist-cpu:v0.17.0
command:
- "python3"
- "/opt/pytorch-mnist/mnist.py"
- "--epochs=1"
- "--lr=${trialParameters.learningRate}"
- "--momentum=${trialParameters.momentum}"
Worker:
replicas: 2
restartPolicy: OnFailure
template:
spec:
containers:
- name: pytorch
image: docker.io/kubeflowkatib/pytorch-mnist-cpu:v0.17.0
command:
- "python3"
- "/opt/pytorch-mnist/mnist.py"
- "--epochs=1"
- "--lr=${trialParameters.learningRate}"
- "--momentum=${trialParameters.momentum}"
@@ -0,0 +1,36 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: experiments.kubeflow.org
spec:
group: kubeflow.org
scope: Namespaced
versions:
- name: v1beta1
served: true
storage: true
additionalPrinterColumns:
- name: Type
type: string
jsonPath: .status.conditions[-1:].type
- name: Status
type: string
jsonPath: .status.conditions[-1:].status
- name: Age
type: date
jsonPath: .metadata.creationTimestamp
subresources:
status: {}
schema:
openAPIV3Schema:
type: object
x-kubernetes-preserve-unknown-fields: true
names:
kind: Experiment
singular: experiment
plural: experiments
categories:
- all
- kubeflow
- katib
@@ -0,0 +1,8 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- experiment.yaml
- suggestion.yaml
- trial.yaml
@@ -0,0 +1,42 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: suggestions.kubeflow.org
spec:
group: kubeflow.org
scope: Namespaced
versions:
- name: v1beta1
served: true
storage: true
additionalPrinterColumns:
- name: Type
type: string
jsonPath: .status.conditions[-1:].type
- name: Status
type: string
jsonPath: .status.conditions[-1:].status
- name: Requested
type: string
jsonPath: .spec.requests
- name: Assigned
type: string
jsonPath: .status.suggestionCount
- name: Age
type: date
jsonPath: .metadata.creationTimestamp
subresources:
status: {}
schema:
openAPIV3Schema:
type: object
x-kubernetes-preserve-unknown-fields: true
names:
kind: Suggestion
singular: suggestion
plural: suggestions
categories:
- all
- kubeflow
- katib
@@ -0,0 +1,36 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: trials.kubeflow.org
spec:
group: kubeflow.org
scope: Namespaced
versions:
- name: v1beta1
served: true
storage: true
additionalPrinterColumns:
- name: Type
type: string
jsonPath: .status.conditions[-1:].type
- name: Status
type: string
jsonPath: .status.conditions[-1:].status
- name: Age
type: date
jsonPath: .metadata.creationTimestamp
subresources:
status: {}
schema:
openAPIV3Schema:
type: object
x-kubernetes-preserve-unknown-fields: true
names:
kind: Trial
singular: trial
plural: trials
categories:
- all
- kubeflow
- katib
@@ -0,0 +1,42 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: katib-db-manager
namespace: kubeflow
labels:
katib.kubeflow.org/component: db-manager
spec:
replicas: 1
selector:
matchLabels:
katib.kubeflow.org/component: db-manager
template:
metadata:
labels:
katib.kubeflow.org/component: db-manager
annotations:
sidecar.istio.io/inject: "false"
spec:
containers:
- name: katib-db-manager
image: docker.io/kubeflowkatib/katib-db-manager
env:
- name: DB_NAME
value: "mysql"
- name: DB_PASSWORD
valueFrom:
secretKeyRef:
name: katib-mysql-secrets
key: MYSQL_ROOT_PASSWORD
command:
- "./katib-db-manager"
ports:
- name: api
containerPort: 6789
livenessProbe:
grpc:
port: 6789
initialDelaySeconds: 10
periodSeconds: 60
failureThreshold: 5
@@ -0,0 +1,7 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- db-manager.yaml
- service.yaml
@@ -0,0 +1,16 @@
---
apiVersion: v1
kind: Service
metadata:
name: katib-db-manager
namespace: kubeflow
labels:
katib.kubeflow.org/component: db-manager
spec:
type: ClusterIP
ports:
- port: 6789
protocol: TCP
name: api
selector:
katib.kubeflow.org/component: db-manager
@@ -0,0 +1,9 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- mysql.yaml
- pvc.yaml
- secret.yaml
- service.yaml
@@ -0,0 +1,74 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: katib-mysql
namespace: kubeflow
labels:
katib.kubeflow.org/component: mysql
spec:
replicas: 1
selector:
matchLabels:
katib.kubeflow.org/component: mysql
strategy:
type: Recreate
template:
metadata:
labels:
katib.kubeflow.org/component: mysql
annotations:
sidecar.istio.io/inject: "false"
spec:
containers:
- name: katib-mysql
image: mysql:8.0.29
args:
- --datadir
- /var/lib/mysql/datadir
env:
- name: MYSQL_ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: katib-mysql-secrets
key: MYSQL_ROOT_PASSWORD
- name: MYSQL_ALLOW_EMPTY_PASSWORD
value: "true"
- name: MYSQL_DATABASE
value: "katib"
ports:
- name: dbapi
containerPort: 3306
readinessProbe:
exec:
command:
- "/bin/bash"
- "-c"
- "mysql -D ${MYSQL_DATABASE} -u root -p${MYSQL_ROOT_PASSWORD} -e 'SELECT 1'"
initialDelaySeconds: 10
periodSeconds: 5
failureThreshold: 10
livenessProbe:
exec:
command:
- "/bin/bash"
- "-c"
- "mysqladmin ping -u root -p${MYSQL_ROOT_PASSWORD}"
initialDelaySeconds: 10
periodSeconds: 5
failureThreshold: 10
startupProbe:
exec:
command:
- "/bin/bash"
- "-c"
- "mysqladmin ping -u root -p${MYSQL_ROOT_PASSWORD}"
periodSeconds: 15
failureThreshold: 60
volumeMounts:
- name: katib-mysql
mountPath: /var/lib/mysql
volumes:
- name: katib-mysql
persistentVolumeClaim:
claimName: katib-mysql
@@ -0,0 +1,12 @@
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: katib-mysql
namespace: kubeflow
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
@@ -0,0 +1,8 @@
---
apiVersion: v1
kind: Secret
type: Opaque
metadata:
name: katib-mysql-secrets
data:
MYSQL_ROOT_PASSWORD: dGVzdA== # "test"
@@ -0,0 +1,16 @@
---
apiVersion: v1
kind: Service
metadata:
name: katib-mysql
namespace: kubeflow
labels:
katib.kubeflow.org/component: mysql
spec:
type: ClusterIP
ports:
- port: 3306
protocol: TCP
name: dbapi
selector:
katib.kubeflow.org/component: mysql
@@ -0,0 +1,7 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: kubeflow
resources:
# Namespace.
- namespace.yaml
@@ -0,0 +1,7 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: kubeflow
labels:
katib.kubeflow.org/metrics-collector-injection: enabled
@@ -0,0 +1,9 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- postgres.yaml
- pvc.yaml
- secret.yaml
- service.yaml
@@ -0,0 +1,42 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: katib-postgres
namespace: kubeflow
labels:
katib.kubeflow.org/component: postgres
spec:
replicas: 1
selector:
matchLabels:
katib.kubeflow.org/component: postgres
strategy:
type: Recreate
template:
metadata:
labels:
katib.kubeflow.org/component: postgres
annotations:
sidecar.istio.io/inject: "false"
spec:
containers:
- name: katib-postgres
image: postgres:14.5-alpine
envFrom:
- secretRef:
name: katib-postgres-secrets
env:
- name: PGDATA
value: /var/lib/postgresql/data/pgdata
ports:
- name: postgres
containerPort: 5432
protocol: TCP
volumeMounts:
- name: katib-postgres
mountPath: /var/lib/postgresql/data
volumes:
- name: katib-postgres
persistentVolumeClaim:
claimName: katib-postgres
@@ -0,0 +1,12 @@
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: katib-postgres
namespace: kubeflow
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 3Gi
@@ -0,0 +1,10 @@
---
apiVersion: v1
kind: Secret
type: Opaque
metadata:
name: katib-postgres-secrets
data:
POSTGRES_USER: a2F0aWI= # katib
POSTGRES_PASSWORD: a2F0aWI= # katib
POSTGRES_DB: a2F0aWI= # katib
@@ -0,0 +1,16 @@
---
apiVersion: v1
kind: Service
metadata:
name: katib-postgres
namespace: kubeflow
labels:
katib.kubeflow.org/component: postgres
spec:
type: ClusterIP
ports:
- port: 5432
protocol: TCP
name: dbapi
selector:
katib.kubeflow.org/component: postgres
@@ -0,0 +1,8 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- rbac.yaml
- service.yaml
- ui.yaml
@@ -0,0 +1,52 @@
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: katib-ui
rules:
- apiGroups:
- ""
resources:
- configmaps
- namespaces
verbs:
- "*"
- apiGroups:
- kubeflow.org
resources:
- experiments
- trials
- suggestions
verbs:
- "*"
- apiGroups:
- ""
resources:
- pods
verbs:
- list
- apiGroups:
- ""
resources:
- pods/log
verbs:
- get
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: katib-ui
namespace: kubeflow
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: katib-ui
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: katib-ui
subjects:
- kind: ServiceAccount
name: katib-ui
namespace: kubeflow
@@ -0,0 +1,17 @@
---
apiVersion: v1
kind: Service
metadata:
name: katib-ui
namespace: kubeflow
labels:
katib.kubeflow.org/component: ui
spec:
type: ClusterIP
ports:
- port: 80
protocol: TCP
name: ui
targetPort: 8080
selector:
katib.kubeflow.org/component: ui
@@ -0,0 +1,36 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: katib-ui
namespace: kubeflow
labels:
katib.kubeflow.org/component: ui
spec:
replicas: 1
selector:
matchLabels:
katib.kubeflow.org/component: ui
template:
metadata:
labels:
katib.kubeflow.org/component: ui
annotations:
sidecar.istio.io/inject: "false"
spec:
containers:
- name: katib-ui
image: docker.io/kubeflowkatib/katib-ui
command:
- "./katib-ui"
args:
- "--port=8080"
env:
- name: KATIB_CORE_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
ports:
- name: ui
containerPort: 8080
serviceAccountName: katib-ui
@@ -0,0 +1,6 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- webhooks.yaml
@@ -0,0 +1,84 @@
---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
name: katib.kubeflow.org
webhooks:
- name: validator.experiment.katib.kubeflow.org
sideEffects: None
admissionReviewVersions:
- v1
clientConfig:
caBundle: Cg==
service:
name: katib-controller
namespace: kubeflow
path: /validate-experiment
rules:
- apiGroups:
- kubeflow.org
apiVersions:
- v1beta1
operations:
- CREATE
- UPDATE
resources:
- experiments
---
apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration
metadata:
name: katib.kubeflow.org
webhooks:
- name: defaulter.experiment.katib.kubeflow.org
sideEffects: None
admissionReviewVersions:
- v1
clientConfig:
caBundle: Cg==
service:
name: katib-controller
namespace: kubeflow
path: /mutate-experiment
rules:
- apiGroups:
- kubeflow.org
apiVersions:
- v1beta1
operations:
- CREATE
- UPDATE
resources:
- experiments
- name: mutator.pod.katib.kubeflow.org
sideEffects: None
admissionReviewVersions:
- v1
clientConfig:
caBundle: Cg==
service:
name: katib-controller
namespace: kubeflow
path: /mutate-pod
namespaceSelector:
matchLabels:
katib.kubeflow.org/metrics-collector-injection: enabled
# Once the AdmissionWebhookMatchConditions feature gate is enabled by default, we should switch to control based on userInfo.
# REF:
# - AdmissionWebhookMatchConditions: https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchconditions
# - Tracking issue: https://github.com/kubeflow/katib/issues/2206
objectSelector:
matchExpressions:
- key: katib.kubeflow.org/metrics-collector-injection
operator: NotIn
values:
- disabled
rules:
- apiGroups:
- ""
apiVersions:
- v1
operations:
- CREATE
resources:
- pods