update kubeflow dip-catalog

This commit is contained in:
ChanghoWoo
2025-01-13 02:31:27 +00:00
parent 1dc1181a03
commit 5451f16d72
1959 changed files with 602337 additions and 0 deletions
@@ -0,0 +1,18 @@
apiVersion: kpt.dev/v1
kind: Kptfile
metadata:
name: metacontroller
upstream:
type: git
git:
repo: https://github.com/kubeflow/pipelines
directory: /manifests/kustomize/third-party/metacontroller
ref: 2.0.0-alpha.3
updateStrategy: resource-merge
upstreamLock:
type: git
git:
repo: https://github.com/kubeflow/pipelines
directory: /manifests/kustomize/third-party/metacontroller
ref: 2.0.0-alpha.3
commit: fe333d49fc08f66ec867d3dd7dbd756927bf390c
@@ -0,0 +1,63 @@
BUILD_DIR=./build
.PHONY: hydrate
hydrate:
rm -r -f "${BUILD_DIR}"
mkdir -p "${BUILD_DIR}"
kustomize build -o "${BUILD_DIR}" ./base
echo "Succesfully built into ${BUILD_DIR}."
.PHONY: clean-build
clean-build:
rm -r -f "${BUILD_DIR}"
echo "Succesfully cleaned ${BUILD_DIR}."
.PHONY: apply
apply: hydrate
kubectl apply -f "${BUILD_DIR}"
echo "Succesfully applied from ${BUILD_DIR}."
.PHONY: delete
delete: hydrate
kubectl delete -f "${BUILD_DIR}"
echo "Succesfully deleted metacontroller."
.PHONY: test
test: test-deploy test-catset-controller test-deploy-cleanup
.PHONY: test-deploy
test-deploy:
kubectl create namespace kubeflow || echo "Found 'kubeflow' namespace"
kustomize build ./base | kubectl apply -f -
sleep 30
# Check CRDs
kubectl -n kubeflow wait --for=condition=established --timeout=60s crd/compositecontrollers.metacontroller.k8s.io
kubectl -n kubeflow wait --for=condition=established --timeout=60s crd/controllerrevisions.metacontroller.k8s.io
kubectl -n kubeflow wait --for=condition=established --timeout=60s crd/decoratorcontrollers.metacontroller.k8s.io
# Check ServiceAccount
kubectl get -n kubeflow serviceaccount/meta-controller-service
kubectl get -n kubeflow clusterrolebinding/meta-controller-cluster-role-binding
# Check StatefulSet
kubectl -n kubeflow rollout status --watch --timeout=60s StatefulSet/metacontroller
echo "[Test] contrib/metacontroller deployment PASSED"
.PHONY: test-deploy-cleanup
test-deploy-cleanup:
# Cleanup
kustomize build ./base | kubectl delete -f -
.PHONY: test-catset-controller
test-catset-controller:
# Create a sample CRD
kubectl apply -f test/catset-controller.yaml
sleep 30
# Check a custom CRD
kubectl -n kubeflow get compositecontrollers/catset-controller
# Cleanup
kubectl delete compositecontrollers/catset-controller
echo "[Test] contrib/metacontroller catset-controller PASSED"
.PHONY: pull
pull:
# Check ./UPGRADE.md for instructions on how to pull updated manifests
kpt pkg update .@${KFP_VERSION}
@@ -0,0 +1,6 @@
approvers:
- zijianjoy
- gkcalat
reviewers:
- zijianjoy
- gkcalat
@@ -0,0 +1,40 @@
# Metacontroller
- [Official documentation](https://metacontroller.github.io/metacontroller/)
- [Official repository](https://github.com/metacontroller/metacontroller)
Metacontroller is an add-on for Kubernetes that makes it easy to write and deploy custom controllers.
## Prerequisites
- Kubernetes v1.16+ (because of maintainability, e2e test suite might not cover all releases).
- You should have `kubectl` available and configured to talk to the desired cluster.
- `kustomize`.
## Compile manifests
```bash
make hydrate
```
## Install Metacontroller
```bash
make apply
```
## Verify deployment
```bash
make test
```
## Uninstall Metacontroller
```bash
make delete
```
## Upgrade Metacontroller
To upgrade to the lates version used in Kubeflow, follow the steps in [UPGRADE.md](./UPDGRADE.md).
@@ -0,0 +1,9 @@
# Upgrade Metacontroller
Metacontroller is pulled from [Kubeflow Pipelines's (KFP) third-party folder](https://github.com/kubeflow/pipelines/tree/master/manifests/kustomize/third-party/metacontroller). To update this component specify the desired Kubeflow Pipelines version as `KFP_VERSION` environment variable and run `make pull` in console:
```bash
KFP_VERSION=2.0.0-alpha.3 make pull
```
Alternatively, you can copy the content from [Kubeflow Pipelines third-party folder](https://github.com/kubeflow/pipelines/tree/master/manifests/kustomize/third-party/metacontroller) by choosing the appropriate `TAG` in that repository.
@@ -0,0 +1,11 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata: # kpt-merge: /meta-controller-cluster-role-binding
name: meta-controller-cluster-role-binding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: meta-controller-service
@@ -0,0 +1,530 @@
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata: # kpt-merge: /compositecontrollers.metacontroller.k8s.io
annotations:
"api-approved.kubernetes.io": "unapproved, request not yet submitted"
name: compositecontrollers.metacontroller.k8s.io
spec:
group: metacontroller.k8s.io
names:
kind: CompositeController
listKind: CompositeControllerList
plural: compositecontrollers
shortNames:
- cc
- cctl
singular: compositecontroller
scope: Cluster
versions:
- name: v1alpha1
schema:
openAPIV3Schema:
properties:
apiVersion:
description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
type: string
kind:
description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
type: string
metadata:
type: object
spec:
properties:
childResources:
items:
properties:
apiVersion:
type: string
resource:
type: string
updateStrategy:
properties:
method:
type: string
statusChecks:
properties:
conditions:
items:
properties:
reason:
type: string
status:
type: string
type:
type: string
required:
- type
type: object
type: array
type: object
type: object
required:
- apiVersion
- resource
type: object
type: array
generateSelector:
type: boolean
hooks:
properties:
customize:
properties:
webhook:
properties:
path:
type: string
service:
properties:
name:
type: string
namespace:
type: string
port:
format: int32
type: integer
protocol:
type: string
required:
- name
- namespace
type: object
timeout:
type: string
url:
type: string
type: object
type: object
finalize:
properties:
webhook:
properties:
path:
type: string
service:
properties:
name:
type: string
namespace:
type: string
port:
format: int32
type: integer
protocol:
type: string
required:
- name
- namespace
type: object
timeout:
type: string
url:
type: string
type: object
type: object
postUpdateChild:
properties:
webhook:
properties:
path:
type: string
service:
properties:
name:
type: string
namespace:
type: string
port:
format: int32
type: integer
protocol:
type: string
required:
- name
- namespace
type: object
timeout:
type: string
url:
type: string
type: object
type: object
preUpdateChild:
properties:
webhook:
properties:
path:
type: string
service:
properties:
name:
type: string
namespace:
type: string
port:
format: int32
type: integer
protocol:
type: string
required:
- name
- namespace
type: object
timeout:
type: string
url:
type: string
type: object
type: object
sync:
properties:
webhook:
properties:
path:
type: string
service:
properties:
name:
type: string
namespace:
type: string
port:
format: int32
type: integer
protocol:
type: string
required:
- name
- namespace
type: object
timeout:
type: string
url:
type: string
type: object
type: object
type: object
parentResource:
properties:
apiVersion:
type: string
resource:
type: string
revisionHistory:
properties:
fieldPaths:
items:
type: string
type: array
type: object
required:
- apiVersion
- resource
type: object
resyncPeriodSeconds:
format: int32
type: integer
required:
- parentResource
type: object
status:
type: object
required:
- metadata
- spec
type: object
served: true
storage: true
subresources:
status: {}
status:
acceptedNames:
kind: ""
plural: ""
conditions: []
storedVersions: []
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata: # kpt-merge: /controllerrevisions.metacontroller.k8s.io
annotations:
"api-approved.kubernetes.io": "unapproved, request not yet submitted"
name: controllerrevisions.metacontroller.k8s.io
spec:
group: metacontroller.k8s.io
names:
kind: ControllerRevision
listKind: ControllerRevisionList
plural: controllerrevisions
singular: controllerrevision
scope: Namespaced
versions:
- name: v1alpha1
schema:
openAPIV3Schema:
properties:
apiVersion:
description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
type: string
children:
items:
properties:
apiGroup:
type: string
kind:
type: string
names:
items:
type: string
type: array
required:
- apiGroup
- kind
- names
type: object
type: array
kind:
description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
type: string
metadata:
type: object
parentPatch:
type: object
required:
- metadata
- parentPatch
type: object
served: true
storage: true
subresources:
status: {}
status:
acceptedNames:
kind: ""
plural: ""
conditions: []
storedVersions: []
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata: # kpt-merge: /decoratorcontrollers.metacontroller.k8s.io
annotations:
"api-approved.kubernetes.io": "unapproved, request not yet submitted"
name: decoratorcontrollers.metacontroller.k8s.io
spec:
group: metacontroller.k8s.io
names:
kind: DecoratorController
listKind: DecoratorControllerList
plural: decoratorcontrollers
shortNames:
- dec
- decorators
singular: decoratorcontroller
scope: Cluster
versions:
- name: v1alpha1
schema:
openAPIV3Schema:
properties:
apiVersion:
description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
type: string
kind:
description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
type: string
metadata:
type: object
spec:
properties:
attachments:
items:
properties:
apiVersion:
type: string
resource:
type: string
updateStrategy:
properties:
method:
type: string
type: object
required:
- apiVersion
- resource
type: object
type: array
hooks:
properties:
customize:
properties:
webhook:
properties:
path:
type: string
service:
properties:
name:
type: string
namespace:
type: string
port:
format: int32
type: integer
protocol:
type: string
required:
- name
- namespace
type: object
timeout:
type: string
url:
type: string
type: object
type: object
finalize:
properties:
webhook:
properties:
path:
type: string
service:
properties:
name:
type: string
namespace:
type: string
port:
format: int32
type: integer
protocol:
type: string
required:
- name
- namespace
type: object
timeout:
type: string
url:
type: string
type: object
type: object
sync:
properties:
webhook:
properties:
path:
type: string
service:
properties:
name:
type: string
namespace:
type: string
port:
format: int32
type: integer
protocol:
type: string
required:
- name
- namespace
type: object
timeout:
type: string
url:
type: string
type: object
type: object
type: object
resources:
items:
properties:
annotationSelector:
properties:
matchAnnotations:
additionalProperties:
type: string
type: object
matchExpressions:
items:
description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
properties:
key:
description: key is the label key that the selector applies to.
type: string
operator:
description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
type: string
values:
description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
items:
type: string
type: array
required:
- key
- operator
type: object
type: array
type: object
apiVersion:
type: string
labelSelector:
description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
items:
description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
properties:
key:
description: key is the label key that the selector applies to.
type: string
operator:
description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
type: string
values:
description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
items:
type: string
type: array
required:
- key
- operator
type: object
type: array
matchLabels:
additionalProperties:
type: string
description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
type: object
type: object
resource:
type: string
required:
- apiVersion
- resource
type: object
type: array
resyncPeriodSeconds:
format: int32
type: integer
required:
- resources
type: object
status:
type: object
required:
- metadata
- spec
type: object
served: true
storage: true
subresources:
status: {}
status:
acceptedNames:
kind: ""
plural: ""
conditions: []
storedVersions: []
@@ -0,0 +1,14 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: kubeflow
resources:
- cluster-role-binding.yaml
- crd.yaml
- service-account.yaml
- stateful-set.yaml
commonLabels:
kustomize.component: metacontroller
# Update metacontroller CRD:
# Copy the upstream file to crd.yaml in this folder.
# Upstream file: https://github.com/metacontroller/metacontroller/blob/master/manifests/production/metacontroller-crds-v1.yaml
@@ -0,0 +1,4 @@
apiVersion: v1
kind: ServiceAccount
metadata: # kpt-merge: /meta-controller-service
name: meta-controller-service
@@ -0,0 +1,45 @@
apiVersion: apps/v1
kind: StatefulSet
metadata: # kpt-merge: /metacontroller
labels:
app: metacontroller
name: metacontroller
spec:
replicas: 1
selector:
matchLabels:
app: metacontroller
serviceName: ""
template:
metadata:
labels:
app: metacontroller
annotations:
sidecar.istio.io/inject: "false"
spec:
containers:
- resources:
limits:
cpu: '1'
memory: 1Gi
requests:
cpu: 100m
memory: 256Mi
command:
- /usr/bin/metacontroller
- --zap-log-level=4
- '--discovery-interval=3600s' # less insane than 10 seconds
securityContext:
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 1000
privileged: false
allowPrivilegeEscalation: false
name: metacontroller
image: 'docker.io/metacontrollerio/metacontroller:v2.0.4'
serviceAccountName: meta-controller-service
# Workaround for https://github.com/kubernetes-sigs/kustomize/issues/677
volumeClaimTemplates: []
@@ -0,0 +1,27 @@
apiVersion: metacontroller.k8s.io/v1alpha1
kind: CompositeController
metadata: # kpt-merge: /catset-controller
name: catset-controller
spec:
parentResource:
apiVersion: ctl.enisoc.com/v1
resource: catsets
revisionHistory:
fieldPaths:
- spec.template
childResources:
- apiVersion: v1
resource: pods
updateStrategy:
method: RollingRecreate
statusChecks:
conditions:
- type: Ready
status: "True"
- apiVersion: v1
resource: persistentvolumeclaims
hooks:
sync:
webhook:
url: http://catset-controller.metacontroller/sync
timeout: 10s