update kubeflow dip-catalog

This commit is contained in:
ChanghoWoo
2025-01-13 02:31:27 +00:00
parent 1dc1181a03
commit 5451f16d72
1959 changed files with 602337 additions and 0 deletions
+3
View File
@@ -0,0 +1,3 @@
approvers:
- Jeffwan
- PatrickXYS
@@ -0,0 +1,56 @@
# Sample installation
1. Create an EKS cluster
Run this command to create EKS cluster
```
eksctl create cluster \
--name AWS-KFP \
--version 1.17 \
--region us-west-2 \
--nodegroup-name linux-nodes \
--node-type m5.xlarge \
--nodes 2 \
--nodes-min 1 \
--nodes-max 4 \
--managed
```
2. Prepare S3
Create S3 bucket. [Console](https://console.aws.amazon.com/s3/home).
Run this command to create S3 bucket by changing `<YOUR_S3_BUCKET_NAME>` to your prefer s3 bucket name.
```
export S3_BUCKET=<YOUR_S3_BUCKET_NAME>
export AWS_REGION=us-west-2
aws s3 mb s3://$S3_BUCKET --region $AWS_REGION
```
3. Prepare RDS
Follow this [doc](https://awslabs.github.io/kubeflow-manifests/docs/deployment/rds-s3/guide/) to set up AWS RDS instance.
4. Customize your values
- Edit [params.env](params.env), [secret.env](secret.env) and [minio-artifact-secret-patch.env](minio-artifact-secret-patch.env)
5. Install
```
kubectl apply -k ../../cluster-scoped-resources
# If upper one action got failed, e.x. you used wrong value, try delete, fix and apply again
# kubectl delete -k ../../cluster-scoped-resources
kubectl wait crd/applications.app.k8s.io --for condition=established --timeout=60s
kubectl apply -k ./
# If upper one action got failed, e.x. you used wrong value, try delete, fix and apply again
# kubectl delete -k ./
kubectl wait applications/pipeline -n kubeflow --for condition=Ready --timeout=1800s
kubectl port-forward -n kubeflow svc/ml-pipeline-ui 8080:80
```
Now you can access via `localhost:8080`
@@ -0,0 +1,32 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: ml-pipeline
spec:
template:
metadata:
labels:
app: ml-pipeline
spec:
containers:
- env:
- name: OBJECTSTORECONFIG_SECURE
value: "true"
- name: OBJECTSTORECONFIG_BUCKETNAME
valueFrom:
configMapKeyRef:
name: pipeline-install-config
key: bucketName
- name: OBJECTSTORECONFIG_HOST
valueFrom:
configMapKeyRef:
name: pipeline-install-config
key: minioServiceHost
- name: OBJECTSTORECONFIG_REGION
valueFrom:
configMapKeyRef:
name: pipeline-install-config
key: minioServiceRegion
- name: OBJECTSTORECONFIG_PORT
value: ""
name: ml-pipeline-api-server
@@ -0,0 +1,27 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: ml-pipeline-ui
spec:
template:
metadata:
labels:
app: ml-pipeline-ui
spec:
volumes:
- name: config-volume
configMap:
name: ml-pipeline-ui-configmap
containers:
- name: ml-pipeline-ui
env:
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: mlpipeline-minio-artifact
key: accesskey
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: mlpipeline-minio-artifact
key: secretkey
+20
View File
@@ -0,0 +1,20 @@
{
artifactRepository:
{
s3: {
bucket: $(kfp-artifact-bucket-name),
keyPrefix: artifacts,
endpoint: s3.amazonaws.com,
insecure: true,
accessKeySecret: {
name: mlpipeline-minio-artifact,
key: accesskey
},
secretKeySecret: {
name: mlpipeline-minio-artifact,
key: secretkey
}
},
archiveLogs: true
}
}
@@ -0,0 +1,34 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: kubeflow
resources:
- ../../env/platform-agnostic
configMapGenerator:
- name: pipeline-install-config
env: params.env
behavior: merge
- name: workflow-controller-configmap
behavior: replace
files:
- config
- name: ml-pipeline-ui-configmap
behavior: replace
files:
- viewer-pod-template.json
secretGenerator:
- name: mysql-secret
env: secret.env
behavior: merge
- name: mlpipeline-minio-artifact
env: minio-artifact-secret-patch.env
behavior: merge
generatorOptions:
disableNameSuffixHash: true
patches:
- path: aws-configuration-pipeline-patch.yaml
- path: aws-configuration-pipeline-ui-patch.yaml
# Identifier for application manager to apply ownerReference.
# The ownerReference ensures the resources get garbage collected
# when application is deleted.
commonLabels:
application-crd-id: kubeflow-pipelines
@@ -0,0 +1,2 @@
accesskey=YOUR_AWS_ACCESS_ID
secretkey=YOUR_AWS_SECRET_KEY
@@ -0,0 +1,5 @@
dbHost=YOUR_RDS_ENDPOINT
bucketName=YOUR_S3_BUCKET_NAME
minioServiceHost=s3.amazonaws.com
minioServiceRegion=YOUR_AWS_REGION
@@ -0,0 +1,2 @@
username=YOUR_RDS_USERNAME
password=YOUR_RDS_PASSWORD
@@ -0,0 +1,37 @@
{
"spec": {
"containers": [
{
"env": [
{
"name": "AWS_ACCESS_KEY_ID",
"valueFrom": {
"secretKeyRef": {
"name": "mlpipeline-minio-artifact",
"key": "accesskey"
}
}
},
{
"name": "AWS_SECRET_ACCESS_KEY",
"valueFrom": {
"secretKeyRef": {
"name": "mlpipeline-minio-artifact",
"key": "secretkey"
}
}
},
{
"name": "AWS_REGION",
"valueFrom": {
"configMapKeyRef": {
"name": "pipeline-install-config",
"key": "minioServiceRegion"
}
}
}
]
}
]
}
}
@@ -0,0 +1,4 @@
approvers:
- dtzar
- eedorenko
- sudivate
@@ -0,0 +1,25 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: kubeflow
resources:
- ../../base/installs/generic
- ../../base/metadata/base
- ../../third-party/argo/installs/namespace
- minio-azure-gateway
configMapGenerator:
- name: pipeline-install-config
env: params.env
behavior: merge
secretGenerator:
- name: mysql-secret
env: mysql-secret.env
behavior: merge
# Identifier for application manager to apply ownerReference.
# The ownerReference ensures the resources get garbage collected
# when application is deleted.
commonLabels:
application-crd-id: kubeflow-pipelines
@@ -0,0 +1,13 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- minio-azure-gateway-deployment.yaml
- minio-azure-gateway-service.yaml
secretGenerator:
- name: mlpipeline-minio-artifact
env: minio-artifact-secret.env
generatorOptions:
# mlpipeline-minio-artifact needs to be referred by exact name
disableNameSuffixHash: true
@@ -0,0 +1,2 @@
accesskey=[STORAGEACCOUNTNAME]
secretkey=[STORAGEACCOUNTKEY]
@@ -0,0 +1,40 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: minio
labels:
app: minio
spec:
selector:
matchLabels:
app: minio
strategy:
type: Recreate
template:
metadata:
labels:
app: minio
spec:
containers:
- name: minio
image: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance
args:
- gateway
- azure
env:
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef:
name: mlpipeline-minio-artifact
key: accesskey
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef:
name: mlpipeline-minio-artifact
key: secretkey
ports:
- containerPort: 9000
resources:
requests:
cpu: 20m
memory: 25Mi
@@ -0,0 +1,11 @@
apiVersion: v1
kind: Service
metadata:
name: minio-service
spec:
ports:
- port: 9000
targetPort: 9000
protocol: TCP
selector:
app: minio
@@ -0,0 +1,2 @@
username=[SQLUSER]@[SQLSERVERNAME]
password=[SQLPASS]
@@ -0,0 +1 @@
dbHost=[SQLSERVERNAME].mysql.database.azure.com
@@ -0,0 +1,15 @@
# KFP customizations for Azure
This template provides a starting point to configure KFP to use an Azure hosted MySQL database, as well as an Azure Blob backed MinIO service.
## MySQL
1. [Create an Azure Database for MySQL](https://docs.microsoft.com/azure/mysql/quickstart-create-mysql-server-database-using-azure-portal). Ensure that it will allow connections from the Kubernetes cluster.
2. Substitute the server name into [params.env](./params.env), and the username and password into [mysql-secret.env](./mysql-secret.env)
## MinIO Gateway for Azure Blobstore
1. [Create an Azure Storage account](https://docs.microsoft.com/azure/storage/common/storage-account-create). Ensure that it will allow connections from the Kubernetes cluster.
2. Substitute the storage name and access key into [minio-artifact-secret.env](./minio-azure-gateway/minio-artifact-secret.env).
@@ -0,0 +1,6 @@
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
name: kfp-cache-selfsigned-issuer
spec:
selfSigned: {}
@@ -0,0 +1,15 @@
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: kfp-cache-cert
spec:
commonName: kfp-cache-cert
isCA: true
dnsNames:
- cache-server
- cache-server.$(kfp-namespace)
- cache-server.$(kfp-namespace).svc
issuerRef:
kind: Issuer
name: kfp-cache-selfsigned-issuer
secretName: webhook-server-tls
@@ -0,0 +1,25 @@
apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration
metadata:
name: cache-webhook-kubeflow
annotations:
cert-manager.io/inject-ca-from: $(kfp-namespace)/kfp-cache-cert
webhooks:
- name: cache-server.$(kfp-namespace).svc
clientConfig:
service:
name: cache-server
namespace: $(kfp-namespace)
path: "/mutate"
failurePolicy: Ignore
rules:
- operations: [ "CREATE" ]
apiGroups: [""]
apiVersions: ["v1"]
resources: ["pods"]
sideEffects: None
timeoutSeconds: 5
objectSelector:
matchLabels:
pipelines.kubeflow.org/cache_enabled: "true"
admissionReviewVersions: ["v1beta1"]
@@ -0,0 +1,13 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: kubeflow
resources:
- cache-cert-issuer.yaml
- cache-cert.yaml
- cache-webhook-config.yaml
commonLabels:
app: cache-server-cert-manager
configurations:
- params.yaml
@@ -0,0 +1,13 @@
varReference:
- path: spec/commonName
kind: Certificate
- path: spec/dnsNames
kind: Certificate
- path: spec/issuerRef/name
kind: Certificate
- path: metadata/annotations
kind: MutatingWebhookConfiguration
- path: webhooks/clientConfig/service/namespace
kind: MutatingWebhookConfiguration
- path: webhooks/name
kind: MutatingWebhookConfiguration
@@ -0,0 +1,7 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../../third-party/application/cluster-scoped
- ../../../third-party/argo/installs/namespace/cluster-scoped
- ../../../base/pipeline/cluster-scoped
@@ -0,0 +1,18 @@
# Delete cache deployer related resources as we use the cert-manager instead
apiVersion: apps/v1
kind: Deployment
metadata:
name: cache-deployer-deployment
$patch: delete
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: kubeflow-pipelines-cache-deployer-role
$patch: delete
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: kubeflow-pipelines-cache-deployer-rolebinding
$patch: delete
@@ -0,0 +1,44 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: kubeflow
resources:
- ../../dev
- ../base
- namespace.yaml
# NOTE: var name must be unique globally to allow composition of multiple kustomize
# packages. Therefore, we added prefix `kfp-dev-` to distinguish it from
# others.
# ml-pipeline sa's metadata.namespace will be first transformed by namespace field in kustomization.yaml
# so that we only need to change kustomization.yaml's namespace field for namespace customization.
vars:
- fieldref:
fieldPath: metadata.namespace
name: kfp-dev-namespace
objref:
apiVersion: v1
kind: ServiceAccount
name: ml-pipeline
configurations:
- params.yaml
# Pass proper arguments to cache-server to use cert-manager certificate
# Delete the cache deployer as we use the cert-manager instead
patches:
- path: ./delete-cache-deployer.yaml
- patch: |-
- op: add
path: /spec/template/spec/containers/0/args/-
value: "--tls_cert_filename=tls.crt"
target:
kind: Deployment
name: cache-server
- patch: |-
- op: add
path: /spec/template/spec/containers/0/args/-
value: "--tls_key_filename=tls.key"
target:
kind: Deployment
name: cache-server
@@ -0,0 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: '$(kfp-dev-namespace)'
@@ -0,0 +1,4 @@
# Allow Kustomize var to replace following fields.
varReference:
- path: metadata/name
kind: Namespace
@@ -0,0 +1,31 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../platform-agnostic-multi-user
- ../base
namespace: kubeflow
# Pass proper arguments to cache-server to use cert-manager certificate
patches:
# Delete the cache deployer as we use the cert-manager instead
- path: patches/delete.clusterrole.cache-deployer.yaml
- path: patches/delete.crb.cache-deployer.yaml
- path: patches/delete.deployment.cache-deployer.yaml
- path: patches/delete.role.cache-deployer.yaml
- path: patches/delete.rolebinding.cache-deployer.yaml
- path: patches/delete.sa.cache-deployer.yaml
- patch: |-
- op: add
path: /spec/template/spec/containers/0/args/-
value: "--tls_cert_filename=tls.crt"
target:
kind: Deployment
name: cache-server
- patch: |-
- op: add
path: /spec/template/spec/containers/0/args/-
value: "--tls_key_filename=tls.key"
target:
kind: Deployment
name: cache-server
@@ -0,0 +1,6 @@
# Delete cache deployer as we use the cert-manager instead
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: kubeflow-pipelines-cache-deployer-clusterrole
$patch: delete
@@ -0,0 +1,6 @@
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kubeflow-pipelines-cache-deployer-clusterrolebinding
$patch: delete
@@ -0,0 +1,6 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: cache-deployer-deployment
$patch: delete
@@ -0,0 +1,6 @@
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: kubeflow-pipelines-cache-deployer-role
$patch: delete
@@ -0,0 +1,6 @@
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: kubeflow-pipelines-cache-deployer-rolebinding
$patch: delete
@@ -0,0 +1,6 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: kubeflow-pipelines-cache-deployer-sa
$patch: delete
@@ -0,0 +1,19 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
# Application controller is used to provide Google Cloud Console integration.
- ../../third-party/application
- ../../base/application
- ../platform-agnostic
- ../gcp/inverse-proxy
# Identifier for application manager to apply ownerReference.
# The ownerReference ensures the resources get garbage collected
# when application is deleted.
commonLabels:
application-crd-id: kubeflow-pipelines
# !!! If you want to customize the namespace,
# please refer sample/cluster-scoped-resources to update the namespace for cluster-scoped-resources
namespace: kubeflow
@@ -0,0 +1,19 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
# Application controller is used to provide Google Cloud Console integration.
- ../../../third-party/application
- ../../../base/application
- ../../platform-agnostic-postgresql
- ../../gcp/inverse-proxy
# Identifier for application manager to apply ownerReference.
# The ownerReference ensures the resources get garbage collected
# when application is deleted.
commonLabels:
application-crd-id: kubeflow-pipelines
# !!! If you want to customize the namespace,
# please refer sample/cluster-scoped-resources to update the namespace for cluster-scoped-resources
namespace: kubeflow
@@ -0,0 +1,89 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: cloudsqlproxy
labels:
app: cloudsqlproxy
spec:
selector:
matchLabels:
app: cloudsqlproxy
replicas: 1
template:
metadata:
labels:
app: cloudsqlproxy
spec:
serviceAccountName: kubeflow-pipelines-cloudsql-proxy
containers:
- image: gcr.io/cloudsql-docker/gce-proxy:1.25.0
name: cloudsqlproxy
env:
- name: GCP_CLOUDSQL_INSTANCE_NAME
valueFrom:
configMapKeyRef:
name: pipeline-install-config
key: gcsCloudSqlInstanceName
command: ["/cloud_sql_proxy",
"-dir=/cloudsql",
"-instances=$(GCP_CLOUDSQL_INSTANCE_NAME)=tcp:0.0.0.0:3306",
"-use_http_health_check",
"term_timeout=10s"]
# set term_timeout if require graceful handling of shutdown
# NOTE: proxy will stop accepting new connections; only wait on existing connections
lifecycle:
preStop:
exec:
# (optional) add a preStop hook so that termination is delayed
# this is required if your server still require new connections (e.g., connection pools)
command: ['sleep', '10']
# Liveness probe configuration is adviced by Cloud SQL Proxy examples:
# https://github.com/GoogleCloudPlatform/cloudsql-proxy/tree/main/examples/k8s-health-check
# For details, see https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
livenessProbe:
httpGet:
path: /liveness
port: 8090
# Number of seconds after the container has started before the first probe is scheduled. Defaults to 0.
# Not necessary when the startup probe is in use.
initialDelaySeconds: 0
# Frequency of the probe.
periodSeconds: 60
# Number of seconds after which the probe times out.
timeoutSeconds: 30
# Number of times the probe is allowed to fail before the transition
# from healthy to failure state.
#
# If periodSeconds = 60, 5 tries will result in five minutes of
# checks. The proxy starts to refresh a certificate five minutes
# before its expiration. If those five minutes lapse without a
# successful refresh, the liveness probe will fail and the pod will be
# restarted.
failureThreshold: 5
readinessProbe:
httpGet:
path: /readiness
port: 8090
initialDelaySeconds: 0
periodSeconds: 10
timeoutSeconds: 5
# Number of times the probe must report success to transition from failure to healthy state.
# Defaults to 1 for readiness probe.
successThreshold: 1
failureThreshold: 1
startupProbe:
httpGet:
path: /startup
port: 8090
periodSeconds: 1
timeoutSeconds: 5
failureThreshold: 20
ports:
- name: mysql
containerPort: 3306
volumeMounts:
- mountPath: /cloudsql
name: cloudsql
volumes:
- name: cloudsql
emptyDir:
@@ -0,0 +1,4 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: kubeflow-pipelines-cloudsql-proxy
@@ -0,0 +1,7 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- cloudsql-proxy-deployment.yaml
- cloudsql-proxy-sa.yaml
- mysql-service.yaml
@@ -0,0 +1,9 @@
apiVersion: v1
kind: Service
metadata:
name: mysql
spec:
ports:
- port: 3306
selector:
app: cloudsqlproxy
@@ -0,0 +1,22 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: ml-pipeline
spec:
template:
spec:
containers:
- name: ml-pipeline-api-server
env:
- name: HAS_DEFAULT_BUCKET
value: 'true'
- name: BUCKET_NAME
valueFrom:
configMapKeyRef:
name: pipeline-install-config
key: bucketName
- name: PROJECT_ID
valueFrom:
configMapKeyRef:
name: pipeline-install-config
key: gcsProjectId
@@ -0,0 +1,11 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
images:
- name: gcr.io/ml-pipeline/inverse-proxy-agent
newTag: 2.2.0
resources:
- proxy-configmap.yaml
- proxy-deployment.yaml
- proxy-role.yaml
- proxy-rolebinding.yaml
- proxy-sa.yaml
@@ -0,0 +1,4 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: inverse-proxy-config
@@ -0,0 +1,21 @@
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: proxy-agent
name: proxy-agent
spec:
selector:
matchLabels:
app: proxy-agent
template:
metadata:
labels:
app: proxy-agent
spec:
hostNetwork: true
containers:
- image: gcr.io/ml-pipeline/inverse-proxy-agent:dummy
imagePullPolicy: IfNotPresent
name: proxy-agent
serviceAccountName: proxy-agent-runner
@@ -0,0 +1,13 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
labels:
app: proxy-agent-runner
name: proxy-agent-runner
rules:
- apiGroups:
- ""
resources:
- configmaps
verbs:
- '*'
@@ -0,0 +1,13 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
app: proxy-agent-runner
name: proxy-agent-runner
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: proxy-agent-runner
subjects:
- kind: ServiceAccount
name: proxy-agent-runner
@@ -0,0 +1,4 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: proxy-agent-runner
@@ -0,0 +1,32 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
# Application controller is used to provide Google Cloud Console integration.
- ../../third-party/application
- ../../base/application
- ../../base/installs/generic
- ../../base/metadata/base
- ../../third-party/argo/installs/namespace
- inverse-proxy
- minio-gcs-gateway
- cloudsql-proxy
# Identifier for application manager to apply ownerReference.
# The ownerReference ensures the resources get garbage collected
# when application is deleted.
commonLabels:
application-crd-id: kubeflow-pipelines
# !!! If you want to customize the namespace,
# please also update base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml
namespace: kubeflow
patches:
- path: gcp-configurations-patch.yaml
# Used by Kustomize
configMapGenerator:
- name: pipeline-install-config
env: params.env
behavior: merge
@@ -0,0 +1,14 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- minio-gcs-gateway-deployment.yaml
- minio-gcs-gateway-sa.yaml
- minio-gcs-gateway-service.yaml
secretGenerator:
- name: mlpipeline-minio-artifact
env: minio-artifact-secret.env
generatorOptions:
# mlpipeline-minio-artifact needs to be referred by exact name
disableNameSuffixHash: true
@@ -0,0 +1,2 @@
accesskey=minio
secretkey=minio123
@@ -0,0 +1,47 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: minio
labels:
app: minio
spec:
selector:
matchLabels:
app: minio
strategy:
type: Recreate
template:
metadata:
labels:
app: minio
spec:
serviceAccountName: kubeflow-pipelines-minio-gcs-gateway
containers:
- name: minio
image: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance
args:
- gateway
- gcs
- $(GCP_PROJECT_ID)
env:
- name: GCP_PROJECT_ID
valueFrom:
configMapKeyRef:
name: pipeline-install-config
key: gcsProjectId
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef:
name: mlpipeline-minio-artifact
key: accesskey
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef:
name: mlpipeline-minio-artifact
key: secretkey
resources:
requests:
cpu: 20m
memory: 25Mi
ports:
- containerPort: 9000
@@ -0,0 +1,4 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: kubeflow-pipelines-minio-gcs-gateway
@@ -0,0 +1,11 @@
apiVersion: v1
kind: Service
metadata:
name: minio-service
spec:
ports:
- port: 9000
targetPort: 9000
protocol: TCP
selector:
app: minio
@@ -0,0 +1,6 @@
pipelineDb=pipelinedb
mlmdDb=metadb
cacheDb=cachedb
bucketName=yourGcsBucketName
gcsProjectId=yourGcsProjectId
gcsCloudSqlInstanceName=yourCloudSqlInstanceName
@@ -0,0 +1,22 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../base/installs/multi-user
- ../../base/metadata/base
- ../../base/metadata/options/istio
- ../../third-party/mysql/base
- ../../third-party/mysql/options/istio
- ../../third-party/minio/base
- ../../third-party/minio/options/istio
- ../../third-party/metacontroller/base
# Identifier for application manager to apply ownerReference.
# The ownerReference ensures the resources get garbage collected
# when application is deleted.
commonLabels:
application-crd-id: kubeflow-pipelines
# !!! If you want to customize the namespace,
# please also update base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml
namespace: kubeflow
@@ -0,0 +1,21 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../cluster-scoped-resources-tekton
- ../../base/installs/generic
- ../../base/metadata/base
- ../../third-party/minio/base
- ../../third-party/mysql/base
# Identifier for application manager to apply ownerReference.
# The ownerReference ensures the resources get garbage collected
# when application is deleted.
labels:
- includeSelectors: true
pairs:
application-crd-id: kubeflow-pipelines
# !!! If you want to customize the namespace,
# please also update base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml
namespace: kubeflow
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../platform-agnostic
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../platform-agnostic-multi-user
@@ -0,0 +1,23 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../base/installs/multi-user
- ../../base/metadata/overlays/db
- ../../base/metadata/options/istio
- ../../third-party/argo/installs/cluster
- ../../third-party/mysql/base
- ../../third-party/mysql/options/istio
- ../../third-party/minio/base
- ../../third-party/minio/options/istio
- ../../third-party/metacontroller/base
# Identifier for application manager to apply ownerReference.
# The ownerReference ensures the resources get garbage collected
# when application is deleted.
commonLabels:
application-crd-id: kubeflow-pipelines
# !!! If you want to customize the namespace,
# please also update base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml
namespace: kubeflow
@@ -0,0 +1,23 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../base/installs/multi-user
- ../../base/metadata/base
- ../../base/metadata/options/istio
- ../../third-party/argo/installs/cluster
- ../../third-party/mysql/base
- ../../third-party/mysql/options/istio
- ../../third-party/minio/base
- ../../third-party/minio/options/istio
- ../../third-party/metacontroller/base
# Identifier for application manager to apply ownerReference.
# The ownerReference ensures the resources get garbage collected
# when application is deleted.
commonLabels:
application-crd-id: kubeflow-pipelines
# !!! If you want to customize the namespace,
# please also update base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml
namespace: kubeflow
@@ -0,0 +1,19 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../base/installs/generic/postgres
- ../../base/metadata/overlays/postgres
- ../../third-party/argo/installs/namespace
- ../../third-party/minio/base
- ../../third-party/postgresql/base
# Identifier for application manager to apply ownerReference.
# The ownerReference ensures the resources get garbage collected
# when application is deleted.
commonLabels:
application-crd-id: kubeflow-pipelines
# !!! If you want to customize the namespace,
# please also update base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml
namespace: kubeflow
@@ -0,0 +1,68 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../third-party/tekton/installs/cluster
- ../../third-party/tekton-custom-task
- ../plain-multi-user
# Identifier for application manager to apply ownerReference.
# The ownerReference ensures the resources get garbage collected
# when application is deleted.
commonLabels:
application-crd-id: kubeflow-pipelines
patches:
- path: tekton-config.yaml
- patch: |-
apiVersion: apps/v1
kind: Deployment
metadata:
name: ml-pipeline
spec:
template:
spec:
containers:
- name: ml-pipeline-api-server
env:
- name: EXECUTIONTYPE
value: PipelineRun
- patch: |-
apiVersion: apps/v1
kind: Deployment
metadata:
name: ml-pipeline-persistenceagent
spec:
template:
spec:
containers:
- name: ml-pipeline-persistenceagent
env:
- name: EXECUTIONTYPE
value: PipelineRun
- patch: |-
apiVersion: apps/v1
kind: Deployment
metadata:
name: ml-pipeline-scheduledworkflow
spec:
template:
spec:
containers:
- name: ml-pipeline-scheduledworkflow
env:
- name: EXECUTIONTYPE
value: PipelineRun
- patch: |-
apiVersion: apps/v1
kind: Deployment
metadata:
name: ml-pipeline-ui
spec:
template:
spec:
containers:
- name: ml-pipeline-ui
env:
- name: POD_LOG_CONTAINER_NAME
value: step-user-main
@@ -0,0 +1,10 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: feature-flags
namespace: tekton-pipelines
labels:
app.kubernetes.io/instance: default
app.kubernetes.io/part-of: tekton-pipelines
data:
running-in-environment-with-injected-sidecars: "true"
@@ -0,0 +1,70 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../third-party/tekton/installs/cluster
- ../../third-party/tekton-custom-task
- ../plain
# Identifier for application manager to apply ownerReference.
# The ownerReference ensures the resources get garbage collected
# when application is deleted.
labels:
- includeSelectors: true
pairs:
application-crd-id: kubeflow-pipelines
patches:
- patch: |-
apiVersion: apps/v1
kind: Deployment
metadata:
name: ml-pipeline
spec:
template:
spec:
containers:
- name: ml-pipeline-api-server
env:
- name: EXECUTIONTYPE
value: PipelineRun
- patch: |-
apiVersion: apps/v1
kind: Deployment
metadata:
name: ml-pipeline-persistenceagent
spec:
template:
spec:
containers:
- name: ml-pipeline-persistenceagent
env:
- name: EXECUTIONTYPE
value: PipelineRun
- patch: |-
apiVersion: apps/v1
kind: Deployment
metadata:
name: ml-pipeline-scheduledworkflow
spec:
template:
spec:
containers:
- name: ml-pipeline-scheduledworkflow
env:
- name: EXECUTIONTYPE
value: PipelineRun
- patch: |-
apiVersion: apps/v1
kind: Deployment
metadata:
name: ml-pipeline-ui
spec:
template:
spec:
containers:
- name: ml-pipeline-ui
env:
- name: POD_LOG_CONTAINER_NAME
value: step-user-main
@@ -0,0 +1,21 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../base/installs/generic
- ../../base/metadata/base
- ../../third-party/argo/installs/namespace
- ../../third-party/minio/base
- ../../third-party/mysql/base
# Identifier for application manager to apply ownerReference.
# The ownerReference ensures the resources get garbage collected
# when application is deleted.
# !!! If you want to customize the namespace,
# please also update base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml
namespace: kubeflow
labels:
- includeSelectors: true
pairs:
application-crd-id: kubeflow-pipelines