update kubeflow dip-catalog

2025-01-13 02:31:27 +00:00
parent 1dc1181a03
commit 5451f16d72
1959 changed files with 602337 additions and 0 deletions
@@ -0,0 +1,89 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cloudsqlproxy
+  labels:
+    app: cloudsqlproxy
+spec:
+  selector:
+    matchLabels:
+      app: cloudsqlproxy
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: cloudsqlproxy
+    spec:
+      serviceAccountName: kubeflow-pipelines-cloudsql-proxy
+      containers:
+        - image: gcr.io/cloudsql-docker/gce-proxy:1.25.0
+          name: cloudsqlproxy
+          env:            
+            - name: GCP_CLOUDSQL_INSTANCE_NAME
+              valueFrom:
+                configMapKeyRef:
+                  name: pipeline-install-config
+                  key: gcsCloudSqlInstanceName
+          command: ["/cloud_sql_proxy",
+                    "-dir=/cloudsql",                    
+                    "-instances=$(GCP_CLOUDSQL_INSTANCE_NAME)=tcp:0.0.0.0:3306",
+                    "-use_http_health_check",
+                    "term_timeout=10s"]
+          # set term_timeout if require graceful handling of shutdown
+          # NOTE: proxy will stop accepting new connections; only wait on existing connections
+          lifecycle:
+            preStop:
+              exec:
+                # (optional) add a preStop hook so that termination is delayed
+                # this is required if your server still require new connections (e.g., connection pools)
+                command: ['sleep', '10']
+          # Liveness probe configuration is adviced by Cloud SQL Proxy examples:
+          # https://github.com/GoogleCloudPlatform/cloudsql-proxy/tree/main/examples/k8s-health-check
+          # For details, see https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
+          livenessProbe:
+            httpGet:
+              path: /liveness
+              port: 8090
+            # Number of seconds after the container has started before the first probe is scheduled. Defaults to 0.
+            # Not necessary when the startup probe is in use.
+            initialDelaySeconds: 0
+            # Frequency of the probe.
+            periodSeconds: 60
+            # Number of seconds after which the probe times out.
+            timeoutSeconds: 30
+            # Number of times the probe is allowed to fail before the transition
+            # from healthy to failure state.
+            #
+            # If periodSeconds = 60, 5 tries will result in five minutes of
+            # checks. The proxy starts to refresh a certificate five minutes
+            # before its expiration. If those five minutes lapse without a
+            # successful refresh, the liveness probe will fail and the pod will be
+            # restarted.
+            failureThreshold: 5
+          readinessProbe:
+            httpGet:
+              path: /readiness
+              port: 8090
+            initialDelaySeconds: 0
+            periodSeconds: 10
+            timeoutSeconds: 5
+            # Number of times the probe must report success to transition from failure to healthy state.
+            # Defaults to 1 for readiness probe.
+            successThreshold: 1
+            failureThreshold: 1
+          startupProbe:
+            httpGet:
+              path: /startup
+              port: 8090
+            periodSeconds: 1
+            timeoutSeconds: 5
+            failureThreshold: 20
+          ports:
+            - name: mysql
+              containerPort: 3306
+          volumeMounts:
+            - mountPath: /cloudsql
+              name: cloudsql
+      volumes:
+        - name: cloudsql
+          emptyDir:
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kubeflow-pipelines-cloudsql-proxy
@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- cloudsql-proxy-deployment.yaml
+- cloudsql-proxy-sa.yaml
+- mysql-service.yaml
@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: mysql
+spec:
+  ports:
+    - port: 3306
+  selector:
+    app: cloudsqlproxy
@@ -0,0 +1,22 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ml-pipeline
+spec:
+  template:
+    spec:
+      containers:
+        - name: ml-pipeline-api-server
+          env:
+            - name: HAS_DEFAULT_BUCKET
+              value: 'true'
+            - name: BUCKET_NAME
+              valueFrom:
+                configMapKeyRef:
+                  name: pipeline-install-config
+                  key: bucketName
+            - name: PROJECT_ID
+              valueFrom:
+                configMapKeyRef:
+                  name: pipeline-install-config
+                  key: gcsProjectId
@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+images:
+  - name: gcr.io/ml-pipeline/inverse-proxy-agent
+    newTag: 2.2.0
+resources:
+  - proxy-configmap.yaml
+  - proxy-deployment.yaml
+  - proxy-role.yaml
+  - proxy-rolebinding.yaml
+  - proxy-sa.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: inverse-proxy-config
@@ -0,0 +1,21 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: proxy-agent
+  name: proxy-agent
+spec:
+  selector:
+    matchLabels:
+      app: proxy-agent
+  template:
+    metadata:
+      labels:
+        app: proxy-agent
+    spec:
+      hostNetwork: true
+      containers:
+      - image: gcr.io/ml-pipeline/inverse-proxy-agent:dummy
+        imagePullPolicy: IfNotPresent
+        name: proxy-agent
+      serviceAccountName: proxy-agent-runner
@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app: proxy-agent-runner
+  name: proxy-agent-runner
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - '*'
@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app: proxy-agent-runner
+  name: proxy-agent-runner
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: proxy-agent-runner
+subjects:
+- kind: ServiceAccount
+  name: proxy-agent-runner
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: proxy-agent-runner
@@ -0,0 +1,32 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  # Application controller is used to provide Google Cloud Console integration.
+  - ../../third-party/application
+  - ../../base/application
+  - ../../base/installs/generic
+  - ../../base/metadata/base
+  - ../../third-party/argo/installs/namespace
+  - inverse-proxy
+  - minio-gcs-gateway
+  - cloudsql-proxy
+
+# Identifier for application manager to apply ownerReference.
+# The ownerReference ensures the resources get garbage collected
+# when application is deleted.
+commonLabels:
+  application-crd-id: kubeflow-pipelines
+
+# !!! If you want to customize the namespace,
+# please also update base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml
+namespace: kubeflow
+
+patches:
+  - path: gcp-configurations-patch.yaml
+
+# Used by Kustomize
+configMapGenerator:
+  - name: pipeline-install-config
+    env: params.env
+    behavior: merge
@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- minio-gcs-gateway-deployment.yaml
+- minio-gcs-gateway-sa.yaml
+- minio-gcs-gateway-service.yaml
+
+secretGenerator:
+- name: mlpipeline-minio-artifact
+  env: minio-artifact-secret.env
+generatorOptions:
+  # mlpipeline-minio-artifact needs to be referred by exact name
+  disableNameSuffixHash: true
@@ -0,0 +1,2 @@
+accesskey=minio
+secretkey=minio123
@@ -0,0 +1,47 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: minio
+  labels:
+    app: minio
+spec:
+  selector:
+    matchLabels:
+      app: minio
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: minio
+    spec:
+      serviceAccountName: kubeflow-pipelines-minio-gcs-gateway
+      containers:
+        - name: minio
+          image: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance
+          args:
+            - gateway
+            - gcs
+            - $(GCP_PROJECT_ID)
+          env:            
+            - name: GCP_PROJECT_ID
+              valueFrom:
+                configMapKeyRef:
+                  name: pipeline-install-config
+                  key: gcsProjectId
+            - name: MINIO_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: mlpipeline-minio-artifact
+                  key: accesskey
+            - name: MINIO_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: mlpipeline-minio-artifact
+                  key: secretkey
+          resources:
+            requests:
+              cpu: 20m
+              memory: 25Mi
+          ports:
+            - containerPort: 9000
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kubeflow-pipelines-minio-gcs-gateway
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio-service
+spec:
+  ports:
+    - port: 9000
+      targetPort: 9000
+      protocol: TCP
+  selector:
+    app: minio
@@ -0,0 +1,6 @@
+pipelineDb=pipelinedb
+mlmdDb=metadb
+cacheDb=cachedb
+bucketName=yourGcsBucketName
+gcsProjectId=yourGcsProjectId
+gcsCloudSqlInstanceName=yourCloudSqlInstanceName