update kubeflow dip-catalog

2025-01-13 02:31:27 +00:00
parent 1dc1181a03
commit 5451f16d72
1959 changed files with 602337 additions and 0 deletions
@@ -0,0 +1,3 @@
+approvers:
+  - Jeffwan
+  - PatrickXYS
@@ -0,0 +1,56 @@
+# Sample installation
+
+1. Create an EKS cluster
+
+Run this command to create EKS cluster
+```
+eksctl create cluster \
+--name AWS-KFP \
+--version 1.17 \
+--region us-west-2 \
+--nodegroup-name linux-nodes \
+--node-type m5.xlarge \
+--nodes 2 \
+--nodes-min 1 \
+--nodes-max 4 \
+--managed
+```
+
+2. Prepare S3
+
+Create S3 bucket. [Console](https://console.aws.amazon.com/s3/home).
+
+Run this command to create S3 bucket by changing `<YOUR_S3_BUCKET_NAME>` to your prefer s3 bucket name.
+
+```
+export S3_BUCKET=<YOUR_S3_BUCKET_NAME>
+export AWS_REGION=us-west-2
+aws s3 mb s3://$S3_BUCKET --region $AWS_REGION
+```
+
+3. Prepare RDS
+
+Follow this [doc](https://awslabs.github.io/kubeflow-manifests/docs/deployment/rds-s3/guide/) to set up AWS RDS instance.
+
+4. Customize your values
+- Edit [params.env](params.env), [secret.env](secret.env) and [minio-artifact-secret-patch.env](minio-artifact-secret-patch.env)
+
+5. Install
+
+```
+kubectl apply -k ../../cluster-scoped-resources
+# If upper one action got failed, e.x. you used wrong value, try delete, fix and apply again
+# kubectl delete -k ../../cluster-scoped-resources
+
+kubectl wait crd/applications.app.k8s.io --for condition=established --timeout=60s
+
+kubectl apply -k ./
+# If upper one action got failed, e.x. you used wrong value, try delete, fix and apply again
+# kubectl delete -k ./
+
+kubectl wait applications/pipeline -n kubeflow --for condition=Ready --timeout=1800s
+
+kubectl port-forward -n kubeflow svc/ml-pipeline-ui 8080:80
+```
+
+Now you can access via `localhost:8080`
@@ -0,0 +1,32 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ml-pipeline
+spec:
+  template:
+    metadata:
+      labels:
+        app: ml-pipeline
+    spec:
+      containers:
+        - env:
+            - name: OBJECTSTORECONFIG_SECURE
+              value: "true"
+            - name: OBJECTSTORECONFIG_BUCKETNAME
+              valueFrom:
+                configMapKeyRef:
+                  name: pipeline-install-config
+                  key: bucketName
+            - name: OBJECTSTORECONFIG_HOST
+              valueFrom:
+                configMapKeyRef:
+                  name: pipeline-install-config
+                  key: minioServiceHost
+            - name: OBJECTSTORECONFIG_REGION
+              valueFrom:
+                configMapKeyRef:
+                  name: pipeline-install-config
+                  key: minioServiceRegion
+            - name: OBJECTSTORECONFIG_PORT
+              value: ""
+          name: ml-pipeline-api-server
@@ -0,0 +1,27 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ml-pipeline-ui
+spec:
+  template:
+    metadata:
+      labels:
+        app: ml-pipeline-ui
+    spec:
+      volumes:
+        - name: config-volume
+          configMap:
+            name: ml-pipeline-ui-configmap
+      containers:
+        - name: ml-pipeline-ui
+          env:
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: mlpipeline-minio-artifact
+                  key: accesskey
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: mlpipeline-minio-artifact
+                  key: secretkey
@@ -0,0 +1,20 @@
+{
+artifactRepository:
+{
+    s3: {
+        bucket: $(kfp-artifact-bucket-name),
+        keyPrefix: artifacts,
+        endpoint: s3.amazonaws.com,
+        insecure: true,
+        accessKeySecret: {
+            name: mlpipeline-minio-artifact,
+            key: accesskey
+        },
+        secretKeySecret: {
+            name: mlpipeline-minio-artifact,
+            key: secretkey
+        }
+    },
+    archiveLogs: true
+}
+}
@@ -0,0 +1,34 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow
+resources:
+  - ../../env/platform-agnostic
+configMapGenerator:
+  - name: pipeline-install-config
+    env: params.env
+    behavior: merge
+  - name: workflow-controller-configmap
+    behavior: replace
+    files:
+      - config
+  - name: ml-pipeline-ui-configmap
+    behavior: replace
+    files:
+      - viewer-pod-template.json
+secretGenerator:
+  - name: mysql-secret
+    env: secret.env
+    behavior: merge
+  - name: mlpipeline-minio-artifact
+    env: minio-artifact-secret-patch.env
+    behavior: merge
+generatorOptions:
+  disableNameSuffixHash: true
+patches:
+  - path: aws-configuration-pipeline-patch.yaml
+  - path: aws-configuration-pipeline-ui-patch.yaml
+# Identifier for application manager to apply ownerReference.
+# The ownerReference ensures the resources get garbage collected
+# when application is deleted.
+commonLabels:
+  application-crd-id: kubeflow-pipelines
@@ -0,0 +1,2 @@
+accesskey=YOUR_AWS_ACCESS_ID
+secretkey=YOUR_AWS_SECRET_KEY
@@ -0,0 +1,5 @@
+dbHost=YOUR_RDS_ENDPOINT
+
+bucketName=YOUR_S3_BUCKET_NAME
+minioServiceHost=s3.amazonaws.com
+minioServiceRegion=YOUR_AWS_REGION
@@ -0,0 +1,2 @@
+username=YOUR_RDS_USERNAME
+password=YOUR_RDS_PASSWORD
@@ -0,0 +1,37 @@
+{
+  "spec": {
+    "containers": [
+      {
+        "env": [
+          {
+            "name": "AWS_ACCESS_KEY_ID",
+            "valueFrom": {
+              "secretKeyRef": {
+                "name": "mlpipeline-minio-artifact",
+                "key": "accesskey"
+              }
+            }
+          },
+          {
+            "name": "AWS_SECRET_ACCESS_KEY",
+            "valueFrom": {
+              "secretKeyRef": {
+                "name": "mlpipeline-minio-artifact",
+                "key": "secretkey"
+              }
+            }
+          },
+          {
+            "name": "AWS_REGION",
+            "valueFrom": {
+              "configMapKeyRef": {
+                "name": "pipeline-install-config",
+                "key": "minioServiceRegion"
+              }
+            }
+          }
+        ]
+      }
+    ]
+  }
+}