Add kubeflow/v1.10.0
This commit is contained in:
wbsong111
@@ -0,0 +1,6 @@
|
||||
approvers:
|
||||
# - pschoen-itsc
|
||||
- juliusvonkohout
|
||||
reviewers:
|
||||
# - pschoen-itsc
|
||||
- juliusvonkohout
|
||||
@@ -0,0 +1,51 @@
|
||||
# SeaweedFS
|
||||
|
||||
- [Official documentation](https://github.com/seaweedfs/seaweedfs/wiki)
|
||||
- [Official repository](https://github.com/seaweedfs/seaweedfs)
|
||||
|
||||
SeaweedFS is a simple and highly scalable distributed file system. It has an S3 interface which makes it usable as an object store for kubeflow.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Kubernetes (any recent Version should work)
|
||||
- You should have `kubectl` available and configured to talk to the desired cluster.
|
||||
- `kustomize`
|
||||
- If you installed kubeflow with minio, use the `istio` dir instead of `base` for the kustomize commands.
|
||||
|
||||
## Compile manifests
|
||||
|
||||
```bash
|
||||
kubectl kustomize ./base/
|
||||
```
|
||||
|
||||
## Install SeaweedFS
|
||||
|
||||
**WARNING**
|
||||
This replaces the service `minio-service` and will redirect the traffic to seaweedfs.
|
||||
|
||||
```bash
|
||||
# Optional, but recommended to backup existing minio-service
|
||||
kubectl get -n kubeflow svc minio-service -o=jsonpath='{.metadata.annotations.kubectl\.kubernetes\.io/last-applied-configuration}' > svc-minio-service-backup.json
|
||||
|
||||
kubectl kustomize ./base/ | kubectl apply -f -
|
||||
```
|
||||
|
||||
## Verify deployment
|
||||
|
||||
Run
|
||||
```bash
|
||||
./test.sh
|
||||
```
|
||||
With the ready check on the container it already verifies that the S3 starts correctly.
|
||||
You can then use it with the endpoint at http://localhost:8333.
|
||||
To create access keys open a shell on the pod and use `weed shell` to configure your instance.
|
||||
Create a user with the command `s3.configure -user <username> -access_key <access-key> -secret-key <secret-key> -actions Read:<my-bucket>/<my-prefix>,Write::<my-bucket>/<my-prefix> -apply`
|
||||
Documentation for this can also be found [here](https://github.com/seaweedfs/seaweedfs/wiki/Amazon-S3-API).
|
||||
|
||||
## Uninstall SeaweedFS
|
||||
|
||||
```bash
|
||||
kubectl kustomize ./base/ | kubectl delete -f -
|
||||
# Restore minio-service from backup
|
||||
kubectl apply -f svc-minio-service-backup.json
|
||||
```
|
||||
@@ -0,0 +1,3 @@
|
||||
# Upgrade SeaweedFS
|
||||
|
||||
Change the image tag in the Deployment to the desired version. You can find the available images [here](https://hub.docker.com/r/chrislusf/seaweedfs).
|
||||
@@ -0,0 +1,29 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: kubeflow
|
||||
|
||||
resources:
|
||||
- seaweedfs/
|
||||
- ../../../apps/pipeline/upstream/env/cert-manager/platform-agnostic-multi-user
|
||||
configMapGenerator:
|
||||
- name: kubeflow-pipelines-profile-controller-code
|
||||
behavior: replace
|
||||
files:
|
||||
- pipeline-profile-controller/sync.py
|
||||
patches:
|
||||
- path: minio-service-patch.yaml
|
||||
- path: pipeline-profile-controller/deployment.yaml
|
||||
- patch: |-
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: ml-pipeline-ui
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: ml-pipeline-ui
|
||||
env:
|
||||
- name: ARTIFACTS_SERVICE_PROXY_ENABLED
|
||||
value: 'false'
|
||||
$patch: merge
|
||||
@@ -0,0 +1,13 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: minio-service
|
||||
namespace: kubeflow
|
||||
spec:
|
||||
ports:
|
||||
- name: http
|
||||
port: 9000
|
||||
protocol: TCP
|
||||
targetPort: 8333
|
||||
selector:
|
||||
app: seaweedfs
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: kubeflow-pipelines-profile-controller
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: profile-controller
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
runAsGroup: 0
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
# We just need an image with the python botocore library installed
|
||||
image: docker.io/alpine/k8s:1.32.3
|
||||
command: ["python", "/hooks/sync.py"]
|
||||
env:
|
||||
- name: KFP_VERSION
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: pipeline-install-config
|
||||
key: appVersion
|
||||
- name: AWS_ENDPOINT_URL
|
||||
value: http://seaweedfs:8111
|
||||
- name: AWS_REGION
|
||||
value: us-east-1
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mlpipeline-minio-artifact
|
||||
key: accesskey
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mlpipeline-minio-artifact
|
||||
key: secretkey
|
||||
@@ -0,0 +1,243 @@
|
||||
# Copyright 2020-2021 The Kubeflow Authors
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
import json
|
||||
import os
|
||||
import base64
|
||||
|
||||
# From awscli installed in alpine/k8s image
|
||||
import botocore.session
|
||||
|
||||
S3_BUCKET_NAME = 'mlpipeline'
|
||||
|
||||
session = botocore.session.get_session()
|
||||
# To interact with seaweedfs user management. Region does not matter.
|
||||
iam = session.create_client('iam', region_name='foobar')
|
||||
|
||||
|
||||
def main():
|
||||
settings = get_settings_from_env()
|
||||
server = server_factory(**settings)
|
||||
server.serve_forever()
|
||||
|
||||
|
||||
def get_settings_from_env(controller_port=None,
|
||||
visualization_server_image=None, frontend_image=None,
|
||||
visualization_server_tag=None, frontend_tag=None, disable_istio_sidecar=None):
|
||||
"""
|
||||
Returns a dict of settings from environment variables relevant to the controller
|
||||
|
||||
Environment settings can be overridden by passing them here as arguments.
|
||||
|
||||
Settings are pulled from the all-caps version of the setting name. The
|
||||
following defaults are used if those environment variables are not set
|
||||
to enable backwards compatibility with previous versions of this script:
|
||||
visualization_server_image: ghcr.io/kubeflow/kfp-visualization-server
|
||||
visualization_server_tag: value of KFP_VERSION environment variable
|
||||
frontend_image: ghcr.io/kubeflow/kfp-frontend
|
||||
frontend_tag: value of KFP_VERSION environment variable
|
||||
disable_istio_sidecar: Required (no default)
|
||||
minio_access_key: Required (no default)
|
||||
minio_secret_key: Required (no default)
|
||||
"""
|
||||
settings = dict()
|
||||
settings["controller_port"] = \
|
||||
controller_port or \
|
||||
os.environ.get("CONTROLLER_PORT", "8080")
|
||||
|
||||
settings["visualization_server_image"] = \
|
||||
visualization_server_image or \
|
||||
os.environ.get("VISUALIZATION_SERVER_IMAGE", "ghcr.io/kubeflow/kfp-visualization-server")
|
||||
|
||||
settings["frontend_image"] = \
|
||||
frontend_image or \
|
||||
os.environ.get("FRONTEND_IMAGE", "ghcr.io/kubeflow/kfp-frontend")
|
||||
|
||||
# Look for specific tags for each image first, falling back to
|
||||
# previously used KFP_VERSION environment variable for backwards
|
||||
# compatibility
|
||||
settings["visualization_server_tag"] = \
|
||||
visualization_server_tag or \
|
||||
os.environ.get("VISUALIZATION_SERVER_TAG") or \
|
||||
os.environ["KFP_VERSION"]
|
||||
|
||||
settings["frontend_tag"] = \
|
||||
frontend_tag or \
|
||||
os.environ.get("FRONTEND_TAG") or \
|
||||
os.environ["KFP_VERSION"]
|
||||
|
||||
settings["disable_istio_sidecar"] = \
|
||||
disable_istio_sidecar if disable_istio_sidecar is not None \
|
||||
else os.environ.get("DISABLE_ISTIO_SIDECAR") == "true"
|
||||
|
||||
return settings
|
||||
|
||||
|
||||
def server_factory(visualization_server_image,
|
||||
visualization_server_tag, frontend_image, frontend_tag,
|
||||
disable_istio_sidecar, url="", controller_port=8080):
|
||||
"""
|
||||
Returns an HTTPServer populated with Handler with customized settings
|
||||
"""
|
||||
class Controller(BaseHTTPRequestHandler):
|
||||
def sync(self, parent, attachments):
|
||||
# parent is a namespace
|
||||
namespace = parent.get("metadata", {}).get("name")
|
||||
|
||||
pipeline_enabled = parent.get("metadata", {}).get(
|
||||
"labels", {}).get("pipelines.kubeflow.org/enabled")
|
||||
|
||||
if pipeline_enabled != "true":
|
||||
return {"status": {}, "attachments": []}
|
||||
|
||||
# Compute status based on observed state.
|
||||
desired_status = {
|
||||
"kubeflow-pipelines-ready":
|
||||
len(attachments["Secret.v1"]) == 1 and
|
||||
len(attachments["ConfigMap.v1"]) == 3 and
|
||||
len(attachments["Deployment.apps/v1"]) == 2 and
|
||||
len(attachments["Service.v1"]) == 2 and
|
||||
len(attachments["DestinationRule.networking.istio.io/v1alpha3"]) == 1 and
|
||||
len(attachments["AuthorizationPolicy.security.istio.io/v1beta1"]) == 1 and
|
||||
"True" or "False"
|
||||
}
|
||||
|
||||
# Generate the desired attachment object(s).
|
||||
desired_resources = [
|
||||
{
|
||||
"apiVersion": "v1",
|
||||
"kind": "ConfigMap",
|
||||
"metadata": {
|
||||
"name": "kfp-launcher",
|
||||
"namespace": namespace,
|
||||
},
|
||||
"data": {
|
||||
"defaultPipelineRoot": f"minio://{S3_BUCKET_NAME}/private-artifacts/{namespace}/v2/artifacts",
|
||||
},
|
||||
},
|
||||
{
|
||||
"apiVersion": "v1",
|
||||
"kind": "ConfigMap",
|
||||
"metadata": {
|
||||
"name": "metadata-grpc-configmap",
|
||||
"namespace": namespace,
|
||||
},
|
||||
"data": {
|
||||
"METADATA_GRPC_SERVICE_HOST":
|
||||
"metadata-grpc-service.kubeflow",
|
||||
"METADATA_GRPC_SERVICE_PORT": "8080",
|
||||
},
|
||||
},
|
||||
{
|
||||
"apiVersion": "v1",
|
||||
"kind": "ConfigMap",
|
||||
"metadata": {
|
||||
"name": "artifact-repositories",
|
||||
"namespace": namespace,
|
||||
"annotations": {
|
||||
"workflows.argoproj.io/default-artifact-repository": "default-namespaced"
|
||||
}
|
||||
},
|
||||
"data": {
|
||||
"default-namespaced": json.dumps({
|
||||
"archiveLogs": True,
|
||||
"s3": {
|
||||
"endpoint": "minio-service.kubeflow:9000",
|
||||
"bucket": S3_BUCKET_NAME,
|
||||
"keyFormat": f"private-artifacts/{namespace}/{{{{workflow.name}}}}/{{{{workflow.creationTimestamp.Y}}}}/{{{{workflow.creationTimestamp.m}}}}/{{{{workflow.creationTimestamp.d}}}}/{{{{pod.name}}}}",
|
||||
"insecure": True,
|
||||
"accessKeySecret": {
|
||||
"name": "mlpipeline-minio-artifact",
|
||||
"key": "accesskey",
|
||||
},
|
||||
"secretKeySecret": {
|
||||
"name": "mlpipeline-minio-artifact",
|
||||
"key": "secretkey",
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
},
|
||||
]
|
||||
print('Received request:\n', json.dumps(parent, sort_keys=True))
|
||||
print('Desired resources except secrets:\n', json.dumps(desired_resources, sort_keys=True))
|
||||
|
||||
# Moved after the print argument because this is sensitive data.
|
||||
|
||||
# Check if secret is already there when the controller made the request. If yes, then
|
||||
# use it. Else create a new credentials on seaweedfs for the namespace.
|
||||
if s3_secret := attachments["Secret.v1"].get(f"{namespace}/mlpipeline-minio-artifact"):
|
||||
desired_resources.append(s3_secret)
|
||||
print('Using existing secret')
|
||||
else:
|
||||
print('Creating new access key.')
|
||||
s3_access_key = iam.create_access_key(UserName=namespace)
|
||||
# Use the AWS IAM API of seaweedfs to manage access policies to bucket.
|
||||
# This policy ensures that a user can only access artifacts from his own profile.
|
||||
iam.put_user_policy(
|
||||
UserName=namespace,
|
||||
PolicyName=f"KubeflowProject{namespace}",
|
||||
PolicyDocument=json.dumps(
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:Put*",
|
||||
"s3:Get*",
|
||||
"s3:List*"
|
||||
],
|
||||
"Resource": [
|
||||
f"arn:aws:s3:::{S3_BUCKET_NAME}/artifacts/*",
|
||||
f"arn:aws:s3:::{S3_BUCKET_NAME}/private-artifacts/{namespace}/*",
|
||||
f"arn:aws:s3:::{S3_BUCKET_NAME}/private/{namespace}/*",
|
||||
f"arn:aws:s3:::{S3_BUCKET_NAME}/shared/*",
|
||||
]
|
||||
}]
|
||||
})
|
||||
)
|
||||
desired_resources.insert(
|
||||
0,
|
||||
{
|
||||
"apiVersion": "v1",
|
||||
"kind": "Secret",
|
||||
"metadata": {
|
||||
"name": "mlpipeline-minio-artifact",
|
||||
"namespace": namespace,
|
||||
},
|
||||
"data": {
|
||||
"accesskey": base64.b64encode(s3_access_key["AccessKey"]["AccessKeyId"].encode('utf-8')).decode("utf-8"),
|
||||
"secretkey": base64.b64encode(s3_access_key["AccessKey"]["SecretAccessKey"].encode('utf-8')).decode("utf-8"),
|
||||
},
|
||||
})
|
||||
|
||||
return {"status": desired_status, "attachments": desired_resources}
|
||||
|
||||
def do_POST(self):
|
||||
# Serve the sync() function as a JSON webhook.
|
||||
observed = json.loads(
|
||||
self.rfile.read(int(self.headers.get("content-length"))))
|
||||
desired = self.sync(observed["object"], observed["attachments"])
|
||||
|
||||
self.send_response(200)
|
||||
self.send_header("Content-type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(bytes(json.dumps(desired), 'utf-8'))
|
||||
|
||||
return HTTPServer((url, int(controller_port)), Controller)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: kubeflow
|
||||
|
||||
resources:
|
||||
- seaweedfs-deployment.yaml
|
||||
- seaweedfs-pvc.yaml
|
||||
- seaweedfs-networkpolicy.yaml
|
||||
- seaweedfs-create-admin-user-job.yaml
|
||||
- seaweedfs-service.yaml
|
||||
- seaweedfs-service-account.yaml
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
kind: Job
|
||||
apiVersion: batch/v1
|
||||
metadata:
|
||||
name: init-seaweedfs
|
||||
spec:
|
||||
template:
|
||||
metadata:
|
||||
name: init-seaweedfs
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: init-seaweedfs
|
||||
image: 'chrislusf/seaweedfs:3.85'
|
||||
env:
|
||||
- name: WEED_CLUSTER_DEFAULT
|
||||
value: "sw"
|
||||
- name: WEED_CLUSTER_SW_MASTER
|
||||
value: "seaweedfs.kubeflow:9333"
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: mlpipeline-minio-artifact
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
- |
|
||||
wait_for_service() {
|
||||
local url=$1
|
||||
local max_attempts=60 # 5 minutes total (5s * 60)
|
||||
local attempt=1
|
||||
|
||||
echo "Waiting for service at $url..."
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if wget -q --spider "$url" >/dev/null 2>&1; then
|
||||
echo "Service at $url is up!"
|
||||
return 0
|
||||
fi
|
||||
echo "Attempt $attempt: Service not ready yet, retrying in 5s..."
|
||||
sleep 5
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
echo "Service at $url failed to become ready within 5 minutes"
|
||||
exit 1
|
||||
}
|
||||
wait_for_service "http://minio-service.kubeflow:9000/status"
|
||||
exec /bin/echo "s3.bucket.create --name mlpipeline" | /usr/bin/weed shell
|
||||
exec /bin/echo \
|
||||
"s3.configure -user kubeflow-admin \
|
||||
-access_key $accesskey \
|
||||
-secret_key $secretkey \
|
||||
-actions Admin \
|
||||
-apply" |\
|
||||
/usr/bin/weed shell
|
||||
securityContext: # Using restricted profile
|
||||
allowPrivilegeEscalation: false
|
||||
privileged: false
|
||||
runAsNonRoot: true
|
||||
# image defaults to root user
|
||||
runAsUser: 1001
|
||||
runAsGroup: 1001
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
add:
|
||||
- NET_BIND_SERVICE
|
||||
serviceAccountName: seaweedfs
|
||||
@@ -0,0 +1,72 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: seaweedfs
|
||||
namespace: kubeflow
|
||||
labels:
|
||||
app: seaweedfs
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: seaweedfs
|
||||
strategy:
|
||||
type: Recreate
|
||||
# Single container setup not scalable
|
||||
replicas: 1
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: seaweedfs
|
||||
application-crd-id: kubeflow-pipelines
|
||||
spec:
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: 'chrislusf/seaweedfs:3.85'
|
||||
args:
|
||||
- 'server'
|
||||
- '-dir=/data'
|
||||
- '-s3'
|
||||
- '-iam'
|
||||
ports:
|
||||
- containerPort: 8333
|
||||
- containerPort: 8111
|
||||
- containerPort: 9333
|
||||
- containerPort: 19333
|
||||
- containerPort: 8888
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /status
|
||||
port: 8333
|
||||
scheme: HTTP
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 15
|
||||
successThreshold: 1
|
||||
failureThreshold: 100
|
||||
timeoutSeconds: 10
|
||||
securityContext: # Using restricted profile
|
||||
allowPrivilegeEscalation: false
|
||||
privileged: false
|
||||
runAsNonRoot: true
|
||||
# image defaults to root user
|
||||
runAsUser: 1001
|
||||
runAsGroup: 1001
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
add:
|
||||
- NET_BIND_SERVICE
|
||||
volumeMounts:
|
||||
- mountPath: /data
|
||||
name: data
|
||||
resources:
|
||||
# Benchmark this, just taken from minio
|
||||
requests:
|
||||
cpu: 20m
|
||||
memory: 100Mi
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: seaweedfs-pvc
|
||||
serviceAccountName: seaweedfs
|
||||
@@ -0,0 +1,43 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default-allow-same-namespace
|
||||
namespace: kubeflow
|
||||
spec:
|
||||
podSelector: {}
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: seaweedfs
|
||||
spec:
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchExpressions:
|
||||
- key: app.kubernetes.io/part-of
|
||||
operator: In
|
||||
values:
|
||||
- kubeflow-profile
|
||||
ports:
|
||||
- port: 8333
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchExpressions:
|
||||
- key: kubernetes.io/metadata.name
|
||||
operator: In
|
||||
values:
|
||||
- istio-system
|
||||
podSelector:
|
||||
matchExpressions:
|
||||
- key: app
|
||||
operator: In
|
||||
values:
|
||||
- seaweedfs
|
||||
policyTypes:
|
||||
- Ingress
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: seaweedfs-pvc
|
||||
namespace: kubeflow
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 20Gi
|
||||
@@ -0,0 +1,4 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: seaweedfs
|
||||
@@ -0,0 +1,33 @@
|
||||
# Separate service for new ports of seaweedfs. If we add them to the existing minio-service there will be a problem
|
||||
# with the mlpipeline api server because it relies on MINIO_SERVICE_SERVICE_PORT pointing to the S3 port.
|
||||
# But with multiple ports on a service that is not really reliable. So we use the existing minio-service for
|
||||
# backwards-compatibility, but everything new, seaweedfs related is here.
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: seaweedfs
|
||||
namespace: kubeflow
|
||||
spec:
|
||||
ports:
|
||||
- name: http-iam
|
||||
port: 8111
|
||||
protocol: TCP
|
||||
targetPort: 8111
|
||||
- name: http-master
|
||||
port: 9333
|
||||
protocol: TCP
|
||||
targetPort: 9333
|
||||
- name: grpc-master
|
||||
port: 19333
|
||||
protocol: TCP
|
||||
targetPort: 19333
|
||||
- name: grpc-filer
|
||||
port: 18888
|
||||
protocol: TCP
|
||||
targetPort: 18888
|
||||
- name: http-filer
|
||||
port: 8888
|
||||
protocol: TCP
|
||||
targetPort: 8888
|
||||
selector:
|
||||
app: seaweedfs
|
||||
@@ -0,0 +1,32 @@
|
||||
apiVersion: security.istio.io/v1beta1
|
||||
kind: AuthorizationPolicy
|
||||
metadata:
|
||||
name: seaweedfs-service
|
||||
spec:
|
||||
action: ALLOW
|
||||
selector:
|
||||
matchLabels:
|
||||
app: seaweedfs
|
||||
rules:
|
||||
- from:
|
||||
- source:
|
||||
principals:
|
||||
- cluster.local/ns/kubeflow/sa/ml-pipeline
|
||||
- from:
|
||||
- source:
|
||||
principals:
|
||||
- cluster.local/ns/kubeflow/sa/ml-pipeline-ui
|
||||
# Allow traffic to s3 endpoint from User Pipeline Pods, which don't have a sidecar.
|
||||
# Also needed for traffic from seaweedfs init pod. Seaweedfs gives the client an ip to connect to. This can not be
|
||||
# handled well by istio (AuthPolicy). Instead, access to the sensitive ports will be limited by the NetworkPolicy.
|
||||
- {}
|
||||
---
|
||||
apiVersion: "networking.istio.io/v1alpha3"
|
||||
kind: DestinationRule
|
||||
metadata:
|
||||
name: ml-pipeline-seaweedfs
|
||||
spec:
|
||||
host: seaweedfs.kubeflow.svc.cluster.local
|
||||
trafficPolicy:
|
||||
tls:
|
||||
mode: ISTIO_MUTUAL
|
||||
@@ -0,0 +1,7 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: kubeflow
|
||||
|
||||
resources:
|
||||
- ../base/
|
||||
- istio-authorization-policy.yaml
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -xe
|
||||
|
||||
kubectl create ns kubeflow || echo "namespace kubeflow already exists"
|
||||
kubectl get -n kubeflow svc minio-service -o=jsonpath='{.metadata.annotations.kubectl\.kubernetes\.io/last-applied-configuration}' > svc-minio-service-backup.json
|
||||
kustomize build istio/ | kubectl apply --server-side -f -
|
||||
kubectl -n kubeflow wait --for=condition=available --timeout=600s deploy/seaweedfs
|
||||
kubectl -n kubeflow exec deployments/seaweedfs -c seaweedfs -- sh -c "echo \"s3.configure -user minio -access_key minio -secret_key minio123 -actions Read,Write,List -apply\" | /usr/bin/weed shell"
|
||||
|
||||
kubectl -n kubeflow port-forward svc/minio-service 8333:9000
|
||||
echo "S3 endpoint available on localhost:8333" &
|
||||
|
||||
function trap_handler {
|
||||
kubectl -n kubeflow logs -l app=seaweedfs --tail=100
|
||||
kustomize build istio/ | kubectl delete -f -
|
||||
kubectl apply -f svc-minio-service-backup.json
|
||||
}
|
||||
|
||||
trap trap_handler EXIT
|
||||
Reference in New Issue
Block a user