update kubeflow dip-catalog

2025-01-13 02:31:27 +00:00
parent 1dc1181a03
commit 5451f16d72
1959 changed files with 602337 additions and 0 deletions
@@ -0,0 +1,25 @@
+# Kustomize Manifest Tests
+
+## E2E
+
+There are 2 goals for the e2e tests:
+1. Verify that core components can be applied and become Ready
+2. Verify that core CRDs can be created and succeed
+
+### Components
+
+The e2e are installing everything by using the [single-install
+command](../README.md#install-with-a-single-command). This means that all of
+the core and common (Istio, Knativ, Cert Manager) are being installed and
+included in the tests.
+
+### Test Suite
+
+The e2e tests are completely independent of the underlying K8s cluster, as well
+as the platform of the cluster. These tests should be able to run in real
+world clusters, as well as ephemeral ones like KinD.
+
+The tests are also explicitly bypassing any authentication system of the
+installation. The goal of these tests are to ensure that a common use case that
+deploys all of Kubeflow's components can succeed. Thus we only want to test
+that the core CRDs can be successfully applied and complete.
@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+kubectl create namespace argo
+# Download the binary
+curl -sLO https://github.com/argoproj/argo-workflows/releases/download/v3.1.3/argo-linux-amd64.gz
+
+# Unzip
+gunzip argo-linux-amd64.gz
+
+# Make binary executable
+chmod +x argo-linux-amd64
+
+# Move binary to path
+mv ./argo-linux-amd64 /usr/local/bin/argo
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+echo "Installing cert-manager ..."
+cd common/cert-manager
+kubectl create namespace cert-manager
+kustomize build cert-manager/base | kubectl apply -f -
+
+echo "Waiting for cert-manager to be ready ..."
+kubectl wait --for=condition=ready pod -l 'app in (cert-manager,webhook)' --timeout=180s -n cert-manager
+kubectl wait --for=jsonpath='{.subsets[0].addresses[0].targetRef.kind}'=Pod endpoints -l 'app in (cert-manager,webhook)' --timeout=180s -n cert-manager
+
+echo "Deploy clusterissuer.cert-manager.io/kubeflow-self-signing-issuer"
+kustomize build kubeflow-issuer/base | kubectl apply -f -
@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+echo "Installing Istio-cni ..."
+cd common/istio-cni-1-22
+kustomize build istio-crds/base | kubectl apply -f -
+kustomize build istio-namespace/base | kubectl apply -f -
+kustomize build istio-install/base | kubectl apply -f -
@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+echo "Installing Istio ..."
+cd common/istio-1-22
+kustomize build istio-crds/base | kubectl apply -f -
+kustomize build istio-namespace/base | kubectl apply -f -
+kustomize build istio-install/base | kubectl apply -f -
+
+echo "Waiting for all Istio Pods to become ready..."
+kubectl wait --for=condition=Ready pods --all -n istio-system --timeout 300s
@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+echo "Installing Istio configured with external authorization..."
+cd common/istio-1-22
+kustomize build istio-crds/base | kubectl apply -f -
+kustomize build istio-namespace/base | kubectl apply -f -
+kustomize build istio-install/overlays/oauth2-proxy | kubectl apply -f -
+cd -
+
+echo "Waiting for all Istio Pods to become ready..."
+kubectl wait --for=condition=Ready pods --all -n istio-system --timeout=300s \
+  --field-selector=status.phase!=Succeeded
+
+echo "Installing oauth2-proxy..."
+cd common/oidc-client
+kustomize build oauth2-proxy/overlays/m2m-self-signed/ | kubectl apply -f -
+kubectl wait --for=condition=ready pod -l 'app.kubernetes.io/name=oauth2-proxy' --timeout=180s -n oauth2-proxy
@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+echo "Fetching KinD executable ..."
+sudo swapoff -a
+
+# This conditional helps running GH Workflows through
+# [act](https://github.com/nektos/act)
+if [ -e /swapfile ]; then
+    sudo rm -f /swapfile
+    sudo mkdir -p /tmp/etcd
+    sudo mount -t tmpfs tmpfs /tmp/etcd
+fi
+curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
+chmod +x ./kind
+sudo mv kind /usr/local/bin
@@ -0,0 +1,14 @@
+#!/bin/bash
+set -euo pipefail
+echo "Installing KNative with istio-cni ..."
+set +e
+kustomize build common/knative/knative-serving/base | kubectl apply -f -
+set -e
+kustomize build common/knative/knative-serving/base | kubectl apply -f -
+
+kustomize build common/istio-cni-1-22/cluster-local-gateway/base | kubectl apply -f -
+kustomize build common/istio-cni-1-22/kubeflow-istio-resources/base | kubectl apply -f -
+
+kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=600s \
+  --field-selector=status.phase!=Succeeded
+kubectl patch cm config-domain --patch '{"data":{"example.com":""}}' -n knative-serving
@@ -0,0 +1,14 @@
+#!/bin/bash
+set -euo pipefail
+echo "Installing KNative ..."
+set +e
+kustomize build common/knative/knative-serving/base | kubectl apply -f -
+set -e
+kustomize build common/knative/knative-serving/base | kubectl apply -f -
+
+kustomize build common/istio-1-22/cluster-local-gateway/base | kubectl apply -f -
+kustomize build common/istio-1-22/kubeflow-istio-resources/base | kubectl apply -f -
+
+kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=600s \
+  --field-selector=status.phase!=Succeeded
+kubectl patch cm config-domain --patch '{"data":{"example.com":""}}' -n knative-serving
@@ -0,0 +1,15 @@
+#!/bin/bash
+set -euo pipefail
+echo "Installing Kserve ..."
+cd contrib/kserve
+set +e
+kustomize build kserve | kubectl apply -f -
+sleep 30
+kustomize build kserve | kubectl apply -f -
+set -e
+echo "Waiting for crd/clusterservingruntimes.serving.kserve.io to be available ..."
+kubectl wait --for condition=established --timeout=30s crd/clusterservingruntimes.serving.kserve.io
+kustomize build kserve | kubectl apply -f -
+kustomize build models-web-app/overlays/kubeflow | kubectl apply -f -
+kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=600s \
+  --field-selector=status.phase!=Succeeded
@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+
+echo "Fetching kubectl executable ..."
+curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+chmod +x ./kubectl
+sudo mv kubectl /usr/local/bin
@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+curl --silent --location --remote-name "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.2.1/kustomize_v5.2.1_linux_amd64.tar.gz"
+tar -xzvf kustomize_v5.2.1_linux_amd64.tar.gz
+chmod a+x kustomize
+sudo mv kustomize /usr/local/bin/kustomize
@@ -0,0 +1,9 @@
+#!/bin/bash
+set -euo pipefail
+
+echo "Installing Profiles Controller"
+kustomize build apps/profiles/upstream/overlays/kubeflow | kubectl apply -f -
+kubectl -n kubeflow wait --for=condition=Ready pods -l kustomize.component=profiles --timeout 180s
+
+echo "Installing Multitenancy Kubeflow Roles"
+kustomize build common/kubeflow-roles/base | kubectl apply -f -
@@ -0,0 +1,12 @@
+#!/bin/bash
+set -euo pipefail
+echo "Installing Pipelines ..."
+cd apps/pipeline/upstream
+kubectl apply -f third-party/metacontroller/base/crd.yaml
+echo "Waiting for crd/compositecontrollers.metacontroller.k8s.io to be available ..."
+kubectl wait --for condition=established --timeout=30s crd/compositecontrollers.metacontroller.k8s.io
+kustomize build env/cert-manager/platform-agnostic-multi-user | kubectl apply -f -
+sleep 60
+kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=600s \
+  --field-selector=status.phase!=Succeeded
+cd -
@@ -0,0 +1,56 @@
+---
+---
+apiVersion: kubeflow.org/v1beta1
+kind: Experiment
+metadata:
+  namespace: kubeflow-user
+  name: grid
+spec:
+  objective:
+    type: minimize
+    goal: 0.1
+    objectiveMetricName: loss
+  algorithm:
+    algorithmName: grid
+  parallelTrialCount: 2
+  maxTrialCount: 2
+  maxFailedTrialCount: 2
+  parameters:
+    - name: lr
+      parameterType: double
+      feasibleSpace:
+        min: "0.01"
+        step: "0.005"
+        max: "0.05"
+    - name: momentum
+      parameterType: double
+      feasibleSpace:
+        min: "0.5"
+        step: "0.1"
+        max: "0.9"
+  trialTemplate:
+    primaryContainerName: training-container
+    trialParameters:
+      - name: learningRate
+        description: Learning rate for the training model
+        reference: lr
+      - name: momentum
+        description: Momentum for the training model
+        reference: momentum
+    trialSpec:
+      apiVersion: batch/v1
+      kind: Job
+      spec:
+        template:
+          spec:
+            containers:
+              - name: training-container
+                image: docker.io/kubeflowkatib/pytorch-mnist-cpu:latest
+                command:
+                  - "python3"
+                  - "/opt/pytorch-mnist/mnist.py"
+                  - "--epochs=1"
+                  - "--batch-size=16"
+                  - "--lr=${trialParameters.learningRate}"
+                  - "--momentum=${trialParameters.momentum}"
+            restartPolicy: Never
@@ -0,0 +1,15 @@
+apiVersion: "serving.kserve.io/v1beta1"
+kind: "InferenceService"
+metadata:
+  name: "sklearn-iris"
+spec:
+  predictor:
+    sklearn:
+      resources:
+          limits:
+            cpu: "1"
+            memory: 2Gi
+          requests:
+            cpu: "0.1"
+            memory: 200M 
+      storageUri: "gs://kfserving-examples/models/sklearn/1.0/model"
@@ -0,0 +1,27 @@
+apiVersion: kubeflow.org/v1
+kind: Notebook
+metadata:
+  annotations:
+    notebooks.kubeflow.org/creator: user@example.com
+    notebooks.kubeflow.org/server-type: jupyter
+  generation: 1
+  labels:
+    access-ml-pipeline: "true"
+    app: test
+  name: test
+  namespace: kubeflow-user-example-com
+spec:
+  template:
+    spec:
+      containers:
+      - name: test
+        image: kubeflownotebookswg/jupyter-scipy:v1.9.0-rc.1
+        imagePullPolicy: IfNotPresent
+        resources:
+          limits:
+            cpu: "0.6"
+            memory: 1.2Gi
+          requests:
+            cpu: "0.5"
+            memory: 1Gi
+      serviceAccountName: default-editor
@@ -0,0 +1,25 @@
+apiVersion: kubeflow.org/v1alpha1
+kind: PodDefault
+metadata:
+  name: access-ml-pipeline
+  namespace: kubeflow-user-example-com
+spec:
+  desc: Allow access to Kubeflow Pipelines
+  selector:
+    matchLabels:
+      access-ml-pipeline: "true"
+  env:
+    - name: KF_PIPELINES_SA_TOKEN_PATH
+      value: /var/run/secrets/kubeflow/pipelines/token
+  volumes:
+    - name: volume-kf-pipeline-token
+      projected:
+        sources:
+          - serviceAccountToken:
+              path: token
+              expirationSeconds: 7200
+              audience: pipelines.kubeflow.org
+  volumeMounts:
+    - mountPath: /var/run/secrets/kubeflow/pipelines
+      name: volume-kf-pipeline-token
+      readOnly: true
@@ -0,0 +1,29 @@
+import kfp
+from kfp import dsl
+import kfp.components as comp
+
+
+@comp.create_component_from_func
+def echo_op():
+    print("Test pipeline")
+
+
+@dsl.pipeline(name="test-pipeline", description="A test pipeline.")
+def hello_world_pipeline():
+    echo_task = echo_op()
+
+
+if __name__ == "__main__":
+    # Run the Kubeflow Pipeline in the user's namespace.
+    kfp_client = kfp.Client(
+        host="http://localhost:3000", namespace="kubeflow-user-example-com"
+    )
+    kfp_client.runs.api_client.default_headers.update(
+        {"kubeflow-userid": "kubeflow-user-example-com"}
+    )
+    # create the KFP run
+    run_id = kfp_client.create_run_from_pipeline_func(
+        hello_world_pipeline,
+        namespace="kubeflow-user-example-com",
+        arguments={},
+    ).run_id
@@ -0,0 +1,21 @@
+apiVersion: "kubeflow.org/v1"
+kind: TFJob
+metadata:
+  name: tfjob-simple
+  namespace: kubeflow
+spec:
+   tfReplicaSpecs:
+     Worker:
+       replicas: 2
+       restartPolicy: OnFailure
+       template:
+         spec:
+           containers:
+             - name: tensorflow
+               image: gcr.io/kubeflow-ci/tf-mnist-with-summaries:1.0
+               command:
+                 - "python"
+                 - "/var/tf_mnist/mnist_with_summaries.py"
+                 - "--log_dir=/train/logs"
+                 - "--learning_rate=0.01"
+                 - "--batch_size=150"
@@ -0,0 +1,26 @@
+apiVersion: kind.x-k8s.io/v1alpha4
+kind: Cluster
+# Configure registry for KinD.
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."$REGISTRY_NAME:$REGISTRY_PORT"]
+    endpoint = ["http://$REGISTRY_NAME:$REGISTRY_PORT"]
+# This is needed in order to support projected volumes with service account tokens.
+# See: https://kubernetes.slack.com/archives/CEKK1KTN2/p1600268272383600
+kubeadmConfigPatches:
+  - |
+    apiVersion: kubeadm.k8s.io/v1beta2
+    kind: ClusterConfiguration
+    metadata:
+      name: config
+    apiServer:
+      extraArgs:
+        "service-account-issuer": "kubernetes.default.svc"
+        "service-account-signing-key-file": "/etc/kubernetes/pki/sa.key"
+nodes:
+- role: control-plane
+  image: kindest/node:v1.29.4@sha256:3abb816a5b1061fb15c6e9e60856ec40d56b7b52bcea5f5f1350bc6e2320b6f8
+- role: worker
+  image: kindest/node:v1.29.4@sha256:3abb816a5b1061fb15c6e9e60856ec40d56b7b52bcea5f5f1350bc6e2320b6f8
+- role: worker
+  image: kindest/node:v1.29.4@sha256:3abb816a5b1061fb15c6e9e60856ec40d56b7b52bcea5f5f1350bc6e2320b6f8
@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+
+from kfp import dsl
+import kfp
+from time import sleep
+import subprocess
+import logging
+import sys
+from datetime import datetime, timezone
+
+logger = logging.getLogger("run_and_wait_for_pipeline")
+logging.basicConfig(
+    stream=sys.stdout,
+    level=logging.DEBUG,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+)
+
+
+client = kfp.Client()
+experiment_name = "my-experiment"
+experiment_namespace = "kubeflow-user-example-com"
+
+
+@dsl.component
+def add(a: float, b: float) -> float:
+    """Calculates sum of two arguments"""
+    return a + b
+
+
+@dsl.pipeline(
+    name="Addition pipeline",
+    description="An example pipeline that performs addition calculations.",
+)
+def add_pipeline(
+    a: float = 1.0,
+    b: float = 7.0,
+):
+    first_add_task = add(a=a, b=4.0)
+    add(a=first_add_task.output, b=b)
+
+
+try:
+    logger.info(
+        f"Trying to get experiment from {experiment_name=} {experiment_namespace=}."
+    )
+    experiment = client.get_experiment(
+        experiment_name=experiment_name, namespace=experiment_namespace
+    )
+    logger.info("Experiment found!")
+except Exception:
+    logger.info("Experiment not found, trying to create experiment.")
+    experiment = client.create_experiment(
+        name=experiment_name, namespace=experiment_namespace
+    )
+    logger.info("Experiment created!")
+
+try:
+    logger.info("Trying to create Pipeline Run.")
+    run = client.create_run_from_pipeline_func(
+        add_pipeline,
+        arguments={"a": 7.0, "b": 8.0},
+        experiment_id=experiment.experiment_id,
+        enable_caching=False,
+    )
+except Exception as e:
+    logger.error(
+        f"Failed to create Pipeline Run. Exception: {e.__class__.__name__}: {str(e)}"
+    )
+    raise SystemExit(1)
+
+# For now being able to start a pipeline is enough.
+# while True:
+#     live_run = client.get_run(run_id=run.run_id)
+#     logger.info(f"Pipeline Run State: {live_run.state}.")
+
+#     minutes_from_pipeline_run_start = (
+#         datetime.now(timezone.utc) - live_run.created_at
+#     ).seconds / 60
+
+#     if minutes_from_pipeline_run_start > 5:
+#         logger.debug(
+#             "Pipeline is running for more than 5 minutes, "
+#             f"showing pod states in {experiment_namespace=}."
+#         )
+#         subprocess.run(["kubectl", "get", "pods"])
+
+#     if live_run.finished_at > live_run.created_at:
+#         logger.info("Finished Pipeline Run!")
+#         logger.info(
+#             f"Pipeline was running for {minutes_from_pipeline_run_start:0.2} minutes."
+#         )
+#         logger.info(f"Pipeline Run finished in state: {live_run.state}.")
+#         logger.info(f"Pipeline Run finished with error: {live_run.error}.")
+
+#         if live_run.state != "SUCCEEDED":
+#             logger.warn("The Pipeline Run finished but has failed...")
+
+#             logger.warn("Running 'kubectl get pods':")
+#             subprocess.run(["kubectl", "get", "pods"])
+
+#             logger.warn("Running 'kubectl describe wf':")
+#             subprocess.run(["kubectl", "describe", "wf"])
+
+#             raise SystemExit(1)
+#         break
+#     else:
+#         logger.info("Waiting for pipeline to finish...")
+#     sleep(5)
@@ -0,0 +1,102 @@
+#!/bin/bash
+
+namespace="kubeflow"
+error_flag=0
+
+# Function to check if 'id' command is available in a container
+has_id_command() {
+  local pod_name="$1"
+  local container_name="$2"
+
+  # Execute 'id' command and capture the output
+  if kubectl exec -it -n "$namespace" "$pod_name" -c "$container_name" -- id -u >/dev/null 2>&1; then
+    return 0  # 'id' command is available
+  else
+    return 1  # 'id' command is not available
+  fi
+}
+
+# Function to check 'securityContext' and 'runAsNonRoot' at the pod or container level
+has_securityContext_and_runAsNonRoot() {
+  local pod_name="$1"
+  local container_name="$2"
+
+  # Use jq to check if 'securityContext' is defined at the pod level
+  local securityContextPod=$(kubectl get pod -n "$namespace" "$pod_name" -o json | jq -r '.spec.securityContext')
+
+  if [ "$securityContextPod" = "null" ]; then
+    : # 'securityContext' is missing at the pod level, continue checking at the container level
+  else
+    # Check 'runAsNonRoot' at the pod level
+    local runAsNonRootPod=$(kubectl get pod -n "$namespace" "$pod_name" -o json | jq -r '.spec.securityContext.runAsNonRoot // "Missing"')
+
+    if [ "$runAsNonRootPod" = "Missing" ]; then
+      : # 'runAsNonRoot' is missing at the pod level, continue checking at the container level
+    else
+      return 0  # 'runAsNonRoot' is present at the pod level (success)
+    fi
+  fi
+  
+  # Use jq to check 'securityContext' at the container level
+  local securityContextContainer=$(kubectl get pod -n "$namespace" "$pod_name" -o json | jq -r '.spec.containers[] | select(.name == "'"$container_name"'").securityContext')
+
+  if [ "$securityContextContainer" = "null" ]; then
+    if [ "$securityContextPod" = "null" ]; then
+      echo "Error: 'securityContext' is missing at the pod and container level in container $container_name of pod $pod_name"
+      return 1
+    else
+      echo "Error: There is no runasnonroot on pod level and 'securityContext' is missing at container level in container $container_name of pod $pod_name"
+      return 1
+    fi
+  fi
+
+  # Check 'runAsNonRoot' at the container level
+  local runAsNonRootContainer=$(kubectl get pod -n "$namespace" "$pod_name" -o json | jq -r '.spec.containers[] | select(.name == "'"$container_name"'").securityContext.runAsNonRoot // "Missing"')
+
+  if [ "$runAsNonRootContainer" = "Missing" ]; then
+    echo "Error: There is no runasnonroot on pod level and'runAsNonRoot' is missing in container $container_name of pod $pod_name"
+    return 1  # 'runAsNonRoot' is missing at the container level (fail)
+  fi
+
+  return 0  # 'securityContext' and 'runAsNonRoot' are defined at the container level
+}
+
+# Get a list of pod names in the specified namespace that are not in the "Completed" state
+pod_names=$(kubectl get pods -n "$namespace" --field-selector=status.phase!=Succeeded,status.phase!=Failed -o json | jq -r '.items[].metadata.name')
+
+# Loop through the pod names and execute checks
+for pod_name in $pod_names; do
+  echo "Entering pod $pod_name in namespace $namespace..."
+
+  container_names=$(kubectl get pod -n "$namespace" "$pod_name" -o json | jq -r '.spec.containers[].name')
+
+  for container_name in $container_names; do
+    if has_securityContext_and_runAsNonRoot "$pod_name" "$container_name"; then
+      error_flag=1
+    fi
+
+    if has_id_command "$pod_name" "$container_name"; then
+      user_id=$(kubectl exec -it -n "$namespace" "$pod_name" -c "$container_name" -- id -u)
+      
+      # Clean up whitespace in the user_id using tr
+      user_id_cleaned=$(echo -n "$user_id" | tr -d '[:space:]')
+
+      if [ "$user_id_cleaned" = "0" ]; then
+        echo "Error: Pod $pod_name contains user ID 0 in container $container_name"
+        error_flag=1
+      else
+        echo "Container: $container_name - User ID: $user_id_cleaned"
+      fi
+    else
+      echo "Warning: 'id' command not available in container $container_name"
+    fi
+  done
+done
+
+# Exit with an error if any pod contains an error condition
+if [ $error_flag -eq 1 ]; then
+  exit 1
+fi
+
+# Exit successfully
+exit 0
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+CRONJOB_NAME=kubeflow-m2m-oidc-configurator
+NAMESPACE=istio-system
+
+# Function to get the latest Job created by the CronJob
+get_latest_job() {
+  kubectl get jobs -n "${NAMESPACE}" \
+    --sort-by=.metadata.creationTimestamp -o json \
+    | jq --arg cronjob_name "${CRONJOB_NAME}" -r '.items[] | select(.metadata.ownerReferences[] | select(.name==$cronjob_name)) | .metadata.name' \
+    | tail -n 1
+}
+
+# Wait until a Job is created
+echo "Waiting for a Job to be created by the ${CRONJOB_NAME} CronJob..."
+while true; do
+  JOB_NAME=$(get_latest_job)
+  if [[ -n "${JOB_NAME}" ]]; then
+    echo "Job ${JOB_NAME} created."
+    break
+  fi
+  sleep 5
+  echo "Waiting..."
+done
+
+# Wait for the Job to complete successfully
+echo "Waiting for the Job ${JOB_NAME} to complete..."
+while true; do
+  STATUS=$(kubectl get job "${JOB_NAME}" -n "${NAMESPACE}" -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}')
+  if [[ "${STATUS}" == "True" ]]; then
+    echo "Job ${JOB_NAME} completed successfully."
+    break
+  fi
+
+  FAILED=$(kubectl get job "${JOB_NAME}" -n "${NAMESPACE}" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}')
+  if [[ "${FAILED}" == "True" ]]; then
+    echo "Job ${JOB_NAME} failed."
+    exit 1
+  fi
+  sleep 5
+done