Add kubeflow/v1.10.0
This commit is contained in:
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env bash
|
||||
# Common functions for Kubeflow manifest synchronization scripts
|
||||
|
||||
setup_error_handling() {
|
||||
set -euxo pipefail
|
||||
IFS=$'\n\t'
|
||||
}
|
||||
|
||||
# Check if the git repository has uncommitted changes
|
||||
check_uncommitted_changes() {
|
||||
if [ -n "$(git status --porcelain)" ]; then
|
||||
echo "WARNING: You have uncommitted changes"
|
||||
fi
|
||||
}
|
||||
|
||||
# Create a new git branch if it doesn't exist
|
||||
create_branch() {
|
||||
local branch="$1"
|
||||
|
||||
check_uncommitted_changes
|
||||
|
||||
if [ $(git branch --list "$branch") ]; then
|
||||
echo "WARNING: Branch $branch already exists."
|
||||
fi
|
||||
|
||||
if ! git show-ref --verify --quiet refs/heads/$branch; then
|
||||
git checkout -b "$branch"
|
||||
else
|
||||
echo "Branch $branch already exists."
|
||||
fi
|
||||
}
|
||||
|
||||
clone_and_checkout() {
|
||||
local source_directory="$1"
|
||||
local repository_url="$2"
|
||||
local repository_directory="$3"
|
||||
local commit="$4"
|
||||
|
||||
echo "Checking out in $source_directory to $commit..."
|
||||
|
||||
mkdir -p "$source_directory"
|
||||
cd "$source_directory"
|
||||
|
||||
# Clone repository if it doesn't exist
|
||||
if [ ! -d "$repository_directory/.git" ]; then
|
||||
git clone "$repository_url" "$repository_directory"
|
||||
fi
|
||||
|
||||
# Checkout to specific commit
|
||||
cd "$source_directory/$repository_directory"
|
||||
if ! git rev-parse --verify --quiet "$commit"; then
|
||||
git checkout -b "$commit"
|
||||
else
|
||||
git checkout "$commit"
|
||||
fi
|
||||
|
||||
check_uncommitted_changes
|
||||
}
|
||||
|
||||
# Copy manifests from source to destination
|
||||
copy_manifests() {
|
||||
local source="$1"
|
||||
local destination="$2"
|
||||
|
||||
echo "Copying manifests..."
|
||||
|
||||
if [ -d "$destination" ]; then
|
||||
rm -r "$destination"
|
||||
fi
|
||||
|
||||
cp "$source" "$destination" -r
|
||||
echo "Successfully copied all manifests."
|
||||
}
|
||||
|
||||
# Update README with new commit reference
|
||||
update_readme() {
|
||||
local manifests_directory="$1"
|
||||
local source_text="$2"
|
||||
local destination_text="$3"
|
||||
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
sed -i "" "s|$source_text|$destination_text|g" "${manifests_directory}/README.md" # BSD sed of Mac OSX
|
||||
else
|
||||
sed -i "s|$source_text|$destination_text|g" "${manifests_directory}/README.md" # GNU sed of Linux
|
||||
fi
|
||||
}
|
||||
|
||||
# Commit changes to git repository
|
||||
commit_changes() {
|
||||
local manifests_directory="$1"
|
||||
local commit_message="$2"
|
||||
local paths_to_add=("${@:3}")
|
||||
|
||||
cd "$manifests_directory"
|
||||
|
||||
for path in "${paths_to_add[@]}"; do
|
||||
git add "$path"
|
||||
done
|
||||
|
||||
git commit -s -m "$commit_message"
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env bash
|
||||
# This script helps to create a PR to update the Istio CNI manifests
|
||||
|
||||
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||
source "${SCRIPT_DIRECTORY}/library.sh"
|
||||
|
||||
setup_error_handling
|
||||
|
||||
COMPONENT_NAME="istio-cni"
|
||||
COMMIT="1.24.3"
|
||||
CURRENT_VERSION="1-24"
|
||||
NEW_VERSION="1-24" # Must be a release
|
||||
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:=/tmp/${COMPONENT_NAME}}
|
||||
BRANCH_NAME=${BRANCH_NAME:=${COMPONENT_NAME}-${COMMIT?}}
|
||||
|
||||
# Path configurations
|
||||
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
|
||||
ISTIO_OLD=$MANIFESTS_DIRECTORY/common/${COMPONENT_NAME}-${CURRENT_VERSION}
|
||||
ISTIO_NEW=$MANIFESTS_DIRECTORY/common/${COMPONENT_NAME}-${NEW_VERSION}
|
||||
|
||||
if [ ! -d "$ISTIO_NEW" ]; then
|
||||
cp -a $ISTIO_OLD $ISTIO_NEW
|
||||
fi
|
||||
|
||||
create_branch "$BRANCH_NAME"
|
||||
|
||||
echo "Checking out in $SOURCE_DIRECTORY to $COMMIT..."
|
||||
mkdir -p $SOURCE_DIRECTORY
|
||||
cd $SOURCE_DIRECTORY
|
||||
if [ ! -d "istio-${COMMIT}" ]; then
|
||||
wget "https://github.com/istio/istio/releases/download/${COMMIT}/istio-${COMMIT}-linux-amd64.tar.gz"
|
||||
tar xvfz istio-${COMMIT}-linux-amd64.tar.gz
|
||||
fi
|
||||
|
||||
ISTIOCTL=$SOURCE_DIRECTORY/istio-${COMMIT}/bin/istioctl
|
||||
cd $ISTIO_NEW
|
||||
|
||||
$ISTIOCTL manifest generate -f profile.yaml -f profile-overlay.yaml --set components.cni.enabled=true --set components.cni.namespace=kube-system > dump.yaml
|
||||
./split-istio-packages -f dump.yaml
|
||||
mv $ISTIO_NEW/crd.yaml $ISTIO_NEW/istio-crds/base
|
||||
mv $ISTIO_NEW/install.yaml $ISTIO_NEW/istio-install/base
|
||||
mv $ISTIO_NEW/cluster-local-gateway.yaml $ISTIO_NEW/cluster-local-gateway/base
|
||||
rm dump.yaml
|
||||
|
||||
check_uncommitted_changes
|
||||
|
||||
SOURCE_TEXT="\[.*\](https://github.com/istio/istio/releases/tag/.*)"
|
||||
DESTINATION_TEXT="\[$COMMIT\](https://github.com/istio/istio/releases/tag/$COMMIT)"
|
||||
|
||||
update_readme "$MANIFESTS_DIRECTORY" "$SOURCE_TEXT" "$DESTINATION_TEXT"
|
||||
|
||||
echo "Synchronizing directory names..."
|
||||
find "$MANIFESTS_DIRECTORY" -type f -not -path '*/.git/*' -exec sed -i "s/istio-cni-${CURRENT_VERSION}/istio-cni-${NEW_VERSION}/g" {} +
|
||||
|
||||
cd "$MANIFESTS_DIRECTORY"
|
||||
if [ "$CURRENT_VERSION" != "$NEW_VERSION" ]; then
|
||||
rm -rf $ISTIO_OLD
|
||||
fi
|
||||
commit_changes "$MANIFESTS_DIRECTORY" "Upgrade istio-cni to v.${COMMIT}" "."
|
||||
|
||||
echo "Synchronization completed successfully."
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env bash
|
||||
# This script helps to create a PR to update the Istio manifests
|
||||
|
||||
# Source the common library functions
|
||||
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||
source "${SCRIPT_DIRECTORY}/library.sh"
|
||||
|
||||
setup_error_handling
|
||||
|
||||
COMPONENT_NAME="istio"
|
||||
COMMIT="1.24.3"
|
||||
CURRENT_VERSION="1-24"
|
||||
NEW_VERSION="1-24" # Must be a release
|
||||
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:=/tmp/${COMPONENT_NAME}}
|
||||
BRANCH_NAME=${BRANCH_NAME:=${COMPONENT_NAME}-${COMMIT?}}
|
||||
|
||||
# Path configurations
|
||||
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
|
||||
ISTIO_OLD=$MANIFESTS_DIRECTORY/common/${COMPONENT_NAME}-${CURRENT_VERSION}
|
||||
ISTIO_NEW=$MANIFESTS_DIRECTORY/common/${COMPONENT_NAME}-${NEW_VERSION}
|
||||
|
||||
if [ ! -d "$ISTIO_NEW" ]; then
|
||||
cp -a $ISTIO_OLD $ISTIO_NEW
|
||||
fi
|
||||
|
||||
create_branch "$BRANCH_NAME"
|
||||
|
||||
echo "Checking out in $SOURCE_DIRECTORY to $COMMIT..."
|
||||
mkdir -p $SOURCE_DIRECTORY
|
||||
cd $SOURCE_DIRECTORY
|
||||
if [ ! -d "istio-${COMMIT}" ]; then
|
||||
wget "https://github.com/istio/istio/releases/download/${COMMIT}/istio-${COMMIT}-linux-amd64.tar.gz"
|
||||
tar xvfz istio-${COMMIT}-linux-amd64.tar.gz
|
||||
fi
|
||||
|
||||
ISTIOCTL=$SOURCE_DIRECTORY/istio-${COMMIT}/bin/istioctl
|
||||
cd $ISTIO_NEW
|
||||
|
||||
$ISTIOCTL manifest generate -f profile.yaml -f profile-overlay.yaml > dump.yaml
|
||||
./split-istio-packages -f dump.yaml
|
||||
mv $ISTIO_NEW/crd.yaml $ISTIO_NEW/istio-crds/base
|
||||
mv $ISTIO_NEW/install.yaml $ISTIO_NEW/istio-install/base
|
||||
mv $ISTIO_NEW/cluster-local-gateway.yaml $ISTIO_NEW/cluster-local-gateway/base
|
||||
rm dump.yaml
|
||||
|
||||
check_uncommitted_changes
|
||||
|
||||
SOURCE_TEXT="\[.*\](https://github.com/istio/istio/releases/tag/.*)"
|
||||
DESTINATION_TEXT="\[$COMMIT\](https://github.com/istio/istio/releases/tag/$COMMIT)"
|
||||
|
||||
update_readme "$MANIFESTS_DIRECTORY" "$SOURCE_TEXT" "$DESTINATION_TEXT"
|
||||
|
||||
find "$MANIFESTS_DIRECTORY" -type f -not -path '*/.git/*' -exec sed -i "s/istio-${CURRENT_VERSION}/istio-${NEW_VERSION}/g" {} +
|
||||
|
||||
cd "$MANIFESTS_DIRECTORY"
|
||||
if [ "$CURRENT_VERSION" != "$NEW_VERSION" ]; then
|
||||
rm -rf $ISTIO_OLD
|
||||
fi
|
||||
commit_changes "$MANIFESTS_DIRECTORY" "Upgrade istio to v.${COMMIT}" "."
|
||||
|
||||
echo "Synchronization completed successfully."
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
# This script helps to create a PR to update the Katib manifests
|
||||
|
||||
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||
source "${SCRIPT_DIRECTORY}/library.sh"
|
||||
|
||||
setup_error_handling
|
||||
|
||||
COMPONENT_NAME="katib"
|
||||
REPOSITORY_NAME="kubeflow/katib"
|
||||
REPOSITORY_URL="https://github.com/kubeflow/katib.git"
|
||||
COMMIT="v0.18.0"
|
||||
REPOSITORY_DIRECTORY="katib"
|
||||
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:=/tmp/kubeflow-${COMPONENT_NAME}}
|
||||
BRANCH_NAME=${BRANCH_NAME:=synchronize-${COMPONENT_NAME}-manifests-${COMMIT?}}
|
||||
|
||||
# Path configurations
|
||||
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
|
||||
SOURCE_MANIFESTS_PATH="manifests/v1beta1"
|
||||
DESTINATION_MANIFESTS_PATH="apps/${COMPONENT_NAME}/upstream"
|
||||
|
||||
# README update patterns
|
||||
SOURCE_TEXT="\[.*\](https://github.com/${REPOSITORY_NAME}/tree/.*/manifests/v1beta1)"
|
||||
DESTINATION_TEXT="\[${COMMIT}\](https://github.com/${REPOSITORY_NAME}/tree/${COMMIT}/manifests/v1beta1)"
|
||||
|
||||
create_branch "$BRANCH_NAME"
|
||||
|
||||
clone_and_checkout "$SOURCE_DIRECTORY" "$REPOSITORY_URL" "$REPOSITORY_DIRECTORY" "$COMMIT"
|
||||
|
||||
copy_manifests "${SOURCE_DIRECTORY}/${REPOSITORY_DIRECTORY}/${SOURCE_MANIFESTS_PATH}" "${MANIFESTS_DIRECTORY}/${DESTINATION_MANIFESTS_PATH}"
|
||||
|
||||
update_readme "$MANIFESTS_DIRECTORY" "$SOURCE_TEXT" "$DESTINATION_TEXT"
|
||||
|
||||
commit_changes "$MANIFESTS_DIRECTORY" "Update ${REPOSITORY_NAME} manifests from ${COMMIT}" \
|
||||
"apps" \
|
||||
"README.md"
|
||||
|
||||
echo "Synchronization completed successfully."
|
||||
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env bash
|
||||
# This script helps to create a PR to update the Knative manifests
|
||||
|
||||
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||
source "${SCRIPT_DIRECTORY}/library.sh"
|
||||
|
||||
setup_error_handling
|
||||
|
||||
COMPONENT_NAME="knative"
|
||||
KN_SERVING_RELEASE="v1.16.2" # Must be a release
|
||||
KN_EXTENSION_RELEASE="v1.16.0" # Must be a release
|
||||
KN_EVENTING_RELEASE="v1.16.4" # Must be a release
|
||||
BRANCH_NAME=${BRANCH_NAME:=synchronize-${COMPONENT_NAME}-manifests-${KN_SERVING_RELEASE?}}
|
||||
|
||||
# Path configurations
|
||||
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
|
||||
DESTINATION_DIRECTORY=$MANIFESTS_DIRECTORY/common/${COMPONENT_NAME}
|
||||
|
||||
create_branch "$BRANCH_NAME"
|
||||
|
||||
check_uncommitted_changes
|
||||
|
||||
# Clean up existing files (keep README and OWNERS)
|
||||
if [ -d "$DESTINATION_DIRECTORY" ]; then
|
||||
rm -r "$DESTINATION_DIRECTORY/knative-serving/base/upstream"
|
||||
rm "$DESTINATION_DIRECTORY/knative-serving-post-install-jobs/base/serving-post-install-jobs.yaml"
|
||||
rm -r "$DESTINATION_DIRECTORY/knative-eventing/base/upstream"
|
||||
rm "$DESTINATION_DIRECTORY/knative-eventing-post-install-jobs/base/eventing-post-install.yaml"
|
||||
fi
|
||||
|
||||
# Create required directories
|
||||
mkdir -p "$DESTINATION_DIRECTORY/knative-serving/base/upstream"
|
||||
mkdir -p "$DESTINATION_DIRECTORY/knative-serving-post-install-jobs/base"
|
||||
mkdir -p "$DESTINATION_DIRECTORY/knative-eventing/base/upstream"
|
||||
mkdir -p "$DESTINATION_DIRECTORY/knative-eventing-post-install-jobs/base"
|
||||
|
||||
echo "Downloading knative-serving manifests..."
|
||||
wget -O $DESTINATION_DIRECTORY/knative-serving/base/upstream/serving-core.yaml "https://github.com/knative/serving/releases/download/knative-$KN_SERVING_RELEASE/serving-core.yaml"
|
||||
wget -O $DESTINATION_DIRECTORY/knative-serving/base/upstream/net-istio.yaml "https://github.com/knative-extensions/net-istio/releases/download/knative-$KN_EXTENSION_RELEASE/net-istio.yaml"
|
||||
wget -O $DESTINATION_DIRECTORY/knative-serving-post-install-jobs/base/serving-post-install-jobs.yaml "https://github.com/knative/serving/releases/download/knative-$KN_SERVING_RELEASE/serving-post-install-jobs.yaml"
|
||||
|
||||
yq eval -i '... comments=""' $DESTINATION_DIRECTORY/knative-serving/base/upstream/serving-core.yaml
|
||||
yq eval -i '... comments=""' $DESTINATION_DIRECTORY/knative-serving/base/upstream/net-istio.yaml
|
||||
yq eval -i '... comments=""' $DESTINATION_DIRECTORY/knative-serving-post-install-jobs/base/serving-post-install-jobs.yaml
|
||||
|
||||
yq eval -i 'explode(.)' $DESTINATION_DIRECTORY/knative-serving/base/upstream/serving-core.yaml
|
||||
yq eval -i 'explode(.)' $DESTINATION_DIRECTORY/knative-serving/base/upstream/net-istio.yaml
|
||||
yq eval -i 'explode(.)' $DESTINATION_DIRECTORY/knative-serving-post-install-jobs/base/serving-post-install-jobs.yaml
|
||||
|
||||
yq eval -i 'select(.kind == "Job" and .metadata.generateName == "storage-version-migration-serving-") | .metadata.name = "storage-version-migration-serving"' $DESTINATION_DIRECTORY/knative-serving-post-install-jobs/base/serving-post-install-jobs.yaml
|
||||
|
||||
echo "Downloading knative-eventing manifests..."
|
||||
wget -O $DESTINATION_DIRECTORY/knative-eventing/base/upstream/eventing-core.yaml "https://github.com/knative/eventing/releases/download/knative-$KN_EVENTING_RELEASE/eventing-core.yaml"
|
||||
wget -O $DESTINATION_DIRECTORY/knative-eventing/base/upstream/in-memory-channel.yaml "https://github.com/knative/eventing/releases/download/knative-$KN_EVENTING_RELEASE/in-memory-channel.yaml"
|
||||
wget -O $DESTINATION_DIRECTORY/knative-eventing/base/upstream/mt-channel-broker.yaml "https://github.com/knative/eventing/releases/download/knative-$KN_EVENTING_RELEASE/mt-channel-broker.yaml"
|
||||
wget -O $DESTINATION_DIRECTORY/knative-eventing-post-install-jobs/base/eventing-post-install.yaml "https://github.com/knative/eventing/releases/download/knative-$KN_EVENTING_RELEASE/eventing-post-install.yaml"
|
||||
|
||||
yq eval -i '... comments=""' $DESTINATION_DIRECTORY/knative-eventing/base/upstream/eventing-core.yaml
|
||||
yq eval -i '... comments=""' $DESTINATION_DIRECTORY/knative-eventing/base/upstream/in-memory-channel.yaml
|
||||
yq eval -i '... comments=""' $DESTINATION_DIRECTORY/knative-eventing/base/upstream/mt-channel-broker.yaml
|
||||
yq eval -i '... comments=""' $DESTINATION_DIRECTORY/knative-eventing-post-install-jobs/base/eventing-post-install.yaml
|
||||
|
||||
yq eval -i 'explode(.)' $DESTINATION_DIRECTORY/knative-eventing/base/upstream/eventing-core.yaml
|
||||
yq eval -i 'explode(.)' $DESTINATION_DIRECTORY/knative-eventing/base/upstream/in-memory-channel.yaml
|
||||
yq eval -i 'explode(.)' $DESTINATION_DIRECTORY/knative-eventing/base/upstream/mt-channel-broker.yaml
|
||||
yq eval -i 'explode(.)' $DESTINATION_DIRECTORY/knative-eventing-post-install-jobs/base/eventing-post-install.yaml
|
||||
|
||||
yq eval -i 'select(.kind == "Job" and .metadata.generateName == "storage-version-migration-eventing-") | .metadata.name = "storage-version-migration-eventing"' $DESTINATION_DIRECTORY/knative-eventing-post-install-jobs/base/eventing-post-install.yaml
|
||||
|
||||
yq eval -i 'select((.kind == "ConfigMap" and .metadata.name == "config-observability") | not)' $DESTINATION_DIRECTORY/knative-eventing/base/upstream/in-memory-channel.yaml
|
||||
yq eval -i 'select((.kind == "ConfigMap" and .metadata.name == "config-tracing") | not)' $DESTINATION_DIRECTORY/knative-eventing/base/upstream/in-memory-channel.yaml
|
||||
|
||||
# Helper function to replace text in files
|
||||
replace_in_file() {
|
||||
local SOURCE_TEXT=$1
|
||||
local DESTINATION_TEXT=$2
|
||||
local FILE=$3
|
||||
sed -i "s|$SOURCE_TEXT|$DESTINATION_TEXT|g" $FILE
|
||||
}
|
||||
|
||||
replace_in_file \
|
||||
"\[.*\](https://github.com/knative/serving/releases/tag/knative-.*) <" \
|
||||
"\[$KN_SERVING_RELEASE\](https://github.com/knative/serving/releases/tag/knative-$KN_SERVING_RELEASE) <" \
|
||||
${MANIFESTS_DIRECTORY}/README.md
|
||||
|
||||
replace_in_file \
|
||||
"> \[.*\](https://github.com/knative/eventing/releases/tag/knative-.*)" \
|
||||
"> \[$KN_EVENTING_RELEASE\](https://github.com/knative/eventing/releases/tag/knative-$KN_EVENTING_RELEASE)" \
|
||||
${MANIFESTS_DIRECTORY}/README.md
|
||||
|
||||
replace_in_file \
|
||||
"\[Knative serving (v.*)\](https://github.com/knative/serving/releases/tag/knative-v.*)" \
|
||||
"\[Knative serving ($KN_SERVING_RELEASE)\](https://github.com/knative/serving/releases/tag/knative-$KN_SERVING_RELEASE)" \
|
||||
$DESTINATION_DIRECTORY/README.md
|
||||
|
||||
replace_in_file \
|
||||
"\[Knative ingress controller for Istio (v.*)\](https://github.com/knative-extensions/net-istio/releases/tag/knative-v.*)" \
|
||||
"\[Knative ingress controller for Istio ($KN_EXTENSION_RELEASE)\](https://github.com/knative-extensions/net-istio/releases/tag/knative-$KN_EXTENSION_RELEASE)" \
|
||||
$DESTINATION_DIRECTORY/README.md
|
||||
|
||||
replace_in_file \
|
||||
"The manifests for Knative Eventing are based off the \[v.* release\](https://github.com/knative/eventing/releases/tag/knative-v.*)" \
|
||||
"The manifests for Knative Eventing are based off the \[$KN_EVENTING_RELEASE release\](https://github.com/knative/eventing/releases/tag/knative-$KN_EVENTING_RELEASE)" \
|
||||
$DESTINATION_DIRECTORY/README.md
|
||||
|
||||
commit_changes "$MANIFESTS_DIRECTORY" "Update common/knative manifests from ${KN_SERVING_RELEASE}/${KN_EVENTING_RELEASE}" \
|
||||
"$DESTINATION_DIRECTORY" \
|
||||
"README.md"
|
||||
|
||||
echo "Synchronization completed successfully."
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bash
|
||||
# This script helps to create a PR to update the KServe manifests
|
||||
|
||||
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||
source "${SCRIPT_DIRECTORY}/library.sh"
|
||||
|
||||
setup_error_handling
|
||||
|
||||
COMPONENT_NAME="kserve"
|
||||
REPOSITORY_NAME="kserve/kserve"
|
||||
REPOSITORY_URL="https://github.com/kserve/kserve.git"
|
||||
KSERVE_VERSION="v0.15.0"
|
||||
COMMIT="v0.15.0"
|
||||
REPOSITORY_DIRECTORY="kserve"
|
||||
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:=/tmp/${COMPONENT_NAME}}
|
||||
BRANCH_NAME=${BRANCH_NAME:=synchronize-${COMPONENT_NAME}-manifests-${COMMIT?}}
|
||||
|
||||
# Path configurations
|
||||
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
|
||||
SOURCE_MANIFESTS_PATH="install/${KSERVE_VERSION}"
|
||||
DESTINATION_MANIFESTS_PATH="apps/${COMPONENT_NAME}/${COMPONENT_NAME}"
|
||||
|
||||
# README update patterns
|
||||
SOURCE_TEXT="\[.*\](https://github.com/${REPOSITORY_NAME}/releases/tag/.*)"
|
||||
DESTINATION_TEXT="\[${COMMIT}\](https://github.com/${REPOSITORY_NAME}/releases/tag/${COMMIT}/install/${KSERVE_VERSION})"
|
||||
|
||||
create_branch "$BRANCH_NAME"
|
||||
|
||||
clone_and_checkout "$SOURCE_DIRECTORY" "$REPOSITORY_URL" "$REPOSITORY_DIRECTORY" "$COMMIT"
|
||||
|
||||
echo "Copying kserve manifests..."
|
||||
DESTINATION_DIRECTORY=$MANIFESTS_DIRECTORY/$DESTINATION_MANIFESTS_PATH
|
||||
if [ -d "$DESTINATION_DIRECTORY" ]; then
|
||||
rm -rf "$DESTINATION_DIRECTORY"/kserve*
|
||||
fi
|
||||
cp $SOURCE_DIRECTORY/$REPOSITORY_DIRECTORY/$SOURCE_MANIFESTS_PATH/* $DESTINATION_DIRECTORY -r
|
||||
|
||||
|
||||
update_readme "$MANIFESTS_DIRECTORY" "$SOURCE_TEXT" "$DESTINATION_TEXT"
|
||||
|
||||
commit_changes "$MANIFESTS_DIRECTORY" "Update ${REPOSITORY_NAME} manifests from ${KSERVE_VERSION}" \
|
||||
"apps/${COMPONENT_NAME}" \
|
||||
"README.md" \
|
||||
"scripts"
|
||||
|
||||
echo "Synchronization completed successfully."
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
# This script helps to create a PR to update the KServe Models Web App manifests
|
||||
|
||||
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||
source "${SCRIPT_DIRECTORY}/library.sh"
|
||||
|
||||
setup_error_handling
|
||||
|
||||
COMPONENT_NAME="models-web-app"
|
||||
REPOSITORY_NAME="kserve/models-web-app"
|
||||
REPOSITORY_URL="https://github.com/kserve/models-web-app.git"
|
||||
COMMIT="v0.14.0"
|
||||
REPOSITORY_DIRECTORY="models-web-app"
|
||||
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:=/tmp/kserve-${COMPONENT_NAME}}
|
||||
BRANCH_NAME=${BRANCH_NAME:=synchronize-kserve-${COMPONENT_NAME}-manifests-${COMMIT?}}
|
||||
|
||||
# Path configurations
|
||||
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
|
||||
SOURCE_MANIFESTS_PATH="config"
|
||||
DESTINATION_MANIFESTS_PATH="apps/kserve/${COMPONENT_NAME}"
|
||||
|
||||
# README update patterns
|
||||
SOURCE_TEXT="\[.*\](https://github.com/${REPOSITORY_NAME}/tree/.*)"
|
||||
DESTINATION_TEXT="\[${COMMIT}\](https://github.com/${REPOSITORY_NAME}/tree/${COMMIT}/${SOURCE_MANIFESTS_PATH})"
|
||||
|
||||
create_branch "$BRANCH_NAME"
|
||||
|
||||
clone_and_checkout "$SOURCE_DIRECTORY" "$REPOSITORY_URL" "$REPOSITORY_DIRECTORY" "$COMMIT"
|
||||
|
||||
echo "Copying manifests"
|
||||
copy_manifests "${SOURCE_DIRECTORY}/${REPOSITORY_DIRECTORY}/${SOURCE_MANIFESTS_PATH}" "${MANIFESTS_DIRECTORY}/${DESTINATION_MANIFESTS_PATH}"
|
||||
|
||||
update_readme "$MANIFESTS_DIRECTORY" "$SOURCE_TEXT" "$DESTINATION_TEXT"
|
||||
|
||||
commit_changes "$MANIFESTS_DIRECTORY" "Update kserve models web application manifests from ${COMMIT}" \
|
||||
"${DESTINATION_MANIFESTS_PATH}" \
|
||||
"README.md"
|
||||
|
||||
echo "Synchronization completed successfully."
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env bash
|
||||
# This script helps to create a PR to update the Kubeflow manifests
|
||||
|
||||
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||
source "${SCRIPT_DIRECTORY}/library.sh"
|
||||
|
||||
setup_error_handling
|
||||
|
||||
COMPONENT_NAME="kubeflow"
|
||||
REPOSITORY_NAME="kubeflow/kubeflow"
|
||||
REPOSITORY_URL="https://github.com/kubeflow/kubeflow.git"
|
||||
COMMIT="v1.10.0"
|
||||
REPOSITORY_DIRECTORY="kubeflow"
|
||||
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:=/tmp/${COMPONENT_NAME}-${COMPONENT_NAME}}
|
||||
BRANCH_NAME=${BRANCH_NAME:=synchronize-${COMPONENT_NAME}-${COMPONENT_NAME}-manifests-${COMMIT?}}
|
||||
|
||||
# Path configurations
|
||||
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
|
||||
|
||||
create_branch "$BRANCH_NAME"
|
||||
|
||||
clone_and_checkout "$SOURCE_DIRECTORY" "$REPOSITORY_URL" "$REPOSITORY_DIRECTORY" "$COMMIT"
|
||||
|
||||
# Function to copy manifests for a specific component
|
||||
copy_component_manifests() {
|
||||
local component_name=$1
|
||||
local source_path=$2
|
||||
local destination_path=$3
|
||||
local readme_path_pattern=$4
|
||||
|
||||
echo "Copying ${component_name} manifests..."
|
||||
|
||||
local destination_directory="${MANIFESTS_DIRECTORY}/${destination_path}"
|
||||
if [ -d "$destination_directory" ]; then
|
||||
rm -r "$destination_directory"
|
||||
fi
|
||||
mkdir -p "$destination_directory"
|
||||
|
||||
cp "${SOURCE_DIRECTORY}/${REPOSITORY_DIRECTORY}/${source_path}/"* "$destination_directory" -r
|
||||
|
||||
echo "Updating README for ${component_name}..."
|
||||
local source_text="\[.*\](https://github.com/${REPOSITORY_NAME}/tree/.*/components/${readme_path_pattern})"
|
||||
local destination_text="\[${COMMIT}\](https://github.com/${REPOSITORY_NAME}/tree/${COMMIT}/components/${readme_path_pattern})"
|
||||
|
||||
update_readme "$MANIFESTS_DIRECTORY" "$source_text" "$destination_text"
|
||||
}
|
||||
|
||||
copy_component_manifests "admission-webhook" \
|
||||
"components/admission-webhook/manifests" \
|
||||
"apps/admission-webhook/upstream" \
|
||||
"admission-webhook/manifests"
|
||||
|
||||
copy_component_manifests "centraldashboard" \
|
||||
"components/centraldashboard/manifests" \
|
||||
"apps/centraldashboard/upstream" \
|
||||
"centraldashboard/manifests"
|
||||
|
||||
copy_component_manifests "jupyter-web-app" \
|
||||
"components/crud-web-apps/jupyter/manifests" \
|
||||
"apps/jupyter/jupyter-web-app/upstream" \
|
||||
"crud-web-apps/jupyter/manifests"
|
||||
|
||||
copy_component_manifests "volumes-web-app" \
|
||||
"components/crud-web-apps/volumes/manifests" \
|
||||
"apps/volumes-web-app/upstream" \
|
||||
"crud-web-apps/volumes/manifests"
|
||||
|
||||
copy_component_manifests "tensorboards-web-app" \
|
||||
"components/crud-web-apps/tensorboards/manifests" \
|
||||
"apps/tensorboard/tensorboards-web-app/upstream" \
|
||||
"crud-web-apps/tensorboards/manifests"
|
||||
|
||||
copy_component_manifests "profile-controller" \
|
||||
"components/profile-controller/config" \
|
||||
"apps/profiles/upstream" \
|
||||
"profile-controller/config"
|
||||
|
||||
copy_component_manifests "notebook-controller" \
|
||||
"components/notebook-controller/config" \
|
||||
"apps/jupyter/notebook-controller/upstream" \
|
||||
"notebook-controller/config"
|
||||
|
||||
copy_component_manifests "tensorboard-controller" \
|
||||
"components/tensorboard-controller/config" \
|
||||
"apps/tensorboard/tensorboard-controller/upstream" \
|
||||
"tensorboard-controller/config"
|
||||
|
||||
copy_component_manifests "pvcviewer-controller" \
|
||||
"components/pvcviewer-controller/config" \
|
||||
"apps/pvcviewer-controller/upstream" \
|
||||
"pvcviewer-controller/config"
|
||||
|
||||
echo "Successfully copied all manifests."
|
||||
|
||||
commit_changes "$MANIFESTS_DIRECTORY" "Update ${REPOSITORY_NAME} manifests from ${COMMIT}" \
|
||||
"apps" \
|
||||
"README.md"
|
||||
|
||||
echo "Synchronization completed successfully."
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env bash
|
||||
# This script helps to create a PR to update the Model Registry manifests
|
||||
|
||||
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||
source "${SCRIPT_DIRECTORY}/library.sh"
|
||||
|
||||
setup_error_handling
|
||||
|
||||
COMPONENT_NAME="model-registry"
|
||||
REPOSITORY_NAME="kubeflow/model-registry"
|
||||
REPOSITORY_URL="https://github.com/kubeflow/model-registry.git"
|
||||
COMMIT="v0.2.13"
|
||||
REPOSITORY_DIRECTORY="model-registry"
|
||||
DEV_MODE=${DEV_MODE:=false}
|
||||
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:=/tmp/kubeflow-${COMPONENT_NAME}}
|
||||
BRANCH_NAME=${BRANCH_NAME:=synchronize-kubeflow-${COMPONENT_NAME}-manifests-${COMMIT?}}
|
||||
|
||||
# Path configurations
|
||||
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
|
||||
SOURCE_MANIFESTS_PATH="manifests/kustomize"
|
||||
DESTINATION_MANIFESTS_PATH="apps/${COMPONENT_NAME}/upstream"
|
||||
|
||||
# README update patterns
|
||||
SOURCE_TEXT="\[.*\](https://github.com/${REPOSITORY_NAME}/tree/.*/manifests/kustomize)"
|
||||
DESTINATION_TEXT="\[${COMMIT}\](https://github.com/${REPOSITORY_NAME}/tree/${COMMIT}/manifests/kustomize)"
|
||||
|
||||
create_branch "$BRANCH_NAME"
|
||||
|
||||
clone_and_checkout "$SOURCE_DIRECTORY" "$REPOSITORY_URL" "$REPOSITORY_DIRECTORY" "$COMMIT"
|
||||
|
||||
echo "Copying ${COMPONENT_NAME} manifests..."
|
||||
DESTINATION_DIRECTORY=$MANIFESTS_DIRECTORY/$DESTINATION_MANIFESTS_PATH
|
||||
if [ -d "$DESTINATION_DIRECTORY" ]; then
|
||||
rm -r "$DESTINATION_DIRECTORY"
|
||||
fi
|
||||
mkdir -p $DESTINATION_DIRECTORY
|
||||
cp -r "$SOURCE_DIRECTORY/$REPOSITORY_DIRECTORY/$SOURCE_MANIFESTS_PATH/"* "$DESTINATION_DIRECTORY"
|
||||
|
||||
echo "Successfully copied all manifests."
|
||||
|
||||
update_readme "$MANIFESTS_DIRECTORY" "$SOURCE_TEXT" "$DESTINATION_TEXT"
|
||||
|
||||
commit_changes "$MANIFESTS_DIRECTORY" "Update ${REPOSITORY_NAME} manifests from ${COMMIT}" \
|
||||
"apps" \
|
||||
"README.md"
|
||||
|
||||
echo "Synchronization completed successfully."
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
# This script helps to create a PR to update the Kubeflow Pipelines manifests
|
||||
|
||||
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||
source "${SCRIPT_DIRECTORY}/library.sh"
|
||||
|
||||
setup_error_handling
|
||||
|
||||
COMPONENT_NAME="pipelines"
|
||||
REPOSITORY_NAME="kubeflow/pipelines"
|
||||
REPOSITORY_URL="https://github.com/kubeflow/pipelines.git"
|
||||
COMMIT="2.5.0"
|
||||
REPOSITORY_DIRECTORY="pipelines"
|
||||
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:=/tmp/kubeflow-${COMPONENT_NAME}}
|
||||
BRANCH_NAME=${BRANCH_NAME:=synchronize-kubeflow-${COMPONENT_NAME}-manifests-${COMMIT?}}
|
||||
|
||||
# Path configurations
|
||||
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
|
||||
SOURCE_MANIFESTS_PATH="manifests/kustomize"
|
||||
DESTINATION_MANIFESTS_PATH="apps/pipeline/upstream"
|
||||
|
||||
# README update patterns
|
||||
SOURCE_TEXT="\[.*\](https://github.com/${REPOSITORY_NAME}/tree/.*/manifests/kustomize)"
|
||||
DESTINATION_TEXT="\[${COMMIT}\](https://github.com/${REPOSITORY_NAME}/tree/${COMMIT}/manifests/kustomize)"
|
||||
|
||||
create_branch "$BRANCH_NAME"
|
||||
|
||||
clone_and_checkout "$SOURCE_DIRECTORY" "$REPOSITORY_URL" "$REPOSITORY_DIRECTORY" "$COMMIT"
|
||||
|
||||
echo "Copying ${COMPONENT_NAME} manifests..."
|
||||
copy_manifests "${SOURCE_DIRECTORY}/${REPOSITORY_DIRECTORY}/${SOURCE_MANIFESTS_PATH}" "${MANIFESTS_DIRECTORY}/${DESTINATION_MANIFESTS_PATH}"
|
||||
|
||||
update_readme "$MANIFESTS_DIRECTORY" "$SOURCE_TEXT" "$DESTINATION_TEXT"
|
||||
|
||||
commit_changes "$MANIFESTS_DIRECTORY" "Update ${REPOSITORY_NAME} manifests from ${COMMIT}" \
|
||||
"apps" \
|
||||
"README.md"
|
||||
|
||||
echo "Synchronization completed successfully."
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env bash
|
||||
# This script helps to create a PR to update the Spark Operator manifests
|
||||
|
||||
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||
source "${SCRIPT_DIRECTORY}/library.sh"
|
||||
|
||||
setup_error_handling
|
||||
|
||||
COMPONENT_NAME="spark-operator"
|
||||
SPARK_OPERATOR_VERSION=${SPARK_OPERATOR_VERSION:="2.1.1"}
|
||||
SPARK_OPERATOR_HELM_CHART_REPO=${SPARK_OPERATOR_HELM_CHART_REPO:="https://kubeflow.github.io/spark-operator"}
|
||||
DEV_MODE=${DEV_MODE:=false}
|
||||
BRANCH_NAME=${BRANCH_NAME:=synchronize-${COMPONENT_NAME}-manifests-${SPARK_OPERATOR_VERSION?}}
|
||||
|
||||
# Path configurations
|
||||
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
|
||||
DESTINATION_MANIFESTS_PATH="apps/spark/${COMPONENT_NAME}/base"
|
||||
|
||||
create_branch "$BRANCH_NAME"
|
||||
|
||||
echo "Generating manifests from Helm chart version ${SPARK_OPERATOR_VERSION}..."
|
||||
|
||||
DESTINATION_DIRECTORY=$MANIFESTS_DIRECTORY/$DESTINATION_MANIFESTS_PATH
|
||||
mkdir -p $DESTINATION_DIRECTORY
|
||||
cd $DESTINATION_DIRECTORY
|
||||
|
||||
# Create a kustomization.yaml file if it doesn't exist
|
||||
if [ ! -f kustomization.yaml ]; then
|
||||
cat > kustomization.yaml << EOF
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- resources.yaml
|
||||
EOF
|
||||
fi
|
||||
|
||||
helm template -n kubeflow --include-crds spark-operator spark-operator \
|
||||
--set "spark.jobNamespaces={}" \
|
||||
--set webhook.enable=true \
|
||||
--set webhook.port=9443 \
|
||||
--version ${SPARK_OPERATOR_VERSION} \
|
||||
--repo ${SPARK_OPERATOR_HELM_CHART_REPO} > resources.yaml
|
||||
|
||||
echo "Successfully generated manifests."
|
||||
|
||||
# Use OS-compatible sed command
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
sed -i '' 's/Spark Operator[^|]*|[^|]*apps\/spark\/spark-operator[^|]*|[^|]*[0-9]\.[0-9]\.[0-9]/Spark Operator | apps\/spark\/spark-operator | '"${SPARK_OPERATOR_VERSION}"'/g' "${MANIFESTS_DIRECTORY}/README.md"
|
||||
else
|
||||
sed -i 's/Spark Operator.*|.*apps\/spark\/spark-operator[^|]*|.*[0-9]\.[0-9]\.[0-9]/Spark Operator | apps\/spark\/spark-operator | '"${SPARK_OPERATOR_VERSION}"'/g' "${MANIFESTS_DIRECTORY}/README.md"
|
||||
fi
|
||||
|
||||
commit_changes "$MANIFESTS_DIRECTORY" "Update kubeflow/${COMPONENT_NAME} manifests to ${SPARK_OPERATOR_VERSION}" \
|
||||
"apps/spark" \
|
||||
"README.md" \
|
||||
"scripts"
|
||||
|
||||
echo "Synchronization completed successfully."
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
# This script helps to create a PR to update the Training Operator manifests
|
||||
|
||||
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||
source "${SCRIPT_DIRECTORY}/library.sh"
|
||||
|
||||
setup_error_handling
|
||||
|
||||
COMPONENT_NAME="training-operator"
|
||||
REPOSITORY_NAME="kubeflow/training-operator"
|
||||
REPOSITORY_URL="https://github.com/kubeflow/training-operator.git"
|
||||
COMMIT="v1.9.2"
|
||||
REPOSITORY_DIRECTORY="training-operator"
|
||||
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:=/tmp/kubeflow-${COMPONENT_NAME}}
|
||||
BRANCH_NAME=${BRANCH_NAME:=synchronize-${COMPONENT_NAME}-manifests-${COMMIT?}}
|
||||
|
||||
# Path configurations
|
||||
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
|
||||
SOURCE_MANIFESTS_PATH="manifests"
|
||||
DESTINATION_MANIFESTS_PATH="apps/${COMPONENT_NAME}/upstream"
|
||||
|
||||
# README update patterns
|
||||
SOURCE_TEXT="\[.*\](https://github.com/${REPOSITORY_NAME}/tree/.*/manifests)"
|
||||
DESTINATION_TEXT="\[${COMMIT}\](https://github.com/${REPOSITORY_NAME}/tree/${COMMIT}/manifests)"
|
||||
|
||||
create_branch "$BRANCH_NAME"
|
||||
|
||||
clone_and_checkout "$SOURCE_DIRECTORY" "$REPOSITORY_URL" "$REPOSITORY_DIRECTORY" "$COMMIT"
|
||||
|
||||
copy_manifests "${SOURCE_DIRECTORY}/${REPOSITORY_DIRECTORY}/${SOURCE_MANIFESTS_PATH}" "${MANIFESTS_DIRECTORY}/${DESTINATION_MANIFESTS_PATH}"
|
||||
|
||||
update_readme "$MANIFESTS_DIRECTORY" "$SOURCE_TEXT" "$DESTINATION_TEXT"
|
||||
|
||||
commit_changes "$MANIFESTS_DIRECTORY" "Update ${REPOSITORY_NAME} manifests from ${COMMIT}" \
|
||||
"apps" \
|
||||
"README.md"
|
||||
|
||||
echo "Synchronization completed successfully."
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
# Template for Kubeflow manifests synchronization scripts
|
||||
# Usage: Copy this file and adjust the variables for your specific component
|
||||
|
||||
# Source the common library functions
|
||||
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||
source "${SCRIPT_DIRECTORY}/library.sh"
|
||||
|
||||
setup_error_handling
|
||||
|
||||
# Configuration variables (adjust these for your specific component)
|
||||
COMPONENT_NAME="component-name" # Name of the component (e.g., katib, training-operator)
|
||||
REPOSITORY_NAME="repo-name" # Repository name (e.g., kubeflow/katib)
|
||||
REPOSITORY_URL="https://github.com/org/repo.git" # Repository URL to clone from
|
||||
COMMIT="v0.0.0" # Version/commit to synchronize
|
||||
REPOSITORY_DIRECTORY="${COMPONENT_NAME}" # Directory name within SOURCE_DIRECTORY
|
||||
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:=/tmp/kubeflow-${COMPONENT_NAME}} # Where to clone the source repo
|
||||
BRANCH_NAME=${BRANCH_NAME:=synchronize-${COMPONENT_NAME}-manifests-${COMMIT?}} # Branch name for the PR
|
||||
|
||||
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
|
||||
SOURCE_MANIFESTS_PATH="manifests" # Path within source repo where manifests are
|
||||
DESTINATION_MANIFESTS_PATH="apps/${COMPONENT_NAME}/upstream" # Destination path within manifests repo
|
||||
|
||||
SOURCE_TEXT="\[.*\](https://github.com/${REPOSITORY_NAME}/tree/.*/manifests)"
|
||||
DESTINATION_TEXT="\[${COMMIT}\](https://github.com/${REPOSITORY_NAME}/tree/${COMMIT}/manifests)"
|
||||
|
||||
create_branch "$BRANCH_NAME"
|
||||
|
||||
clone_and_checkout "$SOURCE_DIRECTORY" "$REPOSITORY_URL" "$REPOSITORY_DIRECTORY" "$COMMIT"
|
||||
|
||||
copy_manifests "${SOURCE_DIRECTORY}/${REPOSITORY_DIRECTORY}/${SOURCE_MANIFESTS_PATH}" "${MANIFESTS_DIRECTORY}/${DESTINATION_MANIFESTS_PATH}"
|
||||
|
||||
update_readme "$MANIFESTS_DIRECTORY" "$SOURCE_TEXT" "$DESTINATION_TEXT"
|
||||
|
||||
commit_changes "$MANIFESTS_DIRECTORY" "Update ${REPOSITORY_NAME} manifests from ${COMMIT}" \
|
||||
"${DESTINATION_MANIFESTS_PATH}" \
|
||||
"README.md"
|
||||
|
||||
echo "Synchronization completed successfully."
|
||||
@@ -0,0 +1,412 @@
|
||||
# The script:
|
||||
# 1. Extract all the images used by the Kubeflow Working Groups
|
||||
# - The reported image lists are saved in respective files under ../image_lists directory
|
||||
# 2. Scan the reported images using Trivy for security vulnerabilities
|
||||
# - Scanned reports will be saved in JSON format inside ../image_lists/security_scan_reports/ folder for each Working Group
|
||||
# 3. The script will also generate a summary of the security scan reports with severity counts for each Working Group with images
|
||||
# - Summary of security counts with images a JSON file inside ../image_lists/summary_of_severity_counts_for_WG folder
|
||||
# 4. Generate a summary of the security scan reports
|
||||
# - The summary will be saved in JSON format inside ../image_lists/summary_of_severity_counts_for_WG folder
|
||||
# 5. Before run this file you have to
|
||||
# 1. Install kustomize
|
||||
# - sudo apt install snapd
|
||||
# - sudo snap install kustomize
|
||||
# 2. Install trivy
|
||||
# - sudo apt install snapd
|
||||
# - sudo snap install trivy
|
||||
# 4. Install Python
|
||||
# 5. Install prettytable
|
||||
# - pip install prettytable
|
||||
|
||||
# The script must be executed from the hack folder as it use relative paths
|
||||
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import re
|
||||
import argparse
|
||||
import json
|
||||
import glob
|
||||
from prettytable import PrettyTable
|
||||
|
||||
# Dictionary mapping Kubeflow workgroups to directories containing kustomization files
|
||||
wg_dirs = {
|
||||
"automl": "../apps/katib/upstream/installs",
|
||||
"pipelines": "../apps/pipeline/upstream/env ../apps/kfp-tekton/upstream/env",
|
||||
"training": "../apps/training-operator/upstream/overlays",
|
||||
"manifests": "../common/cert-manager/cert-manager/base ../common/cert-manager/kubeflow-issuer/base ../common/istio-1-24/istio-crds/base ../common/istio-1-24/istio-namespace/base ../common/istio-1-24/istio-install/overlays/oauth2-proxy ../common/oauth2-proxy/overlays/m2m-self-signed ../common/dex/overlays/oauth2-proxy ../common/knative/knative-serving/overlays/gateways ../common/knative/knative-eventing/base ../common/istio-1-24/cluster-local-gateway/base ../common/kubeflow-namespace/base ../common/kubeflow-roles/base ../common/istio-1-24/kubeflow-istio-resources/base",
|
||||
"workbenches": "../apps/pvcviewer-controller/upstream/base ../apps/admission-webhook/upstream/overlays ../apps/centraldashboard/overlays ../apps/jupyter/jupyter-web-app/upstream/overlays ../apps/volumes-web-app/upstream/overlays ../apps/tensorboard/tensorboards-web-app/upstream/overlays ../apps/profiles/upstream/overlays ../apps/jupyter/notebook-controller/upstream/overlays ../apps/tensorboard/tensorboard-controller/upstream/overlays",
|
||||
"serving": "../apps/kserve - ../apps/kserve/models-web-app/overlays/kubeflow",
|
||||
"model-registry": "../apps/model-registry/upstream",
|
||||
}
|
||||
|
||||
DIRECTORY = "../image_lists"
|
||||
os.makedirs(DIRECTORY, exist_ok=True)
|
||||
SCAN_REPORTS_DIR = os.path.join(DIRECTORY, "security_scan_reports")
|
||||
ALL_SEVERITY_COUNTS = os.path.join(DIRECTORY, "severity_counts_with_images_for_WG")
|
||||
SUMMARY_OF_SEVERITY_COUNTS = os.path.join(
|
||||
DIRECTORY, "summary_of_severity_counts_for_WG"
|
||||
)
|
||||
|
||||
os.makedirs(SCAN_REPORTS_DIR, exist_ok=True)
|
||||
os.makedirs(ALL_SEVERITY_COUNTS, exist_ok=True)
|
||||
os.makedirs(SUMMARY_OF_SEVERITY_COUNTS, exist_ok=True)
|
||||
|
||||
|
||||
def log(*args, **kwargs):
|
||||
# Custom log function that print messages with flush=True by default.
|
||||
kwargs.setdefault("flush", True)
|
||||
print(*args, **kwargs)
|
||||
|
||||
|
||||
def save_images(wg, images, version):
|
||||
# Saves a list of container images to a text file named after the workgroup and version.
|
||||
output_file = f"../image_lists/kf_{version}_{wg}_images.txt"
|
||||
with open(output_file, "w") as f:
|
||||
f.write("\n".join(images))
|
||||
log(f"File {output_file} successfully created")
|
||||
|
||||
|
||||
def validate_semantic_version(version):
|
||||
# Validates a semantic version string (e.g., "0.1.2" or "latest").
|
||||
regex = r"^[0-9]+\.[0-9]+\.[0-9]+$"
|
||||
if re.match(regex, version) or version == "latest":
|
||||
return version
|
||||
else:
|
||||
raise ValueError(f"Invalid semantic version: '{version}'")
|
||||
|
||||
|
||||
def extract_images(version):
|
||||
version = validate_semantic_version(version)
|
||||
log(f"Running the script using Kubeflow version: {version}")
|
||||
|
||||
all_images = set() # Collect all unique images across workgroups
|
||||
|
||||
for wg, dirs in wg_dirs.items():
|
||||
wg_images = set() # Collect unique images for this workgroup
|
||||
for dir_path in dirs.split():
|
||||
for root, _, files in os.walk(dir_path):
|
||||
for file in files:
|
||||
if file in [
|
||||
"kustomization.yaml",
|
||||
"kustomization.yml",
|
||||
"Kustomization",
|
||||
]:
|
||||
full_path = os.path.join(root, file)
|
||||
try:
|
||||
# Execute `kustomize build` to render the kustomization file
|
||||
result = subprocess.run(
|
||||
["kustomize", "build", root],
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
log(
|
||||
f'ERROR:\t Failed "kustomize build" command for directory: {root}. See error above'
|
||||
)
|
||||
continue
|
||||
|
||||
# Use regex to find lines with 'image: <image-name>:<version>' or 'image: <image-name>'
|
||||
# and '- image: <image-name>:<version>' but avoid environment variables
|
||||
kustomize_images = re.findall(
|
||||
r"^\s*-?\s*image:\s*([^$\s:]+(?:\:[^\s]+)?)$",
|
||||
result.stdout,
|
||||
re.MULTILINE,
|
||||
)
|
||||
wg_images.update(kustomize_images)
|
||||
|
||||
# Ensure uniqueness within workgroup images
|
||||
uniq_wg_images = sorted(wg_images)
|
||||
all_images.update(uniq_wg_images)
|
||||
save_images(wg, uniq_wg_images, version)
|
||||
|
||||
# Ensure uniqueness across all workgroups
|
||||
uniq_images = sorted(all_images)
|
||||
save_images("all", uniq_images, version)
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Extract images from Kubeflow kustomizations."
|
||||
)
|
||||
# Define a positional argument 'version' with optional occurrence and default value 'latest'. You can run this file as python3 <filename>.py or python <filename>.py <version>
|
||||
parser.add_argument(
|
||||
"version",
|
||||
nargs="?",
|
||||
type=str,
|
||||
default="latest",
|
||||
help="Kubeflow version to use (defaults to latest).",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
extract_images(args.version)
|
||||
|
||||
|
||||
log("Started scanning images")
|
||||
|
||||
# Get list of text files excluding "kf_latest_all_images.txt"
|
||||
files = [
|
||||
f
|
||||
for f in glob.glob(os.path.join(DIRECTORY, "*.txt"))
|
||||
if not f.endswith("kf_latest_all_images.txt")
|
||||
]
|
||||
|
||||
# Loop through each text file in the specified directory
|
||||
for file in files:
|
||||
log(f"Scanning images in {file}")
|
||||
|
||||
file_base_name = os.path.basename(file).replace(".txt", "")
|
||||
|
||||
# Directory to save reports for this specific file
|
||||
file_reports_dir = os.path.join(SCAN_REPORTS_DIR, file_base_name)
|
||||
os.makedirs(file_reports_dir, exist_ok=True)
|
||||
|
||||
# Directory to save security count
|
||||
severity_count = os.path.join(file_reports_dir, "severity_counts")
|
||||
os.makedirs(severity_count, exist_ok=True)
|
||||
|
||||
with open(file, "r") as f:
|
||||
lines = f.readlines()
|
||||
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
image_name = line.split(":")[0]
|
||||
image_tag = line.split(":")[1] if ":" in line else ""
|
||||
|
||||
image_name_scan = image_name.split("/")[-1]
|
||||
|
||||
if image_tag:
|
||||
image_name_scan = f"{image_name_scan}_{image_tag}"
|
||||
|
||||
scan_output_file = os.path.join(
|
||||
file_reports_dir, f"{image_name_scan}_scan.json"
|
||||
)
|
||||
|
||||
log(f"Scanning ", line)
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"trivy",
|
||||
"image",
|
||||
"--format",
|
||||
"json",
|
||||
"--output",
|
||||
scan_output_file,
|
||||
line,
|
||||
],
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
|
||||
with open(scan_output_file, "r") as json_file:
|
||||
scan_data = json.load(json_file)
|
||||
|
||||
if not scan_data.get("Results"):
|
||||
log(f"No vulnerabilities found in {image_name}:{image_tag}")
|
||||
else:
|
||||
vulnerabilities_list = [
|
||||
result["Vulnerabilities"]
|
||||
for result in scan_data["Results"]
|
||||
if "Vulnerabilities" in result and result["Vulnerabilities"]
|
||||
]
|
||||
|
||||
if not vulnerabilities_list:
|
||||
log(
|
||||
f"The vulnerabilities detection may be insufficient because security updates are not provided for {image_name}:{image_tag}\n"
|
||||
)
|
||||
else:
|
||||
severity_counts = {"LOW": 0, "MEDIUM": 0, "HIGH": 0, "CRITICAL": 0}
|
||||
for vulnerabilities in vulnerabilities_list:
|
||||
for vulnerability in vulnerabilities:
|
||||
severity = vulnerability.get("Severity", "UNKNOWN")
|
||||
if severity == "UNKNOWN":
|
||||
continue
|
||||
elif severity in severity_counts:
|
||||
severity_counts[severity] += 1
|
||||
|
||||
report = {"image": line, "severity_counts": severity_counts}
|
||||
|
||||
image_table = PrettyTable()
|
||||
image_table.field_names = ["Critical", "High", "Medium", "Low"]
|
||||
image_table.add_row(
|
||||
[
|
||||
severity_counts["CRITICAL"],
|
||||
severity_counts["HIGH"],
|
||||
severity_counts["MEDIUM"],
|
||||
severity_counts["LOW"],
|
||||
]
|
||||
)
|
||||
log(f"{image_table}\n")
|
||||
|
||||
severity_report_file = os.path.join(
|
||||
severity_count, f"{image_name_scan}_severity_report.json"
|
||||
)
|
||||
with open(severity_report_file, "w") as report_file:
|
||||
json.dump(report, report_file, indent=4)
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
log(f"Error scanning {image_name}:{image_tag}")
|
||||
log(e.stderr)
|
||||
|
||||
# Combine all the JSON files into a single file with severity counts for all images
|
||||
json_files = glob.glob(os.path.join(severity_count, "*.json"))
|
||||
|
||||
output_file = os.path.join(ALL_SEVERITY_COUNTS, f"{file_base_name}.json")
|
||||
|
||||
if not json_files:
|
||||
log(f"No JSON files found in '{severity_count}'. Skipping combination.")
|
||||
else:
|
||||
combined_data = []
|
||||
for json_file in json_files:
|
||||
with open(json_file, "r") as jf:
|
||||
combined_data.append(json.load(jf))
|
||||
|
||||
with open(output_file, "w") as of:
|
||||
json.dump({"data": combined_data}, of, indent=4)
|
||||
|
||||
log(f"JSON files successfully combined into '{output_file}'")
|
||||
|
||||
# File to save summary of the severity counts for WGs as JSON format.
|
||||
summary_file = os.path.join(
|
||||
SUMMARY_OF_SEVERITY_COUNTS, "severity_summary_in_json_format.json"
|
||||
)
|
||||
|
||||
# Initialize counters
|
||||
unique_images = {} # unique set of images across all WGs
|
||||
total_images = 0
|
||||
total_low = 0
|
||||
total_medium = 0
|
||||
total_high = 0
|
||||
total_critical = 0
|
||||
|
||||
# Initialize a dictionary to hold the final JSON data
|
||||
merged_data = {}
|
||||
|
||||
# Loop through each JSON file in the ALL_SEVERITY_COUNTS
|
||||
for file_path in glob.glob(os.path.join(ALL_SEVERITY_COUNTS, "*.json")):
|
||||
# Split filename based on underscores
|
||||
filename_parts = os.path.basename(file_path).split("_")
|
||||
|
||||
# Check if there are at least 3 parts (prefix, name, _images)
|
||||
if len(filename_parts) >= 4:
|
||||
# Extract name (second part)
|
||||
filename = filename_parts[2]
|
||||
filename = filename.capitalize()
|
||||
|
||||
else:
|
||||
log(f"Skipping invalid filename format: {file_path}")
|
||||
continue
|
||||
|
||||
with open(file_path, "r") as f:
|
||||
data = json.load(f)["data"]
|
||||
|
||||
# Initialize counts for this file
|
||||
image_count = len(data)
|
||||
low = sum(entry["severity_counts"]["LOW"] for entry in data)
|
||||
medium = sum(entry["severity_counts"]["MEDIUM"] for entry in data)
|
||||
high = sum(entry["severity_counts"]["HIGH"] for entry in data)
|
||||
critical = sum(entry["severity_counts"]["CRITICAL"] for entry in data)
|
||||
|
||||
# Update unique_images for the total counts later
|
||||
for d in data:
|
||||
unique_images[d["image"]] = d
|
||||
|
||||
# Create the output for this file
|
||||
file_data = {
|
||||
"images": image_count,
|
||||
"LOW": low,
|
||||
"MEDIUM": medium,
|
||||
"HIGH": high,
|
||||
"CRITICAL": critical,
|
||||
}
|
||||
|
||||
# Update merged_data with filename as key
|
||||
merged_data[filename] = file_data
|
||||
|
||||
|
||||
# Update the total counts
|
||||
unique_images = unique_images.values() # keep the set of values
|
||||
total_images += len(unique_images)
|
||||
total_low += sum(entry["severity_counts"]["LOW"] for entry in unique_images)
|
||||
total_medium += sum(entry["severity_counts"]["MEDIUM"] for entry in unique_images)
|
||||
total_high += sum(entry["severity_counts"]["HIGH"] for entry in unique_images)
|
||||
total_critical += sum(entry["severity_counts"]["CRITICAL"] for entry in unique_images)
|
||||
|
||||
# Add total counts to merged_data
|
||||
merged_data["total"] = {
|
||||
"images": total_images,
|
||||
"LOW": total_low,
|
||||
"MEDIUM": total_medium,
|
||||
"HIGH": total_high,
|
||||
"CRITICAL": total_critical,
|
||||
}
|
||||
|
||||
log("Summary in Json Format:")
|
||||
log(json.dumps(merged_data, indent=4))
|
||||
|
||||
|
||||
# Write the final output to a file
|
||||
with open(summary_file, "w") as summary_f:
|
||||
json.dump(merged_data, summary_f, indent=4)
|
||||
|
||||
log(f"Summary written to: {summary_file} as JSON format")
|
||||
|
||||
# Load JSON content from the file
|
||||
with open(summary_file, "r") as file:
|
||||
data = json.load(file)
|
||||
|
||||
# Define a mapping for working group names
|
||||
groupnames = {
|
||||
"Automl": "AutoML",
|
||||
"Pipelines": "Pipelines",
|
||||
"Workbenches": "Workbenches(Notebooks)",
|
||||
"Serving": "Kserve",
|
||||
"Manifests": "Manifests",
|
||||
"Training": "Training",
|
||||
"Model-registry": "Model Registry",
|
||||
"total": "All Images",
|
||||
}
|
||||
|
||||
# Create PrettyTable
|
||||
table = PrettyTable()
|
||||
table.field_names = [
|
||||
"Working Group",
|
||||
"Images",
|
||||
"Critical CVE",
|
||||
"High CVE",
|
||||
"Medium CVE",
|
||||
"Low CVE",
|
||||
]
|
||||
|
||||
# Populate the table with data
|
||||
for group_name in groupnames:
|
||||
if group_name in data: # Check if group_name exists in data
|
||||
value = data[group_name]
|
||||
table.add_row(
|
||||
[
|
||||
groupnames[group_name],
|
||||
value["images"],
|
||||
value["CRITICAL"],
|
||||
value["HIGH"],
|
||||
value["MEDIUM"],
|
||||
value["LOW"],
|
||||
]
|
||||
)
|
||||
|
||||
# log the table
|
||||
log(table)
|
||||
|
||||
|
||||
# Write the table output to a file in the specified folder
|
||||
output_file = (
|
||||
SUMMARY_OF_SEVERITY_COUNTS + "/summary_of_severity_counts_for_WGs_in_table.txt"
|
||||
)
|
||||
with open(output_file, "w") as f:
|
||||
f.write(str(table))
|
||||
|
||||
log("Output saved to:", output_file)
|
||||
log("Severity counts with images respect to WGs are saved in the",ALL_SEVERITY_COUNTS)
|
||||
log("Scanned Json reports on images are saved in",SCAN_REPORTS_DIR)
|
||||
Reference in New Issue
Block a user