Add kubeflow/v1.10.0

This commit is contained in:
wbsong111
2025-06-24 12:03:10 +09:00
parent 0132496142
commit 6e8dd89e48
1531 changed files with 120984 additions and 262120 deletions
+101
View File
@@ -0,0 +1,101 @@
#!/usr/bin/env bash
# Common functions for Kubeflow manifest synchronization scripts
setup_error_handling() {
set -euxo pipefail
IFS=$'\n\t'
}
# Check if the git repository has uncommitted changes
check_uncommitted_changes() {
if [ -n "$(git status --porcelain)" ]; then
echo "WARNING: You have uncommitted changes"
fi
}
# Create a new git branch if it doesn't exist
create_branch() {
local branch="$1"
check_uncommitted_changes
if [ $(git branch --list "$branch") ]; then
echo "WARNING: Branch $branch already exists."
fi
if ! git show-ref --verify --quiet refs/heads/$branch; then
git checkout -b "$branch"
else
echo "Branch $branch already exists."
fi
}
clone_and_checkout() {
local source_directory="$1"
local repository_url="$2"
local repository_directory="$3"
local commit="$4"
echo "Checking out in $source_directory to $commit..."
mkdir -p "$source_directory"
cd "$source_directory"
# Clone repository if it doesn't exist
if [ ! -d "$repository_directory/.git" ]; then
git clone "$repository_url" "$repository_directory"
fi
# Checkout to specific commit
cd "$source_directory/$repository_directory"
if ! git rev-parse --verify --quiet "$commit"; then
git checkout -b "$commit"
else
git checkout "$commit"
fi
check_uncommitted_changes
}
# Copy manifests from source to destination
copy_manifests() {
local source="$1"
local destination="$2"
echo "Copying manifests..."
if [ -d "$destination" ]; then
rm -r "$destination"
fi
cp "$source" "$destination" -r
echo "Successfully copied all manifests."
}
# Update README with new commit reference
update_readme() {
local manifests_directory="$1"
local source_text="$2"
local destination_text="$3"
if [[ "$OSTYPE" == "darwin"* ]]; then
sed -i "" "s|$source_text|$destination_text|g" "${manifests_directory}/README.md" # BSD sed of Mac OSX
else
sed -i "s|$source_text|$destination_text|g" "${manifests_directory}/README.md" # GNU sed of Linux
fi
}
# Commit changes to git repository
commit_changes() {
local manifests_directory="$1"
local commit_message="$2"
local paths_to_add=("${@:3}")
cd "$manifests_directory"
for path in "${paths_to_add[@]}"; do
git add "$path"
done
git commit -s -m "$commit_message"
}
@@ -0,0 +1,61 @@
#!/usr/bin/env bash
# This script helps to create a PR to update the Istio CNI manifests
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
source "${SCRIPT_DIRECTORY}/library.sh"
setup_error_handling
COMPONENT_NAME="istio-cni"
COMMIT="1.24.3"
CURRENT_VERSION="1-24"
NEW_VERSION="1-24" # Must be a release
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:=/tmp/${COMPONENT_NAME}}
BRANCH_NAME=${BRANCH_NAME:=${COMPONENT_NAME}-${COMMIT?}}
# Path configurations
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
ISTIO_OLD=$MANIFESTS_DIRECTORY/common/${COMPONENT_NAME}-${CURRENT_VERSION}
ISTIO_NEW=$MANIFESTS_DIRECTORY/common/${COMPONENT_NAME}-${NEW_VERSION}
if [ ! -d "$ISTIO_NEW" ]; then
cp -a $ISTIO_OLD $ISTIO_NEW
fi
create_branch "$BRANCH_NAME"
echo "Checking out in $SOURCE_DIRECTORY to $COMMIT..."
mkdir -p $SOURCE_DIRECTORY
cd $SOURCE_DIRECTORY
if [ ! -d "istio-${COMMIT}" ]; then
wget "https://github.com/istio/istio/releases/download/${COMMIT}/istio-${COMMIT}-linux-amd64.tar.gz"
tar xvfz istio-${COMMIT}-linux-amd64.tar.gz
fi
ISTIOCTL=$SOURCE_DIRECTORY/istio-${COMMIT}/bin/istioctl
cd $ISTIO_NEW
$ISTIOCTL manifest generate -f profile.yaml -f profile-overlay.yaml --set components.cni.enabled=true --set components.cni.namespace=kube-system > dump.yaml
./split-istio-packages -f dump.yaml
mv $ISTIO_NEW/crd.yaml $ISTIO_NEW/istio-crds/base
mv $ISTIO_NEW/install.yaml $ISTIO_NEW/istio-install/base
mv $ISTIO_NEW/cluster-local-gateway.yaml $ISTIO_NEW/cluster-local-gateway/base
rm dump.yaml
check_uncommitted_changes
SOURCE_TEXT="\[.*\](https://github.com/istio/istio/releases/tag/.*)"
DESTINATION_TEXT="\[$COMMIT\](https://github.com/istio/istio/releases/tag/$COMMIT)"
update_readme "$MANIFESTS_DIRECTORY" "$SOURCE_TEXT" "$DESTINATION_TEXT"
echo "Synchronizing directory names..."
find "$MANIFESTS_DIRECTORY" -type f -not -path '*/.git/*' -exec sed -i "s/istio-cni-${CURRENT_VERSION}/istio-cni-${NEW_VERSION}/g" {} +
cd "$MANIFESTS_DIRECTORY"
if [ "$CURRENT_VERSION" != "$NEW_VERSION" ]; then
rm -rf $ISTIO_OLD
fi
commit_changes "$MANIFESTS_DIRECTORY" "Upgrade istio-cni to v.${COMMIT}" "."
echo "Synchronization completed successfully."
@@ -0,0 +1,61 @@
#!/usr/bin/env bash
# This script helps to create a PR to update the Istio manifests
# Source the common library functions
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
source "${SCRIPT_DIRECTORY}/library.sh"
setup_error_handling
COMPONENT_NAME="istio"
COMMIT="1.24.3"
CURRENT_VERSION="1-24"
NEW_VERSION="1-24" # Must be a release
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:=/tmp/${COMPONENT_NAME}}
BRANCH_NAME=${BRANCH_NAME:=${COMPONENT_NAME}-${COMMIT?}}
# Path configurations
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
ISTIO_OLD=$MANIFESTS_DIRECTORY/common/${COMPONENT_NAME}-${CURRENT_VERSION}
ISTIO_NEW=$MANIFESTS_DIRECTORY/common/${COMPONENT_NAME}-${NEW_VERSION}
if [ ! -d "$ISTIO_NEW" ]; then
cp -a $ISTIO_OLD $ISTIO_NEW
fi
create_branch "$BRANCH_NAME"
echo "Checking out in $SOURCE_DIRECTORY to $COMMIT..."
mkdir -p $SOURCE_DIRECTORY
cd $SOURCE_DIRECTORY
if [ ! -d "istio-${COMMIT}" ]; then
wget "https://github.com/istio/istio/releases/download/${COMMIT}/istio-${COMMIT}-linux-amd64.tar.gz"
tar xvfz istio-${COMMIT}-linux-amd64.tar.gz
fi
ISTIOCTL=$SOURCE_DIRECTORY/istio-${COMMIT}/bin/istioctl
cd $ISTIO_NEW
$ISTIOCTL manifest generate -f profile.yaml -f profile-overlay.yaml > dump.yaml
./split-istio-packages -f dump.yaml
mv $ISTIO_NEW/crd.yaml $ISTIO_NEW/istio-crds/base
mv $ISTIO_NEW/install.yaml $ISTIO_NEW/istio-install/base
mv $ISTIO_NEW/cluster-local-gateway.yaml $ISTIO_NEW/cluster-local-gateway/base
rm dump.yaml
check_uncommitted_changes
SOURCE_TEXT="\[.*\](https://github.com/istio/istio/releases/tag/.*)"
DESTINATION_TEXT="\[$COMMIT\](https://github.com/istio/istio/releases/tag/$COMMIT)"
update_readme "$MANIFESTS_DIRECTORY" "$SOURCE_TEXT" "$DESTINATION_TEXT"
find "$MANIFESTS_DIRECTORY" -type f -not -path '*/.git/*' -exec sed -i "s/istio-${CURRENT_VERSION}/istio-${NEW_VERSION}/g" {} +
cd "$MANIFESTS_DIRECTORY"
if [ "$CURRENT_VERSION" != "$NEW_VERSION" ]; then
rm -rf $ISTIO_OLD
fi
commit_changes "$MANIFESTS_DIRECTORY" "Upgrade istio to v.${COMMIT}" "."
echo "Synchronization completed successfully."
@@ -0,0 +1,38 @@
#!/usr/bin/env bash
# This script helps to create a PR to update the Katib manifests
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
source "${SCRIPT_DIRECTORY}/library.sh"
setup_error_handling
COMPONENT_NAME="katib"
REPOSITORY_NAME="kubeflow/katib"
REPOSITORY_URL="https://github.com/kubeflow/katib.git"
COMMIT="v0.18.0"
REPOSITORY_DIRECTORY="katib"
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:=/tmp/kubeflow-${COMPONENT_NAME}}
BRANCH_NAME=${BRANCH_NAME:=synchronize-${COMPONENT_NAME}-manifests-${COMMIT?}}
# Path configurations
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
SOURCE_MANIFESTS_PATH="manifests/v1beta1"
DESTINATION_MANIFESTS_PATH="apps/${COMPONENT_NAME}/upstream"
# README update patterns
SOURCE_TEXT="\[.*\](https://github.com/${REPOSITORY_NAME}/tree/.*/manifests/v1beta1)"
DESTINATION_TEXT="\[${COMMIT}\](https://github.com/${REPOSITORY_NAME}/tree/${COMMIT}/manifests/v1beta1)"
create_branch "$BRANCH_NAME"
clone_and_checkout "$SOURCE_DIRECTORY" "$REPOSITORY_URL" "$REPOSITORY_DIRECTORY" "$COMMIT"
copy_manifests "${SOURCE_DIRECTORY}/${REPOSITORY_DIRECTORY}/${SOURCE_MANIFESTS_PATH}" "${MANIFESTS_DIRECTORY}/${DESTINATION_MANIFESTS_PATH}"
update_readme "$MANIFESTS_DIRECTORY" "$SOURCE_TEXT" "$DESTINATION_TEXT"
commit_changes "$MANIFESTS_DIRECTORY" "Update ${REPOSITORY_NAME} manifests from ${COMMIT}" \
"apps" \
"README.md"
echo "Synchronization completed successfully."
@@ -0,0 +1,110 @@
#!/usr/bin/env bash
# This script helps to create a PR to update the Knative manifests
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
source "${SCRIPT_DIRECTORY}/library.sh"
setup_error_handling
COMPONENT_NAME="knative"
KN_SERVING_RELEASE="v1.16.2" # Must be a release
KN_EXTENSION_RELEASE="v1.16.0" # Must be a release
KN_EVENTING_RELEASE="v1.16.4" # Must be a release
BRANCH_NAME=${BRANCH_NAME:=synchronize-${COMPONENT_NAME}-manifests-${KN_SERVING_RELEASE?}}
# Path configurations
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
DESTINATION_DIRECTORY=$MANIFESTS_DIRECTORY/common/${COMPONENT_NAME}
create_branch "$BRANCH_NAME"
check_uncommitted_changes
# Clean up existing files (keep README and OWNERS)
if [ -d "$DESTINATION_DIRECTORY" ]; then
rm -r "$DESTINATION_DIRECTORY/knative-serving/base/upstream"
rm "$DESTINATION_DIRECTORY/knative-serving-post-install-jobs/base/serving-post-install-jobs.yaml"
rm -r "$DESTINATION_DIRECTORY/knative-eventing/base/upstream"
rm "$DESTINATION_DIRECTORY/knative-eventing-post-install-jobs/base/eventing-post-install.yaml"
fi
# Create required directories
mkdir -p "$DESTINATION_DIRECTORY/knative-serving/base/upstream"
mkdir -p "$DESTINATION_DIRECTORY/knative-serving-post-install-jobs/base"
mkdir -p "$DESTINATION_DIRECTORY/knative-eventing/base/upstream"
mkdir -p "$DESTINATION_DIRECTORY/knative-eventing-post-install-jobs/base"
echo "Downloading knative-serving manifests..."
wget -O $DESTINATION_DIRECTORY/knative-serving/base/upstream/serving-core.yaml "https://github.com/knative/serving/releases/download/knative-$KN_SERVING_RELEASE/serving-core.yaml"
wget -O $DESTINATION_DIRECTORY/knative-serving/base/upstream/net-istio.yaml "https://github.com/knative-extensions/net-istio/releases/download/knative-$KN_EXTENSION_RELEASE/net-istio.yaml"
wget -O $DESTINATION_DIRECTORY/knative-serving-post-install-jobs/base/serving-post-install-jobs.yaml "https://github.com/knative/serving/releases/download/knative-$KN_SERVING_RELEASE/serving-post-install-jobs.yaml"
yq eval -i '... comments=""' $DESTINATION_DIRECTORY/knative-serving/base/upstream/serving-core.yaml
yq eval -i '... comments=""' $DESTINATION_DIRECTORY/knative-serving/base/upstream/net-istio.yaml
yq eval -i '... comments=""' $DESTINATION_DIRECTORY/knative-serving-post-install-jobs/base/serving-post-install-jobs.yaml
yq eval -i 'explode(.)' $DESTINATION_DIRECTORY/knative-serving/base/upstream/serving-core.yaml
yq eval -i 'explode(.)' $DESTINATION_DIRECTORY/knative-serving/base/upstream/net-istio.yaml
yq eval -i 'explode(.)' $DESTINATION_DIRECTORY/knative-serving-post-install-jobs/base/serving-post-install-jobs.yaml
yq eval -i 'select(.kind == "Job" and .metadata.generateName == "storage-version-migration-serving-") | .metadata.name = "storage-version-migration-serving"' $DESTINATION_DIRECTORY/knative-serving-post-install-jobs/base/serving-post-install-jobs.yaml
echo "Downloading knative-eventing manifests..."
wget -O $DESTINATION_DIRECTORY/knative-eventing/base/upstream/eventing-core.yaml "https://github.com/knative/eventing/releases/download/knative-$KN_EVENTING_RELEASE/eventing-core.yaml"
wget -O $DESTINATION_DIRECTORY/knative-eventing/base/upstream/in-memory-channel.yaml "https://github.com/knative/eventing/releases/download/knative-$KN_EVENTING_RELEASE/in-memory-channel.yaml"
wget -O $DESTINATION_DIRECTORY/knative-eventing/base/upstream/mt-channel-broker.yaml "https://github.com/knative/eventing/releases/download/knative-$KN_EVENTING_RELEASE/mt-channel-broker.yaml"
wget -O $DESTINATION_DIRECTORY/knative-eventing-post-install-jobs/base/eventing-post-install.yaml "https://github.com/knative/eventing/releases/download/knative-$KN_EVENTING_RELEASE/eventing-post-install.yaml"
yq eval -i '... comments=""' $DESTINATION_DIRECTORY/knative-eventing/base/upstream/eventing-core.yaml
yq eval -i '... comments=""' $DESTINATION_DIRECTORY/knative-eventing/base/upstream/in-memory-channel.yaml
yq eval -i '... comments=""' $DESTINATION_DIRECTORY/knative-eventing/base/upstream/mt-channel-broker.yaml
yq eval -i '... comments=""' $DESTINATION_DIRECTORY/knative-eventing-post-install-jobs/base/eventing-post-install.yaml
yq eval -i 'explode(.)' $DESTINATION_DIRECTORY/knative-eventing/base/upstream/eventing-core.yaml
yq eval -i 'explode(.)' $DESTINATION_DIRECTORY/knative-eventing/base/upstream/in-memory-channel.yaml
yq eval -i 'explode(.)' $DESTINATION_DIRECTORY/knative-eventing/base/upstream/mt-channel-broker.yaml
yq eval -i 'explode(.)' $DESTINATION_DIRECTORY/knative-eventing-post-install-jobs/base/eventing-post-install.yaml
yq eval -i 'select(.kind == "Job" and .metadata.generateName == "storage-version-migration-eventing-") | .metadata.name = "storage-version-migration-eventing"' $DESTINATION_DIRECTORY/knative-eventing-post-install-jobs/base/eventing-post-install.yaml
yq eval -i 'select((.kind == "ConfigMap" and .metadata.name == "config-observability") | not)' $DESTINATION_DIRECTORY/knative-eventing/base/upstream/in-memory-channel.yaml
yq eval -i 'select((.kind == "ConfigMap" and .metadata.name == "config-tracing") | not)' $DESTINATION_DIRECTORY/knative-eventing/base/upstream/in-memory-channel.yaml
# Helper function to replace text in files
replace_in_file() {
local SOURCE_TEXT=$1
local DESTINATION_TEXT=$2
local FILE=$3
sed -i "s|$SOURCE_TEXT|$DESTINATION_TEXT|g" $FILE
}
replace_in_file \
"\[.*\](https://github.com/knative/serving/releases/tag/knative-.*) <" \
"\[$KN_SERVING_RELEASE\](https://github.com/knative/serving/releases/tag/knative-$KN_SERVING_RELEASE) <" \
${MANIFESTS_DIRECTORY}/README.md
replace_in_file \
"> \[.*\](https://github.com/knative/eventing/releases/tag/knative-.*)" \
"> \[$KN_EVENTING_RELEASE\](https://github.com/knative/eventing/releases/tag/knative-$KN_EVENTING_RELEASE)" \
${MANIFESTS_DIRECTORY}/README.md
replace_in_file \
"\[Knative serving (v.*)\](https://github.com/knative/serving/releases/tag/knative-v.*)" \
"\[Knative serving ($KN_SERVING_RELEASE)\](https://github.com/knative/serving/releases/tag/knative-$KN_SERVING_RELEASE)" \
$DESTINATION_DIRECTORY/README.md
replace_in_file \
"\[Knative ingress controller for Istio (v.*)\](https://github.com/knative-extensions/net-istio/releases/tag/knative-v.*)" \
"\[Knative ingress controller for Istio ($KN_EXTENSION_RELEASE)\](https://github.com/knative-extensions/net-istio/releases/tag/knative-$KN_EXTENSION_RELEASE)" \
$DESTINATION_DIRECTORY/README.md
replace_in_file \
"The manifests for Knative Eventing are based off the \[v.* release\](https://github.com/knative/eventing/releases/tag/knative-v.*)" \
"The manifests for Knative Eventing are based off the \[$KN_EVENTING_RELEASE release\](https://github.com/knative/eventing/releases/tag/knative-$KN_EVENTING_RELEASE)" \
$DESTINATION_DIRECTORY/README.md
commit_changes "$MANIFESTS_DIRECTORY" "Update common/knative manifests from ${KN_SERVING_RELEASE}/${KN_EVENTING_RELEASE}" \
"$DESTINATION_DIRECTORY" \
"README.md"
echo "Synchronization completed successfully."
@@ -0,0 +1,46 @@
#!/usr/bin/env bash
# This script helps to create a PR to update the KServe manifests
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
source "${SCRIPT_DIRECTORY}/library.sh"
setup_error_handling
COMPONENT_NAME="kserve"
REPOSITORY_NAME="kserve/kserve"
REPOSITORY_URL="https://github.com/kserve/kserve.git"
KSERVE_VERSION="v0.15.0"
COMMIT="v0.15.0"
REPOSITORY_DIRECTORY="kserve"
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:=/tmp/${COMPONENT_NAME}}
BRANCH_NAME=${BRANCH_NAME:=synchronize-${COMPONENT_NAME}-manifests-${COMMIT?}}
# Path configurations
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
SOURCE_MANIFESTS_PATH="install/${KSERVE_VERSION}"
DESTINATION_MANIFESTS_PATH="apps/${COMPONENT_NAME}/${COMPONENT_NAME}"
# README update patterns
SOURCE_TEXT="\[.*\](https://github.com/${REPOSITORY_NAME}/releases/tag/.*)"
DESTINATION_TEXT="\[${COMMIT}\](https://github.com/${REPOSITORY_NAME}/releases/tag/${COMMIT}/install/${KSERVE_VERSION})"
create_branch "$BRANCH_NAME"
clone_and_checkout "$SOURCE_DIRECTORY" "$REPOSITORY_URL" "$REPOSITORY_DIRECTORY" "$COMMIT"
echo "Copying kserve manifests..."
DESTINATION_DIRECTORY=$MANIFESTS_DIRECTORY/$DESTINATION_MANIFESTS_PATH
if [ -d "$DESTINATION_DIRECTORY" ]; then
rm -rf "$DESTINATION_DIRECTORY"/kserve*
fi
cp $SOURCE_DIRECTORY/$REPOSITORY_DIRECTORY/$SOURCE_MANIFESTS_PATH/* $DESTINATION_DIRECTORY -r
update_readme "$MANIFESTS_DIRECTORY" "$SOURCE_TEXT" "$DESTINATION_TEXT"
commit_changes "$MANIFESTS_DIRECTORY" "Update ${REPOSITORY_NAME} manifests from ${KSERVE_VERSION}" \
"apps/${COMPONENT_NAME}" \
"README.md" \
"scripts"
echo "Synchronization completed successfully."
@@ -0,0 +1,39 @@
#!/usr/bin/env bash
# This script helps to create a PR to update the KServe Models Web App manifests
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
source "${SCRIPT_DIRECTORY}/library.sh"
setup_error_handling
COMPONENT_NAME="models-web-app"
REPOSITORY_NAME="kserve/models-web-app"
REPOSITORY_URL="https://github.com/kserve/models-web-app.git"
COMMIT="v0.14.0"
REPOSITORY_DIRECTORY="models-web-app"
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:=/tmp/kserve-${COMPONENT_NAME}}
BRANCH_NAME=${BRANCH_NAME:=synchronize-kserve-${COMPONENT_NAME}-manifests-${COMMIT?}}
# Path configurations
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
SOURCE_MANIFESTS_PATH="config"
DESTINATION_MANIFESTS_PATH="apps/kserve/${COMPONENT_NAME}"
# README update patterns
SOURCE_TEXT="\[.*\](https://github.com/${REPOSITORY_NAME}/tree/.*)"
DESTINATION_TEXT="\[${COMMIT}\](https://github.com/${REPOSITORY_NAME}/tree/${COMMIT}/${SOURCE_MANIFESTS_PATH})"
create_branch "$BRANCH_NAME"
clone_and_checkout "$SOURCE_DIRECTORY" "$REPOSITORY_URL" "$REPOSITORY_DIRECTORY" "$COMMIT"
echo "Copying manifests"
copy_manifests "${SOURCE_DIRECTORY}/${REPOSITORY_DIRECTORY}/${SOURCE_MANIFESTS_PATH}" "${MANIFESTS_DIRECTORY}/${DESTINATION_MANIFESTS_PATH}"
update_readme "$MANIFESTS_DIRECTORY" "$SOURCE_TEXT" "$DESTINATION_TEXT"
commit_changes "$MANIFESTS_DIRECTORY" "Update kserve models web application manifests from ${COMMIT}" \
"${DESTINATION_MANIFESTS_PATH}" \
"README.md"
echo "Synchronization completed successfully."
@@ -0,0 +1,99 @@
#!/usr/bin/env bash
# This script helps to create a PR to update the Kubeflow manifests
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
source "${SCRIPT_DIRECTORY}/library.sh"
setup_error_handling
COMPONENT_NAME="kubeflow"
REPOSITORY_NAME="kubeflow/kubeflow"
REPOSITORY_URL="https://github.com/kubeflow/kubeflow.git"
COMMIT="v1.10.0"
REPOSITORY_DIRECTORY="kubeflow"
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:=/tmp/${COMPONENT_NAME}-${COMPONENT_NAME}}
BRANCH_NAME=${BRANCH_NAME:=synchronize-${COMPONENT_NAME}-${COMPONENT_NAME}-manifests-${COMMIT?}}
# Path configurations
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
create_branch "$BRANCH_NAME"
clone_and_checkout "$SOURCE_DIRECTORY" "$REPOSITORY_URL" "$REPOSITORY_DIRECTORY" "$COMMIT"
# Function to copy manifests for a specific component
copy_component_manifests() {
local component_name=$1
local source_path=$2
local destination_path=$3
local readme_path_pattern=$4
echo "Copying ${component_name} manifests..."
local destination_directory="${MANIFESTS_DIRECTORY}/${destination_path}"
if [ -d "$destination_directory" ]; then
rm -r "$destination_directory"
fi
mkdir -p "$destination_directory"
cp "${SOURCE_DIRECTORY}/${REPOSITORY_DIRECTORY}/${source_path}/"* "$destination_directory" -r
echo "Updating README for ${component_name}..."
local source_text="\[.*\](https://github.com/${REPOSITORY_NAME}/tree/.*/components/${readme_path_pattern})"
local destination_text="\[${COMMIT}\](https://github.com/${REPOSITORY_NAME}/tree/${COMMIT}/components/${readme_path_pattern})"
update_readme "$MANIFESTS_DIRECTORY" "$source_text" "$destination_text"
}
copy_component_manifests "admission-webhook" \
"components/admission-webhook/manifests" \
"apps/admission-webhook/upstream" \
"admission-webhook/manifests"
copy_component_manifests "centraldashboard" \
"components/centraldashboard/manifests" \
"apps/centraldashboard/upstream" \
"centraldashboard/manifests"
copy_component_manifests "jupyter-web-app" \
"components/crud-web-apps/jupyter/manifests" \
"apps/jupyter/jupyter-web-app/upstream" \
"crud-web-apps/jupyter/manifests"
copy_component_manifests "volumes-web-app" \
"components/crud-web-apps/volumes/manifests" \
"apps/volumes-web-app/upstream" \
"crud-web-apps/volumes/manifests"
copy_component_manifests "tensorboards-web-app" \
"components/crud-web-apps/tensorboards/manifests" \
"apps/tensorboard/tensorboards-web-app/upstream" \
"crud-web-apps/tensorboards/manifests"
copy_component_manifests "profile-controller" \
"components/profile-controller/config" \
"apps/profiles/upstream" \
"profile-controller/config"
copy_component_manifests "notebook-controller" \
"components/notebook-controller/config" \
"apps/jupyter/notebook-controller/upstream" \
"notebook-controller/config"
copy_component_manifests "tensorboard-controller" \
"components/tensorboard-controller/config" \
"apps/tensorboard/tensorboard-controller/upstream" \
"tensorboard-controller/config"
copy_component_manifests "pvcviewer-controller" \
"components/pvcviewer-controller/config" \
"apps/pvcviewer-controller/upstream" \
"pvcviewer-controller/config"
echo "Successfully copied all manifests."
commit_changes "$MANIFESTS_DIRECTORY" "Update ${REPOSITORY_NAME} manifests from ${COMMIT}" \
"apps" \
"README.md"
echo "Synchronization completed successfully."
@@ -0,0 +1,47 @@
#!/usr/bin/env bash
# This script helps to create a PR to update the Model Registry manifests
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
source "${SCRIPT_DIRECTORY}/library.sh"
setup_error_handling
COMPONENT_NAME="model-registry"
REPOSITORY_NAME="kubeflow/model-registry"
REPOSITORY_URL="https://github.com/kubeflow/model-registry.git"
COMMIT="v0.2.13"
REPOSITORY_DIRECTORY="model-registry"
DEV_MODE=${DEV_MODE:=false}
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:=/tmp/kubeflow-${COMPONENT_NAME}}
BRANCH_NAME=${BRANCH_NAME:=synchronize-kubeflow-${COMPONENT_NAME}-manifests-${COMMIT?}}
# Path configurations
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
SOURCE_MANIFESTS_PATH="manifests/kustomize"
DESTINATION_MANIFESTS_PATH="apps/${COMPONENT_NAME}/upstream"
# README update patterns
SOURCE_TEXT="\[.*\](https://github.com/${REPOSITORY_NAME}/tree/.*/manifests/kustomize)"
DESTINATION_TEXT="\[${COMMIT}\](https://github.com/${REPOSITORY_NAME}/tree/${COMMIT}/manifests/kustomize)"
create_branch "$BRANCH_NAME"
clone_and_checkout "$SOURCE_DIRECTORY" "$REPOSITORY_URL" "$REPOSITORY_DIRECTORY" "$COMMIT"
echo "Copying ${COMPONENT_NAME} manifests..."
DESTINATION_DIRECTORY=$MANIFESTS_DIRECTORY/$DESTINATION_MANIFESTS_PATH
if [ -d "$DESTINATION_DIRECTORY" ]; then
rm -r "$DESTINATION_DIRECTORY"
fi
mkdir -p $DESTINATION_DIRECTORY
cp -r "$SOURCE_DIRECTORY/$REPOSITORY_DIRECTORY/$SOURCE_MANIFESTS_PATH/"* "$DESTINATION_DIRECTORY"
echo "Successfully copied all manifests."
update_readme "$MANIFESTS_DIRECTORY" "$SOURCE_TEXT" "$DESTINATION_TEXT"
commit_changes "$MANIFESTS_DIRECTORY" "Update ${REPOSITORY_NAME} manifests from ${COMMIT}" \
"apps" \
"README.md"
echo "Synchronization completed successfully."
@@ -0,0 +1,39 @@
#!/usr/bin/env bash
# This script helps to create a PR to update the Kubeflow Pipelines manifests
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
source "${SCRIPT_DIRECTORY}/library.sh"
setup_error_handling
COMPONENT_NAME="pipelines"
REPOSITORY_NAME="kubeflow/pipelines"
REPOSITORY_URL="https://github.com/kubeflow/pipelines.git"
COMMIT="2.5.0"
REPOSITORY_DIRECTORY="pipelines"
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:=/tmp/kubeflow-${COMPONENT_NAME}}
BRANCH_NAME=${BRANCH_NAME:=synchronize-kubeflow-${COMPONENT_NAME}-manifests-${COMMIT?}}
# Path configurations
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
SOURCE_MANIFESTS_PATH="manifests/kustomize"
DESTINATION_MANIFESTS_PATH="apps/pipeline/upstream"
# README update patterns
SOURCE_TEXT="\[.*\](https://github.com/${REPOSITORY_NAME}/tree/.*/manifests/kustomize)"
DESTINATION_TEXT="\[${COMMIT}\](https://github.com/${REPOSITORY_NAME}/tree/${COMMIT}/manifests/kustomize)"
create_branch "$BRANCH_NAME"
clone_and_checkout "$SOURCE_DIRECTORY" "$REPOSITORY_URL" "$REPOSITORY_DIRECTORY" "$COMMIT"
echo "Copying ${COMPONENT_NAME} manifests..."
copy_manifests "${SOURCE_DIRECTORY}/${REPOSITORY_DIRECTORY}/${SOURCE_MANIFESTS_PATH}" "${MANIFESTS_DIRECTORY}/${DESTINATION_MANIFESTS_PATH}"
update_readme "$MANIFESTS_DIRECTORY" "$SOURCE_TEXT" "$DESTINATION_TEXT"
commit_changes "$MANIFESTS_DIRECTORY" "Update ${REPOSITORY_NAME} manifests from ${COMMIT}" \
"apps" \
"README.md"
echo "Synchronization completed successfully."
@@ -0,0 +1,58 @@
#!/usr/bin/env bash
# This script helps to create a PR to update the Spark Operator manifests
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
source "${SCRIPT_DIRECTORY}/library.sh"
setup_error_handling
COMPONENT_NAME="spark-operator"
SPARK_OPERATOR_VERSION=${SPARK_OPERATOR_VERSION:="2.1.1"}
SPARK_OPERATOR_HELM_CHART_REPO=${SPARK_OPERATOR_HELM_CHART_REPO:="https://kubeflow.github.io/spark-operator"}
DEV_MODE=${DEV_MODE:=false}
BRANCH_NAME=${BRANCH_NAME:=synchronize-${COMPONENT_NAME}-manifests-${SPARK_OPERATOR_VERSION?}}
# Path configurations
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
DESTINATION_MANIFESTS_PATH="apps/spark/${COMPONENT_NAME}/base"
create_branch "$BRANCH_NAME"
echo "Generating manifests from Helm chart version ${SPARK_OPERATOR_VERSION}..."
DESTINATION_DIRECTORY=$MANIFESTS_DIRECTORY/$DESTINATION_MANIFESTS_PATH
mkdir -p $DESTINATION_DIRECTORY
cd $DESTINATION_DIRECTORY
# Create a kustomization.yaml file if it doesn't exist
if [ ! -f kustomization.yaml ]; then
cat > kustomization.yaml << EOF
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- resources.yaml
EOF
fi
helm template -n kubeflow --include-crds spark-operator spark-operator \
--set "spark.jobNamespaces={}" \
--set webhook.enable=true \
--set webhook.port=9443 \
--version ${SPARK_OPERATOR_VERSION} \
--repo ${SPARK_OPERATOR_HELM_CHART_REPO} > resources.yaml
echo "Successfully generated manifests."
# Use OS-compatible sed command
if [[ "$OSTYPE" == "darwin"* ]]; then
sed -i '' 's/Spark Operator[^|]*|[^|]*apps\/spark\/spark-operator[^|]*|[^|]*[0-9]\.[0-9]\.[0-9]/Spark Operator | apps\/spark\/spark-operator | '"${SPARK_OPERATOR_VERSION}"'/g' "${MANIFESTS_DIRECTORY}/README.md"
else
sed -i 's/Spark Operator.*|.*apps\/spark\/spark-operator[^|]*|.*[0-9]\.[0-9]\.[0-9]/Spark Operator | apps\/spark\/spark-operator | '"${SPARK_OPERATOR_VERSION}"'/g' "${MANIFESTS_DIRECTORY}/README.md"
fi
commit_changes "$MANIFESTS_DIRECTORY" "Update kubeflow/${COMPONENT_NAME} manifests to ${SPARK_OPERATOR_VERSION}" \
"apps/spark" \
"README.md" \
"scripts"
echo "Synchronization completed successfully."
@@ -0,0 +1,38 @@
#!/usr/bin/env bash
# This script helps to create a PR to update the Training Operator manifests
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
source "${SCRIPT_DIRECTORY}/library.sh"
setup_error_handling
COMPONENT_NAME="training-operator"
REPOSITORY_NAME="kubeflow/training-operator"
REPOSITORY_URL="https://github.com/kubeflow/training-operator.git"
COMMIT="v1.9.2"
REPOSITORY_DIRECTORY="training-operator"
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:=/tmp/kubeflow-${COMPONENT_NAME}}
BRANCH_NAME=${BRANCH_NAME:=synchronize-${COMPONENT_NAME}-manifests-${COMMIT?}}
# Path configurations
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
SOURCE_MANIFESTS_PATH="manifests"
DESTINATION_MANIFESTS_PATH="apps/${COMPONENT_NAME}/upstream"
# README update patterns
SOURCE_TEXT="\[.*\](https://github.com/${REPOSITORY_NAME}/tree/.*/manifests)"
DESTINATION_TEXT="\[${COMMIT}\](https://github.com/${REPOSITORY_NAME}/tree/${COMMIT}/manifests)"
create_branch "$BRANCH_NAME"
clone_and_checkout "$SOURCE_DIRECTORY" "$REPOSITORY_URL" "$REPOSITORY_DIRECTORY" "$COMMIT"
copy_manifests "${SOURCE_DIRECTORY}/${REPOSITORY_DIRECTORY}/${SOURCE_MANIFESTS_PATH}" "${MANIFESTS_DIRECTORY}/${DESTINATION_MANIFESTS_PATH}"
update_readme "$MANIFESTS_DIRECTORY" "$SOURCE_TEXT" "$DESTINATION_TEXT"
commit_changes "$MANIFESTS_DIRECTORY" "Update ${REPOSITORY_NAME} manifests from ${COMMIT}" \
"apps" \
"README.md"
echo "Synchronization completed successfully."
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env bash
# Template for Kubeflow manifests synchronization scripts
# Usage: Copy this file and adjust the variables for your specific component
# Source the common library functions
SCRIPT_DIRECTORY=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
source "${SCRIPT_DIRECTORY}/library.sh"
setup_error_handling
# Configuration variables (adjust these for your specific component)
COMPONENT_NAME="component-name" # Name of the component (e.g., katib, training-operator)
REPOSITORY_NAME="repo-name" # Repository name (e.g., kubeflow/katib)
REPOSITORY_URL="https://github.com/org/repo.git" # Repository URL to clone from
COMMIT="v0.0.0" # Version/commit to synchronize
REPOSITORY_DIRECTORY="${COMPONENT_NAME}" # Directory name within SOURCE_DIRECTORY
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:=/tmp/kubeflow-${COMPONENT_NAME}} # Where to clone the source repo
BRANCH_NAME=${BRANCH_NAME:=synchronize-${COMPONENT_NAME}-manifests-${COMMIT?}} # Branch name for the PR
MANIFESTS_DIRECTORY=$(dirname $SCRIPT_DIRECTORY)
SOURCE_MANIFESTS_PATH="manifests" # Path within source repo where manifests are
DESTINATION_MANIFESTS_PATH="apps/${COMPONENT_NAME}/upstream" # Destination path within manifests repo
SOURCE_TEXT="\[.*\](https://github.com/${REPOSITORY_NAME}/tree/.*/manifests)"
DESTINATION_TEXT="\[${COMMIT}\](https://github.com/${REPOSITORY_NAME}/tree/${COMMIT}/manifests)"
create_branch "$BRANCH_NAME"
clone_and_checkout "$SOURCE_DIRECTORY" "$REPOSITORY_URL" "$REPOSITORY_DIRECTORY" "$COMMIT"
copy_manifests "${SOURCE_DIRECTORY}/${REPOSITORY_DIRECTORY}/${SOURCE_MANIFESTS_PATH}" "${MANIFESTS_DIRECTORY}/${DESTINATION_MANIFESTS_PATH}"
update_readme "$MANIFESTS_DIRECTORY" "$SOURCE_TEXT" "$DESTINATION_TEXT"
commit_changes "$MANIFESTS_DIRECTORY" "Update ${REPOSITORY_NAME} manifests from ${COMMIT}" \
"${DESTINATION_MANIFESTS_PATH}" \
"README.md"
echo "Synchronization completed successfully."
+412
View File
@@ -0,0 +1,412 @@
# The script:
# 1. Extract all the images used by the Kubeflow Working Groups
# - The reported image lists are saved in respective files under ../image_lists directory
# 2. Scan the reported images using Trivy for security vulnerabilities
# - Scanned reports will be saved in JSON format inside ../image_lists/security_scan_reports/ folder for each Working Group
# 3. The script will also generate a summary of the security scan reports with severity counts for each Working Group with images
# - Summary of security counts with images a JSON file inside ../image_lists/summary_of_severity_counts_for_WG folder
# 4. Generate a summary of the security scan reports
# - The summary will be saved in JSON format inside ../image_lists/summary_of_severity_counts_for_WG folder
# 5. Before run this file you have to
# 1. Install kustomize
# - sudo apt install snapd
# - sudo snap install kustomize
# 2. Install trivy
# - sudo apt install snapd
# - sudo snap install trivy
# 4. Install Python
# 5. Install prettytable
# - pip install prettytable
# The script must be executed from the hack folder as it use relative paths
import os
import subprocess
import re
import argparse
import json
import glob
from prettytable import PrettyTable
# Dictionary mapping Kubeflow workgroups to directories containing kustomization files
wg_dirs = {
"automl": "../apps/katib/upstream/installs",
"pipelines": "../apps/pipeline/upstream/env ../apps/kfp-tekton/upstream/env",
"training": "../apps/training-operator/upstream/overlays",
"manifests": "../common/cert-manager/cert-manager/base ../common/cert-manager/kubeflow-issuer/base ../common/istio-1-24/istio-crds/base ../common/istio-1-24/istio-namespace/base ../common/istio-1-24/istio-install/overlays/oauth2-proxy ../common/oauth2-proxy/overlays/m2m-self-signed ../common/dex/overlays/oauth2-proxy ../common/knative/knative-serving/overlays/gateways ../common/knative/knative-eventing/base ../common/istio-1-24/cluster-local-gateway/base ../common/kubeflow-namespace/base ../common/kubeflow-roles/base ../common/istio-1-24/kubeflow-istio-resources/base",
"workbenches": "../apps/pvcviewer-controller/upstream/base ../apps/admission-webhook/upstream/overlays ../apps/centraldashboard/overlays ../apps/jupyter/jupyter-web-app/upstream/overlays ../apps/volumes-web-app/upstream/overlays ../apps/tensorboard/tensorboards-web-app/upstream/overlays ../apps/profiles/upstream/overlays ../apps/jupyter/notebook-controller/upstream/overlays ../apps/tensorboard/tensorboard-controller/upstream/overlays",
"serving": "../apps/kserve - ../apps/kserve/models-web-app/overlays/kubeflow",
"model-registry": "../apps/model-registry/upstream",
}
DIRECTORY = "../image_lists"
os.makedirs(DIRECTORY, exist_ok=True)
SCAN_REPORTS_DIR = os.path.join(DIRECTORY, "security_scan_reports")
ALL_SEVERITY_COUNTS = os.path.join(DIRECTORY, "severity_counts_with_images_for_WG")
SUMMARY_OF_SEVERITY_COUNTS = os.path.join(
DIRECTORY, "summary_of_severity_counts_for_WG"
)
os.makedirs(SCAN_REPORTS_DIR, exist_ok=True)
os.makedirs(ALL_SEVERITY_COUNTS, exist_ok=True)
os.makedirs(SUMMARY_OF_SEVERITY_COUNTS, exist_ok=True)
def log(*args, **kwargs):
# Custom log function that print messages with flush=True by default.
kwargs.setdefault("flush", True)
print(*args, **kwargs)
def save_images(wg, images, version):
# Saves a list of container images to a text file named after the workgroup and version.
output_file = f"../image_lists/kf_{version}_{wg}_images.txt"
with open(output_file, "w") as f:
f.write("\n".join(images))
log(f"File {output_file} successfully created")
def validate_semantic_version(version):
# Validates a semantic version string (e.g., "0.1.2" or "latest").
regex = r"^[0-9]+\.[0-9]+\.[0-9]+$"
if re.match(regex, version) or version == "latest":
return version
else:
raise ValueError(f"Invalid semantic version: '{version}'")
def extract_images(version):
version = validate_semantic_version(version)
log(f"Running the script using Kubeflow version: {version}")
all_images = set() # Collect all unique images across workgroups
for wg, dirs in wg_dirs.items():
wg_images = set() # Collect unique images for this workgroup
for dir_path in dirs.split():
for root, _, files in os.walk(dir_path):
for file in files:
if file in [
"kustomization.yaml",
"kustomization.yml",
"Kustomization",
]:
full_path = os.path.join(root, file)
try:
# Execute `kustomize build` to render the kustomization file
result = subprocess.run(
["kustomize", "build", root],
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
except subprocess.CalledProcessError as e:
log(
f'ERROR:\t Failed "kustomize build" command for directory: {root}. See error above'
)
continue
# Use regex to find lines with 'image: <image-name>:<version>' or 'image: <image-name>'
# and '- image: <image-name>:<version>' but avoid environment variables
kustomize_images = re.findall(
r"^\s*-?\s*image:\s*([^$\s:]+(?:\:[^\s]+)?)$",
result.stdout,
re.MULTILINE,
)
wg_images.update(kustomize_images)
# Ensure uniqueness within workgroup images
uniq_wg_images = sorted(wg_images)
all_images.update(uniq_wg_images)
save_images(wg, uniq_wg_images, version)
# Ensure uniqueness across all workgroups
uniq_images = sorted(all_images)
save_images("all", uniq_images, version)
parser = argparse.ArgumentParser(
description="Extract images from Kubeflow kustomizations."
)
# Define a positional argument 'version' with optional occurrence and default value 'latest'. You can run this file as python3 <filename>.py or python <filename>.py <version>
parser.add_argument(
"version",
nargs="?",
type=str,
default="latest",
help="Kubeflow version to use (defaults to latest).",
)
args = parser.parse_args()
extract_images(args.version)
log("Started scanning images")
# Get list of text files excluding "kf_latest_all_images.txt"
files = [
f
for f in glob.glob(os.path.join(DIRECTORY, "*.txt"))
if not f.endswith("kf_latest_all_images.txt")
]
# Loop through each text file in the specified directory
for file in files:
log(f"Scanning images in {file}")
file_base_name = os.path.basename(file).replace(".txt", "")
# Directory to save reports for this specific file
file_reports_dir = os.path.join(SCAN_REPORTS_DIR, file_base_name)
os.makedirs(file_reports_dir, exist_ok=True)
# Directory to save security count
severity_count = os.path.join(file_reports_dir, "severity_counts")
os.makedirs(severity_count, exist_ok=True)
with open(file, "r") as f:
lines = f.readlines()
for line in lines:
line = line.strip()
image_name = line.split(":")[0]
image_tag = line.split(":")[1] if ":" in line else ""
image_name_scan = image_name.split("/")[-1]
if image_tag:
image_name_scan = f"{image_name_scan}_{image_tag}"
scan_output_file = os.path.join(
file_reports_dir, f"{image_name_scan}_scan.json"
)
log(f"Scanning ", line)
try:
result = subprocess.run(
[
"trivy",
"image",
"--format",
"json",
"--output",
scan_output_file,
line,
],
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
with open(scan_output_file, "r") as json_file:
scan_data = json.load(json_file)
if not scan_data.get("Results"):
log(f"No vulnerabilities found in {image_name}:{image_tag}")
else:
vulnerabilities_list = [
result["Vulnerabilities"]
for result in scan_data["Results"]
if "Vulnerabilities" in result and result["Vulnerabilities"]
]
if not vulnerabilities_list:
log(
f"The vulnerabilities detection may be insufficient because security updates are not provided for {image_name}:{image_tag}\n"
)
else:
severity_counts = {"LOW": 0, "MEDIUM": 0, "HIGH": 0, "CRITICAL": 0}
for vulnerabilities in vulnerabilities_list:
for vulnerability in vulnerabilities:
severity = vulnerability.get("Severity", "UNKNOWN")
if severity == "UNKNOWN":
continue
elif severity in severity_counts:
severity_counts[severity] += 1
report = {"image": line, "severity_counts": severity_counts}
image_table = PrettyTable()
image_table.field_names = ["Critical", "High", "Medium", "Low"]
image_table.add_row(
[
severity_counts["CRITICAL"],
severity_counts["HIGH"],
severity_counts["MEDIUM"],
severity_counts["LOW"],
]
)
log(f"{image_table}\n")
severity_report_file = os.path.join(
severity_count, f"{image_name_scan}_severity_report.json"
)
with open(severity_report_file, "w") as report_file:
json.dump(report, report_file, indent=4)
except subprocess.CalledProcessError as e:
log(f"Error scanning {image_name}:{image_tag}")
log(e.stderr)
# Combine all the JSON files into a single file with severity counts for all images
json_files = glob.glob(os.path.join(severity_count, "*.json"))
output_file = os.path.join(ALL_SEVERITY_COUNTS, f"{file_base_name}.json")
if not json_files:
log(f"No JSON files found in '{severity_count}'. Skipping combination.")
else:
combined_data = []
for json_file in json_files:
with open(json_file, "r") as jf:
combined_data.append(json.load(jf))
with open(output_file, "w") as of:
json.dump({"data": combined_data}, of, indent=4)
log(f"JSON files successfully combined into '{output_file}'")
# File to save summary of the severity counts for WGs as JSON format.
summary_file = os.path.join(
SUMMARY_OF_SEVERITY_COUNTS, "severity_summary_in_json_format.json"
)
# Initialize counters
unique_images = {} # unique set of images across all WGs
total_images = 0
total_low = 0
total_medium = 0
total_high = 0
total_critical = 0
# Initialize a dictionary to hold the final JSON data
merged_data = {}
# Loop through each JSON file in the ALL_SEVERITY_COUNTS
for file_path in glob.glob(os.path.join(ALL_SEVERITY_COUNTS, "*.json")):
# Split filename based on underscores
filename_parts = os.path.basename(file_path).split("_")
# Check if there are at least 3 parts (prefix, name, _images)
if len(filename_parts) >= 4:
# Extract name (second part)
filename = filename_parts[2]
filename = filename.capitalize()
else:
log(f"Skipping invalid filename format: {file_path}")
continue
with open(file_path, "r") as f:
data = json.load(f)["data"]
# Initialize counts for this file
image_count = len(data)
low = sum(entry["severity_counts"]["LOW"] for entry in data)
medium = sum(entry["severity_counts"]["MEDIUM"] for entry in data)
high = sum(entry["severity_counts"]["HIGH"] for entry in data)
critical = sum(entry["severity_counts"]["CRITICAL"] for entry in data)
# Update unique_images for the total counts later
for d in data:
unique_images[d["image"]] = d
# Create the output for this file
file_data = {
"images": image_count,
"LOW": low,
"MEDIUM": medium,
"HIGH": high,
"CRITICAL": critical,
}
# Update merged_data with filename as key
merged_data[filename] = file_data
# Update the total counts
unique_images = unique_images.values() # keep the set of values
total_images += len(unique_images)
total_low += sum(entry["severity_counts"]["LOW"] for entry in unique_images)
total_medium += sum(entry["severity_counts"]["MEDIUM"] for entry in unique_images)
total_high += sum(entry["severity_counts"]["HIGH"] for entry in unique_images)
total_critical += sum(entry["severity_counts"]["CRITICAL"] for entry in unique_images)
# Add total counts to merged_data
merged_data["total"] = {
"images": total_images,
"LOW": total_low,
"MEDIUM": total_medium,
"HIGH": total_high,
"CRITICAL": total_critical,
}
log("Summary in Json Format:")
log(json.dumps(merged_data, indent=4))
# Write the final output to a file
with open(summary_file, "w") as summary_f:
json.dump(merged_data, summary_f, indent=4)
log(f"Summary written to: {summary_file} as JSON format")
# Load JSON content from the file
with open(summary_file, "r") as file:
data = json.load(file)
# Define a mapping for working group names
groupnames = {
"Automl": "AutoML",
"Pipelines": "Pipelines",
"Workbenches": "Workbenches(Notebooks)",
"Serving": "Kserve",
"Manifests": "Manifests",
"Training": "Training",
"Model-registry": "Model Registry",
"total": "All Images",
}
# Create PrettyTable
table = PrettyTable()
table.field_names = [
"Working Group",
"Images",
"Critical CVE",
"High CVE",
"Medium CVE",
"Low CVE",
]
# Populate the table with data
for group_name in groupnames:
if group_name in data: # Check if group_name exists in data
value = data[group_name]
table.add_row(
[
groupnames[group_name],
value["images"],
value["CRITICAL"],
value["HIGH"],
value["MEDIUM"],
value["LOW"],
]
)
# log the table
log(table)
# Write the table output to a file in the specified folder
output_file = (
SUMMARY_OF_SEVERITY_COUNTS + "/summary_of_severity_counts_for_WGs_in_table.txt"
)
with open(output_file, "w") as f:
f.write(str(table))
log("Output saved to:", output_file)
log("Severity counts with images respect to WGs are saved in the",ALL_SEVERITY_COUNTS)
log("Scanned Json reports on images are saved in",SCAN_REPORTS_DIR)