From aced28b13d22c4b67b9890e466914e5dcb76a479 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Fri, 14 Jul 2023 18:50:48 -0400 Subject: [PATCH 01/20] ignore cached remote samples Signed-off-by: Michael Valdron --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 11ca59650..f94f89466 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ registry-support/ .idea/ devfile-web/ vendor/ -.odo \ No newline at end of file +.odo +samples/.cache From da8dfc7a0b86cc652f850be1878ae528b6692546 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Fri, 14 Jul 2023 18:52:02 -0400 Subject: [PATCH 02/20] nightly script for building sample parent stack list Signed-off-by: Michael Valdron --- tests/build_sample_parents_nightly.sh | 53 +++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 tests/build_sample_parents_nightly.sh diff --git a/tests/build_sample_parents_nightly.sh b/tests/build_sample_parents_nightly.sh new file mode 100644 index 000000000..07477284c --- /dev/null +++ b/tests/build_sample_parents_nightly.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +base_path=$(dirname $0)/.. +samples_file=${base_path}/extraDevfileEntries.yaml +# Cached remote samples directory +samples_dir=${base_path}/samples/.cache + +# Clones remote samples into cache directory +clone_samples() { + samples_len=$(yq eval '.samples | length' ${samples_file}) + + # Removes old cached samples directory + if [ -d ${samples_dir} ]; then + rm -rf ${samples_dir} + fi + + for ((idx=0;idx<${samples_len};idx++)); do + name=$(yq eval .samples.${idx}.name ${samples_file}) + remote_url=$(yq eval .samples.${idx}.git.remotes.origin ${samples_file}) + + git clone --depth=1 ${remote_url} ${samples_dir}/${name} + done +} + +# Builds sample parent dependency file +build_parents() { + samples_len=$(yq eval '.samples | length' ${samples_file}) + + if [ -f parents.yaml ]; then + rm parents.yaml + fi + + for ((idx=0;idx<${samples_len};idx++)); do + name=$(yq eval .samples.${idx}.name ${samples_file}) + devfile=${samples_dir}/${name}/devfile.yaml + parent=$(yq eval .parent.id ${devfile}) + + if [ "${parent}" != "null" ]; then + if [ -f parents.yaml ] && [ "$(yq eval .parents.${parent}.children parents.yaml)" != "null" ]; then + next_idx=$(yq eval ".parents.${parent}.children | length" parents.yaml) + yq eval ".parents.${parent}.children[${next_idx}] = \"${name}\"" -i parents.yaml + elif [ -f parents.yaml ]; then + yq eval ".parents.${parent}.children[0] = \"${name}\"" -i parents.yaml + else + yq eval -n ".parents.${parent}.children[0] = \"${name}\"" > parents.yaml + fi + fi + done +} + +clone_samples + +build_parents \ No newline at end of file From cecd9ca0f69b9a3f237d9759516fa0f6bcbeca80 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Fri, 14 Jul 2023 18:52:39 -0400 Subject: [PATCH 03/20] init check parent stacks script Signed-off-by: Michael Valdron --- tests/check_parent_stacks.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 tests/check_parent_stacks.sh diff --git a/tests/check_parent_stacks.sh b/tests/check_parent_stacks.sh new file mode 100644 index 000000000..911a9aa68 --- /dev/null +++ b/tests/check_parent_stacks.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +base_path=$(dirname $0)/.. + From b19f793a710ffb9d70f7fe898c776ce549ec9c8f Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Tue, 18 Jul 2023 14:46:09 -0400 Subject: [PATCH 04/20] convert nightly parents build to on the fly parents build. Signed-off-by: Michael Valdron --- tests/build_sample_parents_nightly.sh | 53 --------------------------- tests/check_parent_stacks.sh | 49 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 53 deletions(-) delete mode 100644 tests/build_sample_parents_nightly.sh diff --git a/tests/build_sample_parents_nightly.sh b/tests/build_sample_parents_nightly.sh deleted file mode 100644 index 07477284c..000000000 --- a/tests/build_sample_parents_nightly.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -base_path=$(dirname $0)/.. -samples_file=${base_path}/extraDevfileEntries.yaml -# Cached remote samples directory -samples_dir=${base_path}/samples/.cache - -# Clones remote samples into cache directory -clone_samples() { - samples_len=$(yq eval '.samples | length' ${samples_file}) - - # Removes old cached samples directory - if [ -d ${samples_dir} ]; then - rm -rf ${samples_dir} - fi - - for ((idx=0;idx<${samples_len};idx++)); do - name=$(yq eval .samples.${idx}.name ${samples_file}) - remote_url=$(yq eval .samples.${idx}.git.remotes.origin ${samples_file}) - - git clone --depth=1 ${remote_url} ${samples_dir}/${name} - done -} - -# Builds sample parent dependency file -build_parents() { - samples_len=$(yq eval '.samples | length' ${samples_file}) - - if [ -f parents.yaml ]; then - rm parents.yaml - fi - - for ((idx=0;idx<${samples_len};idx++)); do - name=$(yq eval .samples.${idx}.name ${samples_file}) - devfile=${samples_dir}/${name}/devfile.yaml - parent=$(yq eval .parent.id ${devfile}) - - if [ "${parent}" != "null" ]; then - if [ -f parents.yaml ] && [ "$(yq eval .parents.${parent}.children parents.yaml)" != "null" ]; then - next_idx=$(yq eval ".parents.${parent}.children | length" parents.yaml) - yq eval ".parents.${parent}.children[${next_idx}] = \"${name}\"" -i parents.yaml - elif [ -f parents.yaml ]; then - yq eval ".parents.${parent}.children[0] = \"${name}\"" -i parents.yaml - else - yq eval -n ".parents.${parent}.children[0] = \"${name}\"" > parents.yaml - fi - fi - done -} - -clone_samples - -build_parents \ No newline at end of file diff --git a/tests/check_parent_stacks.sh b/tests/check_parent_stacks.sh index 911a9aa68..658c1460f 100644 --- a/tests/check_parent_stacks.sh +++ b/tests/check_parent_stacks.sh @@ -1,4 +1,53 @@ #!/bin/bash base_path=$(dirname $0)/.. +samples_file=${base_path}/extraDevfileEntries.yaml +# Cached remote samples directory +samples_dir=${base_path}/samples/.cache +# Clones remote samples into cache directory +clone_samples() { + samples_len=$(yq eval '.samples | length' ${samples_file}) + + # Removes old cached samples directory + if [ -d ${samples_dir} ]; then + rm -rf ${samples_dir} + fi + + for ((idx=0;idx<${samples_len};idx++)); do + name=$(yq eval .samples.${idx}.name ${samples_file}) + remote_url=$(yq eval .samples.${idx}.git.remotes.origin ${samples_file}) + + git clone --depth=1 ${remote_url} ${samples_dir}/${name} + done +} + +# Builds sample parent dependency file +build_parents() { + samples_len=$(yq eval '.samples | length' ${samples_file}) + + if [ -f parents.yaml ]; then + rm parents.yaml + fi + + for ((idx=0;idx<${samples_len};idx++)); do + name=$(yq eval .samples.${idx}.name ${samples_file}) + devfile=${samples_dir}/${name}/devfile.yaml + parent=$(yq eval .parent.id ${devfile}) + + if [ "${parent}" != "null" ]; then + if [ -f parents.yaml ] && [ "$(yq eval .parents.${parent}.children parents.yaml)" != "null" ]; then + next_idx=$(yq eval ".parents.${parent}.children | length" parents.yaml) + yq eval ".parents.${parent}.children[${next_idx}] = \"${name}\"" -i parents.yaml + elif [ -f parents.yaml ]; then + yq eval ".parents.${parent}.children[0] = \"${name}\"" -i parents.yaml + else + yq eval -n ".parents.${parent}.children[0] = \"${name}\"" > parents.yaml + fi + fi + done +} + +clone_samples + +build_parents From 804853fc8bbc3d8a8babe3dcd7dc4a05e80289d8 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Tue, 18 Jul 2023 20:09:15 -0400 Subject: [PATCH 05/20] multi version support for build parent stacks Signed-off-by: Michael Valdron --- tests/check_parent_stacks.sh | 72 ++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 20 deletions(-) diff --git a/tests/check_parent_stacks.sh b/tests/check_parent_stacks.sh index 658c1460f..a5bc34764 100644 --- a/tests/check_parent_stacks.sh +++ b/tests/check_parent_stacks.sh @@ -1,9 +1,17 @@ #!/bin/bash base_path=$(dirname $0)/.. +# default samples file path samples_file=${base_path}/extraDevfileEntries.yaml # Cached remote samples directory samples_dir=${base_path}/samples/.cache +parents_file=${base_path}/parents.yaml + +# Read samples file as first argument +# if unset use default samples file path +if [ ! -z "${1}" ]; then + samples_file=${1} +fi # Clones remote samples into cache directory clone_samples() { @@ -14,11 +22,22 @@ clone_samples() { rm -rf ${samples_dir} fi - for ((idx=0;idx<${samples_len};idx++)); do - name=$(yq eval .samples.${idx}.name ${samples_file}) - remote_url=$(yq eval .samples.${idx}.git.remotes.origin ${samples_file}) + for ((s_idx=0;s_idx<${samples_len};s_idx++)); do + name=$(yq eval .samples.${s_idx}.name ${samples_file}) + versions=($(yq eval .samples.${s_idx}.versions.[].version ${samples_file})) + + # Iterate through sample versions if sample has multi version support + if [ ${#versions[@]} -ne 0 ]; then + for ((v_idx=0;v_idx<${#versions[@]};v_idx++)); do + remote_url=$(yq eval .samples.${s_idx}.versions.${v_idx}.git.remotes.origin ${samples_file}) + + git clone --depth=1 ${remote_url} ${samples_dir}/${name}/${versions[$v_idx]} + done + else + remote_url=$(yq eval .samples.${s_idx}.git.remotes.origin ${samples_file}) - git clone --depth=1 ${remote_url} ${samples_dir}/${name} + git clone --depth=1 ${remote_url} ${samples_dir}/${name} + fi done } @@ -26,23 +45,36 @@ clone_samples() { build_parents() { samples_len=$(yq eval '.samples | length' ${samples_file}) - if [ -f parents.yaml ]; then - rm parents.yaml + if [ -f ${parents_file} ]; then + rm ${parents_file} fi - for ((idx=0;idx<${samples_len};idx++)); do - name=$(yq eval .samples.${idx}.name ${samples_file}) - devfile=${samples_dir}/${name}/devfile.yaml - parent=$(yq eval .parent.id ${devfile}) - - if [ "${parent}" != "null" ]; then - if [ -f parents.yaml ] && [ "$(yq eval .parents.${parent}.children parents.yaml)" != "null" ]; then - next_idx=$(yq eval ".parents.${parent}.children | length" parents.yaml) - yq eval ".parents.${parent}.children[${next_idx}] = \"${name}\"" -i parents.yaml - elif [ -f parents.yaml ]; then - yq eval ".parents.${parent}.children[0] = \"${name}\"" -i parents.yaml - else - yq eval -n ".parents.${parent}.children[0] = \"${name}\"" > parents.yaml + for ((s_idx=0;s_idx<${samples_len};s_idx++)); do + name=$(yq eval .samples.${s_idx}.name ${samples_file}) + versions=($(yq eval .samples.${s_idx}.versions.[].version ${samples_file})) + + # Iterate through sample versions if sample has multi version support + if [ ${#versions[@]} -ne 0 ]; then + for ((v_idx=0;v_idx<${#versions[@]};v_idx++)); do + devfile=${samples_dir}/${name}/${versions[$v_idx]}/devfile.yaml + parent=$(yq eval .parent.id ${devfile}) + parent_version=$(yq eval .parent.version ${devfile}) + + # TODO: multi version build control logic + done + else + devfile=${samples_dir}/${name}/devfile.yaml + parent=$(yq eval .parent.id ${devfile}) + + if [ "${parent}" != "null" ]; then + if [ -f ${parents_file} ] && [ "$(yq eval .parents.${parent}.children ${parents_file})" != "null" ]; then + next_idx=$(yq eval ".parents.${parent}.children | length" ${parents_file}) + yq eval ".parents.${parent}.children[${next_idx}] = \"${name}\"" -i ${parents_file} + elif [ -f ${parents_file} ]; then + yq eval ".parents.${parent}.children[0] = \"${name}\"" -i ${parents_file} + else + yq eval -n ".parents.${parent}.children[0] = \"${name}\"" > ${parents_file} + fi fi fi done @@ -50,4 +82,4 @@ build_parents() { clone_samples -build_parents +# build_parents From c99a656d818f9d2034fd17218318e49cf89da0ac Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Tue, 25 Jul 2023 18:23:09 -0400 Subject: [PATCH 06/20] ignore any local parents file. Signed-off-by: Michael Valdron --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f94f89466..091572578 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ devfile-web/ vendor/ .odo samples/.cache +parents.yaml From 1d3f552fbd3c2ec0551830b2aeb5eb2650cea234 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Tue, 25 Jul 2023 18:29:43 -0400 Subject: [PATCH 07/20] rename check_parent_stacks.sh to build_parents_file.sh, build_parents_file.sh now builds the parents yaml file and returns the children samples of the parent stacks. Signed-off-by: Michael Valdron --- tests/build_parents_file.sh | 240 +++++++++++++++++++++++++++++++++++ tests/check_parent_stacks.sh | 85 ------------- 2 files changed, 240 insertions(+), 85 deletions(-) create mode 100644 tests/build_parents_file.sh delete mode 100644 tests/check_parent_stacks.sh diff --git a/tests/build_parents_file.sh b/tests/build_parents_file.sh new file mode 100644 index 000000000..865c4077d --- /dev/null +++ b/tests/build_parents_file.sh @@ -0,0 +1,240 @@ +#!/bin/bash + +# default samples file path +samples_file=$(pwd)/extraDevfileEntries.yaml +# Cached remote samples directory +samples_dir=$(pwd)/samples/.cache +# default stacks directory +stacks_dir=${STACKS:-"$(pwd)/stacks"} +parents_file=$(pwd)/parents.yaml + +# YAML query cmd path +YQ_PATH=${YQ_PATH:-yq} + +# Read samples file as first argument +# if unset use default samples file path +if [ ! -z "${1}" ]; then + samples_file=${1} +fi + +# Clones remote samples into cache directory +clone_samples() { + samples_len=$($YQ_PATH eval '.samples | length' ${samples_file}) + + # Removes old cached samples directory + if [ -d ${samples_dir} ]; then + rm -rf ${samples_dir} + fi + + for ((s_idx=0;s_idx<${samples_len};s_idx++)); do + name=$($YQ_PATH eval .samples.${s_idx}.name ${samples_file}) + versions=($($YQ_PATH eval .samples.${s_idx}.versions.[].version ${samples_file})) + + # Iterate through sample versions if sample has multi version support + if [ ${#versions[@]} -ne 0 ]; then + for ((v_idx=0;v_idx<${#versions[@]};v_idx++)); do + remote_url=$($YQ_PATH eval .samples.${s_idx}.versions.${v_idx}.git.remotes.origin ${samples_file}) + + git clone --depth=1 ${remote_url} ${samples_dir}/${name}/${versions[$v_idx]} + done + else + remote_url=$($YQ_PATH eval .samples.${s_idx}.git.remotes.origin ${samples_file}) + + git clone --depth=1 ${remote_url} ${samples_dir}/${name} + fi + done +} + +get_parent_version() { + devfile=$1 + name=$2 + version=$($YQ_PATH eval .parent.version ${devfile}) + + if [ "${version}" == "null" ] && [ -f "${stacks_dir}/${name}/stack.yaml" ]; then + version=$($YQ_PATH eval '.versions | filter(.default) | .[0].version' ${stacks_dir}/${name}/stack.yaml) + fi + + echo ${version} +} + +# Get parent index if exists, else returns -1 +parent_index() { + name=$1 + version=$2 + + if [ -z "${version}" ]; then + result=$($YQ_PATH eval ".parents | to_entries | filter(.value.name == \"${name}\") | .[0].key" ${parents_file}) + else + result=$($YQ_PATH eval ".parents | to_entries | filter(.value.name == \"${name}\" and .value.version == \"${version}\") | .[0].key" ${parents_file}) + fi + + if [ "${result}" == "null" ]; then + echo "-1" + else + echo ${result} + fi +} + +# Get child index if exists, else returns -1 +child_index() { + parent_idx=$1 + name=$2 + version=$3 + + if [ -z "${version}" ]; then + result=$($YQ_PATH eval ".parents.[${parent_idx}].children | to_entries | filter(.value.name == \"${name}\") | .[0].key" ${parents_file}) + else + result=$($YQ_PATH eval ".parents.[${parent_idx}].children | to_entries | filter(.value.name == \"${name}\" and .value.version == \"${version}\") | .[0].key" ${parents_file}) + fi + + if [ "${result}" == "null" ]; then + echo "-1" + else + echo ${result} + fi +} + +# Builds sample parents +build_parents() { + parent_name=$1 + parent_version=$2 + + if [ "${parent_version}" == "null" ]; then + parent_version="" + fi + + if [ "${parent_name}" != "null" ]; then + if [ ! -f ${parents_file} ]; then + $YQ_PATH eval -n ".parents[0].name = \"${parent_name}\"" > ${parents_file} + if [ "${parent_version}" != "" ]; then + $YQ_PATH eval ".parents[0].version = \"${parent_version}\"" -i ${parents_file} + fi + + return + fi + + if [ "$($YQ_PATH eval .parents ${parents_file})" == "null" ]; then + $YQ_PATH eval ".parents[0].name = \"${parent_name}\"" -i ${parents_file} + if [ "${parent_version}" != "" ]; then + $YQ_PATH eval ".parents[0].version = \"${parent_version}\"" -i ${parents_file} + fi + + return + fi + + parent_idx=$(parent_index ${parent_name} ${parent_version}) + if [ "${parent_idx}" == "-1" ]; then + next_idx=$($YQ_PATH eval ".parents | length" ${parents_file}) + $YQ_PATH eval ".parents[${next_idx}].name = \"${parent_name}\"" -i ${parents_file} + if [ "${parent_version}" != "" ]; then + $YQ_PATH eval ".parents[${next_idx}].version = \"${parent_version}\"" -i ${parents_file} + fi + fi + else + return 1 + fi +} + +# Builds children of parent stacks +build_children() { + parent_name=$1 + parent_version=$2 + sample_name=$3 + sample_version=$4 + + parent_idx=$(parent_index ${parent_name} ${parent_version}) + + if [ "$($YQ_PATH eval .parents[${parent_idx}].children ${parents_file})" == "null" ]; then + $YQ_PATH eval ".parents[${parent_idx}].children[0].name = \"${sample_name}\"" -i ${parents_file} + if [ "${sample_version}" != "" ]; then + $YQ_PATH eval ".parents[${parent_idx}].children[0].version = \"${sample_version}\"" -i ${parents_file} + fi + + return + fi + + child_idx=$(child_index ${parent_idx} ${sample_name} ${sample_version}) + if [ "${child_idx}" == "-1" ]; then + next_idx=$($YQ_PATH eval ".parents[${parent_idx}].children | length" ${parents_file}) + $YQ_PATH eval ".parents[${parent_idx}].children[${next_idx}].name = \"${sample_name}\"" -i ${parents_file} + if [ "${sample_version}" != "" ]; then + $YQ_PATH eval ".parents[${parent_idx}].children[${next_idx}].version = \"${sample_version}\"" -i ${parents_file} + fi + fi +} + +build_parents_file() { + samples_len=$($YQ_PATH eval '.samples | length' ${samples_file}) + + for ((s_idx=0;s_idx<${samples_len};s_idx++)); do + sample_name=$($YQ_PATH eval .samples.${s_idx}.name ${samples_file}) + sample_versions=($($YQ_PATH eval .samples.${s_idx}.versions.[].version ${samples_file})) + + # Iterate through sample versions if sample has multi version support + if [ ${#sample_versions[@]} -ne 0 ]; then + for ((v_idx=0;v_idx<${#sample_versions[@]};v_idx++)); do + devfile=${samples_dir}/${sample_name}/${sample_versions[$v_idx]}/devfile.yaml + parent_name=$($YQ_PATH eval .parent.id ${devfile}) + parent_version=$(get_parent_version ${devfile} ${parent_name}) + build_parents ${parent_name} ${parent_version} + + if [ $? -eq 0 ]; then + build_children "${parent_name}" "${parent_version}" "${sample_name}" "${sample_versions[$v_idx]}" + fi + done + else + devfile=${samples_dir}/${sample_name}/devfile.yaml + parent_name=$($YQ_PATH eval .parent.id ${devfile}) + parent_version=$(get_parent_version ${devfile} ${parent_name}) + build_parents ${parent_name} ${parent_version} + + if [ $? -eq 0 ]; then + build_children "${parent_name}" "${parent_version}" "${sample_name}" "" + fi + fi + done +} + +# Gets the children sample paths of parents. +# When TEST_DELTA is set to true, only children of parents +# with changes are returned. +get_children_of_parents() { + stack_dirs=$(bash $(pwd)/tests/get_stacks.sh) + children=() + + for stack_dir in $stack_dirs; do + if [ "$(basename $(dirname $stack_dir))" == "." ]; then + stack_name=$(basename $stack_dir) + + names=($($YQ_PATH eval ".parents | filter(.name == \"${stack_name}\") | .[0].children.[].name" ${parents_file})) + versions=($($YQ_PATH eval ".parents | filter(.name == \"${stack_name}\") | .[0].children.[].version" ${parents_file})) + else + stack_name=$(basename $(dirname $stack_dir)) + stack_version=$(basename $stack_dir) + + names=($($YQ_PATH eval ".parents | filter(.name == \"${stack_name}\" and .version == \"${stack_version}\") | .[0].children.[].name" ${parents_file})) + versions=($($YQ_PATH eval ".parents | filter(.name == \"${stack_name}\" and .version == \"${stack_version}\") | .[0].children.[].version" ${parents_file})) + fi + + + for ((c_idx=0;c_idx<${#names[@]};c_idx++)); do + if [ "${versions[$c_idx]}" == "null" ]; then + children+=("${names[$c_idx]}") + else + children+=("${names[$c_idx]}/${versions[$c_idx]}") + fi + done + done + + echo ${children[@]} +} + +clone_samples + +if [ -f ${parents_file} ]; then + rm ${parents_file} +fi + +build_parents_file + +echo $(get_children_of_parents) diff --git a/tests/check_parent_stacks.sh b/tests/check_parent_stacks.sh deleted file mode 100644 index a5bc34764..000000000 --- a/tests/check_parent_stacks.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash - -base_path=$(dirname $0)/.. -# default samples file path -samples_file=${base_path}/extraDevfileEntries.yaml -# Cached remote samples directory -samples_dir=${base_path}/samples/.cache -parents_file=${base_path}/parents.yaml - -# Read samples file as first argument -# if unset use default samples file path -if [ ! -z "${1}" ]; then - samples_file=${1} -fi - -# Clones remote samples into cache directory -clone_samples() { - samples_len=$(yq eval '.samples | length' ${samples_file}) - - # Removes old cached samples directory - if [ -d ${samples_dir} ]; then - rm -rf ${samples_dir} - fi - - for ((s_idx=0;s_idx<${samples_len};s_idx++)); do - name=$(yq eval .samples.${s_idx}.name ${samples_file}) - versions=($(yq eval .samples.${s_idx}.versions.[].version ${samples_file})) - - # Iterate through sample versions if sample has multi version support - if [ ${#versions[@]} -ne 0 ]; then - for ((v_idx=0;v_idx<${#versions[@]};v_idx++)); do - remote_url=$(yq eval .samples.${s_idx}.versions.${v_idx}.git.remotes.origin ${samples_file}) - - git clone --depth=1 ${remote_url} ${samples_dir}/${name}/${versions[$v_idx]} - done - else - remote_url=$(yq eval .samples.${s_idx}.git.remotes.origin ${samples_file}) - - git clone --depth=1 ${remote_url} ${samples_dir}/${name} - fi - done -} - -# Builds sample parent dependency file -build_parents() { - samples_len=$(yq eval '.samples | length' ${samples_file}) - - if [ -f ${parents_file} ]; then - rm ${parents_file} - fi - - for ((s_idx=0;s_idx<${samples_len};s_idx++)); do - name=$(yq eval .samples.${s_idx}.name ${samples_file}) - versions=($(yq eval .samples.${s_idx}.versions.[].version ${samples_file})) - - # Iterate through sample versions if sample has multi version support - if [ ${#versions[@]} -ne 0 ]; then - for ((v_idx=0;v_idx<${#versions[@]};v_idx++)); do - devfile=${samples_dir}/${name}/${versions[$v_idx]}/devfile.yaml - parent=$(yq eval .parent.id ${devfile}) - parent_version=$(yq eval .parent.version ${devfile}) - - # TODO: multi version build control logic - done - else - devfile=${samples_dir}/${name}/devfile.yaml - parent=$(yq eval .parent.id ${devfile}) - - if [ "${parent}" != "null" ]; then - if [ -f ${parents_file} ] && [ "$(yq eval .parents.${parent}.children ${parents_file})" != "null" ]; then - next_idx=$(yq eval ".parents.${parent}.children | length" ${parents_file}) - yq eval ".parents.${parent}.children[${next_idx}] = \"${name}\"" -i ${parents_file} - elif [ -f ${parents_file} ]; then - yq eval ".parents.${parent}.children[0] = \"${name}\"" -i ${parents_file} - else - yq eval -n ".parents.${parent}.children[0] = \"${name}\"" > ${parents_file} - fi - fi - fi - done -} - -clone_samples - -# build_parents From a3092cb100db21c41326cf93549186ac953ce270 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Tue, 25 Jul 2023 18:31:05 -0400 Subject: [PATCH 08/20] validate devfile schemas test suite parameters now changeable Signed-off-by: Michael Valdron --- tests/validate_devfile_schemas.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/validate_devfile_schemas.sh b/tests/validate_devfile_schemas.sh index 305dc8898..e967ddc14 100755 --- a/tests/validate_devfile_schemas.sh +++ b/tests/validate_devfile_schemas.sh @@ -2,7 +2,8 @@ set -x -stackDirs=$(bash "$(pwd)/tests/get_stacks.sh") +stacksDir=${STACKS_DIR:-"$(pwd)/stacks"} +stackDirs=${STACKS:-"$(bash "$(pwd)/tests/get_stacks.sh")"} ginkgo run --procs 2 \ - tests/validate_devfile_schemas -- -stacksPath "$(pwd)"/stacks -stackDirs "$stackDirs" + tests/validate_devfile_schemas -- -stacksPath ${stacksDir} -stackDirs "$stackDirs" From 50dfb62371fbff2efd4a0d867180b623642d2a25 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Tue, 25 Jul 2023 18:32:10 -0400 Subject: [PATCH 09/20] validate samples workflow Signed-off-by: Michael Valdron --- .github/workflows/validate-samples.yaml | 61 +++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .github/workflows/validate-samples.yaml diff --git a/.github/workflows/validate-samples.yaml b/.github/workflows/validate-samples.yaml new file mode 100644 index 000000000..8a54c9581 --- /dev/null +++ b/.github/workflows/validate-samples.yaml @@ -0,0 +1,61 @@ +# +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Validate child samples + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: 0 5 * * * + +concurrency: + group: ${{ github.workflow }}-${{ github.event.number || github.ref }} + cancel-in-progress: true + +env: + MINIKUBE_VERSION: "v1.29.0" + KUBERNETES_VERSION: "v1.25.2" + TEST_DELTA: false + +jobs: + validate-devfile-schema: + name: validate devfile schemas + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: "1.19" + + - name: Install Ginkgo + run: go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo@v2.5.0 + + - name: Test delta if on a pull request + if: ${{ github.event_name == 'pull_request' }} + run: echo "TEST_DELTA=true" >> $GITHUB_ENV + + - name: Build parents file and get child samples + run: echo "STACKS=$(bash tests/build_parents_file.sh)" >> $GITHUB_ENV + + - name: Validate samples + run: STACKS_DIR=$(pwd)/samples/.cache bash tests/validate_devfile_schemas.sh From 6e8a9819bad8a43b635cf054b9654e822c24db49 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Wed, 26 Jul 2023 10:49:07 -0400 Subject: [PATCH 10/20] add check for if there are samples to be checked under validate samples job. Signed-off-by: Michael Valdron --- .github/workflows/validate-samples.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validate-samples.yaml b/.github/workflows/validate-samples.yaml index 8a54c9581..fbe215d35 100644 --- a/.github/workflows/validate-samples.yaml +++ b/.github/workflows/validate-samples.yaml @@ -58,4 +58,5 @@ jobs: run: echo "STACKS=$(bash tests/build_parents_file.sh)" >> $GITHUB_ENV - name: Validate samples + if: ${{ env.STACKS != '' }} run: STACKS_DIR=$(pwd)/samples/.cache bash tests/validate_devfile_schemas.sh From b1a6ec751ecd665a59dcf1e04104e03581764e2f Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Wed, 26 Jul 2023 11:58:01 -0400 Subject: [PATCH 11/20] add README instruction on dependency check changes Signed-off-by: Michael Valdron --- tests/README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/README.md b/tests/README.md index ddae887a9..fd0a4404f 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,5 +1,29 @@ # Devfile Registry Testing +## Dependency check + +### Prerequisites + +- Ensure `yq 4.3x` is installed + +### Running the build script + +- This script performs three actions + - Clones samples from provided `extraDevfileEntries.yaml` under `samples/.cache` + - Creates a `parents.yaml` which contains the dependency tree for parent stacks + - Outputs the child sample paths of parent stacks, `TEST_DELTA=true` will result in only outputting child samples which have changed parent stacks +- The build script takes one optional argument and works off of the current working directory + - `bash tests/build_parents_file.sh`, default samples file is `extraDevfileEntries.yaml` + - `bash tests/build_parents_file.sh ` + +### Use with testing + +- One can test the child samples using the [validate_devfile_schemas]() test suite by performing the following: +```sh +export STACKS=$(bash tests/build_parents_file.sh) +STACKS_DIR=samples/.cache bash tests/validate_devfile_schemas.sh +``` + ## Validating non-terminating images ### Prerequisites From 19004efe7a58e990d16ab73c7d2bc04ec75f8797 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Mon, 31 Jul 2023 14:50:02 -0400 Subject: [PATCH 12/20] yq 4.x installation guide link added to testing prerequisites list. Signed-off-by: Michael Valdron --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index fd0a4404f..bf642d1d1 100644 --- a/tests/README.md +++ b/tests/README.md @@ -4,7 +4,7 @@ ### Prerequisites -- Ensure `yq 4.3x` is installed +- Ensure [yq 4.x](https://github.com/mikefarah/yq/#install) is installed ### Running the build script From 4a227ca0782449a1d6e0060947d68daffb43fafb Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Mon, 31 Jul 2023 15:04:51 -0400 Subject: [PATCH 13/20] add missing link to test suite Signed-off-by: Michael Valdron --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index bf642d1d1..55ecfb07e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -18,7 +18,7 @@ ### Use with testing -- One can test the child samples using the [validate_devfile_schemas]() test suite by performing the following: +- One can test the child samples using the [validate_devfile_schemas](./validate_devfile_schemas/) test suite by performing the following: ```sh export STACKS=$(bash tests/build_parents_file.sh) STACKS_DIR=samples/.cache bash tests/validate_devfile_schemas.sh From 46193a0ce047ab7578fb3d50e8bea26abd9c0d02 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Mon, 31 Jul 2023 16:33:17 -0400 Subject: [PATCH 14/20] validate_devfile_schemas script accepts relative or absolute paths Signed-off-by: Michael Valdron --- tests/validate_devfile_schemas.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/validate_devfile_schemas.sh b/tests/validate_devfile_schemas.sh index e967ddc14..eac6a657a 100755 --- a/tests/validate_devfile_schemas.sh +++ b/tests/validate_devfile_schemas.sh @@ -2,8 +2,13 @@ set -x -stacksDir=${STACKS_DIR:-"$(pwd)/stacks"} +stacksDir=${STACKS_DIR:-stacks} stackDirs=${STACKS:-"$(bash "$(pwd)/tests/get_stacks.sh")"} +# Use pwd if relative path +if [[ ! ${stacksDir} = /* ]]; then + stacksDir=$(pwd)/${stacksDir} +fi + ginkgo run --procs 2 \ tests/validate_devfile_schemas -- -stacksPath ${stacksDir} -stackDirs "$stackDirs" From f45e89d0f3b1b71c82a4e56fb6b6e2bd9651ec30 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Fri, 17 May 2024 09:51:21 -0400 Subject: [PATCH 15/20] use registry-support cache samples Signed-off-by: Michael Valdron --- tests/build_parents_file.sh | 39 +++++++++---------------------------- 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/tests/build_parents_file.sh b/tests/build_parents_file.sh index 865c4077d..c41ce8600 100644 --- a/tests/build_parents_file.sh +++ b/tests/build_parents_file.sh @@ -7,6 +7,8 @@ samples_dir=$(pwd)/samples/.cache # default stacks directory stacks_dir=${STACKS:-"$(pwd)/stacks"} parents_file=$(pwd)/parents.yaml +# Base directory path +base_path=$(realpath $(dirname $(dirname $0))) # YAML query cmd path YQ_PATH=${YQ_PATH:-yq} @@ -17,34 +19,6 @@ if [ ! -z "${1}" ]; then samples_file=${1} fi -# Clones remote samples into cache directory -clone_samples() { - samples_len=$($YQ_PATH eval '.samples | length' ${samples_file}) - - # Removes old cached samples directory - if [ -d ${samples_dir} ]; then - rm -rf ${samples_dir} - fi - - for ((s_idx=0;s_idx<${samples_len};s_idx++)); do - name=$($YQ_PATH eval .samples.${s_idx}.name ${samples_file}) - versions=($($YQ_PATH eval .samples.${s_idx}.versions.[].version ${samples_file})) - - # Iterate through sample versions if sample has multi version support - if [ ${#versions[@]} -ne 0 ]; then - for ((v_idx=0;v_idx<${#versions[@]};v_idx++)); do - remote_url=$($YQ_PATH eval .samples.${s_idx}.versions.${v_idx}.git.remotes.origin ${samples_file}) - - git clone --depth=1 ${remote_url} ${samples_dir}/${name}/${versions[$v_idx]} - done - else - remote_url=$($YQ_PATH eval .samples.${s_idx}.git.remotes.origin ${samples_file}) - - git clone --depth=1 ${remote_url} ${samples_dir}/${name} - fi - done -} - get_parent_version() { devfile=$1 name=$2 @@ -229,7 +203,12 @@ get_children_of_parents() { echo ${children[@]} } -clone_samples +if [ ! -d ${base_path}/registry-support ] +then + git clone https://github.com/devfile/registry-support ${base_path}/registry-support +fi + +bash ${base_path}/registry-support/build-tools/cache_samples.sh ${samples_file} ${samples_dir} if [ -f ${parents_file} ]; then rm ${parents_file} @@ -237,4 +216,4 @@ fi build_parents_file -echo $(get_children_of_parents) +get_children_of_parents From 959bbced0d5e70b8c626d081d268e95dda469d4e Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Tue, 21 May 2024 18:49:44 -0400 Subject: [PATCH 16/20] use yq v4.44.1 Signed-off-by: Michael Valdron --- .ci/Dockerfile | 3 ++- .ci/Dockerfile.offline | 3 ++- .github/workflows/validate-samples.yaml | 5 +++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.ci/Dockerfile b/.ci/Dockerfile index ba7421f0d..6c6c55057 100644 --- a/.ci/Dockerfile +++ b/.ci/Dockerfile @@ -18,7 +18,8 @@ FROM registry.access.redhat.com/ubi8/go-toolset:1.19 AS builder USER root # Install yq -RUN curl -sL -O https://github.com/mikefarah/yq/releases/download/v4.9.5/yq_linux_amd64 -o /usr/local/bin/yq && mv ./yq_linux_amd64 /usr/local/bin/yq && chmod +x /usr/local/bin/yq +ENV YQ_VERSION=v4.44.1 +RUN curl -sL -O https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -o /usr/local/bin/yq && mv ./yq_linux_amd64 /usr/local/bin/yq && chmod +x /usr/local/bin/yq COPY . /registry diff --git a/.ci/Dockerfile.offline b/.ci/Dockerfile.offline index c04974237..777026e5c 100644 --- a/.ci/Dockerfile.offline +++ b/.ci/Dockerfile.offline @@ -18,7 +18,8 @@ FROM registry.access.redhat.com/ubi8/go-toolset:1.19 AS builder USER root # Install yq -RUN curl -sL -O https://github.com/mikefarah/yq/releases/download/v4.9.5/yq_linux_amd64 -o /usr/local/bin/yq && mv ./yq_linux_amd64 /usr/local/bin/yq && chmod +x /usr/local/bin/yq +ENV YQ_VERSION=v4.44.1 +RUN curl -sL -O https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -o /usr/local/bin/yq && mv ./yq_linux_amd64 /usr/local/bin/yq && chmod +x /usr/local/bin/yq COPY . /registry diff --git a/.github/workflows/validate-samples.yaml b/.github/workflows/validate-samples.yaml index fbe215d35..ba132fe79 100644 --- a/.github/workflows/validate-samples.yaml +++ b/.github/workflows/validate-samples.yaml @@ -29,7 +29,9 @@ concurrency: env: MINIKUBE_VERSION: "v1.29.0" + MINIKUBE_RESOURCES: "--memory 14gb --cpus 4" KUBERNETES_VERSION: "v1.25.2" + YQ_VERSION: "v4.44.1" TEST_DELTA: false jobs: @@ -50,6 +52,9 @@ jobs: - name: Install Ginkgo run: go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo@v2.5.0 + - name: Install yq + run: curl -sL -O https://github.com/mikefarah/yq/releases/download/${{ env.YQ_VERSION }}/yq_linux_amd64 -o /usr/local/bin/yq && mv ./yq_linux_amd64 /usr/local/bin/yq && chmod +x /usr/local/bin/yq + - name: Test delta if on a pull request if: ${{ github.event_name == 'pull_request' }} run: echo "TEST_DELTA=true" >> $GITHUB_ENV From 3b386510e72af76aa5b3c76c2395a9d8ece7bda8 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Wed, 22 May 2024 16:37:14 -0400 Subject: [PATCH 17/20] validate-samples actions match revision from other workflows Signed-off-by: Michael Valdron --- .github/workflows/validate-samples.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validate-samples.yaml b/.github/workflows/validate-samples.yaml index ba132fe79..82250bc1c 100644 --- a/.github/workflows/validate-samples.yaml +++ b/.github/workflows/validate-samples.yaml @@ -40,17 +40,17 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 with: fetch-depth: 0 - name: Install Go - uses: actions/setup-go@v3 + uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 with: go-version: "1.19" - name: Install Ginkgo - run: go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo@v2.5.0 + run: go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo@v2.13.0 - name: Install yq run: curl -sL -O https://github.com/mikefarah/yq/releases/download/${{ env.YQ_VERSION }}/yq_linux_amd64 -o /usr/local/bin/yq && mv ./yq_linux_amd64 /usr/local/bin/yq && chmod +x /usr/local/bin/yq From 6ef053db0837559957c0340d314b1ed7a49d8b5f Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Thu, 23 May 2024 13:36:13 -0400 Subject: [PATCH 18/20] create verbose mode for build_parents_file.sh script, defaults to false Signed-off-by: Michael Valdron --- tests/build_parents_file.sh | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/tests/build_parents_file.sh b/tests/build_parents_file.sh index c41ce8600..51066789a 100644 --- a/tests/build_parents_file.sh +++ b/tests/build_parents_file.sh @@ -1,5 +1,27 @@ #!/bin/bash +POSITIONAL_ARGS=() +VERBOSE="false" + +while [[ $# -gt 0 ]]; do + case $1 in + -v|--verbose) + VERBOSE="true" + shift # past argument + ;; + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + # default samples file path samples_file=$(pwd)/extraDevfileEntries.yaml # Cached remote samples directory @@ -205,10 +227,20 @@ get_children_of_parents() { if [ ! -d ${base_path}/registry-support ] then - git clone https://github.com/devfile/registry-support ${base_path}/registry-support + if [ "$VERBOSE" == "true" ] + then + git clone https://github.com/devfile/registry-support ${base_path}/registry-support + else + git clone -q https://github.com/devfile/registry-support ${base_path}/registry-support + fi fi -bash ${base_path}/registry-support/build-tools/cache_samples.sh ${samples_file} ${samples_dir} +if [ "$VERBOSE" == "true" ] +then + bash ${base_path}/registry-support/build-tools/cache_samples.sh ${samples_file} ${samples_dir} +else + bash ${base_path}/registry-support/build-tools/cache_samples.sh ${samples_file} ${samples_dir} > /dev/null 2>&- echo +fi if [ -f ${parents_file} ]; then rm ${parents_file} From a4ac51ce09028ef111b0f823aeeaad0b12b52721 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Thu, 23 May 2024 14:26:53 -0400 Subject: [PATCH 19/20] validate_devfile_schemas.sh script unzips resource files if samples Signed-off-by: Michael Valdron --- .github/workflows/validate-samples.yaml | 2 +- tests/validate_devfile_schemas.sh | 28 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate-samples.yaml b/.github/workflows/validate-samples.yaml index 82250bc1c..6787940e5 100644 --- a/.github/workflows/validate-samples.yaml +++ b/.github/workflows/validate-samples.yaml @@ -64,4 +64,4 @@ jobs: - name: Validate samples if: ${{ env.STACKS != '' }} - run: STACKS_DIR=$(pwd)/samples/.cache bash tests/validate_devfile_schemas.sh + run: STACKS_DIR=$(pwd)/samples/.cache bash tests/validate_devfile_schemas.sh --samples diff --git a/tests/validate_devfile_schemas.sh b/tests/validate_devfile_schemas.sh index eac6a657a..e987874d7 100755 --- a/tests/validate_devfile_schemas.sh +++ b/tests/validate_devfile_schemas.sh @@ -1,5 +1,26 @@ #!/usr/bin/env bash +POSITIONAL_ARGS=() +SAMPLES="false" + +while [[ $# -gt 0 ]]; do + case $1 in + -s|--samples) + SAMPLES="true" + shift # past argument + ;; + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters set -x stacksDir=${STACKS_DIR:-stacks} @@ -10,5 +31,12 @@ if [[ ! ${stacksDir} = /* ]]; then stacksDir=$(pwd)/${stacksDir} fi +# Unzip resource files if samples +if [ "${SAMPLES}" == "true" ]; then + for sample_dir in $(ls $stacksDir); do + unzip -n $stacksDir/$sample_dir/sampleName.zip -d $stacksDir + done +fi + ginkgo run --procs 2 \ tests/validate_devfile_schemas -- -stacksPath ${stacksDir} -stackDirs "$stackDirs" From da82868a253aff3c623081d9441a48ba607da17f Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Wed, 26 Jul 2023 11:01:54 -0400 Subject: [PATCH 20/20] test update on go 1.2.0 Signed-off-by: Michael Valdron --- stacks/go/1.2.0/devfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/go/1.2.0/devfile.yaml b/stacks/go/1.2.0/devfile.yaml index fe0ca80d2..af999c647 100644 --- a/stacks/go/1.2.0/devfile.yaml +++ b/stacks/go/1.2.0/devfile.yaml @@ -1,4 +1,4 @@ -schemaVersion: 2.1.0 +schemaVersion: 2.2.0 metadata: name: go displayName: Go Runtime