ROX-11715: Error better when deployment fails (stackrox#2477)

Author: gavin-stackrox

.openshift-ci/post_tests.py

@@ -290,6 +290,7 @@ def run(self, test_output_dirs=None):
         self.fixup_artifacts_content_type()
         self.make_artifacts_help()
         self.handle_run_failure()
+        self.handle_e2e_progress_failures()
 
     def fixup_artifacts_content_type(self):
         self.run_with_best_effort(
@@ -302,3 +303,12 @@ def make_artifacts_help(self):
             ["scripts/ci/store-artifacts.sh", "make_artifacts_help"],
             timeout=PostTestsConstants.FIXUP_TIMEOUT,
         )
+
+    def handle_e2e_progress_failures(self):
+        self.run_with_best_effort(
+            [
+                "tests/e2e/lib.sh",
+                "handle_e2e_progress_failures",
+            ],
+            timeout=PostTestsConstants.CHECK_TIMEOUT,
+        )

qa-tests-backend/scripts/run-part-2.sh

@@ -4,12 +4,18 @@
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd)"
 source "$ROOT/scripts/ci/lib.sh"
+source "$ROOT/tests/e2e/lib.sh"
 
 set -euo pipefail
 
 run_tests_part_2() {
     info "QA Automation Platform Part 2"
 
+    if [[ ! -f "${STATE_DEPLOYED}" ]]; then
+        info "Skipping part 2 tests due to earlier failure"
+        exit 0
+    fi
+
     export CLUSTER="${ORCHESTRATOR_FLAVOR^^}"
 
     rm -f FAIL

scripts/ci/lib.sh

@@ -1219,6 +1219,29 @@ send_slack_notice_for_failures_on_merge() {
     curl -XPOST -d @- -H 'Content-Type: application/json' "$webhook_url"
 }
 
+save_junit_failure() {
+    if [[ "$#" -ne 3 ]]; then
+        die "missing args. usage: save_junit_failure <class> <description> <details>"
+    fi
+
+    if [[ -z "${ARTIFACT_DIR}" ]]; then
+        info "Warning: save_junit_failure() requires an ARTIFACT_DIR"
+        return
+    fi
+
+    local class="$1"
+    local description="$2"
+    local details="$3"
+
+    cat << EOF > "${ARTIFACT_DIR}/junit-${class}.xml"
+<testsuite name="${class}" tests="1" skipped="0" failures="1" errors="0">
+    <testcase name="${description}" classname="${class}">
+        <failure>${details}</failure>
+    </testcase>
+</testsuite>
+EOF
+}
+
 if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
     if [[ "$#" -lt 1 ]]; then
         die "When invoked at the command line a method is required."

tests/e2e/lib.sh

@@ -7,6 +7,9 @@ set -euo pipefail
 
 TEST_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd)"
 
+# State
+STATE_DEPLOYED="/tmp/state_deployed"
+
 source "$TEST_ROOT/scripts/lib.sh"
 source "$TEST_ROOT/scripts/ci/lib.sh"
 
@@ -24,6 +27,8 @@ deploy_stackrox() {
     kubectl -n stackrox delete pod -l app=collector --grace-period=0
 
     sensor_wait
+
+    touch "${STATE_DEPLOYED}"
 }
 
 # export_test_environment() - Persist environment variables for the remainder of
@@ -400,6 +405,14 @@ db_backup_and_restore_test() {
     [[ ! -f DB_TEST_FAIL ]] || die "The DB test failed"
 }
 
+handle_e2e_progress_failures() {
+    info "Checking for deployment failure"
+
+    if [[ ! -f "${STATE_DEPLOYED}" ]]; then
+        save_junit_failure "Stackrox_Deployment" "Could not deploy StackRox" "Check the build log" || true
+    fi
+}
+
 if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
     if [[ "$#" -lt 1 ]]; then
         usage