Testing consul-1.21-dev + consul-k8s-1.7.x with ocp-4.16

- Testing consul-1.21-dev + consul-k8s-1.7.x  with ocp-4.16 + k8s-1.30.0 + kubectl 1.30.0
- Added OCP logging for error debugging and cluster stuck issues and finalizer issues

Author: anandmukul93

.github/workflows/pr-openshift-accepance.yml

@@ -0,0 +1,25 @@
+# Dispatch to the consul-k8s-workflows with a nightly cron
+name: pr-openshift-acceptance
+on:
+  pull_request:
+    branches:
+      - release/1.7.0-rc1
+
+# these should be the only settings that you will ever need to change
+env:
+  BRANCH: ${{ github.event.pull_request.head.ref }}
+  CONTEXT: "pr"
+
+jobs:
+  openshift-acceptance:
+    name: openshift-acceptance
+    runs-on: ubuntu-latest
+    steps:
+      - uses: benc-uk/workflow-dispatch@25b02cc069be46d637e8fe2f1e8484008e9e9609 # v1.2.3
+        name: cloud
+        with:
+          workflow: cloud.yml
+          repo: hashicorp/consul-k8s-workflows
+          ref: mukul/testing-ocp-compatibility
+          token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
+          inputs: '{ "context":"${{ env.CONTEXT }}", "repository":"${{ github.repository }}", "branch":"${{ env.BRANCH }}", "sha":"${{ github.sha }}", "token":"${{ secrets.ELEVATED_GITHUB_TOKEN }}", "test-integrations": "ocp" }'

.github/workflows/pr.yml

@@ -2,6 +2,8 @@
 name: pr
 on:
   pull_request:
+    branches:
+      - release/1.7.0-rc1
 
 # these should be the only settings that you will ever need to change
 env:
@@ -24,7 +26,7 @@ jobs:
       with:
         workflow: test.yml
         repo: hashicorp/consul-k8s-workflows
-        ref: main
+        ref: mukul/testing-ocp-compatibility
         token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
         inputs: '{ "context":"${{ env.CONTEXT }}", "actor":"${{ github.actor }}", "repository":"${{ github.repository }}", "branch":"${{ env.BRANCH }}", "sha":"${{ env.SHA }}", "token":"${{ secrets.ELEVATED_GITHUB_TOKEN }}" }'
 

acceptance/ci-inputs/kind-inputs.yaml

@@ -2,5 +2,6 @@
 # SPDX-License-Identifier: MPL-2.0
 
 kindVersion: v0.23.0
-kindNodeImage: kindest/node:v1.30.2@sha256:ecfe5841b9bee4fe9690f49c118c33629fa345e3350a0c67a5a34482a99d6bba
-kubectlVersion: v1.30.2
+# digest for kindest/node:v1.30.0
+kindNodeImage: kindest/node:v1.30.0
+kubectlVersion: v1.30.0

acceptance/tests/openshift/openshift_test_runner.go

@@ -13,20 +13,67 @@ import (
 func newOpenshiftCluster(t *testing.T, cfg *config.TestConfig, secure, namespaceMirroring bool) {
 	cmd := exec.Command("helm", "repo", "add", "hashicorp", "https://helm.releases.hashicorp.com")
 	output, err := cmd.CombinedOutput()
-	require.NoErrorf(t, err, "failed to add hashicorp helm repo: %s", string(output))
+	require.NoErrorf(t, err, "failed to add hashicorp helm repo : %s", string(output))
+	// Add diagnostic logging before attempting cleanup
+	logCmd := exec.Command("kubectl", "get", "all", "-n", "consul")
+	logOutput, _ := logCmd.CombinedOutput()
+	t.Logf("Resources in consul namespace before cleanup:\n%s", string(logOutput))
+
+	// Check if namespace exists and its status
+	nsCheckCmd := exec.Command("kubectl", "get", "namespace", "consul", "-o", "json")
+	nsOutput, _ := nsCheckCmd.CombinedOutput()
+	t.Logf("Consul namespace status before cleanup:\n%s", string(nsOutput))
 
 	// FUTURE for some reason NewHelmCluster creates a consul server pod that runs as root which
 	//   isn't allowed in OpenShift. In order to test OpenShift properly, we have to call helm and k8s
 	//   directly to bypass. Ideally we would just fix the framework that is running the pod as root.
+	// First, try to delete the namespace if it exists (cleanup from previous runs)
+	t.Log("Attempting to delete consul namespace if it exists...")
+	cleanupCmd := exec.Command("kubectl", "delete", "namespace", "consul", "--ignore-not-found=true")
+	cleanupOutput, cleanupErr := cleanupCmd.CombinedOutput()
+	// We don't check error here since it's just precautionary cleanup
+	t.Logf("Namespace deletion attempt result: %v\nOutput: %s", cleanupErr, string(cleanupOutput))
+
+	// Wait for namespace to be fully deleted before proceeding
+	t.Log("Waiting for consul namespace to be fully deleted...")
+	waitCmd := exec.Command("kubectl", "wait", "--for=delete", "namespace/consul", "--timeout=120s")
+	waitOutput, waitErr := waitCmd.CombinedOutput() // Ignore errors, as this will error if the namespace doesn't exist at all
+	t.Logf("Wait result: %v\nOutput: %s", waitErr, string(waitOutput))
+
+	// Force cleanup of any stuck resources in the namespace (if it still exists)
+	t.Log("Checking for any stuck resources...")
+	forceCleanupCmd := exec.Command("bash", "-c", `
+		# Try to find finalizers on the namespace
+		FINALIZERS=$(kubectl get namespace consul -o json 2>/dev/null | jq '.spec.finalizers' 2>/dev/null)
+		if [ ! -z "$FINALIZERS" ] && [ "$FINALIZERS" != "null" ]; then
+			echo "Found finalizers on namespace, attempting to remove them..."
+			# Remove finalizers from namespace to force deletion
+			kubectl get namespace consul -o json | jq '.spec.finalizers = []' | kubectl replace --raw "/api/v1/namespaces/consul/finalize" -f -
+		fi
+	`)
+	forceOutput, _ := forceCleanupCmd.CombinedOutput()
+	t.Logf("Force cleanup result:\n%s", string(forceOutput))
+
+	// Now create the namespace
 	cmd = exec.Command("kubectl", "create", "namespace", "consul")
 	output, err = cmd.CombinedOutput()
+
 	helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() {
+		if t.Failed() {
+			t.Log("Test already failed, skipping cleanup assertions")
+			cmd = exec.Command("kubectl", "delete", "namespace", "consul")
+			if err := cmd.Run(); err != nil {
+				t.Logf("Failed to delete namespace: %v", err)
+			}
+			return
+		}
+
 		cmd = exec.Command("kubectl", "delete", "namespace", "consul")
 		output, err = cmd.CombinedOutput()
 		assert.NoErrorf(t, err, "failed to delete namespace: %s", string(output))
 	})
 
-	require.NoErrorf(t, err, "failed to add hashicorp helm repo: %s", string(output))
+	require.NoErrorf(t, err, "failed to create namespace: %s", string(output))
 
 	cmd = exec.Command("kubectl", "create", "secret", "generic",
 		"consul-ent-license",