test(mc): Add function to override logger for sensitive outputs (#1286)

# Description

* add fetchSensitiveOutputs function to override default logger to
prevent sensitive outputs from being logged
* add checkRetinaPodsRunning function to check if Retina pods are
running
* add make target for formatting both OpenTofu and go code
* modify make test target to run test ONLY for TestRetinaKindIntegration
till we get cloud creds
* refactor tests to use new func

## Related Issue

#1267 

## Checklist

- [x] I have read the [contributing
documentation](https://retina.sh/docs/contributing).
- [x] I signed and signed-off the commits (`git commit -S -s ...`). See
[this
documentation](https://docs.github.com/en/authentication/managing-commit-signature-verification/about-commit-signature-verification)
on signing commits.
- [x] I have correctly attributed the author(s) of the code.
- [x] I have tested the changes locally.
- [x] I have followed the project's style guidelines.
- [x] I have updated the documentation, if necessary.
- [x] I have added tests, if applicable.

## Screenshots (if applicable) or Testing Completed


![image](https://github.com/user-attachments/assets/fd711fa8-4064-4c57-87cc-2a7acc088f6f)

## Additional Notes

Add any additional notes or context about the pull request here.

---

Please refer to the [CONTRIBUTING.md](../CONTRIBUTING.md) file for more
information on how to contribute to this project.

test/multicloud/Makefile

@@ -5,7 +5,7 @@ STACK_NAME ?= $(PREFIX)-aks
 
 plan:
 	cd live/$(STACK_NAME) && \
-		tofu fmt && tofu init && tofu plan
+		tofu init && tofu plan
 
 apply:
 	cd live/$(STACK_NAME) && \
@@ -38,5 +38,13 @@ clean: destroy
 kind-kubeconfig:
 	@kubectl config set-context live/$(PREFIX)-kind/mc-kind-config
 
+# For now we only want to run the retina-kind integration
+# since we do not have credentials for the other cloud providers
+# Once we do this targets will be updated to
+# @cd test && go test -v -count=1 -timeout 30m ./...
 test:
-	@cd test && go test -v -count=1 -timeout 30m ./...
+	@cd test && go test -run TestRetinaKindIntegration -count=1 -timeout 10m
+
+fmt:
+	@tofu fmt -recursive
+	@cd test && go fmt ./...

test/multicloud/examples/aks/outputs.tf

@@ -0,0 +1,19 @@
+output "host" {
+  value     = module.aks.host
+  sensitive = true
+}
+
+output "client_certificate" {
+  value     = module.aks.client_certificate
+  sensitive = true
+}
+
+output "client_key" {
+  value     = module.aks.client_key
+  sensitive = true
+}
+
+output "cluster_ca_certificate" {
+  value     = module.aks.cluster_ca_certificate
+  sensitive = true
+}

test/multicloud/live/retina-aks/outputs.tf

@@ -0,0 +1,19 @@
+output "host" {
+  value     = module.aks.host
+  sensitive = true
+}
+
+output "client_certificate" {
+  value     = module.aks.client_certificate
+  sensitive = true
+}
+
+output "client_key" {
+  value     = module.aks.client_key
+  sensitive = true
+}
+
+output "cluster_ca_certificate" {
+  value     = module.aks.cluster_ca_certificate
+  sensitive = true
+}

test/multicloud/modules/aks/outputs.tf

@@ -4,21 +4,21 @@ output "azure_get_kubeconfig" {
 }
 
 output "host" {
-  value = azurerm_kubernetes_cluster.aks.kube_config.0.host
+  value     = azurerm_kubernetes_cluster.aks.kube_config.0.host
   sensitive = true
 }
 
 output "client_certificate" {
-  value = azurerm_kubernetes_cluster.aks.kube_config.0.client_certificate
+  value     = azurerm_kubernetes_cluster.aks.kube_config.0.client_certificate
   sensitive = true
 }
 
 output "client_key" {
-  value = azurerm_kubernetes_cluster.aks.kube_config.0.client_key
+  value     = azurerm_kubernetes_cluster.aks.kube_config.0.client_key
   sensitive = true
 }
 
 output "cluster_ca_certificate" {
-  value = azurerm_kubernetes_cluster.aks.kube_config.0.cluster_ca_certificate
+  value     = azurerm_kubernetes_cluster.aks.kube_config.0.cluster_ca_certificate
   sensitive = true
 }

test/multicloud/modules/gke/output.tf

@@ -4,11 +4,11 @@ output "gcloud_get_kubeconfig" {
 }
 
 output "host" {
-  value = "https://${google_container_cluster.gke.endpoint}"
+  value     = "https://${google_container_cluster.gke.endpoint}"
   sensitive = true
 }
 
 output "cluster_ca_certificate" {
-  value = google_container_cluster.gke.master_auth.0.cluster_ca_certificate
+  value     = google_container_cluster.gke.master_auth.0.cluster_ca_certificate
   sensitive = true
 }

test/multicloud/modules/kind/output.tf

@@ -1,24 +1,24 @@
 output "kubeconfig" {
-  value = kind_cluster.kind.kubeconfig
+  value     = kind_cluster.kind.kubeconfig
   sensitive = true
 }
 
 output "host" {
-  value = kind_cluster.kind.endpoint
+  value     = kind_cluster.kind.endpoint
   sensitive = true
 }
 
 output "client_certificate" {
-  value = kind_cluster.kind.client_certificate
+  value     = kind_cluster.kind.client_certificate
   sensitive = true
 }
 
 output "client_key" {
-  value = kind_cluster.kind.client_key
+  value     = kind_cluster.kind.client_key
   sensitive = true
 }
 
 output "cluster_ca_certificate" {
-  value = kind_cluster.kind.cluster_ca_certificate
+  value     = kind_cluster.kind.cluster_ca_certificate
   sensitive = true
 }

test/multicloud/test/example_aks_test.go

@@ -13,11 +13,9 @@ func TestAKSExample(t *testing.T) {
 		TerraformDir: "../examples/aks",
 
 		Vars: map[string]interface{}{
-			"prefix":              "test",
+			"prefix":              "test-mc",
 			"location":            "uksouth",
-			"subscription_id":     "d6050d84-e4dd-463d-afc7-a6ab3dc33ab7", // TODO: replace with actual project once we get azure "public" access
-			"tenant_id":           "ac8a4ccd-35f1-4f95-a688-f68e3d89adfc",
-			"resource_group_name": "test",
+			"resource_group_name": "test-mc",
 			"labels": map[string]string{
 				"environment": "test",
 				"owner":       "test",
@@ -28,11 +26,28 @@ func TestAKSExample(t *testing.T) {
 
 	// clean up at the end of the test
 	defer terraform.Destroy(t, opts)
+	terraform.InitAndApply(t, opts)
+
+	// get outputs
+	caCert := fetchSensitiveOutput(t, opts, "cluster_ca_certificate")
+	clientCert := fetchSensitiveOutput(t, opts, "client_certificate")
+	clientKey := fetchSensitiveOutput(t, opts, "client_key")
+	host := fetchSensitiveOutput(t, opts, "host")
+
+	// decode the base64 encoded strings
+	caCertDecoded := decodeBase64(t, caCert)
+	clientCertDecoded := decodeBase64(t, clientCert)
+	clientKeyDecoded := decodeBase64(t, clientKey)
+
+	// build the REST config
+	restConfig := createRESTConfigWithClientCert(caCertDecoded, clientCertDecoded, clientKeyDecoded, host)
+
+	// create a Kubernetes clientset
+	clientSet, err := buildClientSet(restConfig)
+	if err != nil {
+		t.Fatalf("Failed to create Kubernetes clientset: %v", err)
+	}
 
-	terraform.Init(t, opts)
-
-	// TODO: uncomment once we get creds for azure "public"
-	// terraform.Apply(t, opts)
-
-	// TODO: add actual tests here
+	// test the cluster is accessible
+	testClusterAccess(t, clientSet)
 }

test/multicloud/test/example_gke_test.go

@@ -22,33 +22,25 @@ func TestGKEExample(t *testing.T) {
 
 	// clean up at the end of the test
 	defer terraform.Destroy(t, opts)
+	terraform.InitAndApply(t, opts)
 
-	terraform.Init(t, opts)
+	// get outputs
+	caCert := fetchSensitiveOutput(t, opts, "cluster_ca_certificate")
+	host := fetchSensitiveOutput(t, opts, "host")
+	token := fetchSensitiveOutput(t, opts, "access_token")
 
-	// TODO: uncomment once we get creds for gcloud
-	// terraform.Apply(t, opts)
+	// decode the base64 encoded cert
+	caCertString := decodeBase64(t, caCert)
 
-	// // get outputs
-	// caCert := terraform.Output(t, opts, "cluster_ca_certificate")
-	// host := terraform.Output(t, opts, "host")
-	// token := terraform.Output(t, opts, "access_token")
+	// build the REST config
+	restConfig := createRESTConfigWithBearer(caCertString, token, host)
 
-	// caCertString, err := decodeBase64(caCert)
-	// if err != nil {
-	// 	t.Fatalf("Failed to decode ca cert: %v", err)
-	// }
-
-	// // build the REST config
-	// restConfig := createRESTConfigWithBearer(caCertString, token, host)
-
-	// // create a Kubernetes clientset
-	// clientSet, err := buildClientSet(restConfig)
-	// if err != nil {
-	// 	t.Fatalf("Failed to create Kubernetes clientset: %v", err)
-	// }
-
-	// // test the cluster is accessible
-	// testClusterAccess(t, clientSet)
+	// create a Kubernetes clientset
+	clientSet, err := buildClientSet(restConfig)
+	if err != nil {
+		t.Fatalf("Failed to create Kubernetes clientset: %v", err)
+	}
 
-	// // TODO: add more tests here
+	// test the cluster is accessible
+	testClusterAccess(t, clientSet)
 }

test/multicloud/test/example_kind_test.go

@@ -19,15 +19,13 @@ func TestKindExample(t *testing.T) {
 
 	// clean up at the end of the test
 	defer terraform.Destroy(t, opts)
-
-	terraform.Init(t, opts)
-	terraform.Apply(t, opts)
+	terraform.InitAndApply(t, opts)
 
 	// get outputs
-	caCert := terraform.Output(t, opts, "cluster_ca_certificate")
-	clientCert := terraform.Output(t, opts, "client_certificate")
-	clientKey := terraform.Output(t, opts, "client_key")
-	host := terraform.Output(t, opts, "host")
+	caCert := fetchSensitiveOutput(t, opts, "cluster_ca_certificate")
+	clientCert := fetchSensitiveOutput(t, opts, "client_certificate")
+	clientKey := fetchSensitiveOutput(t, opts, "client_key")
+	host := fetchSensitiveOutput(t, opts, "host")
 
 	// build the REST config
 	restConfig := createRESTConfigWithClientCert(caCert, clientCert, clientKey, host)

test/multicloud/test/integration_retina_kind_test.go

@@ -20,15 +20,13 @@ func TestRetinaKindIntegration(t *testing.T) {
 
 	// clean up at the end of the test
 	defer terraform.Destroy(t, opts)
-
-	terraform.Init(t, opts)
-	terraform.Apply(t, opts)
+	terraform.InitAndApply(t, opts)
 
 	// get outputs
-	caCert := terraform.Output(t, opts, "cluster_ca_certificate")
-	clientCert := terraform.Output(t, opts, "client_certificate")
-	clientKey := terraform.Output(t, opts, "client_key")
-	host := terraform.Output(t, opts, "host")
+	caCert := fetchSensitiveOutput(t, opts, "cluster_ca_certificate")
+	clientCert := fetchSensitiveOutput(t, opts, "client_certificate")
+	clientKey := fetchSensitiveOutput(t, opts, "client_key")
+	host := fetchSensitiveOutput(t, opts, "host")
 
 	// build the REST config
 	restConfig := createRESTConfigWithClientCert(caCert, clientCert, clientKey, host)
@@ -39,6 +37,9 @@ func TestRetinaKindIntegration(t *testing.T) {
 		t.Fatalf("Failed to create Kubernetes clientset: %v", err)
 	}
 
+	// check the retina pods are running
+	checkRetinaPodsRunning(t, clientSet)
+
 	// test the cluster is accessible
 	testClusterAccess(t, clientSet)
 

test/multicloud/test/testUtils.go

@@ -8,9 +8,13 @@ import (
 	"io"
 	"strings"
 	"testing"
+	"time"
 
+	"github.com/gruntwork-io/terratest/modules/logger"
+	"github.com/gruntwork-io/terratest/modules/terraform"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 )
@@ -109,12 +113,44 @@ func checkRetinaLogs(t *testing.T, clientset *kubernetes.Clientset) {
 }
 
 // function to convert base64 encoded string to plain text
-func decodeBase64(encoded string) (string, error) {
+func decodeBase64(t *testing.T, encoded string) string {
 	// decode the base64 encoded string
 	decoded, err := base64.StdEncoding.DecodeString(encoded)
 	if err != nil {
-		return "", err
+		t.Fatalf("Failed to decode base64 string %v:", err)
 	}
 	// return the decoded string
-	return string(decoded), nil
+	return string(decoded)
+}
+
+// fetch the sensitive output from OpenTofu
+func fetchSensitiveOutput(t *testing.T, options *terraform.Options, name string) string {
+	defer func() {
+		options.Logger = nil
+	}()
+	options.Logger = logger.Discard
+	return terraform.Output(t, options, name)
+}
+
+func checkRetinaPodsRunning(t *testing.T, clientset *kubernetes.Clientset) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+
+	err := wait.PollUntilContextTimeout(ctx, 3*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
+		pods, err := clientset.CoreV1().Pods("kube-system").List(ctx, metav1.ListOptions{
+			LabelSelector: "k8s-app=retina",
+		})
+		if err != nil {
+			return false, err
+		}
+		for _, pod := range pods.Items {
+			if pod.Status.Phase != v1.PodRunning {
+				return false, nil
+			}
+		}
+		return true, nil
+	})
+	if err != nil {
+		t.Fatalf("Retina pods did not start in time: %v\n", err)
+	}
 }