From ca3788be929199d2a8303b6709aa80aff797e462 Mon Sep 17 00:00:00 2001 From: Simone Rodigari Date: Thu, 30 Jan 2025 11:52:29 +0000 Subject: [PATCH] test(mc): Add function to override logger for sensitive outputs --- test/multicloud/Makefile | 12 +++++- test/multicloud/examples/aks/outputs.tf | 19 +++++++++ test/multicloud/live/retina-aks/outputs.tf | 19 +++++++++ test/multicloud/modules/aks/outputs.tf | 8 ++-- test/multicloud/modules/gke/output.tf | 4 +- test/multicloud/modules/kind/output.tf | 10 ++--- test/multicloud/test/example_aks_test.go | 35 +++++++++++----- test/multicloud/test/example_gke_test.go | 40 +++++++----------- test/multicloud/test/example_kind_test.go | 12 +++--- .../test/integration_retina_kind_test.go | 15 +++---- test/multicloud/test/testUtils.go | 42 +++++++++++++++++-- 11 files changed, 152 insertions(+), 64 deletions(-) create mode 100644 test/multicloud/examples/aks/outputs.tf create mode 100644 test/multicloud/live/retina-aks/outputs.tf diff --git a/test/multicloud/Makefile b/test/multicloud/Makefile index 715e86ec2f..fac2583b44 100644 --- a/test/multicloud/Makefile +++ b/test/multicloud/Makefile @@ -5,7 +5,7 @@ STACK_NAME ?= $(PREFIX)-aks plan: cd live/$(STACK_NAME) && \ - tofu fmt && tofu init && tofu plan + tofu init && tofu plan apply: cd live/$(STACK_NAME) && \ @@ -38,5 +38,13 @@ clean: destroy kind-kubeconfig: @kubectl config set-context live/$(PREFIX)-kind/mc-kind-config +# For now we only want to run the retina-kind integration +# since we do not have credentials for the other cloud providers +# Once we do this targets will be updated to +# @cd test && go test -v -count=1 -timeout 30m ./... test: - @cd test && go test -v -count=1 -timeout 30m ./... + @cd test && go test -run TestRetinaKindIntegration -count=1 -timeout 10m + +fmt: + @tofu fmt -recursive + @cd test && go fmt ./... diff --git a/test/multicloud/examples/aks/outputs.tf b/test/multicloud/examples/aks/outputs.tf new file mode 100644 index 0000000000..154ce0721b --- /dev/null +++ b/test/multicloud/examples/aks/outputs.tf @@ -0,0 +1,19 @@ +output "host" { + value = module.aks.host + sensitive = true +} + +output "client_certificate" { + value = module.aks.client_certificate + sensitive = true +} + +output "client_key" { + value = module.aks.client_key + sensitive = true +} + +output "cluster_ca_certificate" { + value = module.aks.cluster_ca_certificate + sensitive = true +} \ No newline at end of file diff --git a/test/multicloud/live/retina-aks/outputs.tf b/test/multicloud/live/retina-aks/outputs.tf new file mode 100644 index 0000000000..154ce0721b --- /dev/null +++ b/test/multicloud/live/retina-aks/outputs.tf @@ -0,0 +1,19 @@ +output "host" { + value = module.aks.host + sensitive = true +} + +output "client_certificate" { + value = module.aks.client_certificate + sensitive = true +} + +output "client_key" { + value = module.aks.client_key + sensitive = true +} + +output "cluster_ca_certificate" { + value = module.aks.cluster_ca_certificate + sensitive = true +} \ No newline at end of file diff --git a/test/multicloud/modules/aks/outputs.tf b/test/multicloud/modules/aks/outputs.tf index e71ca414ff..c2b7e548ad 100644 --- a/test/multicloud/modules/aks/outputs.tf +++ b/test/multicloud/modules/aks/outputs.tf @@ -4,21 +4,21 @@ output "azure_get_kubeconfig" { } output "host" { - value = azurerm_kubernetes_cluster.aks.kube_config.0.host + value = azurerm_kubernetes_cluster.aks.kube_config.0.host sensitive = true } output "client_certificate" { - value = azurerm_kubernetes_cluster.aks.kube_config.0.client_certificate + value = azurerm_kubernetes_cluster.aks.kube_config.0.client_certificate sensitive = true } output "client_key" { - value = azurerm_kubernetes_cluster.aks.kube_config.0.client_key + value = azurerm_kubernetes_cluster.aks.kube_config.0.client_key sensitive = true } output "cluster_ca_certificate" { - value = azurerm_kubernetes_cluster.aks.kube_config.0.cluster_ca_certificate + value = azurerm_kubernetes_cluster.aks.kube_config.0.cluster_ca_certificate sensitive = true } diff --git a/test/multicloud/modules/gke/output.tf b/test/multicloud/modules/gke/output.tf index 692cb7eddf..25af8af7e6 100644 --- a/test/multicloud/modules/gke/output.tf +++ b/test/multicloud/modules/gke/output.tf @@ -4,11 +4,11 @@ output "gcloud_get_kubeconfig" { } output "host" { - value = "https://${google_container_cluster.gke.endpoint}" + value = "https://${google_container_cluster.gke.endpoint}" sensitive = true } output "cluster_ca_certificate" { - value = google_container_cluster.gke.master_auth.0.cluster_ca_certificate + value = google_container_cluster.gke.master_auth.0.cluster_ca_certificate sensitive = true } \ No newline at end of file diff --git a/test/multicloud/modules/kind/output.tf b/test/multicloud/modules/kind/output.tf index 1b5b5f3698..7cacd76d32 100644 --- a/test/multicloud/modules/kind/output.tf +++ b/test/multicloud/modules/kind/output.tf @@ -1,24 +1,24 @@ output "kubeconfig" { - value = kind_cluster.kind.kubeconfig + value = kind_cluster.kind.kubeconfig sensitive = true } output "host" { - value = kind_cluster.kind.endpoint + value = kind_cluster.kind.endpoint sensitive = true } output "client_certificate" { - value = kind_cluster.kind.client_certificate + value = kind_cluster.kind.client_certificate sensitive = true } output "client_key" { - value = kind_cluster.kind.client_key + value = kind_cluster.kind.client_key sensitive = true } output "cluster_ca_certificate" { - value = kind_cluster.kind.cluster_ca_certificate + value = kind_cluster.kind.cluster_ca_certificate sensitive = true } \ No newline at end of file diff --git a/test/multicloud/test/example_aks_test.go b/test/multicloud/test/example_aks_test.go index 7bbdf6fa4d..f0ddcdcc69 100644 --- a/test/multicloud/test/example_aks_test.go +++ b/test/multicloud/test/example_aks_test.go @@ -13,11 +13,9 @@ func TestAKSExample(t *testing.T) { TerraformDir: "../examples/aks", Vars: map[string]interface{}{ - "prefix": "test", + "prefix": "test-mc", "location": "uksouth", - "subscription_id": "d6050d84-e4dd-463d-afc7-a6ab3dc33ab7", // TODO: replace with actual project once we get azure "public" access - "tenant_id": "ac8a4ccd-35f1-4f95-a688-f68e3d89adfc", - "resource_group_name": "test", + "resource_group_name": "test-mc", "labels": map[string]string{ "environment": "test", "owner": "test", @@ -28,11 +26,28 @@ func TestAKSExample(t *testing.T) { // clean up at the end of the test defer terraform.Destroy(t, opts) + terraform.InitAndApply(t, opts) + + // get outputs + caCert := fetchSensitiveOutput(t, opts, "cluster_ca_certificate") + clientCert := fetchSensitiveOutput(t, opts, "client_certificate") + clientKey := fetchSensitiveOutput(t, opts, "client_key") + host := fetchSensitiveOutput(t, opts, "host") + + // decode the base64 encoded strings + caCertDecoded := decodeBase64(t, caCert) + clientCertDecoded := decodeBase64(t, clientCert) + clientKeyDecoded := decodeBase64(t, clientKey) + + // build the REST config + restConfig := createRESTConfigWithClientCert(caCertDecoded, clientCertDecoded, clientKeyDecoded, host) + + // create a Kubernetes clientset + clientSet, err := buildClientSet(restConfig) + if err != nil { + t.Fatalf("Failed to create Kubernetes clientset: %v", err) + } - terraform.Init(t, opts) - - // TODO: uncomment once we get creds for azure "public" - // terraform.Apply(t, opts) - - // TODO: add actual tests here + // test the cluster is accessible + testClusterAccess(t, clientSet) } diff --git a/test/multicloud/test/example_gke_test.go b/test/multicloud/test/example_gke_test.go index 2f9980d0e8..3d05eca1dd 100644 --- a/test/multicloud/test/example_gke_test.go +++ b/test/multicloud/test/example_gke_test.go @@ -22,33 +22,25 @@ func TestGKEExample(t *testing.T) { // clean up at the end of the test defer terraform.Destroy(t, opts) + terraform.InitAndApply(t, opts) - terraform.Init(t, opts) + // get outputs + caCert := fetchSensitiveOutput(t, opts, "cluster_ca_certificate") + host := fetchSensitiveOutput(t, opts, "host") + token := fetchSensitiveOutput(t, opts, "access_token") - // TODO: uncomment once we get creds for gcloud - // terraform.Apply(t, opts) + // decode the base64 encoded cert + caCertString := decodeBase64(t, caCert) - // // get outputs - // caCert := terraform.Output(t, opts, "cluster_ca_certificate") - // host := terraform.Output(t, opts, "host") - // token := terraform.Output(t, opts, "access_token") + // build the REST config + restConfig := createRESTConfigWithBearer(caCertString, token, host) - // caCertString, err := decodeBase64(caCert) - // if err != nil { - // t.Fatalf("Failed to decode ca cert: %v", err) - // } - - // // build the REST config - // restConfig := createRESTConfigWithBearer(caCertString, token, host) - - // // create a Kubernetes clientset - // clientSet, err := buildClientSet(restConfig) - // if err != nil { - // t.Fatalf("Failed to create Kubernetes clientset: %v", err) - // } - - // // test the cluster is accessible - // testClusterAccess(t, clientSet) + // create a Kubernetes clientset + clientSet, err := buildClientSet(restConfig) + if err != nil { + t.Fatalf("Failed to create Kubernetes clientset: %v", err) + } - // // TODO: add more tests here + // test the cluster is accessible + testClusterAccess(t, clientSet) } diff --git a/test/multicloud/test/example_kind_test.go b/test/multicloud/test/example_kind_test.go index f16affa4dd..5534d73d11 100644 --- a/test/multicloud/test/example_kind_test.go +++ b/test/multicloud/test/example_kind_test.go @@ -19,15 +19,13 @@ func TestKindExample(t *testing.T) { // clean up at the end of the test defer terraform.Destroy(t, opts) - - terraform.Init(t, opts) - terraform.Apply(t, opts) + terraform.InitAndApply(t, opts) // get outputs - caCert := terraform.Output(t, opts, "cluster_ca_certificate") - clientCert := terraform.Output(t, opts, "client_certificate") - clientKey := terraform.Output(t, opts, "client_key") - host := terraform.Output(t, opts, "host") + caCert := fetchSensitiveOutput(t, opts, "cluster_ca_certificate") + clientCert := fetchSensitiveOutput(t, opts, "client_certificate") + clientKey := fetchSensitiveOutput(t, opts, "client_key") + host := fetchSensitiveOutput(t, opts, "host") // build the REST config restConfig := createRESTConfigWithClientCert(caCert, clientCert, clientKey, host) diff --git a/test/multicloud/test/integration_retina_kind_test.go b/test/multicloud/test/integration_retina_kind_test.go index 110f6dc191..f7767b7937 100644 --- a/test/multicloud/test/integration_retina_kind_test.go +++ b/test/multicloud/test/integration_retina_kind_test.go @@ -20,15 +20,13 @@ func TestRetinaKindIntegration(t *testing.T) { // clean up at the end of the test defer terraform.Destroy(t, opts) - - terraform.Init(t, opts) - terraform.Apply(t, opts) + terraform.InitAndApply(t, opts) // get outputs - caCert := terraform.Output(t, opts, "cluster_ca_certificate") - clientCert := terraform.Output(t, opts, "client_certificate") - clientKey := terraform.Output(t, opts, "client_key") - host := terraform.Output(t, opts, "host") + caCert := fetchSensitiveOutput(t, opts, "cluster_ca_certificate") + clientCert := fetchSensitiveOutput(t, opts, "client_certificate") + clientKey := fetchSensitiveOutput(t, opts, "client_key") + host := fetchSensitiveOutput(t, opts, "host") // build the REST config restConfig := createRESTConfigWithClientCert(caCert, clientCert, clientKey, host) @@ -39,6 +37,9 @@ func TestRetinaKindIntegration(t *testing.T) { t.Fatalf("Failed to create Kubernetes clientset: %v", err) } + // check the retina pods are running + checkRetinaPodsRunning(t, clientSet) + // test the cluster is accessible testClusterAccess(t, clientSet) diff --git a/test/multicloud/test/testUtils.go b/test/multicloud/test/testUtils.go index dd1e24924e..9c119bf7d1 100644 --- a/test/multicloud/test/testUtils.go +++ b/test/multicloud/test/testUtils.go @@ -8,9 +8,13 @@ import ( "io" "strings" "testing" + "time" + "github.com/gruntwork-io/terratest/modules/logger" + "github.com/gruntwork-io/terratest/modules/terraform" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ) @@ -109,12 +113,44 @@ func checkRetinaLogs(t *testing.T, clientset *kubernetes.Clientset) { } // function to convert base64 encoded string to plain text -func decodeBase64(encoded string) (string, error) { +func decodeBase64(t *testing.T, encoded string) string { // decode the base64 encoded string decoded, err := base64.StdEncoding.DecodeString(encoded) if err != nil { - return "", err + t.Fatalf("Failed to decode base64 string %v:", err) } // return the decoded string - return string(decoded), nil + return string(decoded) +} + +// fetch the sensitive output from OpenTofu +func fetchSensitiveOutput(t *testing.T, options *terraform.Options, name string) string { + defer func() { + options.Logger = nil + }() + options.Logger = logger.Discard + return terraform.Output(t, options, name) +} + +func checkRetinaPodsRunning(t *testing.T, clientset *kubernetes.Clientset) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + err := wait.PollUntilContextTimeout(ctx, 3*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + pods, err := clientset.CoreV1().Pods("kube-system").List(ctx, metav1.ListOptions{ + LabelSelector: "k8s-app=retina", + }) + if err != nil { + return false, err + } + for _, pod := range pods.Items { + if pod.Status.Phase != v1.PodRunning { + return false, nil + } + } + return true, nil + }) + if err != nil { + t.Fatalf("Retina pods did not start in time: %v\n", err) + } }