Tests for containerstatus and livenssprobe

.pipelines/azure_pipeline_testframework.yaml

@@ -0,0 +1,94 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+#
+
+trigger:
+  batch: true
+  branches:
+    include:
+    - ci_prod
+
+pr:
+  branches:
+    include:
+    - ci_prod
+
+jobs:
+- deployment: Testkube
+  environment: container-insights
+  displayName: "Test: run testkube tests"
+  pool:
+    name: Azure-Pipelines-CI-Test-EO
+  variables:
+    skipComponentGovernanceDetection: true
+  strategy:
+    runOnce:
+      deploy:
+        steps:
+        - checkout: self
+          persistCredentials: true
+
+        - bash: |
+            wget -qO - https://repo.testkube.io/key.pub | sudo apt-key add -
+            echo "deb https://repo.testkube.io/linux linux main" | sudo tee -a /etc/apt/sources.list
+            sudo apt-get update
+            sudo apt-get install -y testkube=1.14.2
+          workingDirectory: $(Build.SourcesDirectory)
+          displayName: "Install testkube CLI"
+
+        - task: AzureCLI@1
+          displayName: Get kubeconfig
+          inputs:
+            azureSubscription: 'ContainerInsights_Build_Subscription'
+            scriptLocation: 'inlineScript'
+            inlineScript: 'az aks get-credentials -g $(RESOURCE_GROUP) -n $(CLUSTER_NAME)'
+
+        - bash: |
+            envsubst < ./testkube/testkube-test-crs.yaml > ./testkube/testkube-test-crs-$(CLUSTER_NAME).yaml
+            kubectl apply -f ./testkube/api-server-permissions.yaml
+            kubectl apply -f ./testkube/testkube-test-crs-$(CLUSTER_NAME).yaml
+            exit 0
+          workingDirectory: $(Build.SourcesDirectory)/test/
+          displayName: "Apply TestKube CRs and pod/service monitors"
+
+        - bash: |
+            sleep 120
+          displayName: "Wait for cluster to be ready"
+
+        - bash: |
+            # Run the full test suite
+            kubectl testkube run testsuite e2e-tests-merge --verbose
+
+            # Get the current id of the test suite now running
+            execution_id=$(kubectl testkube get testsuiteexecutions --test-suite e2e-tests-merge --limit 1 | grep e2e-tests | awk '{print $1}')
+
+            # Watch until the all the tests in the test suite finish
+            kubectl testkube watch testsuiteexecution $execution_id
+
+            # Get the results as a formatted json file
+            kubectl testkube get testsuiteexecution $execution_id --output json > testkube-results.json
+
+            # For any test that has failed, print out the Ginkgo logs
+            if [[ $(jq -r '.status' testkube-results.json) == "failed" ]]; then
+
+              # Get each test name and id that failed
+              jq -r '.executeStepResults[].execute[] | select(.execution.executionResult.status=="failed") | "\(.execution.testName) \(.execution.id)"' testkube-results.json | while read line; do
+                testName=$(echo $line | cut -d ' ' -f 1)
+                id=$(echo $line | cut -d ' ' -f 2)
+                echo "Test $testName failed. Test ID: $id"
+
+                # Get the Ginkgo logs of the test
+                kubectl testkube get execution $id > out 2>error.log
+
+                # Remove superfluous logs of everything before the last occurence of 'go downloading'.
+                # The actual errors can be viewed from the ADO run, instead of needing to view the testkube dashboard.
+                cat error.log | tac | awk '/go: downloading/ {exit} 1' | tac
+              done
+
+              # Explicitly fail the ADO task since at least one test failed
+              exit 1
+            fi
+          workingDirectory: $(Build.SourcesDirectory)
+          displayName: "Run tests"

test/ginkgo-e2e/containerstatus/containerstatus_suite_test.go

@@ -0,0 +1,34 @@
+package containerstatus_test
+
+import (
+	"testing"
+
+	"docker-provider/test/utils"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+// These tests use Ginkgo (BDD-style Go testing framework). Refer to
+// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
+
+var K8sClient *kubernetes.Clientset
+var Cfg *rest.Config
+
+func TestContainerStatus(t *testing.T) {
+	RegisterFailHandler(Fail)
+
+	RunSpecs(t, "Container Status Test Suite")
+}
+
+var _ = BeforeSuite(func() {
+	var err error
+	K8sClient, Cfg, err = utils.SetupKubernetesClient()
+	Expect(err).NotTo(HaveOccurred())
+})
+
+var _ = AfterSuite(func() {
+	By("tearing down the test environment")
+})

test/ginkgo-e2e/containerstatus/containerstatus_test.go

@@ -0,0 +1,119 @@
+package containerstatus_test
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"docker-provider/test/utils"
+)
+
+/*
+ * For each of the pods that we deploy, ensure each container within that pod has status 'Running'.
+ * The replicaset, and daemonset are always deployed.
+ * The label and values are provided to get a list of pods only with that label.
+ */
+var _ = DescribeTable("The containers should be running",
+	func(namespace string, controllerLabelName string, controllerLabelValue string) {
+		err := utils.CheckIfAllContainersAreRunning(K8sClient, namespace, controllerLabelName, controllerLabelValue)
+		Expect(err).NotTo(HaveOccurred())
+	},
+	Entry("when checking the ama-logs", "kube-system", "component", "ama-logs-agent"),
+	Entry("when checking the ama-logs-rs replica pod(s)", "kube-system", "rsName", "ama-logs-rs"),
+)
+
+/*
+ * For each of the DS pods that we deploy, ensure that all nodes have been used to schedule these pods.
+ * The label and values are provided to get a list of pods only with that label.
+ * The osLabel is provided to check on all DS pods based on the OS.
+ */
+var _ = DescribeTable("The pods should be scheduled in all nodes",
+	func(namespace string, controllerLabelName string, controllerLabelValue string, osLabel string) {
+		err := utils.CheckIfAllPodsScheduleOnNodes(K8sClient, namespace, controllerLabelName, controllerLabelValue, osLabel)
+		Expect(err).NotTo(HaveOccurred())
+	},
+	Entry("when checking the ama-logs", "kube-system", "component", "ama-logs-agent", "linux"),
+	Entry("when checking the ama-logs-win pod", "kube-system", "component", "ama-logs-agent-windows", "windows", Label(utils.WindowsLabel)),
+)
+
+/*
+* For each of the DS pods that we deploy, ensure that all specific nodes like ARM64, FIPS have been used to schedule these pods.
+* The label and values are provided to get a list of pods only with that label.
+ */
+var _ = DescribeTable("The pods should be scheduled in all Fips and ARM64 nodes",
+	func(namespace string, controllerLabelName string, controllerLabelValue string, nodeLabelKey string, nodeLabelValue string) {
+		err := utils.CheckIfAllPodsScheduleOnSpecificNodesLabels(K8sClient, namespace, controllerLabelName, controllerLabelValue, nodeLabelKey, nodeLabelValue)
+		Expect(err).NotTo(HaveOccurred())
+	},
+	Entry("when checking the ama-logs", "kube-system", "component", "ama-logs-agent", "kubernetes.azure.com/fips_enabled", "true", Label(utils.FIPSLabel)),
+	Entry("when checking the ama-logs-win pod", "kube-system", "component", "ama-logs-agent-windows", "kubernetes.azure.com/fips_enabled", "true", Label(utils.WindowsLabel), Label(utils.FIPSLabel)),
+	Entry("when checking the ama-logs", "kube-system", "component", "ama-logs-agent", "kubernetes.io/arch", "arm64", Label(utils.ARM64Label)),
+)
+
+/*
+* For each of the pods that have the ama-logs container, check all expected processes are running.
+* The linux replicaset and daemonset should have the same processes running.
+ */
+var _ = DescribeTable("All processes are running",
+	func(namespace, labelName, labelValue, containerName string, processes []string) {
+		err := utils.CheckAllProcessesRunning(K8sClient, Cfg, labelName, labelValue, namespace, containerName, processes)
+		Expect(err).NotTo(HaveOccurred())
+	},
+	Entry("when checking the ama-logs-rs replica pod", "kube-system", "rsName", "ama-logs-rs", "ama-logs",
+		[]string{
+			"fluent-bit",
+			"fluentd",
+			"mdsd -a -A -r",
+			"inotifywait /etc/config/settings",
+			"crond",
+		},
+	),
+	Entry("when checking the ama-logs daemonset pods", "kube-system", "component", "ama-logs-agent", "ama-logs",
+		[]string{
+			"fluent-bit",
+			"fluentd",
+			"mdsd -a -A -r",
+			"inotifywait /etc/config/settings",
+			"crond",
+			"telegraf",
+		},
+	),
+)
+
+/*
+* For windows daemonset pods that have the ama-logs-windows container, check all expected processes are running.
+ */
+var _ = DescribeTable("All processes are running",
+	func(namespace, labelName, labelValue, containerName string, processes []string) {
+		err := utils.CheckAllWindowsProcessesRunning(K8sClient, Cfg, labelName, labelValue, namespace, containerName, processes)
+		Expect(err).NotTo(HaveOccurred())
+	},
+	Entry("when checking the ama-logs-windows daemonset pods", "kube-system", "component", "ama-logs-agent-windows", "ama-logs-windows",
+		[]string{
+			"fluent-bit",
+			"MonAgentLauncher",
+			"MonAgentHost",
+			"MonAgentManager",
+			"MonAgentCore",
+			"telegraf",
+		},
+		Label(utils.WindowsLabel),
+		FlakeAttempts(3),
+	),
+)
+
+/*
+- For each of the pods that we deploy, ensure each container within that pod doesn't have errors in the logs.
+- The replicaset and daemonset are always deployed.
+- The label and values are provided to get a list of pods only with that label.
+*/
+var _ = DescribeTable("The container logs should not contain errors",
+	func(namespace string, controllerLabelName string, controllerLabelValue string) {
+		err := utils.CheckContainerLogsForErrors(K8sClient, namespace, controllerLabelName, controllerLabelValue)
+		Expect(err).NotTo(HaveOccurred())
+	},
+	Entry("when checking the ama-logs-rs pods", "kube-system", "rsName", "ama-logs-rs"),
+	Entry("when checking the ama-logs daemonset pods", "kube-system", "component", "ama-logs-agent"),
+	Entry("when checking the ama-logs-rs pods", "kube-system", "rsName", "ama-logs-rs", Label(utils.ARM64Label)),
+	Entry("when checking the ama-logs daemonset pods", "kube-system", "component", "ama-logs-agent", Label(utils.ARM64Label)),
+	Entry("when checking the ama-logs-windows daemonset pods", "kube-system", "component", "ama-logs-agent-windows", Label(utils.WindowsLabel)),
+)

test/ginkgo-e2e/containerstatus/go.mod

@@ -0,0 +1,60 @@
+module docker-provider/test/containerstatus
+
+go 1.20
+
+replace docker-provider/test/utils => ../utils
+
+require (
+	docker-provider/test/utils v0.0.0
+	github.com/onsi/ginkgo/v2 v2.13.1
+	github.com/onsi/gomega v1.30.0
+	k8s.io/client-go v0.28.4
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/imdario/mergo v0.3.6 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/testify v1.9.0 // indirect
+	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/oauth2 v0.16.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/term v0.19.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.14.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/api v0.28.4 // indirect
+	k8s.io/apimachinery v0.28.4 // indirect
+	k8s.io/klog/v2 v2.100.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
+	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)