working e2e tests for happy case scenario

make webhook e2e test work on linux

fix test on linux

Author: amrmahdi

Diff for: .gitignore

@@ -24,3 +24,5 @@ go.work.sum
 # env file
 .env
 bin/
+
+test/logs/

Diff for: go.mod

@@ -18,7 +18,6 @@ require (
 	github.com/stretchr/testify v1.9.0
 	go.uber.org/mock v0.4.0
 	go.uber.org/zap v1.27.0
-	golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc
 	k8s.io/api v0.31.1
 	k8s.io/apimachinery v0.31.1
 	k8s.io/client-go v0.31.0
@@ -69,6 +68,7 @@ require (
 	github.com/vladimirvivien/gexe v0.2.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
 	golang.org/x/mod v0.18.0 // indirect
 	golang.org/x/net v0.27.0 // indirect
 	golang.org/x/oauth2 v0.21.0 // indirect

Diff for: test/controller_test.go

@@ -0,0 +1,291 @@
+package test
+
+import (
+	"context"
+	"os/exec"
+	"stagger/pkg/blocker"
+	"stagger/pkg/controller"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var _ = Describe("Happy Case Scenario", func() {
+	Context("When creating a deployment", func() {
+		It("should create deployment and verify pacing", func() {
+			// Create a Deployment with the StaggerGroup label
+			deploymentName := "test-deployment"
+
+			k8sClient, err := client.New(testEnv.Config, client.Options{})
+			Expect(err).ToNot(HaveOccurred())
+
+			replicas := int32(10)
+
+			logger.Info("Creating the Deployment", "name", deploymentName, "namespace", Namespace)
+			deployment := &appsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      deploymentName,
+					Namespace: Namespace,
+				},
+				Spec: appsv1.DeploymentSpec{
+					Strategy: appsv1.DeploymentStrategy{
+						Type: appsv1.RollingUpdateDeploymentStrategyType,
+					},
+					Replicas: &replicas,
+					Selector: &metav1.LabelSelector{
+						MatchLabels: map[string]string{
+							"app": "test-app",
+						},
+					},
+					Template: corev1.PodTemplateSpec{
+						ObjectMeta: metav1.ObjectMeta{
+							Labels: map[string]string{
+								"app":                         "test-app",
+								controller.DefaultEnableLabel: "1",
+							},
+						},
+						Spec: corev1.PodSpec{
+							Containers: []corev1.Container{
+								{
+									Name:    "busybox",
+									Image:   "busybox",
+									Command: []string{"sleep", "3600"},
+									ReadinessProbe: &corev1.Probe{
+										ProbeHandler: corev1.ProbeHandler{
+											Exec: &corev1.ExecAction{
+												Command: []string{
+													// check if the file /tmp/ready exists
+													"test", "-f", "/tmp/ready",
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			}
+
+			By("Creating the Deployment")
+			ctx := context.Background()
+			err = k8sClient.Create(ctx, deployment)
+			Expect(err).ToNot(HaveOccurred())
+
+			labels := map[string]string{"app": "test-app"}
+
+			// Step 1: 0 ready, 1 starting, 9 blocked
+			starting := waitForPodsConditionAndReturnStartingPods(
+				ctx,
+				k8sClient,
+				Namespace,
+				labels,
+				0,                    // expectedReady
+				1,                    // expectedStarting
+				9,                    // expectedBlocked
+				"busybox",            // containerName
+				time.Minute,          // timeout
+				500*time.Millisecond, // interval
+				"All must be pending, expect 1 should be starting", // description
+			)
+
+			// Make the starting pod ready
+			makePodsReady(ctx, starting)
+
+			// Step 2: 1 ready, 1 starting, 8 blocked
+			starting = waitForPodsConditionAndReturnStartingPods(
+				ctx,
+				k8sClient,
+				Namespace,
+				labels,
+				1,                    // expectedReady
+				1,                    // expectedStarting
+				8,                    // expectedBlocked
+				"busybox",            // containerName
+				time.Minute,          // timeout
+				500*time.Millisecond, // interval
+				"1 pod should be ready, 1 starting, 8 blocked", // description
+			)
+
+			// Make the starting pods ready
+			makePodsReady(ctx, starting)
+
+			// Step 3: 2 ready, 2 starting, 6 blocked
+			starting = waitForPodsConditionAndReturnStartingPods(
+				ctx,
+				k8sClient,
+				Namespace,
+				labels,
+				2,                    // expectedReady
+				2,                    // expectedStarting
+				6,                    // expectedBlocked
+				"busybox",            // containerName
+				time.Minute,          // timeout
+				500*time.Millisecond, // interval
+				"2 pods should be ready, 2 starting, 6 blocked", // description
+			)
+
+			// Make the starting pods ready
+			makePodsReady(ctx, starting)
+
+			// Step 4: 4 ready, 4 starting, 2 blocked
+			starting = waitForPodsConditionAndReturnStartingPods(
+				ctx,
+				k8sClient,
+				Namespace,
+				labels,
+				4,                    // expectedReady
+				4,                    // expectedStarting
+				2,                    // expectedBlocked
+				"busybox",            // containerName
+				time.Minute,          // timeout
+				500*time.Millisecond, // interval
+				"4 pods should be ready, 4 starting, 2 blocked", // description
+			)
+
+			// Make the starting pods ready
+			makePodsReady(ctx, starting)
+
+			// Step 5: 10 ready, 0 starting, 0 blocked
+			starting = waitForPodsConditionAndReturnStartingPods(
+				ctx,
+				k8sClient,
+				Namespace,
+				labels,
+				8,                    // expectedReady
+				2,                    // expectedStarting
+				0,                    // expectedBlocked
+				"busybox",            // containerName
+				time.Minute,          // timeout
+				500*time.Millisecond, // interval
+				"8 pods should be ready, 2 starting, 0 blocked", // description
+			)
+
+			makePodsReady(ctx, starting)
+
+			// Step 6: 10 ready, 0 starting, 0 blocked
+			waitForPodsConditionAndReturnStartingPods(
+				ctx,
+				k8sClient,
+				Namespace,
+				labels,
+				10,                        // expectedReady
+				0,                         // expectedStarting
+				0,                         // expectedBlocked
+				"busybox",                 // containerName
+				time.Minute,               // timeout
+				500*time.Millisecond,      // interval
+				"10 pods should be ready", // description
+			)
+		})
+	})
+})
+
+// waitForPodsReady waits until all Pods matching the label selector are running and ready.
+func getPodCounts(ctx context.Context, c client.Client, namespace string, labelSelector map[string]string) (ready, starting, blocked []corev1.Pod, err error) {
+	listOpts := []client.ListOption{
+		client.InNamespace(namespace),
+		client.MatchingLabels(labelSelector),
+	}
+
+	blocker := blocker.NewNodeSelectorPodBlocker()
+
+	pods := &corev1.PodList{}
+	err = c.List(ctx, pods, listOpts...)
+	if err != nil {
+		logger.Error(err, "Failed to list pods", "namespace", namespace, "labelSelector", labelSelector)
+		return ready, starting, blocked, err
+	}
+
+	if len(pods.Items) == 0 {
+		logger.Info("No pods matched the labelSelector", "namespace", namespace, "labelSelector", labelSelector)
+		return ready, starting, blocked, nil
+	}
+
+	for _, pod := range pods.Items {
+		if isPodReady(&pod) {
+			ready = append(ready, pod)
+		} else if blocker.IsBlocked(&pod.Spec) {
+			blocked = append(blocked, pod)
+		} else {
+			starting = append(starting, pod)
+		}
+	}
+
+	return ready, starting, blocked, nil
+}
+
+// waitForPodsConditionAndReturnStartingPods waits until the pods match the expected ready, starting and blocked counts.
+// It also checks if the specified container in the starting pods has started.
+func waitForPodsConditionAndReturnStartingPods(
+	ctx context.Context,
+	k8sClient client.Client,
+	namespace string,
+	labels map[string]string,
+	expectedReady int,
+	expectedStarting int,
+	expectedBlocked int,
+	containerName string,
+	timeout time.Duration,
+	interval time.Duration,
+	description string,
+) (startingPods []corev1.Pod) {
+	By(description)
+	Eventually(func() bool {
+		ready, starting, blocked, err := getPodCounts(ctx, k8sClient, namespace, labels)
+		Expect(err).ToNot(HaveOccurred())
+		if len(ready) != expectedReady || len(starting) != expectedStarting || len(blocked) != expectedBlocked {
+			return false
+		}
+		if expectedStarting > 0 {
+			if isContainerStarted(containerName, starting...) {
+				startingPods = starting
+				return true
+			}
+			return false
+		}
+		return true
+	}, timeout, interval).Should(BeTrue())
+
+	return startingPods
+}
+
+// makePodsReady marks each pod in the provided list as ready.
+func makePodsReady(ctx context.Context, pods []corev1.Pod) {
+	for _, pod := range pods {
+		makePodReady(ctx, &pod)
+	}
+}
+
+func isContainerStarted(containerName string, pod ...corev1.Pod) bool {
+	started := 0
+	for _, p := range pod {
+		for _, status := range p.Status.ContainerStatuses {
+			if status.Name == containerName && status.Started != nil && *status.Started {
+				started++
+			}
+		}
+	}
+
+	return started == len(pod)
+}
+
+func isPodReady(pod *corev1.Pod) bool {
+	for _, cond := range pod.Status.Conditions {
+		if cond.Type == corev1.PodReady && cond.Status == corev1.ConditionTrue {
+			return true
+		}
+	}
+	return false
+}
+
+func makePodReady(ctx context.Context, pod *corev1.Pod) {
+	execCommand := exec.CommandContext(ctx, "kubectl", "--kubeconfig", kubeConfigPath, "exec", "-n", pod.Namespace, pod.Name, "--", "touch", "/tmp/ready")
+	out, err := execCommand.CombinedOutput()
+	Expect(err).ToNot(HaveOccurred(), "Output: %s", out)
+}