expiration test updates

kubernetes-sigs · Apr 17, 2024 · e95d76b · e95d76b
1 parent 4064537
commit e95d76b
Showing 1 changed file with 60 additions and 60 deletions.
diff --git a/pkg/controllers/disruption/expiration_test.go b/pkg/controllers/disruption/expiration_test.go
@@ -121,10 +121,9 @@ var _ = Describe("Expiration", func() {
 			// inform cluster state about nodes and nodeclaims
 			ExpectMakeNodesAndNodeClaimsInitializedAndStateUpdated(ctx, env.Client, nodeStateController, nodeClaimStateController, nodes, nodeClaims)
 
-			var wg sync.WaitGroup
-			ExpectTriggerVerifyAction(&wg)
-			ExpectReconcileSucceeded(ctx, disruptionController, client.ObjectKey{})
-			wg.Wait()
+			DecorateWithClockIncrementer(func() {
+				ExpectReconcileSucceeded(ctx, disruptionController, client.ObjectKey{})
+			})
 
 			metric, found := FindMetricWithLabelValues("karpenter_disruption_budgets_allowed_disruptions", map[string]string{
 				"nodepool": nodePool.Name,
@@ -168,10 +167,9 @@ var _ = Describe("Expiration", func() {
 			// inform cluster state about nodes and nodeclaims
 			ExpectMakeNodesAndNodeClaimsInitializedAndStateUpdated(ctx, env.Client, nodeStateController, nodeClaimStateController, nodes, nodeClaims)
 
-			var wg sync.WaitGroup
-			ExpectTriggerVerifyAction(&wg)
-			ExpectReconcileSucceeded(ctx, disruptionController, client.ObjectKey{})
-			wg.Wait()
+			DecorateWithClockIncrementer(func() {
+				ExpectReconcileSucceeded(ctx, disruptionController, client.ObjectKey{})
+			})
 
 			metric, found := FindMetricWithLabelValues("karpenter_disruption_budgets_allowed_disruptions", map[string]string{
 				"nodepool": nodePool.Name,
@@ -215,10 +213,9 @@ var _ = Describe("Expiration", func() {
 			// inform cluster state about nodes and nodeclaims
 			ExpectMakeNodesAndNodeClaimsInitializedAndStateUpdated(ctx, env.Client, nodeStateController, nodeClaimStateController, nodes, nodeClaims)
 
-			var wg sync.WaitGroup
-			ExpectTriggerVerifyAction(&wg)
-			ExpectReconcileSucceeded(ctx, disruptionController, client.ObjectKey{})
-			wg.Wait()
+			DecorateWithClockIncrementer(func() {
+				ExpectReconcileSucceeded(ctx, disruptionController, client.ObjectKey{})
+			})
 
 			metric, found := FindMetricWithLabelValues("karpenter_disruption_budgets_allowed_disruptions", map[string]string{
 				"nodepool": nodePool.Name,
@@ -287,13 +284,12 @@ var _ = Describe("Expiration", func() {
 			// inform cluster state about nodes and nodeclaims
 			ExpectMakeNodesAndNodeClaimsInitializedAndStateUpdated(ctx, env.Client, nodeStateController, nodeClaimStateController, nodes, nodeClaims)
 
-			var wg sync.WaitGroup
-			ExpectTriggerVerifyAction(&wg)
-			// Reconcile 5 times, enqueuing 3 commands total.
-			for i := 0; i < 5; i++ {
-				ExpectReconcileSucceeded(ctx, disruptionController, client.ObjectKey{})
-			}
-			wg.Wait()
+			DecorateWithClockIncrementer(func() {
+				// Reconcile 5 times, enqueuing 3 commands total.
+				for i := 0; i < 5; i++ {
+					ExpectReconcileSucceeded(ctx, disruptionController, client.ObjectKey{})
+				}
+			})
 
 			nodes = ExpectNodes(ctx, env.Client)
 			Expect(len(lo.Filter(nodes, func(nc *v1.Node, _ int) bool {
@@ -354,10 +350,9 @@ var _ = Describe("Expiration", func() {
 			// inform cluster state about nodes and nodeclaims
 			ExpectMakeNodesAndNodeClaimsInitializedAndStateUpdated(ctx, env.Client, nodeStateController, nodeClaimStateController, nodes, nodeClaims)
 
-			var wg sync.WaitGroup
-			ExpectTriggerVerifyAction(&wg)
-			ExpectReconcileSucceeded(ctx, disruptionController, client.ObjectKey{})
-			wg.Wait()
+			DecorateWithClockIncrementer(func() {
+				ExpectReconcileSucceeded(ctx, disruptionController, client.ObjectKey{})
+			})
 
 			for _, np := range nps {
 				metric, found := FindMetricWithLabelValues("karpenter_disruption_budgets_allowed_disruptions", map[string]string{
@@ -419,10 +414,9 @@ var _ = Describe("Expiration", func() {
 			// inform cluster state about nodes and nodeclaims
 			ExpectMakeNodesAndNodeClaimsInitializedAndStateUpdated(ctx, env.Client, nodeStateController, nodeClaimStateController, nodes, nodeClaims)
 
-			var wg sync.WaitGroup
-			ExpectTriggerVerifyAction(&wg)
-			ExpectReconcileSucceeded(ctx, disruptionController, client.ObjectKey{})
-			wg.Wait()
+			DecorateWithClockIncrementer(func() {
+				ExpectReconcileSucceeded(ctx, disruptionController, client.ObjectKey{})
+			})
 
 			for _, np := range nps {
 				metric, found := FindMetricWithLabelValues("karpenter_disruption_budgets_allowed_disruptions", map[string]string{
@@ -555,11 +549,12 @@ var _ = Describe("Expiration", func() {
 			ExpectMakeNodesAndNodeClaimsInitializedAndStateUpdated(ctx, env.Client, nodeStateController, nodeClaimStateController, []*v1.Node{node2}, []*v1beta1.NodeClaim{nodeClaim2})
 
 			// disruption won't delete the old node until the new node is ready
-			var wg sync.WaitGroup
-			ExpectTriggerVerifyAction(&wg)
-			ExpectMakeNewNodeClaimsReady(ctx, env.Client, &wg, cluster, cloudProvider, 1)
-			ExpectReconcileSucceeded(ctx, disruptionController, types.NamespacedName{})
-			wg.Wait()
+			DecorateWithClockIncrementer(func() {
+				wg := sync.WaitGroup{}
+				ExpectMakeNewNodeClaimsReady(ctx, env.Client, &wg, cluster, cloudProvider, 1)
+				ExpectReconcileSucceeded(ctx, disruptionController, types.NamespacedName{})
+				wg.Wait()
+			})
 
 			// Process the item so that the nodes can be deleted.
 			ExpectReconcileSucceeded(ctx, queue, types.NamespacedName{})
@@ -591,10 +586,9 @@ var _ = Describe("Expiration", func() {
 			// inform cluster state about nodes and nodeclaims
 			ExpectMakeNodesAndNodeClaimsInitializedAndStateUpdated(ctx, env.Client, nodeStateController, nodeClaimStateController, []*v1.Node{node}, []*v1beta1.NodeClaim{nodeClaim})
 
-			var wg sync.WaitGroup
-			ExpectTriggerVerifyAction(&wg)
-			ExpectReconcileSucceeded(ctx, disruptionController, types.NamespacedName{})
-			wg.Wait()
+			DecorateWithClockIncrementer(func() {
+				ExpectReconcileSucceeded(ctx, disruptionController, types.NamespacedName{})
+			})
 
 			// Process the item so that the nodes can be deleted.
 			ExpectReconcileSucceeded(ctx, queue, types.NamespacedName{})
@@ -635,10 +629,9 @@ var _ = Describe("Expiration", func() {
 			// inform cluster state about nodes and nodeClaims
 			ExpectMakeNodesAndNodeClaimsInitializedAndStateUpdated(ctx, env.Client, nodeStateController, nodeClaimStateController, nodes, nodeClaims)
 
-			var wg sync.WaitGroup
-			ExpectTriggerVerifyAction(&wg)
-			ExpectReconcileSucceeded(ctx, disruptionController, types.NamespacedName{})
-			wg.Wait()
+			DecorateWithClockIncrementer(func() {
+				ExpectReconcileSucceeded(ctx, disruptionController, types.NamespacedName{})
+			})
 
 			// Process the item so that the nodes can be deleted.
 			ExpectReconcileSucceeded(ctx, queue, types.NamespacedName{})
@@ -705,11 +698,12 @@ var _ = Describe("Expiration", func() {
 			ExpectMakeNodesAndNodeClaimsInitializedAndStateUpdated(ctx, env.Client, nodeStateController, nodeClaimStateController, []*v1.Node{node, node2}, []*v1beta1.NodeClaim{nodeClaim, nodeClaim2})
 
 			// disruption won't delete the old node until the new node is ready
-			var wg sync.WaitGroup
-			ExpectTriggerVerifyAction(&wg)
-			ExpectMakeNewNodeClaimsReady(ctx, env.Client, &wg, cluster, cloudProvider, 1)
-			ExpectReconcileSucceeded(ctx, disruptionController, types.NamespacedName{})
-			wg.Wait()
+			DecorateWithClockIncrementer(func() {
+				wg := sync.WaitGroup{}
+				ExpectMakeNewNodeClaimsReady(ctx, env.Client, &wg, cluster, cloudProvider, 1)
+				ExpectReconcileSucceeded(ctx, disruptionController, types.NamespacedName{})
+				wg.Wait()
+			})
 
 			// Process the item so that the nodes can be deleted.
 			ExpectReconcileSucceeded(ctx, queue, types.NamespacedName{})
@@ -754,11 +748,12 @@ var _ = Describe("Expiration", func() {
 			ExpectMakeNodesAndNodeClaimsInitializedAndStateUpdated(ctx, env.Client, nodeStateController, nodeClaimStateController, []*v1.Node{node}, []*v1beta1.NodeClaim{nodeClaim})
 
 			// disruption won't delete the old node until the new node is ready
-			var wg sync.WaitGroup
-			ExpectTriggerVerifyAction(&wg)
-			ExpectMakeNewNodeClaimsReady(ctx, env.Client, &wg, cluster, cloudProvider, 1)
-			ExpectReconcileSucceeded(ctx, disruptionController, types.NamespacedName{})
-			wg.Wait()
+			DecorateWithClockIncrementer(func() {
+				wg := sync.WaitGroup{}
+				ExpectMakeNewNodeClaimsReady(ctx, env.Client, &wg, cluster, cloudProvider, 1)
+				ExpectReconcileSucceeded(ctx, disruptionController, types.NamespacedName{})
+				wg.Wait()
+			})
 
 			// Process the item so that the nodes can be deleted.
 			ExpectReconcileSucceeded(ctx, queue, types.NamespacedName{})
@@ -807,11 +802,15 @@ var _ = Describe("Expiration", func() {
 			// inform cluster state about nodes and nodeclaims
 			ExpectMakeNodesAndNodeClaimsInitializedAndStateUpdated(ctx, env.Client, nodeStateController, nodeClaimStateController, []*v1.Node{node}, []*v1beta1.NodeClaim{nodeClaim})
 
-			var wg sync.WaitGroup
-			ExpectTriggerVerifyAction(&wg)
-			ExpectNewNodeClaimsDeleted(ctx, env.Client, &wg, 1)
-			ExpectReconcileSucceeded(ctx, disruptionController, types.NamespacedName{})
-			wg.Wait()
+			DecorateWithClockIncrementer(func() {
+				wg := sync.WaitGroup{}
+				ExpectNewNodeClaimsDeleted(ctx, env.Client, &wg, 1)
+				ExpectReconcileSucceeded(ctx, disruptionController, types.NamespacedName{})
+				wg.Wait()
+			})
+
+			// step the clock to cover the eventual consistency delay, ensuring the queue belikeves the nodeclaim was deleted
+			fakeClock.Step(6 * time.Second)
 
 			ExpectReconcileSucceeded(ctx, queue, types.NamespacedName{})
 			// We should have tried to create a new nodeClaim but failed to do so; therefore, we uncordoned the existing node
@@ -896,11 +895,12 @@ var _ = Describe("Expiration", func() {
 			ExpectMakeNodesAndNodeClaimsInitializedAndStateUpdated(ctx, env.Client, nodeStateController, nodeClaimStateController, []*v1.Node{node}, []*v1beta1.NodeClaim{nodeClaim})
 
 			// disruption won't delete the old nodeClaim until the new nodeClaim is ready
-			var wg sync.WaitGroup
-			ExpectTriggerVerifyAction(&wg)
-			ExpectMakeNewNodeClaimsReady(ctx, env.Client, &wg, cluster, cloudProvider, 3)
-			ExpectReconcileSucceeded(ctx, disruptionController, types.NamespacedName{})
-			wg.Wait()
+			DecorateWithClockIncrementer(func() {
+				wg := sync.WaitGroup{}
+				ExpectMakeNewNodeClaimsReady(ctx, env.Client, &wg, cluster, cloudProvider, 3)
+				ExpectReconcileSucceeded(ctx, disruptionController, types.NamespacedName{})
+				wg.Wait()
+			})
 
 			// Process the item so that the nodes can be deleted.
 			ExpectReconcileSucceeded(ctx, queue, types.NamespacedName{})