diff --git a/apis/placement/v1alpha1/stagedupdate_types.go b/apis/placement/v1alpha1/stagedupdate_types.go index 661389dde..defc947ac 100644 --- a/apis/placement/v1alpha1/stagedupdate_types.go +++ b/apis/placement/v1alpha1/stagedupdate_types.go @@ -18,8 +18,8 @@ import ( // +kubebuilder:resource:scope=Cluster,categories={fleet,fleet-placement},shortName=crsur // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:printcolumn:JSONPath=`.spec.placementName`,name="Placement",type=string -// +kubebuilder:printcolumn:JSONPath=`.spec.resourceSnapshotIndex`,name="Resource-Snapshot",type=string -// +kubebuilder:printcolumn:JSONPath=`.status.policySnapshotIndexUsed`,name="Policy-Snapshot",type=string +// +kubebuilder:printcolumn:JSONPath=`.spec.resourceSnapshotIndex`,name="Resource-Snapshot-Index",type=string +// +kubebuilder:printcolumn:JSONPath=`.status.policySnapshotIndexUsed`,name="Policy-Snapshot-Index",type=string // +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="Initialized")].status`,name="Initialized",type=string // +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="Succeeded")].status`,name="Succeeded",type=string // +kubebuilder:printcolumn:JSONPath=`.metadata.creationTimestamp`,name="Age",type=date diff --git a/apis/placement/v1beta1/stageupdate_types.go b/apis/placement/v1beta1/stageupdate_types.go index b6bb96e76..2d7e52070 100644 --- a/apis/placement/v1beta1/stageupdate_types.go +++ b/apis/placement/v1beta1/stageupdate_types.go @@ -17,8 +17,8 @@ import ( // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:storageversion // +kubebuilder:printcolumn:JSONPath=`.spec.placementName`,name="Placement",type=string -// +kubebuilder:printcolumn:JSONPath=`.spec.resourceSnapshotIndex`,name="Resource-Snapshot",type=string -// +kubebuilder:printcolumn:JSONPath=`.status.policySnapshotIndexUsed`,name="Policy-Snapshot",type=string +// +kubebuilder:printcolumn:JSONPath=`.spec.resourceSnapshotIndex`,name="Resource-Snapshot-Index",type=string +// +kubebuilder:printcolumn:JSONPath=`.status.policySnapshotIndexUsed`,name="Policy-Snapshot-Index",type=string // +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="Initialized")].status`,name="Initialized",type=string // +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="Succeeded")].status`,name="Succeeded",type=string // +kubebuilder:printcolumn:JSONPath=`.metadata.creationTimestamp`,name="Age",type=date diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml index 2bc676558..89a55d5f4 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml @@ -24,10 +24,10 @@ spec: name: Placement type: string - jsonPath: .spec.resourceSnapshotIndex - name: Resource-Snapshot + name: Resource-Snapshot-Index type: string - jsonPath: .status.policySnapshotIndexUsed - name: Policy-Snapshot + name: Policy-Snapshot-Index type: string - jsonPath: .status.conditions[?(@.type=="Initialized")].status name: Initialized @@ -1244,10 +1244,10 @@ spec: name: Placement type: string - jsonPath: .spec.resourceSnapshotIndex - name: Resource-Snapshot + name: Resource-Snapshot-Index type: string - jsonPath: .status.policySnapshotIndexUsed - name: Policy-Snapshot + name: Policy-Snapshot-Index type: string - jsonPath: .status.conditions[?(@.type=="Initialized")].status name: Initialized diff --git a/pkg/controllers/updaterun/execution.go b/pkg/controllers/updaterun/execution.go index b4c468bac..2ce101ee4 100644 --- a/pkg/controllers/updaterun/execution.go +++ b/pkg/controllers/updaterun/execution.go @@ -200,7 +200,8 @@ func (r *Reconciler) executeDeleteStage( for i := range existingDeleteStageStatus.Clusters { existingDeleteStageClusterMap[existingDeleteStageStatus.Clusters[i].ClusterName] = &existingDeleteStageStatus.Clusters[i] } - deletingBinding := 0 + // Mark the delete stage as started in case it's not. + markStageUpdatingStarted(updateRun.Status.DeletionStageStatus, updateRun.Generation) for _, binding := range toBeDeletedBindings { curCluster, exist := existingDeleteStageClusterMap[binding.Spec.TargetCluster] if !exist { @@ -225,7 +226,6 @@ func (r *Reconciler) executeDeleteStage( klog.ErrorS(unexpectedErr, "The binding should be deleting before we mark a cluster deleting", "clusterStatus", curCluster, "clusterStagedUpdateRun", updateRunRef) return false, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) } - deletingBinding++ continue } // The cluster status is not deleting yet @@ -235,10 +235,6 @@ func (r *Reconciler) executeDeleteStage( } klog.V(2).InfoS("Deleted a binding pointing to a to be deleted cluster", "binding", klog.KObj(binding), "cluster", curCluster.ClusterName, "clusterStagedUpdateRun", updateRunRef) markClusterUpdatingStarted(curCluster, updateRun.Generation) - if deletingBinding == 0 { - markStageUpdatingStarted(updateRun.Status.DeletionStageStatus, updateRun.Generation) - } - deletingBinding++ } // The rest of the clusters in the stage are not in the toBeDeletedBindings so it should be marked as delete succeeded. for _, clusterStatus := range existingDeleteStageClusterMap { diff --git a/pkg/controllers/updaterun/initialization.go b/pkg/controllers/updaterun/initialization.go index 86a99eff2..60ff3d234 100644 --- a/pkg/controllers/updaterun/initialization.go +++ b/pkg/controllers/updaterun/initialization.go @@ -126,26 +126,23 @@ func (r *Reconciler) determinePolicySnapshot( } updateRun.Status.PolicySnapshotIndexUsed = policyIndex - // Get the cluster count from the policy snapshot. - if latestPolicySnapshot.Spec.Policy == nil { - nopolicyErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("policy snapshot `%s` does not have a policy", latestPolicySnapshot.Name)) - klog.ErrorS(nopolicyErr, "Failed to get the policy from the latestPolicySnapshot", "clusterResourcePlacement", placementName, "latestPolicySnapshot", latestPolicySnapshot.Name, "clusterStagedUpdateRun", updateRunRef) - // no more retries here. - return nil, -1, fmt.Errorf("%w: %s", errInitializedFailed, nopolicyErr.Error()) - } - // for pickAll policy, the observed cluster count is not included in the policy snapshot. We set it to -1. It will be validated in the binding stages. + // For pickAll policy, the observed cluster count is not included in the policy snapshot. + // We set it to -1. It will be validated in the binding stages. + // If policy is nil, it's default to pickAll. clusterCount := -1 - if latestPolicySnapshot.Spec.Policy.PlacementType == placementv1beta1.PickNPlacementType { - count, err := annotations.ExtractNumOfClustersFromPolicySnapshot(&latestPolicySnapshot) - if err != nil { - annErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("%w: the policy snapshot `%s` doesn't have valid cluster count annotation", err, latestPolicySnapshot.Name)) - klog.ErrorS(annErr, "Failed to get the cluster count from the latestPolicySnapshot", "clusterResourcePlacement", placementName, "latestPolicySnapshot", latestPolicySnapshot.Name, "clusterStagedUpdateRun", updateRunRef) - // no more retries here. - return nil, -1, fmt.Errorf("%w, %s", errInitializedFailed, annErr.Error()) + if latestPolicySnapshot.Spec.Policy != nil { + if latestPolicySnapshot.Spec.Policy.PlacementType == placementv1beta1.PickNPlacementType { + count, err := annotations.ExtractNumOfClustersFromPolicySnapshot(&latestPolicySnapshot) + if err != nil { + annErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("%w: the policy snapshot `%s` doesn't have valid cluster count annotation", err, latestPolicySnapshot.Name)) + klog.ErrorS(annErr, "Failed to get the cluster count from the latestPolicySnapshot", "clusterResourcePlacement", placementName, "latestPolicySnapshot", latestPolicySnapshot.Name, "clusterStagedUpdateRun", updateRunRef) + // no more retries here. + return nil, -1, fmt.Errorf("%w, %s", errInitializedFailed, annErr.Error()) + } + clusterCount = count + } else if latestPolicySnapshot.Spec.Policy.PlacementType == placementv1beta1.PickFixedPlacementType { + clusterCount = len(latestPolicySnapshot.Spec.Policy.ClusterNames) } - clusterCount = count - } else if latestPolicySnapshot.Spec.Policy.PlacementType == placementv1beta1.PickFixedPlacementType { - clusterCount = len(latestPolicySnapshot.Spec.Policy.ClusterNames) } updateRun.Status.PolicyObservedClusterCount = clusterCount klog.V(2).InfoS("Found the latest policy snapshot", "latestPolicySnapshot", latestPolicySnapshot.Name, "observedClusterCount", updateRun.Status.PolicyObservedClusterCount, "clusterStagedUpdateRun", updateRunRef) @@ -209,6 +206,7 @@ func (r *Reconciler) collectScheduledClusters( if updateRun.Status.PolicyObservedClusterCount == -1 { // For pickAll policy, the observed cluster count is not included in the policy snapshot. We set it to the number of selected bindings. + // TODO (wantjian): refactor this part to update PolicyObservedClusterCount in one place. updateRun.Status.PolicyObservedClusterCount = len(selectedBindings) } else if updateRun.Status.PolicyObservedClusterCount != len(selectedBindings) { countErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the number of selected bindings %d is not equal to the observed cluster count %d", len(selectedBindings), updateRun.Status.PolicyObservedClusterCount)) diff --git a/pkg/controllers/updaterun/initialization_integration_test.go b/pkg/controllers/updaterun/initialization_integration_test.go index 405594c65..581003eb8 100644 --- a/pkg/controllers/updaterun/initialization_integration_test.go +++ b/pkg/controllers/updaterun/initialization_integration_test.go @@ -232,18 +232,6 @@ var _ = Describe("Updaterun initialization tests", func() { validateFailedInitCondition(ctx, updateRun, "does not have a policy index label") }) - It("Should fail to initialize if the latest policy snapshot has a nil policy", func() { - By("Creating scheduling policy snapshot with nil policy") - policySnapshot.Spec.Policy = nil - Expect(k8sClient.Create(ctx, policySnapshot)).To(Succeed()) - - By("Creating a new clusterStagedUpdateRun") - Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) - - By("Validating the initialization failed") - validateFailedInitCondition(ctx, updateRun, "does not have a policy") - }) - It("Should fail to initialize if the latest policy snapshot does not have valid cluster count annotation", func() { By("Creating scheduling policy snapshot with invalid cluster count annotation") delete(policySnapshot.Annotations, placementv1beta1.NumberOfClustersAnnotation) diff --git a/test/e2e/actuals_test.go b/test/e2e/actuals_test.go index 0f25fdd0b..8d52306e3 100644 --- a/test/e2e/actuals_test.go +++ b/test/e2e/actuals_test.go @@ -653,6 +653,7 @@ func crpStatusWithOverrideUpdatedFailedActual( return nil } } + func crpStatusWithWorkSynchronizedUpdatedFailedActual( wantSelectedResourceIdentifiers []placementv1beta1.ResourceIdentifier, wantSelectedClusters []string, @@ -949,3 +950,184 @@ func validateCRPSnapshotRevisions(crpName string, wantPolicySnapshotRevision, wa } return nil } + +func updateRunClusterRolloutSucceedConditions(generation int64) []metav1.Condition { + return []metav1.Condition{ + { + Type: string(placementv1beta1.ClusterUpdatingConditionStarted), + Status: metav1.ConditionTrue, + Reason: condition.ClusterUpdatingStartedReason, + ObservedGeneration: generation, + }, + { + Type: string(placementv1beta1.ClusterUpdatingConditionSucceeded), + Status: metav1.ConditionTrue, + Reason: condition.ClusterUpdatingSucceededReason, + ObservedGeneration: generation, + }, + } +} + +func updateRunStageRolloutSucceedConditions(generation int64, wait bool) []metav1.Condition { + startedCond := metav1.Condition{ + Type: string(placementv1beta1.StageUpdatingConditionProgressing), + Status: metav1.ConditionTrue, + Reason: condition.StageUpdatingStartedReason, + ObservedGeneration: generation, + } + if wait { + startedCond.Status = metav1.ConditionFalse + startedCond.Reason = condition.StageUpdatingWaitingReason + } + return []metav1.Condition{ + startedCond, + { + Type: string(placementv1beta1.StageUpdatingConditionSucceeded), + Status: metav1.ConditionTrue, + Reason: condition.StageUpdatingSucceededReason, + ObservedGeneration: generation, + }, + } +} + +func updateRunAfterStageTaskSucceedConditions(generation int64, taskType placementv1beta1.AfterStageTaskType) []metav1.Condition { + if taskType == placementv1beta1.AfterStageTaskTypeApproval { + return []metav1.Condition{ + { + Type: string(placementv1beta1.AfterStageTaskConditionApprovalRequestCreated), + Status: metav1.ConditionTrue, + Reason: condition.AfterStageTaskApprovalRequestCreatedReason, + ObservedGeneration: generation, + }, + { + Type: string(placementv1beta1.AfterStageTaskConditionApprovalRequestApproved), + Status: metav1.ConditionTrue, + Reason: condition.AfterStageTaskApprovalRequestApprovedReason, + ObservedGeneration: generation, + }, + } + } + return []metav1.Condition{ + { + Type: string(placementv1beta1.AfterStageTaskConditionWaitTimeElapsed), + Status: metav1.ConditionTrue, + Reason: condition.AfterStageTaskWaitTimeElapsedReason, + ObservedGeneration: generation, + }, + } +} + +func updateRunSucceedConditions(generation int64) []metav1.Condition { + return []metav1.Condition{ + { + Type: string(placementv1beta1.StagedUpdateRunConditionInitialized), + Status: metav1.ConditionTrue, + Reason: condition.UpdateRunInitializeSucceededReason, + ObservedGeneration: generation, + }, + { + Type: string(placementv1beta1.StagedUpdateRunConditionProgressing), + Status: metav1.ConditionTrue, + Reason: condition.UpdateRunStartedReason, + ObservedGeneration: generation, + }, + { + Type: string(placementv1beta1.StagedUpdateRunConditionSucceeded), + Status: metav1.ConditionTrue, + Reason: condition.UpdateRunSucceededReason, + ObservedGeneration: generation, + }, + } +} + +func updateRunStatusSucceededActual( + updateRunName string, + wantPolicyIndex string, + wantClusterCount int, + wantApplyStrategy *placementv1beta1.ApplyStrategy, + wantStrategySpec *placementv1beta1.StagedUpdateStrategySpec, + wantSelectedClusters [][]string, + wantUnscheduledClusters []string, + wantCROs map[string][]string, + wantROs map[string][]placementv1beta1.NamespacedName, +) func() error { + return func() error { + updateRun := &placementv1beta1.ClusterStagedUpdateRun{} + if err := hubClient.Get(ctx, types.NamespacedName{Name: updateRunName}, updateRun); err != nil { + return err + } + + wantStatus := placementv1beta1.StagedUpdateRunStatus{ + PolicySnapshotIndexUsed: wantPolicyIndex, + PolicyObservedClusterCount: wantClusterCount, + ApplyStrategy: wantApplyStrategy.DeepCopy(), + StagedUpdateStrategySnapshot: wantStrategySpec, + } + stagesStatus := make([]placementv1beta1.StageUpdatingStatus, len(wantStrategySpec.Stages)) + for i, stage := range wantStrategySpec.Stages { + stagesStatus[i].StageName = stage.Name + stagesStatus[i].Clusters = make([]placementv1beta1.ClusterUpdatingStatus, len(wantSelectedClusters[i])) + for j := range stagesStatus[i].Clusters { + stagesStatus[i].Clusters[j].ClusterName = wantSelectedClusters[i][j] + stagesStatus[i].Clusters[j].ClusterResourceOverrideSnapshots = wantCROs[wantSelectedClusters[i][j]] + stagesStatus[i].Clusters[j].ResourceOverrideSnapshots = wantROs[wantSelectedClusters[i][j]] + stagesStatus[i].Clusters[j].Conditions = updateRunClusterRolloutSucceedConditions(updateRun.Generation) + } + stagesStatus[i].AfterStageTaskStatus = make([]placementv1beta1.AfterStageTaskStatus, len(stage.AfterStageTasks)) + for j, task := range stage.AfterStageTasks { + stagesStatus[i].AfterStageTaskStatus[j].Type = task.Type + if task.Type == placementv1beta1.AfterStageTaskTypeApproval { + stagesStatus[i].AfterStageTaskStatus[j].ApprovalRequestName = fmt.Sprintf(placementv1beta1.ApprovalTaskNameFmt, updateRun.Name, stage.Name) + } + stagesStatus[i].AfterStageTaskStatus[j].Conditions = updateRunAfterStageTaskSucceedConditions(updateRun.Generation, task.Type) + } + stagesStatus[i].Conditions = updateRunStageRolloutSucceedConditions(updateRun.Generation, true) + } + + deleteStageStatus := &placementv1beta1.StageUpdatingStatus{ + StageName: "kubernetes-fleet.io/deleteStage", + } + deleteStageStatus.Clusters = make([]placementv1beta1.ClusterUpdatingStatus, len(wantUnscheduledClusters)) + for i := range deleteStageStatus.Clusters { + deleteStageStatus.Clusters[i].ClusterName = wantUnscheduledClusters[i] + deleteStageStatus.Clusters[i].Conditions = updateRunClusterRolloutSucceedConditions(updateRun.Generation) + } + deleteStageStatus.Conditions = updateRunStageRolloutSucceedConditions(updateRun.Generation, false) + + wantStatus.StagesStatus = stagesStatus + wantStatus.DeletionStageStatus = deleteStageStatus + wantStatus.Conditions = updateRunSucceedConditions(updateRun.Generation) + if diff := cmp.Diff(updateRun.Status, wantStatus, updateRunStatusCmpOption...); diff != "" { + return fmt.Errorf("CRP status diff (-got, +want): %s", diff) + } + return nil + } +} + +func updateRunAndApprovalRequestsRemovedActual(updateRunName string) func() error { + return func() error { + if err := hubClient.Get(ctx, types.NamespacedName{Name: updateRunName}, &placementv1beta1.ClusterStagedUpdateRun{}); !errors.IsNotFound(err) { + return fmt.Errorf("UpdateRun still exists or an unexpected error occurred: %w", err) + } + + appReqList := &placementv1beta1.ClusterApprovalRequestList{} + if err := hubClient.List(ctx, appReqList, client.MatchingLabels{ + placementv1beta1.TargetUpdateRunLabel: updateRunName, + }); err != nil { + return fmt.Errorf("failed to list ClusterApprovalRequests: %w", err) + } + if len(appReqList.Items) > 0 { + return fmt.Errorf("ClusterApprovalRequests still exist: %v", appReqList.Items) + } + return nil + } +} + +func updateRunStrategyRemovedActual(strategyName string) func() error { + return func() error { + if err := hubClient.Get(ctx, types.NamespacedName{Name: strategyName}, &placementv1beta1.ClusterStagedUpdateStrategy{}); !errors.IsNotFound(err) { + return fmt.Errorf("ClusterStagedUpdateStrategy still exists or an unexpected error occurred: %w", err) + } + return nil + } +} diff --git a/test/e2e/resources_test.go b/test/e2e/resources_test.go index 0f5bfd769..0df723b2f 100644 --- a/test/e2e/resources_test.go +++ b/test/e2e/resources_test.go @@ -34,6 +34,8 @@ const ( internalServiceImportNameTemplate = "isi-%d" endpointSliceExportNameTemplate = "ep-%d" crpEvictionNameTemplate = "crpe-%d" + updateRunStrategyNameTemplate = "curs-%d" + updateRunNameWithSubIndexTemplate = "cur-%d-%d" customDeletionBlockerFinalizer = "custom-deletion-blocker-finalizer" workNamespaceLabelName = "process" diff --git a/test/e2e/setup_test.go b/test/e2e/setup_test.go index 1c1d268fd..e1085ea48 100644 --- a/test/e2e/setup_test.go +++ b/test/e2e/setup_test.go @@ -211,6 +211,12 @@ var ( ignoreClusterNameField, cmpopts.EquateEmpty(), } + + updateRunStatusCmpOption = cmp.Options{ + utils.IgnoreConditionLTTAndMessageFields, + cmpopts.IgnoreFields(placementv1beta1.StageUpdatingStatus{}, "StartTime", "EndTime"), + cmpopts.EquateEmpty(), + } ) // TestMain sets up the E2E test environment. diff --git a/test/e2e/updaterun_test.go b/test/e2e/updaterun_test.go new file mode 100644 index 000000000..28fec1608 --- /dev/null +++ b/test/e2e/updaterun_test.go @@ -0,0 +1,597 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package e2e + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + placementv1alpha1 "go.goms.io/fleet/apis/placement/v1alpha1" + placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + "go.goms.io/fleet/pkg/utils/condition" + "go.goms.io/fleet/test/e2e/framework" +) + +const ( + // The current stage wait duration is 1 minute. + // It might not be enough to wait for the updateRun to reach the wanted state with existing eventuallyDurtation, 2 minutes. + // Here we extend it to 3 minutes. + updateRunEventuallyDuration = time.Minute * 3 + + resourceSnapshotIndex1st = "0" + resourceSnapshotIndex2nd = "1" + policySnapshotIndex1st = "0" + policySnapshotIndex2nd = "1" + policySnapshotIndex3rd = "2" +) + +// Note that this container will run in parallel with other containers. +var _ = Describe("test CRP rollout with staged update run", func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + strategyName := fmt.Sprintf(updateRunStrategyNameTemplate, GinkgoParallelProcess()) + + Context("Test resource rollout and rollback with staged update run", Ordered, func() { + updateRunNames := []string{} + var strategy *placementv1beta1.ClusterStagedUpdateStrategy + var oldConfigMap, newConfigMap corev1.ConfigMap + + BeforeAll(func() { + // Create a test namespace and a configMap inside it on the hub cluster. + createWorkResources() + + // Create the CRP with external rollout strategy. + crp := &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.ClusterResourcePlacementSpec{ + ResourceSelectors: workResourceSelector(), + Strategy: placementv1beta1.RolloutStrategy{ + Type: placementv1beta1.ExternalRolloutStrategyType, + }, + }, + } + Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") + + // Create the clusterStagedUpdateStrategy. + strategy = createStagedUpdateStrategySucceed(strategyName) + + for i := 0; i < 3; i++ { + updateRunNames = append(updateRunNames, fmt.Sprintf(updateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), i)) + } + + oldConfigMap = appConfigMap() + newConfigMap = appConfigMap() + newConfigMap.Data["data"] = "new" + }) + + AfterAll(func() { + // Remove the custom deletion blocker finalizer from the CRP. + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + + // Remove all the clusterStagedUpdateRuns. + for _, name := range updateRunNames { + ensureUpdateRunDeletion(name) + } + + // Delete the clusterStagedUpdateStrategy. + ensureUpdateRunStrategyDeletion(strategyName) + }) + + It("Should not rollout any resources to member clusters as there's no update run yet", checkIfRemovedWorkResourcesFromAllMemberClustersConsistently) + + It("Should have the latest resource snapshot", func() { + validateLatestResourceSnapshot(crpName, resourceSnapshotIndex1st) + }) + + It("Should successfully schedule the crp", func() { + validateLatestPolicySnapshot(crpName, policySnapshotIndex1st) + }) + + It("Should create a staged update run successfully", func() { + createStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) + }) + + It("Should rollout resources to member-cluster-2 only and completes stage canary", func() { + checkIfPlacedWorkResourcesOnMemberClusters([]*framework.Cluster{allMemberClusters[1]}) + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) + validateAndApproveClusterApprovalRequests(updateRunNames[0], envLabelValue2) + }) + + It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { + checkIfPlacedWorkResourcesOnMemberClusters(allMemberClusters) + updateRunSucceededActual := updateRunStatusSucceededActual(updateRunNames[0], policySnapshotIndex1st, len(allMemberClusters), nil, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) + }) + + It("Should update the configmap successfully on hub but not change member clusters", func() { + Eventually(func() error { return hubClient.Update(ctx, &newConfigMap) }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update configmap on hub") + + for _, cluster := range allMemberClusters { + configMapActual := configMapPlacedOnClusterActual(cluster, &oldConfigMap) + Consistently(configMapActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep configmap %s data as expected", oldConfigMap.Name) + } + }) + + It("Should create a new latest resource snapshot", func() { + crsList := &placementv1beta1.ClusterResourceSnapshotList{} + Eventually(func() error { + if err := hubClient.List(ctx, crsList, client.MatchingLabels{placementv1beta1.CRPTrackingLabel: crpName, placementv1beta1.IsLatestSnapshotLabel: "true"}); err != nil { + return fmt.Errorf("Failed to list the resourcesnapshot: %w", err) + } + if len(crsList.Items) != 1 { + return fmt.Errorf("Got %d latest resourcesnapshots, want 1", len(crsList.Items)) + } + if crsList.Items[0].Labels[placementv1beta1.ResourceIndexLabel] != resourceSnapshotIndex2nd { + return fmt.Errorf("Got resource snapshot index %s, want %s", crsList.Items[0].Labels[placementv1beta1.ResourceIndexLabel], resourceSnapshotIndex2nd) + } + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed get the new latest resourcensnapshot") + }) + + It("Should create a new staged update run successfully", func() { + createStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex2nd, strategyName) + }) + + It("Should rollout resources to member-cluster-2 only and completes stage canary", func() { + checkIfPlacedWorkResourcesOnMemberClusters([]*framework.Cluster{allMemberClusters[1]}) + for _, cluster := range []*framework.Cluster{allMemberClusters[0], allMemberClusters[2]} { + configMapActual := configMapPlacedOnClusterActual(cluster, &oldConfigMap) + Consistently(configMapActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep configmap %s data as expected", oldConfigMap.Name) + } + validateAndApproveClusterApprovalRequests(updateRunNames[1], envLabelValue2) + }) + + It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { + checkIfPlacedWorkResourcesOnMemberClusters(allMemberClusters) + updateRunSucceededActual := updateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, len(allMemberClusters), nil, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) + }) + + It("Should create a new staged update run with old resourceSnapshotIndex successfully to rollback", func() { + createStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName) + }) + + It("Should rollback resources to member-cluster-2 only and completes stage canary", func() { + checkIfPlacedWorkResourcesOnMemberClusters([]*framework.Cluster{allMemberClusters[1]}) + configMapActual := configMapPlacedOnClusterActual(allMemberClusters[1], &oldConfigMap) + Eventually(configMapActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to rollback the configmap change on cluster %s", allMemberClusterNames[1]) + checkIfPlacedWorkResourcesOnMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) + validateAndApproveClusterApprovalRequests(updateRunNames[2], envLabelValue2) + }) + + It("Should rollback resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { + for idx := range allMemberClusters { + configMapActual := configMapPlacedOnClusterActual(allMemberClusters[idx], &oldConfigMap) + Eventually(configMapActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to rollback the configmap %s data on cluster %s as expected", oldConfigMap.Name, allMemberClusterNames[idx]) + } + updateRunSucceededActual := updateRunStatusSucceededActual(updateRunNames[2], policySnapshotIndex1st, len(allMemberClusters), nil, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) + }) + }) + + Context("Test cluster scale out and shrink with staged update run", Ordered, func() { + var strategy *placementv1beta1.ClusterStagedUpdateStrategy + updateRunNames := []string{} + + BeforeAll(func() { + // Create a test namespace and a configMap inside it on the hub cluster. + createWorkResources() + + // Create the CRP with external rollout strategy and pick fixed policy. + crp := &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.ClusterResourcePlacementSpec{ + ResourceSelectors: workResourceSelector(), + Policy: &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickFixedPlacementType, + ClusterNames: []string{allMemberClusterNames[0], allMemberClusterNames[1]}, // member-cluster-1 and member-cluster-2 + }, + Strategy: placementv1beta1.RolloutStrategy{ + Type: placementv1beta1.ExternalRolloutStrategyType, + }, + }, + } + Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") + + // Create the clusterStagedUpdateStrategy. + strategy = createStagedUpdateStrategySucceed(strategyName) + + for i := 0; i < 3; i++ { + updateRunNames = append(updateRunNames, fmt.Sprintf(updateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), i)) + } + }) + + AfterAll(func() { + // Remove the custom deletion blocker finalizer from the CRP. + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + + // Remove all the clusterStagedUpdateRuns. + for _, name := range updateRunNames { + ensureUpdateRunDeletion(name) + } + + // Delete the clusterStagedUpdateStrategy. + ensureUpdateRunStrategyDeletion(strategyName) + }) + + It("Should not rollout any resources to member clusters as there's no update run yet", checkIfRemovedWorkResourcesFromAllMemberClustersConsistently) + + It("Should have the latest resource snapshot", func() { + validateLatestResourceSnapshot(crpName, resourceSnapshotIndex1st) + }) + + It("Should successfully schedule the crp", func() { + validateLatestPolicySnapshot(crpName, policySnapshotIndex1st) + }) + + It("Should create a staged update run successfully", func() { + createStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) + }) + + It("Should rollout resources to member-cluster-2 only and completes stage canary", func() { + checkIfPlacedWorkResourcesOnMemberClusters([]*framework.Cluster{allMemberClusters[1]}) + validateAndApproveClusterApprovalRequests(updateRunNames[0], envLabelValue2) + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) + }) + + It("Should rollout resources to member-cluster-1 too but not member-cluster-3 and complete the staged update run successfully", func() { + checkIfPlacedWorkResourcesOnMemberClusters([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) + updateRunSucceededActual := updateRunStatusSucceededActual(updateRunNames[0], policySnapshotIndex1st, 2, nil, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil) + Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) + }) + + It("Should update the crp to pick member-cluster-3 too", func() { + Eventually(func() error { + crp := &placementv1beta1.ClusterResourcePlacement{} + if err := hubClient.Get(ctx, client.ObjectKey{Name: crpName}, crp); err != nil { + return fmt.Errorf("Failed to get the crp: %w", err) + } + crp.Spec.Policy.ClusterNames = allMemberClusterNames + return hubClient.Update(ctx, crp) + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update the crp to pick member-cluster-3 too") + }) + + It("Should successfully schedule the crp", func() { + validateLatestPolicySnapshot(crpName, policySnapshotIndex2nd) + }) + + It("Should create a staged update run successfully", func() { + createStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) + }) + + It("Should still have resources on member-cluster-1 and member-cluster-2 only and completes stage canary", func() { + checkIfPlacedWorkResourcesOnMemberClusters([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) + validateAndApproveClusterApprovalRequests(updateRunNames[1], envLabelValue2) + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) + }) + + It("Should rollout resources to member-cluster-3 too and complete the staged update run successfully", func() { + checkIfPlacedWorkResourcesOnMemberClusters(allMemberClusters) + updateRunSucceededActual := updateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex2nd, 3, nil, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) + }) + + It("Should update the crp to only keep member-cluster-3", func() { + Eventually(func() error { + crp := &placementv1beta1.ClusterResourcePlacement{} + if err := hubClient.Get(ctx, client.ObjectKey{Name: crpName}, crp); err != nil { + return fmt.Errorf("Failed to get the crp: %w", err) + } + crp.Spec.Policy.ClusterNames = []string{allMemberClusterNames[2]} + return hubClient.Update(ctx, crp) + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update the crp to only keep member-cluster-3") + }) + + It("Should successfully schedule the crp", func() { + validateLatestPolicySnapshot(crpName, policySnapshotIndex3rd) + }) + + It("Should create a staged update run successfully", func() { + createStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName) + }) + + It("Should still have resources on all member clusters and complete stage canary", func() { + checkIfPlacedWorkResourcesOnMemberClustersConsistently(allMemberClusters) + validateAndApproveClusterApprovalRequests(updateRunNames[2], envLabelValue2) + }) + + It("Should remove resources on member-cluster-1 and member-cluster-2 and complete the staged update run successfully", func() { + checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) + checkIfPlacedWorkResourcesOnMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) + updateRunSucceededActual := updateRunStatusSucceededActual(updateRunNames[2], policySnapshotIndex3rd, 1, nil, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil) + Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[2]) + }) + }) + + Context("Test staged update run with overrides", Ordered, func() { + var strategy *placementv1beta1.ClusterStagedUpdateStrategy + updateRunName := fmt.Sprintf(updateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), 0) + croName := fmt.Sprintf(croNameTemplate, GinkgoParallelProcess()) + roName := fmt.Sprintf(roNameTemplate, GinkgoParallelProcess()) + roNamespace := fmt.Sprintf(workNamespaceNameTemplate, GinkgoParallelProcess()) + + BeforeAll(func() { + // Create a test namespace and a configMap inside it on the hub cluster. + createWorkResources() + + // Create the cro before crp so that the observed resource index is predictable. + cro := &placementv1alpha1.ClusterResourceOverride{ + ObjectMeta: metav1.ObjectMeta{ + Name: croName, + }, + Spec: placementv1alpha1.ClusterResourceOverrideSpec{ + ClusterResourceSelectors: workResourceSelector(), + Policy: &placementv1alpha1.OverridePolicy{ + OverrideRules: []placementv1alpha1.OverrideRule{ + { + ClusterSelector: &placementv1beta1.ClusterSelector{ + ClusterSelectorTerms: []placementv1beta1.ClusterSelectorTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{regionLabelName: regionLabelValue1, envLabelName: envLabelValue1}, // member-cluster-1 + }, + }, + }, + }, + JSONPatchOverrides: []placementv1alpha1.JSONPatchOverride{ + { + Operator: placementv1alpha1.JSONPatchOverrideOpAdd, + Path: "/metadata/annotations", + Value: apiextensionsv1.JSON{Raw: []byte(fmt.Sprintf(`{"%s": "%s-0"}`, croTestAnnotationKey, croTestAnnotationValue))}, + }, + }, + }, + }, + }, + }, + } + Expect(hubClient.Create(ctx, cro)).To(Succeed(), "Failed to create clusterResourceOverride %s", croName) + + // Create the ro before crp so that the observed resource index is predictable. + ro := &placementv1alpha1.ResourceOverride{ + ObjectMeta: metav1.ObjectMeta{ + Name: roName, + Namespace: roNamespace, + }, + Spec: placementv1alpha1.ResourceOverrideSpec{ + ResourceSelectors: configMapSelector(), + Policy: &placementv1alpha1.OverridePolicy{ + OverrideRules: []placementv1alpha1.OverrideRule{ + { + ClusterSelector: &placementv1beta1.ClusterSelector{ + ClusterSelectorTerms: []placementv1beta1.ClusterSelectorTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{regionLabelName: regionLabelValue1, envLabelName: envLabelValue2}, + }, + }, + }, + }, + JSONPatchOverrides: []placementv1alpha1.JSONPatchOverride{ + { + Operator: placementv1alpha1.JSONPatchOverrideOpAdd, + Path: "/metadata/annotations", + Value: apiextensionsv1.JSON{Raw: []byte(fmt.Sprintf(`{"%s": "%s-1"}`, roTestAnnotationKey, roTestAnnotationValue))}, + }, + }, + }, + }, + }, + }, + } + Expect(hubClient.Create(ctx, ro)).To(Succeed(), "Failed to create resourceOverride %s", roName) + + // Create the CRP with external rollout strategy and pick fixed policy. + crp := &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.ClusterResourcePlacementSpec{ + ResourceSelectors: workResourceSelector(), + Policy: &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickAllPlacementType, + }, + Strategy: placementv1beta1.RolloutStrategy{ + Type: placementv1beta1.ExternalRolloutStrategyType, + }, + }, + } + Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") + + // Create the clusterStagedUpdateStrategy. + strategy = createStagedUpdateStrategySucceed(strategyName) + }) + + AfterAll(func() { + // Remove the custom deletion blocker finalizer from the CRP. + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + + // Delete the clusterStagedUpdateRun. + ensureUpdateRunDeletion(updateRunName) + + // Delete the clusterStagedUpdateStrategy. + ensureUpdateRunStrategyDeletion(strategyName) + + // Delete the clusterResourceOverride. + cleanupClusterResourceOverride(croName) + + // Delete the resourceOverride. + cleanupResourceOverride(roName, roNamespace) + }) + + It("Should not rollout any resources to member clusters as there's no update run yet", checkIfRemovedWorkResourcesFromAllMemberClustersConsistently) + + It("Should have the latest resource snapshot", func() { + validateLatestResourceSnapshot(crpName, resourceSnapshotIndex1st) + }) + + It("Should successfully schedule the crp", func() { + validateLatestPolicySnapshot(crpName, policySnapshotIndex1st) + }) + + It("Should create a staged update run successfully", func() { + createStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + }) + + It("Should rollout resources to member-cluster-2 only and completes stage canary", func() { + checkIfPlacedWorkResourcesOnMemberClusters([]*framework.Cluster{allMemberClusters[1]}) + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) + validateAndApproveClusterApprovalRequests(updateRunName, envLabelValue2) + }) + + It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { + checkIfPlacedWorkResourcesOnMemberClusters(allMemberClusters) + wantCROs := map[string][]string{allMemberClusterNames[0]: {croName + "-0"}} // with override snapshot index 0 + wantROs := map[string][]placementv1beta1.NamespacedName{allMemberClusterNames[1]: {placementv1beta1.NamespacedName{Namespace: roNamespace, Name: roName + "-0"}}} // with override snapshot index 0 + updateRunSucceededActual := updateRunStatusSucceededActual(updateRunName, policySnapshotIndex1st, len(allMemberClusters), nil, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, wantCROs, wantROs) + Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) + }) + + It("should have override annotations on the member cluster 1 and member cluster 2", func() { + wantCROAnnotations := map[string]string{croTestAnnotationKey: fmt.Sprintf("%s-%d", croTestAnnotationValue, 0)} + wantROAnnotations := map[string]string{roTestAnnotationKey: fmt.Sprintf("%s-%d", roTestAnnotationValue, 1)} + Expect(validateAnnotationOfWorkNamespaceOnCluster(allMemberClusters[0], wantCROAnnotations)).Should(Succeed(), "Failed to override the annotation of work namespace on %s", allMemberClusters[0].ClusterName) + Expect(validateOverrideAnnotationOfConfigMapOnCluster(allMemberClusters[0], wantCROAnnotations)).Should(Succeed(), "Failed to override the annotation of configmap on %s", allMemberClusters[0].ClusterName) + Expect(validateOverrideAnnotationOfConfigMapOnCluster(allMemberClusters[1], wantROAnnotations)).Should(Succeed(), "Failed to override the annotation of configmap on %s", allMemberClusters[1].ClusterName) + }) + }) +}) + +func createStagedUpdateStrategySucceed(strategyName string) *placementv1beta1.ClusterStagedUpdateStrategy { + strategy := &placementv1beta1.ClusterStagedUpdateStrategy{ + ObjectMeta: metav1.ObjectMeta{ + Name: strategyName, + }, + Spec: placementv1beta1.StagedUpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: envLabelValue2, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + envLabelName: envLabelValue2, // member-cluster-2 + }, + }, + AfterStageTasks: []placementv1beta1.AfterStageTask{ + { + Type: placementv1beta1.AfterStageTaskTypeApproval, + }, + { + Type: placementv1beta1.AfterStageTaskTypeTimedWait, + WaitTime: metav1.Duration{ + Duration: time.Second * 5, + }, + }, + }, + }, + { + Name: envLabelValue1, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + envLabelName: envLabelValue1, // member-cluster-1 and member-cluster-3 + }, + }, + }, + }, + }, + } + Expect(hubClient.Create(ctx, strategy)).To(Succeed(), "Failed to create ClusterStagedUpdateStrategy") + return strategy +} + +func validateLatestPolicySnapshot(crpName, wantPolicySnapshotIndex string) { + Eventually(func() (string, error) { + var policySnapshotList placementv1beta1.ClusterSchedulingPolicySnapshotList + if err := hubClient.List(ctx, &policySnapshotList, client.MatchingLabels{ + placementv1beta1.CRPTrackingLabel: crpName, + placementv1beta1.IsLatestSnapshotLabel: "true", + }); err != nil { + return "", fmt.Errorf("Failed to list the latest scheduling policy snapshot: %w", err) + } + if len(policySnapshotList.Items) != 1 { + return "", fmt.Errorf("Failed to find the latest scheduling policy snapshot") + } + latestPolicySnapshot := policySnapshotList.Items[0] + if !condition.IsConditionStatusTrue(latestPolicySnapshot.GetCondition(string(placementv1beta1.PolicySnapshotScheduled)), latestPolicySnapshot.Generation) { + return "", fmt.Errorf("The latest scheduling policy snapshot is not scheduled yet") + } + return latestPolicySnapshot.Labels[placementv1beta1.PolicyIndexLabel], nil + }, eventuallyDuration, eventuallyInterval).Should(Equal(wantPolicySnapshotIndex), "Policy snapshot index does not match") +} + +func validateLatestResourceSnapshot(crpName, wantResourceSnapshotIndex string) { + Eventually(func() (string, error) { + crsList := &placementv1beta1.ClusterResourceSnapshotList{} + if err := hubClient.List(ctx, crsList, client.MatchingLabels{ + placementv1beta1.CRPTrackingLabel: crpName, + placementv1beta1.IsLatestSnapshotLabel: "true", + }); err != nil { + return "", fmt.Errorf("Failed to list the latestresourcesnapshot: %w", err) + } + if len(crsList.Items) != 1 { + return "", fmt.Errorf("Got %d resourcesnapshots, want 1", len(crsList.Items)) + } + return crsList.Items[0].Labels[placementv1beta1.ResourceIndexLabel], nil + }, eventuallyDuration, eventuallyInterval).Should(Equal(wantResourceSnapshotIndex), "Resource snapshot index does not match") +} + +func createStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex, strategyName string) { + updateRun := &placementv1beta1.ClusterStagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: updateRunName, + }, + Spec: placementv1beta1.StagedUpdateRunSpec{ + PlacementName: crpName, + ResourceSnapshotIndex: resourceSnapshotIndex, + StagedUpdateStrategyName: strategyName, + }, + } + Expect(hubClient.Create(ctx, updateRun)).To(Succeed(), "Failed to create ClusterStagedUpdateRun %s", updateRunName) +} + +func validateAndApproveClusterApprovalRequests(updateRunName, stageName string) { + Eventually(func() error { + appReqList := &placementv1beta1.ClusterApprovalRequestList{} + if err := hubClient.List(ctx, appReqList, client.MatchingLabels{ + placementv1beta1.TargetUpdatingStageNameLabel: stageName, + placementv1beta1.TargetUpdateRunLabel: updateRunName, + }); err != nil { + return fmt.Errorf("Failed to list approval requests: %w", err) + } + + if len(appReqList.Items) != 1 { + return fmt.Errorf("Got %d approval requests, want 1", len(appReqList.Items)) + } + appReq := &appReqList.Items[0] + meta.SetStatusCondition(&appReq.Status.Conditions, metav1.Condition{ + Status: metav1.ConditionTrue, + Type: string(placementv1beta1.ApprovalRequestConditionApproved), + ObservedGeneration: appReq.GetGeneration(), + Reason: "lgtm", + }) + return hubClient.Status().Update(ctx, appReq) + }, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to get or approve approval request") +} diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go index 04f39cb92..00c73a538 100644 --- a/test/e2e/utils_test.go +++ b/test/e2e/utils_test.go @@ -716,8 +716,12 @@ func checkIfAllMemberClustersHaveLeft() { } func checkIfPlacedWorkResourcesOnAllMemberClusters() { - for idx := range allMemberClusters { - memberCluster := allMemberClusters[idx] + checkIfPlacedWorkResourcesOnMemberClusters(allMemberClusters) +} + +func checkIfPlacedWorkResourcesOnMemberClusters(clusters []*framework.Cluster) { + for idx := range clusters { + memberCluster := clusters[idx] workResourcesPlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(memberCluster) Eventually(workResourcesPlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place work resources on member cluster %s", memberCluster.ClusterName) @@ -725,11 +729,15 @@ func checkIfPlacedWorkResourcesOnAllMemberClusters() { } func checkIfPlacedWorkResourcesOnAllMemberClustersConsistently() { - for idx := range allMemberClusters { - memberCluster := allMemberClusters[idx] + checkIfPlacedWorkResourcesOnMemberClustersConsistently(allMemberClusters) +} + +func checkIfPlacedWorkResourcesOnMemberClustersConsistently(clusters []*framework.Cluster) { + for idx := range clusters { + memberCluster := clusters[idx] workResourcesPlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(memberCluster) - Consistently(workResourcesPlacedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to place work resources on member cluster %s", memberCluster.ClusterName) + Consistently(workResourcesPlacedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to place work resources on member cluster %s consistently", memberCluster.ClusterName) } } @@ -755,6 +763,19 @@ func checkIfRemovedWorkResourcesFromMemberClusters(clusters []*framework.Cluster } } +func checkIfRemovedWorkResourcesFromAllMemberClustersConsistently() { + checkIfRemovedWorkResourcesFromMemberClustersConsistently(allMemberClusters) +} + +func checkIfRemovedWorkResourcesFromMemberClustersConsistently(clusters []*framework.Cluster) { + for idx := range clusters { + memberCluster := clusters[idx] + + workResourcesRemovedActual := workNamespaceRemovedFromClusterActual(memberCluster) + Consistently(workResourcesRemovedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to remove work resources from member cluster %s consistently", memberCluster.ClusterName) + } +} + // cleanupCRP deletes the CRP and waits until the resources are not found. func cleanupCRP(name string) { // TODO(Arvindthiru): There is a conflict which requires the Eventually block, not sure of series of operations that leads to it yet. @@ -1259,3 +1280,28 @@ func createCRPWithApplyStrategy(crpName string, applyStrategy *placementv1beta1. func createCRP(crpName string) { createCRPWithApplyStrategy(crpName, nil) } + +// ensureUpdateRunDeletion deletes the update run with the given name and checks all related approval requests are also deleted. +func ensureUpdateRunDeletion(updateRunName string) { + updateRun := &placementv1beta1.ClusterStagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: updateRunName, + }, + } + Expect(client.IgnoreNotFound(hubClient.Delete(ctx, updateRun))).Should(Succeed(), "Failed to delete ClusterStagedUpdateRun %s", updateRunName) + + removedActual := updateRunAndApprovalRequestsRemovedActual(updateRunName) + Eventually(removedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "ClusterStagedUpdateRun or ClusterApprovalRequests still exists") +} + +// ensureUpdateRunStrategyDeletion deletes the update run strategy with the given name. +func ensureUpdateRunStrategyDeletion(strategyName string) { + strategy := &placementv1beta1.ClusterStagedUpdateStrategy{ + ObjectMeta: metav1.ObjectMeta{ + Name: strategyName, + }, + } + Expect(client.IgnoreNotFound(hubClient.Delete(ctx, strategy))).Should(Succeed(), "Failed to delete ClusterStagedUpdateStrategy %s", strategyName) + removedActual := updateRunStrategyRemovedActual(strategyName) + Eventually(removedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "ClusterStagedUpdateStrategy still exists") +}