From af2af33578f782ecc0c4dbedfa6ba6022f815648 Mon Sep 17 00:00:00 2001 From: Sergi Castro Date: Wed, 13 Nov 2024 15:55:05 +0100 Subject: [PATCH] Failover volume mount should also use custom map if configured --- .../failover/PrimaryToReplicaFailOver.go | 7 +- .../statefulset_spec/VolumeSpecEnforcer.go | 4 +- test/primary_failure_and_recovery_test.go | 81 ++++++++++++++++++- test/resourceConfigs/ConfigForTest.go | 3 + ...configMap_with_promote_replica_script.yaml | 33 ++++++++ test/spec_customConfig_test.go | 2 +- test/util/TestResourceCreator.go | 7 ++ 7 files changed, 130 insertions(+), 7 deletions(-) create mode 100644 test/resourceConfigs/customConfig/configMap_with_promote_replica_script.yaml diff --git a/controllers/spec/enforcer/resources_count_spec/statefulset/failover/PrimaryToReplicaFailOver.go b/controllers/spec/enforcer/resources_count_spec/statefulset/failover/PrimaryToReplicaFailOver.go index 3552459..74cada4 100644 --- a/controllers/spec/enforcer/resources_count_spec/statefulset/failover/PrimaryToReplicaFailOver.go +++ b/controllers/spec/enforcer/resources_count_spec/statefulset/failover/PrimaryToReplicaFailOver.go @@ -22,13 +22,14 @@ package failover import ( "errors" + "strconv" + core "k8s.io/api/core/v1" v1 "reactive-tech.io/kubegres/api/v1" "reactive-tech.io/kubegres/controllers/ctx" "reactive-tech.io/kubegres/controllers/operation" "reactive-tech.io/kubegres/controllers/states" "reactive-tech.io/kubegres/controllers/states/statefulset" - "strconv" ) type PrimaryToReplicaFailOver struct { @@ -244,9 +245,9 @@ func (r *PrimaryToReplicaFailOver) promoteReplicaToPrimary(newPrimary statefulse newPrimary.StatefulSet.Labels["replicationRole"] = ctx.PrimaryRoleName newPrimary.StatefulSet.Spec.Template.Labels["replicationRole"] = ctx.PrimaryRoleName volumeMount := core.VolumeMount{ - Name: "base-config", + Name: r.resourcesStates.Config.ConfigLocations.PromoteReplica, MountPath: "/tmp/promote_replica_to_primary.sh", - SubPath: "promote_replica_to_primary.sh", + SubPath: states.ConfigMapDataKeyPromoteReplica, } initContainer := &newPrimary.StatefulSet.Spec.Template.Spec.InitContainers[0] diff --git a/controllers/spec/enforcer/statefulset_spec/VolumeSpecEnforcer.go b/controllers/spec/enforcer/statefulset_spec/VolumeSpecEnforcer.go index 202d33d..75fe756 100644 --- a/controllers/spec/enforcer/statefulset_spec/VolumeSpecEnforcer.go +++ b/controllers/spec/enforcer/statefulset_spec/VolumeSpecEnforcer.go @@ -59,7 +59,7 @@ func (r *VolumeSpecEnforcer) CheckForSpecDifference(statefulSet *apps.StatefulSe if hasInitContainer && !r.compareVolumeMounts(currentCustomVolumeMountsInit, expectedCustomVolumeMounts) { return StatefulSetSpecDifference{ - SpecName: "Volume.VolumeMounts", + SpecName: "Volume.VolumeMounts[initContainer]", Current: r.volumeMountsToString(currentCustomVolumeMountsInit), Expected: r.volumeMountsToString(expectedCustomVolumeMounts), } @@ -67,7 +67,7 @@ func (r *VolumeSpecEnforcer) CheckForSpecDifference(statefulSet *apps.StatefulSe if !r.compareVolumeMounts(currentCustomVolumeMounts, expectedCustomVolumeMounts) { return StatefulSetSpecDifference{ - SpecName: "Volume.VolumeMounts", + SpecName: "Volume.VolumeMounts[container]", Current: r.volumeMountsToString(currentCustomVolumeMounts), Expected: r.volumeMountsToString(expectedCustomVolumeMounts), } diff --git a/test/primary_failure_and_recovery_test.go b/test/primary_failure_and_recovery_test.go index 5163ed8..12054c1 100644 --- a/test/primary_failure_and_recovery_test.go +++ b/test/primary_failure_and_recovery_test.go @@ -43,6 +43,7 @@ var _ = Describe("Primary instances is not available, checking recovery works", namespace := resourceConfigs.DefaultNamespace test.resourceRetriever = util.CreateTestResourceRetriever(k8sClientTest, namespace) test.resourceCreator = util.CreateTestResourceCreator(k8sClientTest, test.resourceRetriever, namespace) + test.resourceCreator.CreateConfigMapWithPromoteReplicaScript() test.connectionPrimaryDb = util.InitDbConnectionDbUtil(test.resourceCreator, resourceConfigs.KubegresResourceName, resourceConfigs.ServiceToSqlQueryPrimaryDbNodePort, true) test.connectionReplicaDb = util.InitDbConnectionDbUtil(test.resourceCreator, resourceConfigs.KubegresResourceName, resourceConfigs.ServiceToSqlQueryReplicaDbNodePort, false) }) @@ -119,7 +120,7 @@ var _ = Describe("Primary instances is not available, checking recovery works", Context("GIVEN Kubegres with 1 primary and 2 replicas AND primary is deleted", func() { - It("THEN the failover should take place with a replica becoming primary AND a new replica created AND existing data available", func() { + It("THEN the failover should take place with a replica becoming primary AND a new replica created AND existing data available, twice", func() { log.Print("START OF: Test 'GIVEN Kubegres with 1 primary and 2 replicas AND primary is deleted'") @@ -137,6 +138,23 @@ var _ = Describe("Primary instances is not available, checking recovery works", test.GivenUserAddedInPrimaryDb() expectedNbreUsers++ + // First failover + + test.whenPrimaryIsDeleted() + + test.thenPodsStatesShouldBe(1, 2) + + test.ThenPrimaryDbContainsExpectedNbreUsers(expectedNbreUsers) + test.ThenReplicaDbContainsExpectedNbreUsers(expectedNbreUsers) + + test.GivenUserAddedInPrimaryDb() + expectedNbreUsers++ + + test.ThenPrimaryDbContainsExpectedNbreUsers(expectedNbreUsers) + test.ThenReplicaDbContainsExpectedNbreUsers(expectedNbreUsers) + + // Second failover + test.whenPrimaryIsDeleted() test.thenPodsStatesShouldBe(1, 2) @@ -154,6 +172,61 @@ var _ = Describe("Primary instances is not available, checking recovery works", }) }) + Context("GIVEN Kubegres with 1 primary and 2 replicas using custom-configs map AND primary is deleted", func() { + + It("THEN the failover should take place with a replica becoming primary AND a new replica created AND existing data available, twice", func() { + + log.Print("START OF: Test 'GIVEN Kubegres with 1 primary and 2 replicas using custom-configs map AND primary is deleted'") + + test.givenNewKubegresSpecIsSetToWithCustomConfigs(3) + + test.whenKubegresIsCreated() + + test.thenPodsStatesShouldBe(1, 2) + + expectedNbreUsers := 0 + + test.GivenUserAddedInPrimaryDb() + expectedNbreUsers++ + + test.GivenUserAddedInPrimaryDb() + expectedNbreUsers++ + + // First failover + + test.whenPrimaryIsDeleted() + + test.thenPodsStatesShouldBe(1, 2) + + test.ThenPrimaryDbContainsExpectedNbreUsers(expectedNbreUsers) + test.ThenReplicaDbContainsExpectedNbreUsers(expectedNbreUsers) + + test.GivenUserAddedInPrimaryDb() + expectedNbreUsers++ + + test.ThenPrimaryDbContainsExpectedNbreUsers(expectedNbreUsers) + test.ThenReplicaDbContainsExpectedNbreUsers(expectedNbreUsers) + + // Second failover + + test.whenPrimaryIsDeleted() + + test.thenPodsStatesShouldBe(1, 2) + + test.ThenPrimaryDbContainsExpectedNbreUsers(expectedNbreUsers) + test.ThenReplicaDbContainsExpectedNbreUsers(expectedNbreUsers) + + test.GivenUserAddedInPrimaryDb() + expectedNbreUsers++ + + test.ThenPrimaryDbContainsExpectedNbreUsers(expectedNbreUsers) + test.ThenReplicaDbContainsExpectedNbreUsers(expectedNbreUsers) + + log.Print("END OF: Test 'GIVEN Kubegres with 1 primary and 2 replicas using custom-configs map AND primary is deleted'") + }) + + }) + }) type PrimaryFailureAndRecoveryTest struct { @@ -171,6 +244,12 @@ func (r *PrimaryFailureAndRecoveryTest) givenNewKubegresSpecIsSetTo(specNbreRepl r.kubegresResource.Spec.Replicas = &specNbreReplicas } +func (r *PrimaryFailureAndRecoveryTest) givenNewKubegresSpecIsSetToWithCustomConfigs(specNbreReplicas int32) { + r.kubegresResource = resourceConfigs.LoadKubegresYaml() + r.kubegresResource.Spec.Replicas = &specNbreReplicas + r.kubegresResource.Spec.CustomConfig = resourceConfigs.CustomConfigMapWithPromoteReplicaScriptResourceName +} + func (r *PrimaryFailureAndRecoveryTest) whenKubegresIsCreated() { r.resourceCreator.CreateKubegres(r.kubegresResource) } diff --git a/test/resourceConfigs/ConfigForTest.go b/test/resourceConfigs/ConfigForTest.go index aee2537..a9a22fd 100644 --- a/test/resourceConfigs/ConfigForTest.go +++ b/test/resourceConfigs/ConfigForTest.go @@ -75,4 +75,7 @@ const ( CustomConfigMapWithPostgresConfAndWalLevelSetToLogicalResourceName = "config-with-postgres-conf-wal-level-to-logical" CustomConfigMapWithPostgresConfAndWalLevelSetToLogicalYamlFile = "resourceConfigs/customConfig/configMap_with_postgres_conf_and_wal_level_to_logical.yaml" + + CustomConfigMapWithPromoteReplicaScriptResourceName = "config-with-promote-replica-script" + CustomConfigMapWithPromoteReplicaScriptYamlFile = "resourceConfigs/customConfig/configMap_with_promote_replica_script.yaml" ) diff --git a/test/resourceConfigs/customConfig/configMap_with_promote_replica_script.yaml b/test/resourceConfigs/customConfig/configMap_with_promote_replica_script.yaml new file mode 100644 index 0000000..31fb2ce --- /dev/null +++ b/test/resourceConfigs/customConfig/configMap_with_promote_replica_script.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-with-promote-replica-script + namespace: default + labels: + environment: acceptancetesting + +data: + + promote_replica_to_primary.sh: | + #!/bin/bash + set -e + + dt=$(date '+%d/%m/%Y %H:%M:%S'); + echo "$dt - Attempting to promote a Replica PostgreSql to Primary..."; + + standbyFilePath="$PGDATA/standby.signal" + + if [ ! -f "$standbyFilePath" ]; then + echo "$dt - Skipping as this PostgreSql is already a Primary since the file '$standbyFilePath' does not exist." + exit 0 + fi + + promotionTriggerFilePath="$PGDATA/promote_replica_to_primary.log" + + if [ -f "$promotionTriggerFilePath" ]; then + echo "$dt - Skipping as the promotion trigger file '$promotionTriggerFilePath' already exists" + exit 0 + fi + + echo "$dt - Promoting by creating the promotion trigger file: '$promotionTriggerFilePath'" + touch $promotionTriggerFilePath diff --git a/test/spec_customConfig_test.go b/test/spec_customConfig_test.go index fab8919..31c9d70 100644 --- a/test/spec_customConfig_test.go +++ b/test/spec_customConfig_test.go @@ -55,6 +55,7 @@ var _ = Describe("Setting Kubegres specs 'customConfig'", Label("group:2"), func test.resourceCreator.CreateConfigMapWithPgHbaConf() test.resourceCreator.CreateConfigMapWithPostgresConf() test.resourceCreator.CreateConfigMapWithPrimaryInitScript() + test.resourceCreator.CreateConfigMapWithPromoteReplicaScript() }) AfterEach(func() { @@ -344,7 +345,6 @@ var _ = Describe("Setting Kubegres specs 'customConfig'", Label("group:2"), func log.Print("END OF: Test 'GIVEN new Kubegres is created with backUp enabled and spec 'customConfig' set to base-config AND later it is updated to a configMap containing data-key 'backup_database.sh''") }) }) - }) type SpecCustomConfigTest struct { diff --git a/test/util/TestResourceCreator.go b/test/util/TestResourceCreator.go index 70402dc..7cac0b8 100644 --- a/test/util/TestResourceCreator.go +++ b/test/util/TestResourceCreator.go @@ -145,6 +145,13 @@ func (r *TestResourceCreator) CreateConfigMapWithPostgresConfAndWalLevelSetToLog r.createResourceFromYaml("Custom ConfigMap with postgres conf and wal_level=logical", resourceConfigs2.CustomConfigMapWithPostgresConfAndWalLevelSetToLogicalResourceName, &existingResource, &resourceToCreate) } +func (r *TestResourceCreator) CreateConfigMapWithPromoteReplicaScript() { + existingResource := v1.ConfigMap{} + resourceToCreate := resourceConfigs2.LoadCustomConfigMapYaml(resourceConfigs2.CustomConfigMapWithPromoteReplicaScriptYamlFile) + resourceToCreate.Namespace = r.namespace + r.createResourceFromYaml("Custom ConfigMap with promote replica script", resourceConfigs2.CustomConfigMapWithPromoteReplicaScriptResourceName, &existingResource, &resourceToCreate) +} + func (r *TestResourceCreator) CreateNamespace() { if r.namespace == resourceConfigs2.DefaultNamespace { return