From dace6ffbb3b553c097c0d915fb98c3544a8b0aa4 Mon Sep 17 00:00:00 2001
From: Mayank <33252549+mynktl@users.noreply.github.com>
Date: Mon, 30 Sep 2019 12:10:41 +0530
Subject: [PATCH] fix(ci): adding check for healthy CVR (#42)

Signed-off-by: mayank <mayank.patel@mayadata.io>
---
 script/install-velero.sh         |  6 +++---
 tests/openebs/logs.go            | 21 ++++++++++++++++++---
 tests/openebs/storage_install.go |  8 +++++++-
 tests/openebs/storage_status.go  |  3 ++-
 tests/sanity/backup_test.go      |  7 +++++++
 5 files changed, 37 insertions(+), 8 deletions(-)

diff --git a/script/install-velero.sh b/script/install-velero.sh
index 635e585e..070bde97 100755
--- a/script/install-velero.sh
+++ b/script/install-velero.sh
@@ -55,7 +55,7 @@ export MAPI_ADDR="http://${MAPI_SVC_ADDR}:5656"
 export KUBERNETES_SERVICE_HOST="127.0.0.1"
 export KUBECONFIG=$HOME/.kube/config
 
-wget -O velero.tar.gz https://github.com/heptio/velero/releases/download/${VELERO_RELEASE}/velero-${VELERO_RELEASE}-linux-amd64.tar.gz
+wget -nv -O velero.tar.gz https://github.com/heptio/velero/releases/download/${VELERO_RELEASE}/velero-${VELERO_RELEASE}-linux-amd64.tar.gz
 mkdir velero
 tar xf velero.tar.gz -C velero
 velero=$PWD/velero/velero-${VELERO_RELEASE}-linux-amd64/velero
@@ -65,11 +65,11 @@ if [ ! -f ${velero} ]; then
 fi
 
 if [ ! -f minio ]; then
-	wget https://dl.min.io/server/minio/release/linux-amd64/minio
+	wget -nv https://dl.min.io/server/minio/release/linux-amd64/minio
 fi
 
 if [ ! -f mc ]; then
-	wget https://dl.min.io/client/mc/release/linux-amd64/mc
+	wget -nv https://dl.min.io/client/mc/release/linux-amd64/mc
 fi
 chmod +x minio
 chmod +x mc
diff --git a/tests/openebs/logs.go b/tests/openebs/logs.go
index 6b06d724..78162b6d 100644
--- a/tests/openebs/logs.go
+++ b/tests/openebs/logs.go
@@ -17,8 +17,6 @@ limitations under the License.
 package openebs
 
 import (
-	"fmt"
-
 	k8s "github.com/openebs/velero-plugin/tests/k8s"
 	corev1 "k8s.io/api/core/v1"
 )
@@ -26,12 +24,14 @@ import (
 const (
 	mayaAPIPodLabel = "openebs.io/component-name=maya-apiserver"
 	cstorPodLabel   = "app=cstor-pool"
+	pvcPodLabel     = "openebs.io/target=cstor-target"
 )
 
 // DumpLogs will dump openebs logs
 func (c *ClientSet) DumpLogs() error {
 	mayaPod := c.getMayaAPIServerPodName()
 	spcPod := c.getSPCPodName()
+	pvcPod := c.getPVCPodName()
 
 	for _, v := range mayaPod {
 		_ = k8s.Client.DumpLogs(OpenEBSNs, v[0], v[1])
@@ -39,6 +39,10 @@ func (c *ClientSet) DumpLogs() error {
 	for _, v := range spcPod {
 		_ = k8s.Client.DumpLogs(OpenEBSNs, v[0], v[1])
 	}
+	for _, v := range pvcPod {
+		_ = k8s.Client.DumpLogs(OpenEBSNs, v[0], v[1])
+	}
+
 	return nil
 }
 
@@ -66,6 +70,18 @@ func (c *ClientSet) getSPCPodName() [][]string {
 	return getPodContainerList(podList)
 }
 
+// getPVCPodName return PVC pod name and container
+// {{"pod1","container1"},{"pod2","container2"},}
+func (c *ClientSet) getPVCPodName() [][]string {
+	podList, err := k8s.Client.GetPodList(OpenEBSNs,
+		pvcPodLabel,
+	)
+	if err != nil {
+		return [][]string{}
+	}
+	return getPodContainerList(podList)
+}
+
 // returns {{"pod1","container1"},{"pod2","container2"},}
 func getPodContainerList(podList *corev1.PodList) [][]string {
 	pod := make([][]string, 0)
@@ -73,7 +89,6 @@ func getPodContainerList(podList *corev1.PodList) [][]string {
 	for _, p := range podList.Items {
 		for _, c := range p.Spec.Containers {
 			pod = append(pod, []string{p.Name, c.Name})
-			fmt.Println(c.Name)
 		}
 	}
 	return pod
diff --git a/tests/openebs/storage_install.go b/tests/openebs/storage_install.go
index 8ee66ad0..a71f3a95 100644
--- a/tests/openebs/storage_install.go
+++ b/tests/openebs/storage_install.go
@@ -42,6 +42,9 @@ var (
 
 	// PVCName for PVC
 	PVCName string
+
+	// AppPVC created by openebs
+	AppPVC *corev1.PersistentVolumeClaim
 )
 
 const (
@@ -105,7 +108,10 @@ func (c *ClientSet) CreateVolume(pvcYAML, pvcNs string, wait bool) error {
 	time.Sleep(5 * time.Second)
 	PVCName = pvc.Name
 	if wait {
-		err = c.waitForHealthyCVR(&pvc)
+		err = c.WaitForHealthyCVR(&pvc)
+	}
+	if err == nil {
+		AppPVC = &pvc
 	}
 	return err
 }
diff --git a/tests/openebs/storage_status.go b/tests/openebs/storage_status.go
index 7198add0..4cc8d42b 100644
--- a/tests/openebs/storage_status.go
+++ b/tests/openebs/storage_status.go
@@ -36,7 +36,8 @@ const (
 	CVRMaxRetry = 5
 )
 
-func (c *ClientSet) waitForHealthyCVR(pvc *v1.PersistentVolumeClaim) error {
+// WaitForHealthyCVR wait till CVR for given PVC becomes healthy
+func (c *ClientSet) WaitForHealthyCVR(pvc *v1.PersistentVolumeClaim) error {
 	dumpLog := 0
 	for {
 		if healthy := c.CheckCVRStatus(pvc.Name,
diff --git a/tests/sanity/backup_test.go b/tests/sanity/backup_test.go
index 0edfe39e..07c61b1e 100644
--- a/tests/sanity/backup_test.go
+++ b/tests/sanity/backup_test.go
@@ -18,6 +18,7 @@ package sanity
 
 import (
 	"testing"
+	"time"
 
 	v1 "github.com/heptio/velero/pkg/apis/velero/v1"
 	. "github.com/onsi/ginkgo"
@@ -72,6 +73,12 @@ var _ = Describe("Backup/Restore Test", func() {
 		It("Backup Test 1", func() {
 			var status v1.BackupPhase
 			By("Creating a backup")
+
+			err = openebs.Client.WaitForHealthyCVR(openebs.AppPVC)
+			Expect(err).NotTo(HaveOccurred())
+			// There are chances that istgt is not updated, but replica is healthy
+			time.Sleep(30 * time.Second)
+
 			backupName, status, err = velero.Client.CreateBackup(AppNs)
 			if ((err != nil) || status != v1.BackupPhaseCompleted) &&
 				len(backupName) != 0 {