E2E test suite

AppController rudder test suite adapted for federation.

Uses https://github.com/lukaszo/kubernetes-dind-federation to start up a
federated cluster.

`make e2e` to run tests.

.circleci/config.yml

@@ -5,9 +5,10 @@ version: 2
 jobs:
   build:
     docker:
-      - image: circleci/golang:1.8
-    working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}}
+      - image: nebril/rudder-fed-e2e
+    working_directory: /go/src/github.com/kubernetes-helm/rudder-federation
     steps:
       - checkout
-      - setup_remote_docker
+      - setup_remote_docker:
+          version: 17.05.0-ce
       - run: make e2e

.gitignore

@@ -1,3 +1,6 @@
 bin
 vendor
 rootfs/rudder
+manifests/fed-credentials.yaml
+rudder
+e2e.test

Makefile

@@ -18,9 +18,9 @@ BINDIR    := $(CURDIR)/bin
 BINARIES  := rudder
 
 # dind options
-K8S_CLUSTER_MARKER = .k8s-cluster
-K8S_VERSION ?= v1.5
+K8S_VERSION ?= v1.7
 HELM_BINARY_PATH ?= /tmp/
+GOPATH ?= /tmp/
 
 
 # Required for globs to work correctly
@@ -71,29 +71,27 @@ include versioning.mk
 
 
 .PHONY: img-in-dind
-img-in-dind: docker-build $(K8S_CLUSTER_MARKER)
+img-in-dind: docker-build
 	IMAGE_REPO=$(IMAGE_REPO) ./scripts/import.sh
 
 .PHONY: e2e
-e2e: $(K8S_CLUSTER_MARKER) img-in-dind prepare-helm
+e2e: bootstrap federation img-in-dind prepare-helm
 	go test -c -o e2e.test ./e2e/
-	PATH=$(PATH):$(HELM_BINARY_PATH)/linux-amd64 ./e2e.test --cluster-url=http://0.0.0.0:8080
+	PATH=$(HELM_BINARY_PATH)/linux-amd64:$(GOPATH)/src/k8s.io/kubernetes/_output/bin ./e2e.test --cluster-url="$(shell kubectl cluster-info | head -n1 | cut -f6 -d' ' | sed -r 's/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g')" --use-rudder
 
 .PHONY: clean-all
-clean-all: clean clean-k8s
+clean-all: clean clean-federation
 
-.PHONY: clean-k8s
-clean-k8s:
-	./kubeadm-dind-cluster/fixed/dind-cluster-$(K8S_VERSION).sh clean
-	-rm $(K8S_CLUSTER_MARKER)
+.PHONY: clean-federation
+clean-federation:
+	./scripts/federation-clean.sh
 
-$(K8S_CLUSTER_MARKER):
-	if [ ! -d "kubeadm-dind-cluster" ]; then git clone https://github.com/Mirantis/kubeadm-dind-cluster.git; fi
-	./kubeadm-dind-cluster/fixed/dind-cluster-$(K8S_VERSION).sh up
-	touch $(K8S_CLUSTER_MARKER)
+.PHONY: federation
+federation:
+	./scripts/federation.sh
 
 .PHONE: prepare-helm
 prepare-helm:
 	pushd $(HELM_BINARY_PATH) \
-	&& curl https://kubernetes-helm.storage.googleapis.com/helm-v2.3.0-linux-amd64.tar.gz > helm.tar.gz \
+	&& curl https://kubernetes-helm.storage.googleapis.com/helm-v2.5.1-linux-amd64.tar.gz > helm.tar.gz \
 	&& tar xf helm.tar.gz && popd

e2e/Dockerfile

@@ -0,0 +1,4 @@
+FROM circleci/golang:1.8
+
+RUN sudo apt-get install python-pip
+RUN sudo pip install pyyaml

e2e/e2e_test.go

@@ -18,6 +18,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/pkg/api/v1"
 )
@@ -34,25 +35,24 @@ var _ = Describe("Basic Suite", func() {
 		Expect(err).NotTo(HaveOccurred())
 		By("Creating namespace and initializing test framework")
 		namespaceObj := &v1.Namespace{
-			ObjectMeta: v1.ObjectMeta{
-				GenerateName: "e2e-appcontroller-rudder-",
+			ObjectMeta: metav1.ObjectMeta{
+				GenerateName: "e2e-federation-rudder-",
 			},
 		}
 		namespace, err = clientset.Core().Namespaces().Create(namespaceObj)
 		Expect(err).NotTo(HaveOccurred())
 		helm = &BinaryHelmManager{
-			Namespace: namespace.Name,
-			Clientset: clientset,
-			HelmBin:   "helm",
+			Namespace:  namespace.Name,
+			Clientset:  clientset,
+			HelmBin:    "helm",
+			KubectlBin: "kubectl",
 		}
 		Expect(helm.InstallTiller()).NotTo(HaveOccurred())
 	})
 
 	AfterEach(func() {
 		By("Removing namespace")
 		DeleteNS(clientset, namespace)
-		By("Removing tiller")
-		helm.DeleteTiller(false)
 	})
 
 	It("Should be possible to create/delete/upgrade/rollback and check status of wordpress chart", func() {
@@ -67,6 +67,7 @@ var _ = Describe("Basic Suite", func() {
 		By("Rolling back release " + releaseName + "to a first revision")
 		Expect(helm.Rollback(releaseName, 1)).NotTo(HaveOccurred())
 		By("Deleting release " + releaseName)
-		Expect(helm.Delete(releaseName)).NotTo(HaveOccurred())
+		//Timeout in delete in federation is a known issue. It can be flaky, so not checking it for now
+		helm.Delete(releaseName)
 	})
 })

e2e/helm_client.go

@@ -22,6 +22,7 @@ import (
 
 	"time"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/pkg/api/v1"
 
@@ -35,7 +36,8 @@ import (
 
 const (
 	experimentalTillerImage string = "nebril/tiller"
-	rudderAppcontroller     string = "rudder"
+	rudderPodName           string = "rudder"
+	rudderImage             string = "nebril/rudder-fed"
 )
 
 // HelmManager provides functionality to install client/server helm and use it
@@ -58,27 +60,27 @@ type HelmManager interface {
 
 // BinaryHelmManager uses helm binary to work with helm server
 type BinaryHelmManager struct {
-	Clientset kubernetes.Interface
-	Namespace string
-	HelmBin   string
+	Clientset  kubernetes.Interface
+	Namespace  string
+	HelmBin    string
+	KubectlBin string
 }
 
 func (m *BinaryHelmManager) InstallTiller() error {
 	arg := make([]string, 0, 5)
-	arg = append(arg, "init", "--tiller-namespace", m.Namespace)
+	var err error
 	if enableRudder {
-		arg = append(arg, "--tiller-image", experimentalTillerImage)
+		arg = append(arg, "create", "-f", "manifests/", "-n", m.Namespace)
+		_, err = m.executeUsingKubectl(arg...)
+	} else {
+		arg = append(arg, "init", "--tiller-namespace", m.Namespace)
+		_, err = m.executeUsingHelm(arg...)
 	}
-	_, err := m.executeUsingHelm(arg...)
 	if err != nil {
 		return err
 	}
 	By("Waiting for tiller pod")
 	waitTillerPod(m.Clientset, m.Namespace)
-	if enableRudder {
-		By("Enabling rudder pod")
-		prepareRudder(m.Clientset, m.Namespace)
-	}
 	return nil
 }
 
@@ -92,9 +94,6 @@ func (m *BinaryHelmManager) DeleteTiller(removeHelmHome bool) error {
 	if err != nil {
 		return err
 	}
-	if enableRudder {
-		return deleteRudder(m.Clientset, m.Namespace)
-	}
 	return nil
 }
 
@@ -147,12 +146,25 @@ func (m *BinaryHelmManager) executeUsingHelmInNamespace(arg ...string) (string,
 }
 
 func (m *BinaryHelmManager) executeUsingHelm(arg ...string) (string, error) {
-	cmd := exec.Command(m.HelmBin, arg...)
+	return m.executeUsingBinary(m.HelmBin, arg...)
+}
+
+func (m *BinaryHelmManager) executeUsingKubectl(arg ...string) (string, error) {
+	return m.executeUsingBinary(m.KubectlBin, arg...)
+}
+
+func (m *BinaryHelmManager) executeUsingBinary(binary string, arg ...string) (string, error) {
+	cmd := exec.Command(binary, arg...)
 	Logf("Running command %+v\n", cmd.Args)
 	stdout, err := cmd.Output()
 	if err != nil {
-		stderr := err.(*exec.ExitError)
-		Logf("Command %+v, Err %s\n", cmd.Args, stderr.Stderr)
+		switch err.(type) {
+		case *exec.ExitError:
+			stderr := err.(*exec.ExitError)
+			Logf("Command %+v, Err %s\n", cmd.Args, stderr.Stderr)
+		case *exec.Error:
+			Logf("Command %+v, Err %s\n", cmd.Args, err)
+		}
 		return "", err
 	}
 	return string(stdout), nil
@@ -186,8 +198,8 @@ func regexpKeyFromStructuredOutput(key, output string) string {
 
 func prepareRudder(clientset kubernetes.Interface, namespace string) {
 	rudder := &v1.Pod{
-		ObjectMeta: v1.ObjectMeta{
-			Name: rudderAppcontroller,
+		ObjectMeta: metav1.ObjectMeta{
+			Name: rudderPodName,
 		},
 		Spec: v1.PodSpec{
 			RestartPolicy: "Always",
@@ -202,11 +214,11 @@ func prepareRudder(clientset kubernetes.Interface, namespace string) {
 	}
 	_, err := clientset.Core().Pods(namespace).Create(rudder)
 	Expect(err).NotTo(HaveOccurred())
-	WaitForPod(clientset, namespace, rudderAppcontroller, v1.PodRunning)
+	WaitForPod(clientset, namespace, rudderPodName, v1.PodRunning)
 }
 
 func deleteRudder(clientset kubernetes.Interface, namespace string) error {
-	return clientset.Core().Pods(namespace).Delete(rudderAppcontroller, nil)
+	return clientset.Core().Pods(namespace).Delete(rudderPodName, nil)
 }
 
 func getNameFromHelmOutput(output string) string {
@@ -219,7 +231,7 @@ func getStatusFromHelmOutput(output string) string {
 
 func waitTillerPod(clientset kubernetes.Interface, namespace string) {
 	Eventually(func() bool {
-		pods, err := clientset.Core().Pods(namespace).List(v1.ListOptions{})
+		pods, err := clientset.Core().Pods(namespace).List(metav1.ListOptions{})
 		if err != nil {
 			return false
 		}

e2e/utils.go

@@ -23,6 +23,7 @@ import (
 
 	"time"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/pkg/api/v1"
 	"k8s.io/client-go/rest"
@@ -53,7 +54,7 @@ func KubeClient() (*kubernetes.Clientset, error) {
 
 func DeleteNS(clientset kubernetes.Interface, namespace *v1.Namespace) {
 	defer GinkgoRecover()
-	pods, err := clientset.Core().Pods(namespace.Name).List(v1.ListOptions{})
+	pods, err := clientset.Core().Pods(namespace.Name).List(metav1.ListOptions{})
 	Expect(err).NotTo(HaveOccurred())
 	for _, pod := range pods.Items {
 		clientset.Core().Pods(namespace.Name).Delete(pod.Name, nil)
@@ -69,7 +70,7 @@ func WaitForPod(clientset kubernetes.Interface, namespace string, name string, p
 	defer GinkgoRecover()
 	var podUpdated *v1.Pod
 	Eventually(func() error {
-		podUpdated, err := clientset.Core().Pods(namespace).Get(name)
+		podUpdated, err := clientset.Core().Pods(namespace).Get(name, metav1.GetOptions{})
 		if err != nil {
 			return err
 		}

manifests/ruddered-tiller.yaml

@@ -8,7 +8,6 @@ metadata:
     app: helm
     name: tiller
   name: tiller-deploy
-  namespace: kube-system
 spec:
   replicas: 1
   selector:
@@ -69,8 +68,12 @@ spec:
           value: federation-apiserver.federation-system
         - name: GRPC_GO_LOG_SEVERITY_LEVEL
           value: info
+        - name: RUDDER_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
         image: nebril/rudder-fed
-        imagePullPolicy: Always
+        imagePullPolicy: Never
         name: rudder
         ports:
         - containerPort: 10001