chore(testing): update readme w.r.t programmatic vs declarative testing (#188)

This commit makes changes to clearly indicate programmatic as well
declarative way to test D-operators' Kubernetes controllers.

It is important to note that the same model can be followed by other
Operators repo if needed. This model has worked fine so far. It makes
use of GitHub VM as launching machine & same VM used as a single node
K3s cluster to run the tests (both declarative as well as programmatic).

Signed-off-by: AmitKumarDas <[email protected]>

Author: Amit Kumar Das

.github/workflows/test-release.yaml

@@ -1,24 +1,21 @@
 ---
-name: Unit Test, E to E Test and Github Release (if on master)
+name: Testing and Github Release (if on master)
 on: [push, pull_request]
 jobs:
   unittest:
     runs-on: ubuntu-18.04
-    strategy:
-      matrix:
-        test: ['test']
-    name: ${{ matrix.test }}
+    name: Unit Test
     steps:
       - name: Checkout Code
         uses: actions/checkout@v2
       - name: Setup Golang
         uses: actions/setup-go@v2
         with:
           go-version: 1.13.5
-      - run: make ${{ matrix.test }}
+      - run: make test
   integrationtest:
     runs-on: ubuntu-18.04
-    name: integrationtest
+    name: Integration Test
     steps:
       - name: Checkout Code
         uses: actions/checkout@v2
@@ -27,24 +24,21 @@ jobs:
         with:
           go-version: 1.13.5
       - run: sudo make integration-test-suite
-  e2etest:
+  declarativetest:
     runs-on: ubuntu-18.04
-    strategy:
-      matrix:
-        test: ['e2e-test']
-    name: ${{ matrix.test }}
+    name: Declarative Test
     steps:
       - name: Checkout Code
         uses: actions/checkout@v2
       - name: Setup Golang
         uses: actions/setup-go@v2
         with:
           go-version: 1.13.5
-      - run: sudo make ${{ matrix.test }}
+      - run: sudo make declarative-test-suite
   release:
     name: Make Github Release
     runs-on: ubuntu-18.04
-    needs: ['unittest', 'e2etest', 'integrationtest']
+    needs: ['unittest', 'declarativetest', 'integrationtest']
     steps:
       - name: Checkout Code
         uses: actions/checkout@v1

.gitignore

@@ -3,7 +3,7 @@ tools/d-action/vendor/
 d-operators
 test/bin/
 test/kubebin/
-test/e2e/uninstall-k3s.txt
+test/declarative/uninstall-k3s.txt
 test/integration/uninstall-k3s.txt
 uninstall-k3s.txt
 dope

Makefile

@@ -40,9 +40,9 @@ integration-test:
 	# @go test ./... -cover --tags=integration -v -args --logtostderr -v=1
 	@go test ./... -cover --tags=integration
 
-.PHONY: e2e-test
-e2e-test: 
-	@cd test/e2e && ./suite.sh
+.PHONY: declarative-test-suite
+declarative-test-suite: 
+	@cd test/declarative && ./suite.sh
 
 .PHONY: integration-test-suite
 integration-test-suite:

README.md

@@ -54,34 +54,40 @@ It is important to understand that these declarative patterns are built upon pro
 ### When to use D-operators
 D-operators is not meant to build complex controller logic like Deployment, StatefulSet or Pod in a declarative yaml. However, if one needs to use available Kubernetes resources to build new k8s controller(s) then d-operators should be considered to build one. D-operators helps implement the last mile automation needed to manage applications & infrastructure in Kubernetes clusters.
 
-### E to E testing
+### Declarative Testing
 D-operators make use of its custom resource(s) to test its controllers. One can imagine these custom resources acting as the building blocks to implement a custom CI framework. One of the primary advantages with this approach, is to let custom resources remove the need to write code to implement test cases.
 
-_NOTE: One can make use of these YAMLs (kind: Recipe) to test their own Kubernetes controllers declaratively_
+_NOTE: One can make use of these YAMLs (kind: Recipe) to test any Kubernetes controllers declaratively_
 
-Navigate to test/experiments to learn more on these YAMLs.
+Navigate to test/declarative/experiments to learn more on these YAMLs.
 
 ```sh
-# Following runs the e2e test suite
+# Following runs the declarative test suite
 #
-# NOTE: test/e2e/suite.sh does the following:
+# NOTE: test/declarative/suite.sh does the following:
 # - d-operators' image known as 'dope' is built
 # - a docker container is started & acts as the image registry
 # - dope image is pushed to this image registry
 # - k3s is installed with above image registry
 # - d-operators' manifests are applied
 # - experiments _(i.e. test code written as YAMLs)_ are applied
 # - experiments are asserted
-# - if all experiments pass then e2e is a success else it failed
+# - if all experiments pass then this testing is a success else it failed
 # - k3s is un-installed
 # - local image registry is stopped
-sudo make e2e-test
+sudo make declarative-test-suite
 ```
 
-### Deploying experiments using D-operators
-To deploy experiments using d-operators apply the below commands:
-- kubectl apply -f deploy/
-- kubectl apply -f test/experiments/<experiment-yaml>
+### Programmatic Testing
+D-operators also lets one to write testing Kubernetes controllers using Golang. This involves building the docker image (refer Dockerfile.testing) of the entire codebase and letting it run as a Kubernetes pod (refer test/integration/it.yaml). The setup required run these tests can be found at test/integration folder. Actual test logic are regular _test.go files found in respective packages. These _test.go files need to be tagged appropriately. These mode of testing has the additional advantage of getting the code coverage.
+
+```go
+// +build integration
+```
+
+```sh
+make integration-test-suite
+```
 
 ### Available Kubernetes controllers
 - [x] kind: Recipe

pkg/recipe/list_int_items_test.go

@@ -52,6 +52,9 @@ func TestListItemsRun(t *testing.T) {
 						"metadata": map[string]interface{}{
 							"name":      "cm-one",
 							"namespace": "list-items-integration-testing",
+							"labels": map[string]interface{}{
+								"ns": "list-items-integration-testing",
+							},
 						},
 					},
 				},
@@ -67,6 +70,9 @@ func TestListItemsRun(t *testing.T) {
 						"metadata": map[string]interface{}{
 							"name":      "cm-two",
 							"namespace": "list-items-integration-testing",
+							"labels": map[string]interface{}{
+								"ns": "list-items-integration-testing",
+							},
 						},
 					},
 				},
@@ -81,6 +87,9 @@ func TestListItemsRun(t *testing.T) {
 						"kind":       "ConfigMap",
 						"metadata": map[string]interface{}{
 							"namespace": "list-items-integration-testing",
+							"labels": map[string]interface{}{
+								"ns": "list-items-integration-testing",
+							},
 						},
 					},
 				},
@@ -135,6 +144,9 @@ func TestListItemsRun(t *testing.T) {
 					"kind":       "ConfigMap",
 					"metadata": map[string]interface{}{
 						"namespace": "list-items-integration-testing",
+						"labels": map[string]interface{}{
+							"ns": "list-items-integration-testing",
+						},
 					},
 				},
 			},

pkg/recipe/recipe_int_list_test.go

@@ -52,6 +52,9 @@ func TestListSimpleRun(t *testing.T) {
 						"metadata": map[string]interface{}{
 							"name":      "cm-one",
 							"namespace": "list-simple-integration-testing",
+							"labels": map[string]interface{}{
+								"ns": "list-simple-integration-testing",
+							},
 						},
 					},
 				},
@@ -67,6 +70,9 @@ func TestListSimpleRun(t *testing.T) {
 						"metadata": map[string]interface{}{
 							"name":      "cm-two",
 							"namespace": "list-simple-integration-testing",
+							"labels": map[string]interface{}{
+								"ns": "list-simple-integration-testing",
+							},
 						},
 					},
 				},
@@ -110,6 +116,9 @@ func TestListSimpleRun(t *testing.T) {
 						"kind":       "ConfigMap",
 						"metadata": map[string]interface{}{
 							"namespace": "list-simple-integration-testing",
+							"labels": map[string]interface{}{
+								"ns": "list-simple-integration-testing",
+							},
 						},
 					},
 				},

test/e2e/BEST_PRACTICES.md test/declarative/BEST_PRACTICES.md

@@ -94,7 +94,7 @@ echo -e "\n Apply d-operators based ci to K3s cluster"
 k3s kubectl apply -f ci.yaml
 
 echo -e "\n Apply test experiments to K3s cluster"
-k3s kubectl apply -f ./../experiments/
+k3s kubectl apply -f ./experiments/
 
 echo -e "\n Apply ci inference to K3s cluster"
 k3s kubectl apply -f inference.yaml