feat(http): add http as a custom resource (#107)

This commit adds HTTP as a custom resource. This can currently
be used to invoke http requests that do not need any
authentication. Few e2e experiments added to verify HTTP custom
resource.

This change also includes a minor bug fix w.r.t StateCheck assert
action. Before this fix any timeout error resulted in Recipe's
status.phase being set as 'Error'. With current changes any timeout
error will result in Recipe's status.phase as 'Failed'.

In addition, ci related artifacts including experiments & yaml
comments have been updated to make the ci run faster as well as
make it more readable for contributors.

Signed-off-by: AmitKumarDas <[email protected]>

Author: Amit Kumar Das

.gitignore

@@ -3,3 +3,5 @@ d-operators
 test/bin/
 test/kubebin/
 test/e2e/uninstall-k3s.txt
+uninstall-k3s.txt
+dope

Makefile

@@ -1,13 +1,8 @@
-PWD := ${CURDIR}
-
-OS = $(shell uname)
-
-PACKAGE_VERSION ?= $(shell git describe --always --tags)
-GIT_TAGS = $(shell git fetch --all --tags)
+# Fetch the latest tags & then set the package version
+PACKAGE_VERSION ?= $(shell git fetch --all --tags | echo "" | git describe --always --tags)
 ALL_SRC = $(shell find . -name "*.go" | grep -v -e "vendor")
 
 # We are using docker hub as the default registry
-#IMG_REGISTRY ?= quay.io
 IMG_NAME ?= dope
 IMG_REPO ?= mayadataio/dope
 
@@ -22,8 +17,6 @@ $(IMG_NAME): $(ALL_SRC)
 
 $(ALL_SRC): ;
 
-$(GIT_TAGS): ;
-
 # go mod download modules to local cache
 # make vendored copy of dependencies
 # install other go binaries for code generation
@@ -46,9 +39,9 @@ e2e-test:
 	@cd test/e2e && ./suite.sh
 
 .PHONY: image
-image: $(GIT_TAGS)
+image:
 	docker build -t $(IMG_REPO):$(PACKAGE_VERSION) .
 
 .PHONY: push
 push: image
-	docker push $(IMG_REPO):$(PACKAGE_VERSION
+	docker push $(IMG_REPO):$(PACKAGE_VERSION)

config/metac.yaml

@@ -1,14 +1,30 @@
-apiVersion: metac.openebs.io/v1alpha1
+apiVersion: dope/v1
 kind: GenericController
 metadata:
   name: sync-recipe
   namespace: dope
 spec:
-  watch: # kind: Recipe custom resource is watched
+  watch: 
+    # kind: Recipe custom resource is watched
     apiVersion: dope.metacontroller.io/v1
     resource: recipes
   hooks:
     sync:
       inline:
         funcName: sync/recipe
+---
+apiVersion: dope/v1
+kind: GenericController
+metadata:
+  name: sync-http
+  namespace: dope
+spec:
+  watch: 
+    # kind: HTTP custom resource is watched
+    apiVersion: dope.metacontroller.io/v1
+    resource: https
+  hooks:
+    sync:
+      inline:
+        funcName: sync/http
 ---

controller/http/reconciler.go

@@ -74,17 +74,24 @@ func (r *Reconciler) evalObservedHTTP() {
 	}
 	r.observedHTTP = &http
 
-	// validate presence of secret
+	// extract secret name if set
 	r.observedSecretName = http.Spec.SecretName
-	if r.observedSecretName == "" {
-		r.Err = errors.Errorf("Missing spec.secretName")
-		return
-	}
+
+	// validate presence of secret
+	// if r.observedSecretName == "" {
+	// 	r.Err = errors.Errorf("Missing spec.secretName")
+	// 	return
+	// }
 }
 
 // evalObservedSecret parses the relevant secret found in Kubernetes
 // cluster. This secret is used to authenticate the request invocation.
 func (r *Reconciler) evalObservedSecret() {
+	if r.observedSecretName == "" {
+		// absence of secret name implies http invocation does not
+		// require authentication
+		return
+	}
 	r.observedSecret = r.HookRequest.Attachments.FindByGroupKindName(
 		"v1",
 		"Secret",
@@ -199,7 +206,7 @@ func (r *Reconciler) handleRuntimeError() {
 // NOTE:
 //	Status forms the core business logic of reconciling a HTTP
 // custom resource.
-func (r *Reconciler) updateWatchStatus() {
+func (r *Reconciler) updateHTTPStatus() {
 	// check for runtime errors
 	if r.Err != nil {
 		r.handleRuntimeError()
@@ -277,7 +284,7 @@ func Sync(request *generic.SyncHookRequest, response *generic.SyncHookResponse)
 	// NOTE:
 	//	HTTP custom resource is the watch here
 	r.DesiredWatchFns = []func(){
-		r.updateWatchStatus,
+		r.updateHTTPStatus,
 	}
 
 	// run reconcile

controller/recipe/reconciler.go

@@ -74,7 +74,7 @@ func (r *Reconciler) setSyncResponse() {
 	}
 }
 
-func (r *Reconciler) setWatchStatusAsError() {
+func (r *Reconciler) setRecipeStatusAsError() {
 	r.HookResponse.Status = map[string]interface{}{
 		"phase":  "Error",
 		"reason": r.Err.Error(),
@@ -84,7 +84,7 @@ func (r *Reconciler) setWatchStatusAsError() {
 	}
 }
 
-func (r *Reconciler) setWatchStatusFromRecipeStatus() {
+func (r *Reconciler) setRecipeStatus() {
 	r.HookResponse.Status = map[string]interface{}{
 		"phase":           string(r.RecipeStatus.Phase),
 		"reason":          r.RecipeStatus.Reason,
@@ -110,15 +110,15 @@ func (r *Reconciler) setWatchStatus() {
 			r.HookResponse.ResyncAfterSeconds =
 				*r.ObservedRecipe.Spec.Refresh.OnErrorResyncAfterSeconds
 		}
-		r.setWatchStatusAsError()
+		r.setRecipeStatusAsError()
 		return
 	}
 	if r.RecipeStatus.Phase == types.RecipeStatusLocked {
 		// nothing needs to be done
 		// old status will persist
 		return
 	}
-	r.setWatchStatusFromRecipeStatus()
+	r.setRecipeStatus()
 }
 
 // Sync implements the idempotent logic to sync Recipe resource

manifests/crd.yaml

@@ -18,6 +18,29 @@ spec:
     status: {}
   version: v1
   versions:
+  - name: v1
+    served: true
+    storage: true
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+  name: https.dope.metacontroller.io
+spec:
+  group: dope.metacontroller.io
+  names:
+    kind: HTTP
+    listKind: HTTPList
+    plural: https
+    shortNames:
+    - http
+    singular: http
+  scope: Namespaced
+  subresources:
+    status: {}
+  version: v1
+  versions:
   - name: v1
     served: true
     storage: true

pkg/http/http.go

@@ -80,16 +80,20 @@ func (i *Invocable) buildStatus(response *resty.Response) types.HTTPResponse {
 // Invoke executes the http request
 func (i *Invocable) Invoke() (types.HTTPResponse, error) {
 	req := resty.New().R().
-		SetBasicAuth(i.Username, i.Password).
 		SetBody(i.Body).
 		SetHeaders(i.Headers).
 		SetQueryParams(i.QueryParams).
 		SetPathParams(i.PathParams)
 
+	// set credentials only if it was provided
+	if i.Username != "" || i.Password != "" {
+		req.SetBasicAuth(i.Username, i.Password)
+	}
+
 	var response *resty.Response
 	var err error
 
-	switch strings.ToLower(i.HTTPMethod) {
+	switch strings.ToUpper(i.HTTPMethod) {
 	case types.POST:
 		response, err = req.Post(i.URL)
 	case types.GET:

pkg/recipe/recipe.go

@@ -212,7 +212,7 @@ func (r *Runner) buildLockRunner() *LockRunner {
 }
 
 // evalAll evaluates all tasks
-func (r *Runner) evalAll() error {
+func (r *Runner) evalAllTasks() error {
 	for _, task := range r.Recipe.Spec.Tasks {
 		err := r.eval(task)
 		if err != nil {
@@ -257,7 +257,7 @@ func (r *Runner) addRecipeElapsedTimeInSeconds(elapsedtime float64) {
 }
 
 // runAll runs all the tasks
-func (r *Runner) runAll() (status *types.RecipeStatus, err error) {
+func (r *Runner) runAllTasks() (status *types.RecipeStatus, err error) {
 	defer func() {
 		r.fixture.TearDown()
 	}()
@@ -280,6 +280,9 @@ func (r *Runner) runAll() (status *types.RecipeStatus, err error) {
 		}
 		got, err := tr.Run()
 		if err != nil {
+			// We discontinue executing next tasks
+			// if current task execution resulted in
+			// error
 			return nil, errors.Wrapf(
 				err,
 				"Failed to run task [%d] %q",
@@ -289,6 +292,8 @@ func (r *Runner) runAll() (status *types.RecipeStatus, err error) {
 		}
 		r.RecipeStatus.TaskListStatus[task.Name] = got
 		if got.Phase == types.TaskStatusFailed {
+			// We run subsequent tasks even if current task
+			// failed
 			failedTasks++
 		}
 	}
@@ -360,7 +365,7 @@ func (r *Runner) Run() (status *types.RecipeStatus, err error) {
 	}()
 	r.RecipeStatus.TaskListStatus[r.Recipe.GetName()+"-lock"] = lockstatus
 
-	err = r.evalAll()
+	err = r.evalAllTasks()
 	if err != nil {
 		return nil, err
 	}
@@ -380,5 +385,5 @@ func (r *Runner) Run() (status *types.RecipeStatus, err error) {
 		}, nil
 	}
 
-	return r.runAll()
+	return r.runAllTasks()
 }

pkg/recipe/recipe_test.go

@@ -322,8 +322,8 @@ func TestRunnerRunAllTasks(t *testing.T) {
 				},
 				fixture: f,
 			}
-			r.initEnabled()        // init to avoid nil pointers
-			got, err := r.runAll() // method under test
+			r.initEnabled()             // init to avoid nil pointers
+			got, err := r.runAllTasks() // method under test
 			if mock.isErr && err == nil {
 				t.Fatal("Expected error got none")
 			}

pkg/recipe/state_check.go

@@ -184,8 +184,13 @@ func (sc *StateChecking) assertEquals() {
 	sc.retryOnDiff = true
 	success, err := sc.isMergeEqualsObserved(message)
 	if err != nil {
-		sc.err = err
-		return
+		// verify if this was a timeout error
+		if _, ok := err.(*RetryTimeout); !ok {
+			sc.err = err
+			return
+		}
+		// set the timeout error against corresponding status field
+		sc.result.Timeout = err.Error()
 	}
 	// init phase as failed
 	sc.result.Phase = types.StateCheckResultFailed
@@ -209,8 +214,14 @@ func (sc *StateChecking) assertNotEquals() {
 	sc.retryOnEqual = true
 	success, err := sc.isMergeEqualsObserved(message)
 	if err != nil {
-		sc.err = err
-		return
+		// verify if this is a timeout error
+		if _, ok := err.(*RetryTimeout); !ok {
+			// this is a runtime error
+			sc.err = err
+			return
+		}
+		// set timeout error against corresponding status field
+		sc.result.Timeout = err.Error()
 	}
 	// init phase as failed
 	sc.result.Phase = types.StateCheckResultFailed
@@ -251,21 +262,22 @@ func (sc *StateChecking) assertNotFound() {
 				if apierrors.IsNotFound(err) {
 					// phase is set to Passed here
 					phase = types.StateCheckResultPassed
-					// Stop retrying
+					// Stop retrying since resource is not found in the cluster
 					return true, nil
 				}
-				// Keep retrying
+				// Keep retrying since get call errored out
 				return false, err
 			}
 			if len(got.GetFinalizers()) == 0 && got.GetDeletionTimestamp() != nil {
 				phase = types.StateCheckResultWarning
 				warning = fmt.Sprintf(
-					"Marking StateCheck %q to passed: Finalizer count %d: Deletion timestamp %s",
+					"Marking StateCheck %q to passed: Finalizer count %d: Deletion timestamp %q",
 					sc.TaskName,
 					len(got.GetFinalizers()),
 					got.GetDeletionTimestamp(),
 				)
-				// Stop retrying
+				// Stop retrying since Kubernetes has marked the resource
+				// to be deleted
 				return true, nil
 			}
 			// Keep retrying
@@ -274,10 +286,13 @@ func (sc *StateChecking) assertNotFound() {
 		message,
 	)
 	if err != nil {
+		// verify if this is a timeout error
 		if _, ok := err.(*RetryTimeout); !ok {
+			// this is a runtime error
 			sc.err = err
 			return
 		}
+		// set timeout error against corresponding status field
 		sc.result.Timeout = err.Error()
 	}
 	sc.result.Phase = phase
@@ -335,15 +350,18 @@ func (sc *StateChecking) assertListCountEquals() {
 		message,
 	)
 	if err != nil {
+		// verify if this is a timeout error
 		if _, ok := err.(*RetryTimeout); !ok {
+			// this is a runtime error
 			sc.err = err
 			return
 		}
+		// set timeout error against corresponding status field
 		sc.result.Timeout = err.Error()
 	}
 	sc.result.Phase = phase
 	sc.result.Message = message
-	sc.result.Warning = fmt.Sprintf(
+	sc.result.Verbose = fmt.Sprintf(
 		"Expected count %d got %d",
 		*sc.StateCheck.Count,
 		sc.actualListCount,
@@ -377,15 +395,18 @@ func (sc *StateChecking) assertListCountNotEquals() {
 		message,
 	)
 	if err != nil {
+		// verify if this this a timeout error
 		if _, ok := err.(*RetryTimeout); !ok {
+			// this is a runtime error
 			sc.err = err
 			return
 		}
+		// set timeout error against corresponding status field
 		sc.result.Timeout = err.Error()
 	}
 	sc.result.Phase = phase
 	sc.result.Message = message
-	sc.result.Warning = fmt.Sprintf(
+	sc.result.Verbose = fmt.Sprintf(
 		"Expected count %d got %d",
 		*sc.StateCheck.Count,
 		sc.actualListCount,