Preliminary support for @SUMMONDOCKERARGS

Author: doodlesbykumbi

Dockerfile

@@ -9,6 +9,7 @@ COPY go.mod go.sum ./
 
 RUN apk add --no-cache bash \
                        build-base \
+                       docker-cli \
                        git && \
     go mod download && \
     go get -u github.com/jstemmer/go-junit-report && \

Dockerfile.acceptance

@@ -5,7 +5,8 @@ RUN apk add --no-cache bash \
                        git \
                        libffi-dev \
                        ruby-bundler \
-                       ruby-dev
+                       ruby-dev \
+                       docker-cli
 
 # Install summon prerequisites
 WORKDIR /summon

docs/_includes/docker.md

@@ -16,8 +16,66 @@ Since Summon has pluggable providers, you aren't locked into any one solution fo
 managing your secrets.
 
 Summon makes it easy to inject secrets as environment variables into your Docker
-containers by taking advantage of Docker's `--env-file` argument. This is done
-on-demand by using the variable `@SUMMONENVFILE` in the arguments of the process
+containers by taking advantage of Docker's CLI arguments (`--env-file` or, `--env` and `--volume`. There are two options available. It's possible to mix and match as you see fit.
+
+## --env and --volume arguments
+This is done on-demand by using the variable `@SUMMONDOCKERARGS` in the arguments of the process
+you are running with Summon. This variable is replaced by combinations of the Docker arguments `--env` and `--volume` such that the secrets injected by summon are passed into the Docker container. The `--volume` arguments allow memory-mapped temporary files from variables with the `!file` tag to be resolvable inside the container.
+
+**NOTE:** Using the `!file` tag with `@SUMMONDOCKERARGS` assumes that the Docker CLI is being run on the host that is used to create volume mounts to the container. For when this is not the case simply avoid using the `!file` tag, but be mindful that in that case you lose the benefits of memory-mapped temporary files.
+
+```bash
+$ summon -p keyring.py -D env=dev docker run @SUMMONDOCKERARGS deployer
+Checking credentials
+Deploying application
+```
+
+### Example
+The example below demonstrates the use @SUMMONDOCKERARGS. For the sake of brevity
+we use an inline `secrets.yml` and the `/bin/echo` provider. Some points to note:
+1. `summon` is
+invoking docker as the child process.
+2. `@SUMMONDOCKERARGS` is replaced with a combination of `--env` and `--volume`
+arguments.
+3. Variable `D` uses the `!file` tag and therefore is the only one that
+results in a `--volume` argument. The path to this variable inside the container
+is as it is on the host.
+
+```bash
+secretsyml='
+A: |-
+  A_value with
+  multiple lines
+B: B_value
+C: !var C_value
+D: !var:file D_value
+'
+
+# The substitution of @SUMMONDOCKERARGS the docker run command below results in
+# something of the form:
+#
+# docker run --rm \
+#  --env A --env B --env C --env D \
+#  --volume /path/to/D:/path/to/D
+#  alpine ...
+#
+# The output from the command is shown below the command.
+
+summon --provider /bin/echo --yaml "${secretsyml}" \
+ docker run --rm @SUMMONDOCKERARGS alpine sh -c '
+printenv A;
+printenv B;
+printenv C;
+cat $(printenv D);
+'
+# A_value with
+# multiple lines
+# B_value
+# C_value
+# D_value
+```
+## --env-file argument
+This is done on-demand by using the variable `@SUMMONENVFILE` in the arguments of the process
 you are running with Summon. This variable points to a memory-mapped file containing
 the variables and values from secrets.yml in VAR=VAL format.
 
@@ -27,7 +85,7 @@ Checking credentials
 Deploying application
 ```
 
-## Example
+### Example
 
 Let's say we have a deploy script that needs to access our application servers on
 AWS and pull the latest version of our code. It should record the outcome of the

go.mod

@@ -2,6 +2,9 @@ module github.com/cyberark/summon
 
 require (
 	github.com/codegangsta/cli v1.20.0
+	github.com/docker/docker v20.10.2+incompatible
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
 	github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect
 	github.com/jtolds/gls v4.20.0+incompatible // indirect
 	github.com/kr/pretty v0.1.0 // indirect

go.sum

@@ -2,6 +2,13 @@ github.com/codegangsta/cli v1.20.0 h1:iX1FXEgwzd5+XN6wk5cVHOGQj6Q3Dcp20lUeS4lHNT
 github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
+github.com/docker/docker v20.10.2+incompatible h1:vFgEHPqWBTp4pTjdLwjAA4bSo3gvIGOYwuJTlEjVBCw=
+github.com/docker/docker v20.10.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
+github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=

internal/command/action.go

@@ -3,6 +3,7 @@ package command
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -11,13 +12,17 @@ import (
 	"syscall"
 
 	"github.com/codegangsta/cli"
+
 	prov "github.com/cyberark/summon/provider"
 	"github.com/cyberark/summon/secretsyml"
 )
 
 // ActionConfig is an object that holds all the info needed to run
 // a Summon instance
 type ActionConfig struct {
+	StdIn                io.Reader
+	StdOut               io.Writer
+	StdErr               io.Writer
 	Args                 []string
 	Provider             string
 	Filepath             string
@@ -31,6 +36,7 @@ type ActionConfig struct {
 }
 
 const ENV_FILE_MAGIC = "@SUMMONENVFILE"
+const DOCKER_ARGS_MAGIC = "@SUMMONDOCKERARGS"
 const SUMMON_ENV_KEY_NAME = "SUMMON_ENV"
 
 // Action is the runner for the main program logic
@@ -122,6 +128,9 @@ func runAction(ac *ActionConfig) error {
 	results := make(chan Result, len(secrets))
 	var wg sync.WaitGroup
 
+	var dockerArgs []string
+	var dockerArgsMutex sync.Mutex
+
 	for key, spec := range secrets {
 		wg.Add(1)
 		go func(key string, spec secretsyml.SecretSpec) {
@@ -144,6 +153,15 @@ func runAction(ac *ActionConfig) error {
 			}
 
 			k, v := formatForEnv(key, value, spec, &tempFactory)
+
+			// Generate @SUMMONDOCKERARGS
+			dockerArgsMutex.Lock()
+			defer dockerArgsMutex.Unlock()
+			if spec.IsFile() {
+				dockerArgs = append(dockerArgs, "--volume", v+":"+v)
+			}
+			dockerArgs = append(dockerArgs, "--env", k)
+
 			results <- Result{k, v, nil}
 			wg.Done()
 		}(key, spec)
@@ -176,12 +194,52 @@ EnvLoop:
 
 	setupEnvFile(ac.Args, env, &tempFactory)
 
+	// Setup Docker args
+	var argsWithDockerArgs []string
+	for _, arg := range ac.Args {
+		if arg == DOCKER_ARGS_MAGIC {
+			// Replace argument with slice of docker options
+			argsWithDockerArgs = append(argsWithDockerArgs, dockerArgs...)
+			continue
+		}
+
+		// TODO: we need to decide which of these if we want to support (2)
+		// 1. summon [...] docker run @SUMMONDOCKERARGS [...], replace only entire top-level arg. The
+		// top-level arg is replaced by is replaced by N>0 args equating to @SUMMONDOCKERARGS.
+		// 2. summon ... sh -c "docker run @SUMMONDOCKERARGS [...]", also replace substrings
+		// inside args but the replacement is as a single string.
+		//
+		// The code below should support (2). There'll be some ambiguity though...
+		// e.g. summon ... echo "@SUMMONDOCKERARGS" will fall under both (1) and (2), though (1)
+		// takes precedence. I'm not sure if the behaviors of (1) and (2) are equivalent
+		// when there's such ambiguity.
+		//
+		//idx := strings.Index(arg, DOCKER_ARGS_MAGIC)
+		//if idx >= 0 {
+		//	// Replace argument with slice of docker options
+		//	argsWithDockerArgs = append(
+		//		argsWithDockerArgs,
+		//		strings.Replace(arg, DOCKER_ARGS_MAGIC, strings.Join(dockerArgs, " "), -1),
+		//	)
+		//	continue
+		//}
+
+		argsWithDockerArgs = append(argsWithDockerArgs, arg)
+	}
+	ac.Args = argsWithDockerArgs
+
 	var e []string
 	for k, v := range env {
 		e = append(e, fmt.Sprintf("%s=%s", k, v))
 	}
 
-	return runSubcommand(ac.Args, append(os.Environ(), e...))
+	return runSubcommand(
+		ac.Args,
+		append(os.Environ(), e...),
+		ac.StdIn,
+		ac.StdOut,
+		ac.StdErr,
+	)
 }
 
 // formatForEnv returns a string in %k=%v format, where %k=namespace of the secret and
@@ -240,7 +298,7 @@ func findInParentTree(secretsFile string, leafDir string) (string, error) {
 	}
 }
 
-// scans arguments for the magic string; if found,
+// scans arguments for the envfile magic string; if found,
 // creates a tempfile to which all the environment mappings are dumped
 // and replaces the magic string with its path.
 // Returns the path if so, returns an empty string otherwise.

internal/command/action_test.go

@@ -1,19 +1,27 @@
 package command
 
 import (
+	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
-	"github.com/cyberark/summon/secretsyml"
+	"github.com/docker/docker/api/types/container"
 	. "github.com/smartystreets/goconvey/convey"
 	_ "golang.org/x/net/context"
+
+	"github.com/cyberark/summon/secretsyml"
 )
 
 func TestConvertSubsToMap(t *testing.T) {
@@ -120,6 +128,127 @@ func TestRunAction(t *testing.T) {
 
 		So(string(content), ShouldEqual, expectedValue)
 	})
+
+	Convey("Docker options correctly injected", t, func() {
+		// This is a test case for @SUMMONDOCKERARGS. It exercises Docker CLI pointed to a mock
+		// server. It asserts on the request payload received on the container creation
+		// endpoint, the volume mounts and environment variables injected by summon are
+		// expected to be present.
+
+		expected := map[string]string{
+			"A": "A's multiple line\nvalue",
+			"B": "B_value",
+			"C": "C_value",
+			"D": "D_value",
+		}
+		volumeBinds := map[string]struct{
+			ContainerPath string
+			FileContents string
+		}{}
+		envvars := map[string]string{}
+
+		// Mock server for handling API calls by `docker run`
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			var payload struct {
+				*container.Config
+				HostConfig       *container.HostConfig
+			}
+
+			if !regexp.MustCompile("/.*/containers/create").MatchString(r.URL.Path) {
+				// Mock response to all the other endpoints called as part of `docker run`
+				w.WriteHeader(200)
+				fmt.Fprintln(w,  "{}")
+				return
+			}
+			payloadBytes, err := ioutil.ReadAll(r.Body)
+
+			if err != nil {
+				t.Errorf("failure reading payload from docker cli: %s", err)
+				return
+			}
+			err = json.Unmarshal(payloadBytes, &payload)
+			if err != nil {
+				t.Errorf("payload from docker cli could not be parsed: %s", err)
+				return
+			}
+
+			for _, env := range payload.Env {
+				nameAndValue := strings.SplitN(env, "=", 2)
+				name := nameAndValue[0]
+				value := nameAndValue[1]
+
+				envvars[name] = value
+			}
+
+			for _, volumeBind := range payload.HostConfig.Binds {
+				fromAndTo := strings.SplitN(volumeBind, ":", 2)
+				from := fromAndTo[0]
+				to := fromAndTo[1]
+
+				fileContents, _ := ioutil.ReadFile(from)
+				volumeBinds[from] = struct {
+					ContainerPath string
+					FileContents  string
+				}{
+					ContainerPath: to,
+					FileContents: string(fileContents),
+				}
+			}
+
+			w.WriteHeader(201)
+
+			// Mock response to container create endpoint
+			fmt.Fprintln(w,  `{"Id": "e90e34656806", "Warnings": []}`)
+		}))
+		defer ts.Close()
+
+		var buff bytes.Buffer
+		var dockerCommand = []string{
+			"docker",
+			"-H", strings.Replace(ts.URL, "http://", "tcp://", 1),
+			"run",
+			"--rm", "-d", "@SUMMONDOCKERARGS",
+			"alpine",
+		}
+		err := runAction(&ActionConfig{
+			Provider: "/bin/echo",
+			StdOut:   &buff,
+			Args: dockerCommand,
+			YamlInline: `
+A: |-
+ A's multiple line
+ value
+B: !var B_value
+C: !file C_value
+D: !var:file D_value
+`,
+		})
+
+		code, err := returnStatusOfError(err)
+		So(err, ShouldBeNil)
+		So(code, ShouldEqual, 0)
+
+		if err != nil || code != 0 {
+			return
+		}
+
+		So(err, ShouldBeNil)
+		if err != nil {
+			return
+		}
+
+		for from, volumeBind := range volumeBinds {
+			// The volume mount are expected to take the form
+			// 'host_path:container_path', where host_path is equal to container_path
+			So(from, ShouldEqual, volumeBind.ContainerPath)
+		}
+
+		// Ensure envvars and volumemounts passed to Docker match expectations
+		So(envvars["A"], ShouldEqual, expected["A"])
+		So(envvars["B"], ShouldEqual, expected["B"])
+		So(volumeBinds[envvars["C"]].FileContents, ShouldEqual, expected["C"])
+		So(volumeBinds[envvars["D"]].FileContents, ShouldEqual, expected["D"])
+	})
 }
 
 func TestDefaultVariableResolution(t *testing.T) {