Add ejson support (#8)

Adds support to `sync` command for Shopify EJSON format

Author: Drew MacInnis

.circleci/config.yml

@@ -6,10 +6,11 @@ jobs:
   build:
     docker:
       # specify the version
-      - image: circleci/golang:1.9
+      - image: circleci/golang:1.10
     working_directory: /go/src/github.com/drmdrew/syncrets
     steps:
       - checkout
+      - setup_remote_docker
       - run:
           name: Update VERSION file
           command: |
@@ -18,6 +19,6 @@ jobs:
               (cat VERSION.orig | sed s/BUILD/${CIRCLE_BUILD_NUM}/g) > VERSION
               echo "Updating VERSION: $(cat VERSION)"
             fi
-      - run: go get -v -t -d ./...
-      - run: make build
-      - run: go test -v ./...
+      - run: docker build -f Dockerfile.build -t drmdrew/syncrets-build:latest .
+      - run: mkdir testoutput/
+      - run: docker-compose up integration-test

.gitignore

@@ -1,6 +1,9 @@
 syncrets
 debug
 debug.*
+secrets.json
+secrets.ejson
+testoutput/*
 .vault*
 .vscode/
 

CODEOWNERS

@@ -0,0 +1 @@
+*       @drmdrew

Dockerfile.build

@@ -0,0 +1,6 @@
+FROM golang:1.10-alpine AS build
+RUN apk --no-cache add git make
+WORKDIR /go/src/github.com/drmdrew/syncrets
+COPY . .
+RUN go get -v ./...
+RUN make build

Dockerfile.runtime

@@ -0,0 +1,3 @@
+FROM scratch
+COPY --from=drmdrew/syncrets-build /go/src/github.com/drmdrew/syncrets/syncrets /syncrets
+CMD ["/syncrets"]

Dockerfile.test

@@ -0,0 +1,9 @@
+FROM drmdrew/syncrets-build
+
+WORKDIR /go/src/github.com/drmdrew/syncrets
+
+COPY integration_test.go .
+COPY testdata/syncrets-integration.yml syncrets.yml
+RUN go get -t ./...
+
+CMD go test -tags=integration -run ^TestIntegration

Makefile

@@ -2,5 +2,5 @@
 VERSION=$(shell cat VERSION)
 
 build:
-	go build -ldflags "-X github.com/drmdrew/syncrets/cmd.version=$(VERSION)"
+	CGO_ENABLED=0 go build -ldflags "-X github.com/drmdrew/syncrets/cmd.version=$(VERSION)"
 

README.md

@@ -1,23 +1,28 @@
 # syncrets
-A utility for synchronizing secrets between systems like
-[Hashicorp vault][VAULT] and formats like ejson/eyaml. Think of it like an
+*WIP*: This project is a *WORK IN PROGRESS*, so consider it useful for
+experimentation but not ready for production use. Use at your own risk
+but if you *do* use it I would love to hear what you think so please
+log issues for anything you would like to see fixed/improved.
+
+syncrets is a little utility for synchronizing secrets between systems like
+[Hashicorp vault][VAULT] and formats like [ejson][EJSON]. Think of it like an
 _rsync for secrets_. Secrets need to be handled carefully and syncrets can
 help transfer, list, export, and otherwise manage secrets between systems
-and formats.
+and formats. The name `syncrets` is a portmanteau of `secrets` and `sync` ...
+obligatory [xkcd][XKCD-739].
 
 Here is a simple example of using syncrets to copy secrets between two
 vault servers running locally:
 ```
-syncrets sync vault://localhost:8200/secrets/ vault://localhost:8201/secrets/
+syncrets sync vault://vault-a/secrets/ vault://vault-b/secrets/
 ```
-*NOTE*: This project is a *WORK IN PROGRESS*, so consider it useful for
-experimentation but not ready for production use. Use at your own risk.
 
 ## syncrets config file
-Outside of test scenarios, it isn't likely that two instances of vault would
-be running on localhost.  To make working with multiple vaults easier,
-syncrets supports a `~/.syncrets/syncrets.yml` configuration file, for
-example:
+
+To faciliate working with multiple vaults, syncrets looks for a `syncrets.yml`
+configuration file in the working directory as well as `~/.syncrets/syncrets.yml`.
+Here is an example:
+
 ```
 vault:
     vault-a:
@@ -34,17 +39,42 @@ vault:
             file: ~/.syncrets/.vault-b-token
 ```
 Using a configuration file allows you to refer to servers using the name
-(alias) present in their section of the configuration file.
+(alias) present in their section of the configuration file, so you can
+refer to `vault://vault-a/secrets` rather than `http://localhost:8200/secrets`.
 
-For example, using the configuration above we can now rewrite the
-previous syncrets example like so:
-```
-syncrets sync vault://vault-a/secrets/foo/ vault://vault-b/secrets/bar/
-```
-Using our configuration file syncrets will now know to reach `vault-a` using
+This example configuration file configures syncrets to reach `vault-a` using
 `http://localhost:8200` and to reach `vault-b` using `http://localhost:8201`
 which saves you from having to type out the full scheme, hostname, and port
-when building URLs to pass to syncrets.
+when building URLs to pass to syncrets. The configuration also tells syncrets to
+load vault auth tokens from file (assuming that these tokens have been obtained
+previously).
+
+## syncrets ejson
+
+syncrets can directly `sync` secrets between two vault servers but can also
+be used to `sync` secrets to a local file (preferrably in ejson format ...
+these are _secrets_ after all).
+
+If the source or target of a syncrets `sync` ends with `.ejson` then
+syncrets will use the `ejson` configuration section of `syncrets.yml` to
+configure the default encryption public key to use:
+```
+ejson:
+    public_key:   a9d52487a1232e5c292a9680f4a44a84ea302ba05ff12d2e9d11662d20fc0139
+```
+
+For both encryption and decryption syncrets assumes that the ejson `EJSON_KEYDIR`
+environment has been set if the ejson keys are not present in their default location.
+
+*Example*:
+```
+syncrets sync vault://vault-a/secret/ ./secrets.ejson
+```
+
+Note: syncrets will write _unencrypted_ secrets to files ending with `.json` but
+this regular JSON format is included primarily for testing/debugging purposes and
+shouldn't be used for anything that is sensitive if the underlying filesystem isn't
+trustworthy.
 
 ## syncrets commands
 ### auth
@@ -75,5 +105,6 @@ syncrets rm vault://localhost:8200/secrets/
 ```
 *CAUTION*: Use the `rm` command _carefully_, it can be a potent footgun.
 
-
 [VAULT]: https://www.vaultproject.io/
+[EJSON]: https://github.com/Shopify/ejson
+[XKCD-739]: https://xkcd.com/739/

backend/ejson.go

@@ -0,0 +1,45 @@
+package backend
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"log"
+
+	"github.com/Shopify/ejson"
+	"github.com/drmdrew/syncrets/core"
+	"github.com/spf13/viper"
+)
+
+type EJSONEndpoint struct {
+	kv map[string]interface{}
+}
+
+// NewEJSONEndpoint ...
+func NewEJSONEndpoint() *EJSONEndpoint {
+	return &EJSONEndpoint{make(map[string]interface{})}
+}
+
+// Visit ...
+func (j *EJSONEndpoint) Visit(s core.Secret) {
+	AddSecretToKV(s, j.kv)
+}
+
+// Marshal ...
+func (j *EJSONEndpoint) Marshal(out io.Writer) error {
+	var jsonBytes []byte
+	var err error
+	ejsonPubkey := viper.GetString("ejson.public_key")
+	j.kv["_public_key"] = ejsonPubkey
+	if jsonBytes, err = json.Marshal(j.kv); err != nil {
+		log.Printf("ERROR: %v\n", err)
+		return err
+	}
+	in := bytes.NewReader(jsonBytes)
+	_, err = ejson.Encrypt(in, out)
+	if err != nil {
+		log.Printf("ERROR: %v\n", err)
+		return err
+	}
+	return nil
+}

backend/json.go

@@ -0,0 +1,60 @@
+package backend
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"strings"
+
+	"github.com/drmdrew/syncrets/core"
+)
+
+// JSONEndpoint ...
+type JSONEndpoint struct {
+	kv map[string]interface{}
+}
+
+// NewJSONEndpoint ...
+func NewJSONEndpoint() *JSONEndpoint {
+	return &JSONEndpoint{make(map[string]interface{})}
+}
+
+// AddSecretToKV ...
+func AddSecretToKV(s core.Secret, kv map[string]interface{}) {
+	steps := strings.Split(s.Path, "/")
+	for _, step := range steps[:len(steps)-1] {
+		if step == "" {
+			continue
+		}
+		if _, ok := kv[step]; !ok {
+			kv[step] = make(map[string]interface{})
+		}
+		if m, ok := kv[step].(map[string]interface{}); !ok {
+			m = make(map[string]interface{})
+			m["."] = kv[step]
+			kv[step] = m
+			kv = m
+		} else {
+			kv = m
+		}
+	}
+	lastStep := steps[len(steps)-1]
+	kv[lastStep] = s.Value
+}
+
+// Visit ...
+func (j *JSONEndpoint) Visit(s core.Secret) {
+	AddSecretToKV(s, j.kv)
+}
+
+// Marshal ...
+func (j *JSONEndpoint) Marshal(out io.Writer) error {
+	b, err := json.Marshal(j.kv)
+	if err != nil {
+		log.Print(err)
+		return err
+	}
+	fmt.Fprintf(out, "%s\n", string(b[:]))
+	return nil
+}

backend/json_test.go

@@ -0,0 +1,37 @@
+package backend
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/drmdrew/syncrets/core"
+)
+
+var jsonTests = []struct {
+	secrets  []core.Secret
+	expected string
+}{
+	{[]core.Secret{core.Secret{"secret/citizen", "four"}},
+		`{"secret":{"citizen":"four"}}`},
+	{[]core.Secret{core.Secret{"secret/citizen/kane", "Rosebud"}},
+		`{"secret":{"citizen":{"kane":"Rosebud"}}}`},
+	{[]core.Secret{core.Secret{"secret/citizen", "four"}, core.Secret{"secret/citizen/kane", "Rosebud"}},
+		`{"secret":{"citizen":{".":"four","kane":"Rosebud"}}}`},
+}
+
+func TestJSON_Marshal(t *testing.T) {
+	for _, testcase := range jsonTests {
+		buf := new(bytes.Buffer)
+		j := NewJSONEndpoint()
+		for _, s := range testcase.secrets {
+			j.Visit(s)
+		}
+		j.Marshal(buf)
+		result := strings.TrimSpace(buf.String())
+		if result != testcase.expected {
+			t.Fail()
+			t.Fatalf("Expected: '%s' but result was: '%s'\n", testcase.expected, result)
+		}
+	}
+}