Allow using a custom KV store to keep consistent JSON ordering

Author: alecsammon

README.md

@@ -217,6 +217,48 @@ func main() {
 // ]
 ```
 
+## Output ordering
+
+Sheriff converts the input struct into a basic structure using `map[string]interface{}`. This means that the generated 
+JSON will not have the same ordering as the input struct. If you need to have a specific ordering then a custom 
+implementation of the `KVStoreFactory` can be passed as an option.
+```go
+package main
+
+import (
+	"github.com/liip/sheriff/v2"
+	orderedmap "github.com/wk8/go-ordered-map/v2"
+)
+
+type OrderedMap struct {
+	*orderedmap.OrderedMap[string, interface{}]
+}
+
+func NewOrderedMap() *OrderedMap {
+	return &OrderedMap{orderedmap.New[string, interface{}]()}
+}
+
+func (om *OrderedMap) Set(k string, v interface{}) {
+	om.OrderedMap.Set(k, v)
+}
+
+func (om *OrderedMap) Each(f func(k string, v interface{})) {
+	for pair := om.Newest(); pair != nil; pair = pair.Prev() {
+		f(pair.Key, pair.Value)
+	}
+}
+
+func main() {
+	opt := &sheriff.Options{
+		KVStoreFactory: func() sheriff.KVStore {
+			return NewOrderedMap()
+		},
+	}
+
+	// ...
+}
+```
+
 ## Benchmarks
 
 There's a simple benchmark in `bench_test.go` which compares running sheriff -> JSON versus just marshalling into JSON 

kvstore.go

@@ -0,0 +1,17 @@
+package sheriff
+
+// kvStore is the default implementation of the KVStore interface that sheriff converts a struct into.
+// It is the fastest option, but does result in a re-ordering of the final JSON properties.
+type kvStore map[string]interface{}
+
+// Set inserts the value into the map at the given key.
+func (m kvStore) Set(k string, v interface{}) {
+	m[k] = v
+}
+
+// Each applies the callback function to each element in the map.
+func (m kvStore) Each(f func(k string, v interface{})) {
+	for k, v := range m {
+		f(k, v)
+	}
+}

sheriff.go

@@ -10,6 +10,15 @@ import (
 	"github.com/hashicorp/go-version"
 )
 
+// A KVStore is a simple key-value store.
+// The default implementation uses a map[string]interface{}.
+// This is fast, however will lead to inconsistent ordering of the keys in the generated JSON.
+// A custom implementation can be used to maintain the order of the keys.
+type KVStore interface {
+	Set(k string, v interface{})
+	Each(f func(k string, v interface{}))
+}
+
 // A FieldFilter is a function that decides whether a field should be marshalled or not.
 // If it returns true, the field will be marshalled, otherwise it will be skipped.
 type FieldFilter func(field reflect.StructField) (bool, error)
@@ -38,6 +47,12 @@ type Options struct {
 	// This option is false by default.
 	IncludeEmptyTag bool
 
+	// The KVStoreFactory is a function that returns a new KVStore.
+	// The default implementation uses a map[string]interface{}, which is fast but does not maintain the order of the
+	// keys.
+	// A custom implementation can be used to maintain the order of the keys, i.e. using github.com/wk8/go-ordered-map
+	KVStoreFactory func() KVStore
+
 	// This is used internally so that we can propagate anonymous fields groups tag to all child field.
 	nestedGroupsMap map[string][]string
 }
@@ -81,6 +96,12 @@ func Marshal(options *Options, data interface{}) (interface{}, error) {
 		options.FieldFilter = createDefaultFieldFilter(options)
 	}
 
+	if options.KVStoreFactory == nil {
+		options.KVStoreFactory = func() KVStore {
+			return kvStore{}
+		}
+	}
+
 	if t.Kind() == reflect.Ptr {
 		// follow pointer
 		t = t.Elem()
@@ -94,7 +115,7 @@ func Marshal(options *Options, data interface{}) (interface{}, error) {
 		return marshalValue(options, v)
 	}
 
-	dest := make(map[string]interface{})
+	dest := options.KVStoreFactory()
 
 	for i := 0; i < t.NumField(); i++ {
 		field := t.Field(i)
@@ -172,13 +193,13 @@ func Marshal(options *Options, data interface{}) (interface{}, error) {
 
 		// when a composition field we want to bring the child
 		// nodes to the top
-		nestedVal, ok := v.(map[string]interface{})
+		nestedVal, ok := v.(KVStore)
 		if isEmbeddedField && ok {
-			for key, value := range nestedVal {
-				dest[key] = value
-			}
+			nestedVal.Each(func(k string, v interface{}) {
+				dest.Set(k, v)
+			})
 		} else {
-			dest[jsonTag] = v
+			dest.Set(jsonTag, v)
 		}
 	}
 
@@ -303,13 +324,14 @@ func marshalValue(options *Options, v reflect.Value) (interface{}, error) {
 		if mapKeys[0].Kind() != reflect.String {
 			return nil, MarshalInvalidTypeError{t: mapKeys[0].Kind(), data: val}
 		}
-		dest := make(map[string]interface{})
+
+		dest := options.KVStoreFactory()
 		for _, key := range mapKeys {
 			d, err := marshalValue(options, v.MapIndex(key))
 			if err != nil {
 				return nil, err
 			}
-			dest[key.String()] = d
+			dest.Set(key.String(), d)
 		}
 		return dest, nil
 	}

sheriff_test.go

@@ -846,13 +846,12 @@ func TestMarshal_User(t *testing.T) {
 		TestS: "test",
 	}
 
-	v, err := Marshal(&Options{}, j)
+	m, err := Marshal(&Options{}, j)
 	assert.NoError(t, err)
-	assert.Equal(t, map[string]interface{}{"test": "12", "testb": "true", "testf": "12", "tests": "test"}, v)
 
-	d, err := json.Marshal(j)
+	d, err := json.Marshal(m)
 	assert.NoError(t, err)
-	assert.Equal(t, `{"test":"12","testb":"true","testf":"12","tests":"\"test\""}`, string(d))
+	assert.Equal(t, `{"test":"12","testb":"true","testf":"12","tests":"test"}`, string(d))
 }
 
 func TestMarshal_CustomFieldFilter(t *testing.T) {
@@ -873,6 +872,7 @@ func TestMarshal_CustomFieldFilter(t *testing.T) {
 	m, err := Marshal(o, v)
 	assert.NoError(t, err)
 
-	// ensure the "secret" value is not present in the marshalled map
-	assert.Equal(t, map[string]interface{}{"test": "teststring"}, m)
+	d, err := json.Marshal(m)
+	assert.NoError(t, err)
+	assert.Equal(t, `{"test":"teststring"}`, string(d))
 }