Fix edge cases around special floats in JSONified keys

Author: sclevine

README.md

@@ -43,3 +43,6 @@ Preserves map order.
 -h     Show this help message
 -v     Show version
 ```
+
+Packages contained in this repo may also be used to convert all supported data formats to a normalized tree of ordered Go objects.
+See [godoc](https://godoc.org/github.com/sclevine/yj) for details.

convert/encoding.go

@@ -1,9 +1,56 @@
 package convert
 
-import "io"
+import (
+	"io"
+	"math"
+)
 
 type Encoding interface {
 	String() string
 	Encode(w io.Writer, in interface{}) error
 	Decode(r io.Reader) (interface{}, error)
 }
+
+type SpecialFloats int
+
+const (
+	FloatsReal SpecialFloats = iota
+	FloatsString
+	FloatsNumber
+)
+
+func (s SpecialFloats) NaN() interface{} {
+	switch s {
+	case FloatsReal:
+		return math.NaN()
+	case FloatsString:
+		return "NaN"
+	case FloatsNumber:
+		return (*float64)(nil)
+	}
+	panic("NaN: invalid special float type")
+}
+
+func (s SpecialFloats) PosInf() interface{} {
+	switch s {
+	case FloatsReal:
+		return math.Inf(1)
+	case FloatsString:
+		return "Infinity"
+	case FloatsNumber:
+		return math.MaxFloat64
+	}
+	panic("PosInf: invalid special float type")
+}
+
+func (s SpecialFloats) NegInf() interface{} {
+	switch s {
+	case FloatsReal:
+		return math.Inf(-1)
+	case FloatsString:
+		return "-Infinity"
+	case FloatsNumber:
+		return -math.MaxFloat64
+	}
+	panic("NegInf: invalid special float type")
+}

convert/toml.go

@@ -9,8 +9,8 @@ import (
 )
 
 type TOML struct {
-	FloatStrings bool
-	Indent       bool
+	SpecialFloats
+	Indent bool
 }
 
 func (TOML) String() string {
@@ -23,7 +23,11 @@ func (t TOML) Encode(w io.Writer, in interface{}) error {
 	if !t.Indent {
 		tomlEnc.Indentation("")
 	}
-	enc := toml.Encoder{FloatStrings: t.FloatStrings}
+	enc := toml.Encoder{
+		NaN:    t.NaN(),
+		PosInf: t.PosInf(),
+		NegInf: t.NegInf(),
+	}
 	out, err := enc.Encode(in)
 	if err != nil {
 		return err
@@ -57,8 +61,10 @@ func (t TOML) Decode(r io.Reader) (interface{}, error) {
 	if err != nil {
 		return nil, err
 	}
-	dec := toml.Decoder{FloatStrings: t.FloatStrings}
+	dec := toml.Decoder{
+		NaN:    t.NaN(),
+		PosInf: t.PosInf(),
+		NegInf: t.NegInf(),
+	}
 	return dec.Decode(tree)
 }
-
-

convert/yaml.go

@@ -2,29 +2,31 @@ package convert
 
 import (
 	"io"
-	"math"
 
 	goyaml "gopkg.in/yaml.v3"
 
 	"github.com/sclevine/yj/yaml"
 )
 
 type YAML struct {
-	FloatStrings bool
-	JSONKeys     bool
-	EscapeHTML   bool
+	SpecialFloats
+	KeySpecialFloats SpecialFloats
+	JSONKeys         bool
+	EscapeHTML       bool
 }
 
 func (YAML) String() string {
 	return "YAML"
 }
 
 func (y YAML) Encode(w io.Writer, in interface{}) error {
-	enc := &yaml.Encoder{}
-	if y.FloatStrings {
-		enc.NaN = "NaN"
-		enc.PosInf = "Infinity"
-		enc.NegInf = "-Infinity"
+	enc := &yaml.Encoder{
+		NaN:       y.NaN(),
+		PosInf:    y.PosInf(),
+		NegInf:    y.NegInf(),
+		KeyNaN:    y.KeySpecialFloats.NaN(),
+		KeyPosInf: y.KeySpecialFloats.PosInf(),
+		KeyNegInf: y.KeySpecialFloats.NegInf(),
 	}
 	if y.JSONKeys {
 		enc.KeyUnmarshal = (&yaml.KeyJSON{}).Unmarshal
@@ -45,14 +47,12 @@ func (y YAML) Decode(r io.Reader) (interface{}, error) {
 	}
 	dec := &yaml.Decoder{
 		KeyMarshal: (&yaml.KeyJSON{EscapeHTML: y.EscapeHTML}).Marshal, // FIXME: double-check map-keys
-		NaN:    (*float64)(nil),
-		PosInf: math.MaxFloat64,
-		NegInf: -math.MaxFloat64,
-	}
-	if y.FloatStrings {
-		dec.NaN = "NaN"
-		dec.PosInf = "Infinity"
-		dec.NegInf = "-Infinity"
+		NaN:        y.NaN(),
+		PosInf:     y.PosInf(),
+		NegInf:     y.NegInf(),
+		KeyNaN:     y.KeySpecialFloats.NaN(),
+		KeyPosInf:  y.KeySpecialFloats.PosInf(),
+		KeyNegInf:  y.KeySpecialFloats.NegInf(),
 	}
 	return dec.Decode(&node)
 }

flags.go

@@ -61,17 +61,15 @@ func flagFilter(r rune) rune {
 func transform(s string) (from, to convert.Encoding, err error) {
 	escapeHTML := strings.ContainsRune(s, FlagEscapeHTML)
 	indent := strings.ContainsRune(s, FlagIndent)
-	floatStrings := !strings.ContainsRune(s, FlagNoFloatStrings)
 	jsonKeys := strings.ContainsRune(s, FlagJSONKeys)
+	floatStrings := !strings.ContainsRune(s, FlagNoFloatStrings)
 
 	yaml := &convert.YAML{
-		FloatStrings: floatStrings,
-		JSONKeys:     jsonKeys,
-		EscapeHTML:   escapeHTML,
+		JSONKeys:   jsonKeys,
+		EscapeHTML: escapeHTML,
 	}
 	toml := &convert.TOML{
-		FloatStrings: floatStrings,
-		Indent:       indent,
+		Indent: indent,
 	}
 	json := &convert.JSON{
 		EscapeHTML: escapeHTML,
@@ -100,6 +98,44 @@ func transform(s string) (from, to convert.Encoding, err error) {
 		from, to = to, json
 	}
 
+	setKeyFloats(to, func(setTo func(v convert.SpecialFloats)) {
+		if floatStrings {
+			setTo(convert.FloatsString)
+		} else {
+			// never convert number to nan, inf, -inf
+			setTo(convert.FloatsReal)
+		}
+	})
+
+	setKeyFloats(from, func(setFrom func(v convert.SpecialFloats)) {
+		if floatStrings {
+			setFrom(convert.FloatsString)
+		} else {
+			setFrom(convert.FloatsNumber)
+		}
+	})
+
+	setFloats(to, func(setTo func(v convert.SpecialFloats)) {
+		if floatStrings {
+			setTo(convert.FloatsString)
+		} else {
+			// never convert number to nan, inf, -inf
+			setTo(convert.FloatsReal)
+		}
+	})
+
+	setFloats(from, func(setFrom func(v convert.SpecialFloats)) {
+		if floatStrings {
+			setFrom(convert.FloatsString)
+		} else {
+			setFrom(convert.FloatsNumber)
+		}
+		setFloats(to, func(setTo func(v convert.SpecialFloats)) {
+			setFrom(convert.FloatsReal)
+			setTo(convert.FloatsReal)
+		})
+	})
+
 	if _, toYAML := to.(*convert.YAML); jsonKeys && !toYAML {
 		err = fmt.Errorf("flag -%c only valid for YAML output", FlagJSONKeys)
 		return
@@ -118,23 +154,23 @@ func transform(s string) (from, to convert.Encoding, err error) {
 		}
 	}
 
-	floatOff(to, func(toOff func()) {
-		floatOff(from, func(fromOff func()) {
-			toOff()
-			fromOff()
-		})
-	})
-
-	// FIXME: validate -n isn't used between inapplicable types
+	// TODO: validate -n isn't used between inapplicable types
 
 	return
 }
 
-func floatOff(e convert.Encoding, f func(off func())) {
+func setFloats(e convert.Encoding, f func(set func(v convert.SpecialFloats))) {
 	switch e := e.(type) {
 	case *convert.YAML:
-		f(func() { e.FloatStrings = false })
+		f(func(v convert.SpecialFloats) { e.SpecialFloats = v })
 	case *convert.TOML:
-		f(func() { e.FloatStrings = false })
+		f(func(v convert.SpecialFloats) { e.SpecialFloats = v })
+	}
+}
+
+func setKeyFloats(e convert.Encoding, f func(set func(v convert.SpecialFloats))) {
+	switch e := e.(type) {
+	case *convert.YAML:
+		f(func(v convert.SpecialFloats) { e.KeySpecialFloats = v })
 	}
 }

flags_test.go

@@ -10,32 +10,33 @@ import (
 func TestParse(t *testing.T) {
 	config, err := main.Parse("-t", "y\tk-", "kn-k ", "h h", "")
 	assertEq(t, err, nil)
-	toml, ok := config.From.(convert.TOML)
+	toml, ok := config.From.(*convert.TOML)
 	assertEq(t, ok, true)
-	assertEq(t, toml, convert.TOML{
-		FloatStrings: false,
+	assertEq(t, toml, &convert.TOML{
+		SpecialFloats: convert.FloatsReal,
 	})
-	yaml, ok := config.To.(convert.YAML)
+	yaml, ok := config.To.(*convert.YAML)
 	assertEq(t, ok, true)
-	assertEq(t, yaml, convert.YAML{
-		EscapeHTML:   false,
-		FloatStrings: false,
-		JSONKeys:     true,
+	assertEq(t, yaml, &convert.YAML{
+		SpecialFloats: convert.FloatsReal,
+		EscapeHTML:    false,
+		JSONKeys:      true,
 	})
 	assertEq(t, config.Help, true)
 
 	config, err = main.Parse("--\t\te  ", "")
 	assertEq(t, err, nil)
-	yaml, ok = config.From.(convert.YAML)
+	yaml, ok = config.From.(*convert.YAML)
 	assertEq(t, ok, true)
-	assertEq(t, yaml, convert.YAML{
-		EscapeHTML:   true,
-		FloatStrings: true,
-		JSONKeys:     false,
+	assertEq(t, yaml, &convert.YAML{
+		SpecialFloats:    convert.FloatsString,
+		KeySpecialFloats: convert.FloatsString,
+		EscapeHTML:       true,
+		JSONKeys:         false,
 	})
-	json, ok := config.To.(convert.JSON)
+	json, ok := config.To.(*convert.JSON)
 	assertEq(t, ok, true)
-	assertEq(t, json, convert.JSON{
+	assertEq(t, json, &convert.JSON{
 		EscapeHTML: true,
 	})
 	assertEq(t, config.Help, false)

testdata/case3_out_tyk.yml

@@ -0,0 +1,32 @@
+0: 0
+1: 1
+1.1: 1.1
+key: '"value"'
+<: '>'
+? <: '>'
+: '>'
+? key: value
+: value
+.inf: .inf
+-.inf: -.inf
++Infinity: .inf
+.nan: .nan
+-NaN: .nan
++NaN: .nan
+? .inf: .inf
+: .inf
+? -.inf: -.inf
+: -.inf
+? .nan: .nan
+: .nan
+key-number-list:
+  - 0
+  - 1
+  - 1.1
+key-string-list:
+  - a
+  - b
+date: 2006-01-02T15:04:05-07:00
+key-obj-list:
+  - key-string: value-string
+    key-number: 2

toml/decoder.go

@@ -10,7 +10,8 @@ import (
 )
 
 type Decoder struct {
-	FloatStrings bool
+	// If set, NaN, Inf, etc. are replaced by the set values
+	NaN, PosInf, NegInf interface{}
 }
 
 func (d *Decoder) Decode(toml interface{}) (normal interface{}, err error) {
@@ -75,25 +76,15 @@ func (d Decoder) tomlToSimple(v interface{}) (out interface{}) {
 func (d Decoder) postprocess(in interface{}) interface{} {
 	switch in := in.(type) {
 	case float64:
-		if d.FloatStrings {
-			switch {
-			case math.IsNaN(in):
-				return "NaN"
-			case math.IsInf(in, 1):
-				return "Infinity"
-			case math.IsInf(in, -1):
-				return "-Infinity"
-			}
-		} else {
-			switch {
-			case math.IsNaN(in):
-				return (*float64)(nil)
-			case math.IsInf(in, 1):
-				return math.MaxFloat64
-			case math.IsInf(in, -1):
-				return -math.MaxFloat64
-			}
+		switch {
+		case d.NaN != nil && math.IsNaN(in):
+			return d.NaN
+		case d.PosInf != nil && math.IsInf(in, 1):
+			return d.PosInf
+		case d.NegInf != nil && math.IsInf(in, -1):
+			return d.NegInf
 		}
+		return in
 	}
 	return in
 }