feat: add Always Encrypted support

This commit adds partial support for the Microsoft SQL
"Always Encrypted" feature (basically, E2E encryption).

The current implementation is to be consider a __preview__
since it might not be perfectly implemented.

Supported features:
- PFX "keystore"
- Seamless encryption

Missing features:
- Support for Private Keys that are not RSA
- Encryption support (only Decryption is possible at the moment)

The most probably needs to be improved a bit, but so far it's
working for some of the use cases that I needed it for.
Feel free to test the feature and open an issue if you find any
problem: my goal is to have enough testers to spot eventual bugs.

fix: lint issues

README.md

@@ -28,6 +28,12 @@ Other supported formats are listed below.
   * `false` - Data sent between client and server is not encrypted beyond the login packet. (Default)
   * `true` - Data sent between client and server is encrypted.
 * `app name` - The application name (default is go-mssqldb)
+* `columnEncryption` - Set to "true" if you want to use [Always Encrypted](https://docs.microsoft.com/en-us/sql/relational-databases/security/encryption/always-encrypted-database-engine?view=sql-server-ver15)
+* `keyStoreAuthentication`
+  * `pfx` - Use a PFX file as a key store to authenticate and perform Always Encrypted operations, used when `columnEncryption` is enabled
+* `keyStoreLocation` - The location of the key store file (e.g: `./resources/test/always-encrypted/ae-1.pfx`), used when `columnEncryption` is enabled
+* `keyStoreSecret` - The password of the key store file provided in `keyStoreLocation`, used when `columnEncryption` is enabled
+
 
 ### Connection parameters for ODBC and ADO style connection strings:
 
@@ -126,6 +132,80 @@ Where `tokenProvider` is a function that returns a fresh access token or an erro
 actually trigger the retrieval of a token, this happens when the first statment is issued and a connection
 is created.
 
+
+### Always Encrypted support (preview)
+
+`go-mssql` supports a client-side decryption of the column encrypted values for those databases
+that are using the [Always Encrypted](https://docs.microsoft.com/en-us/sql/relational-databases/security/encryption/always-encrypted-database-engine?view=sql-server-ver15)
+feature.  
+
+To start using the feature, you have to use the following parameters in your DSN:
+
+* `columnEncryption=true`
+* `keyStoreAuthentication=pfx` - Only `pfx` is supported at the moment
+* `keyStoreLocation=/path/to/your/keystore.pfx` - The location of the key store file (e.g: `./resources/test/always-encrypted/ae-1.pfx`), used when `columnEncryption` is enabled
+* `keyStoreSecret=secret` - The password of your keystore (`keyStoreLocation`)
+
+#### Usage
+
+Using the Always Encrypted feature should be transparent in the driver:
+```go
+query := url.Values{}
+query.Add("database", "dbname")
+query.Add("columnEncryption", "true")
+query.Add("keyStoreAuthentication", "pfx")
+query.Add("keyStoreLocation", "./resources/test/always-encrypted/ae-1.pfx")
+query.Add("keyStoreSecret", "password")
+
+
+hostname := "172.20.0.2"
+port:= 1433
+
+u := &url.URL{
+    Scheme:   "sqlserver",
+    User:     url.UserPassword("sa", "superSecurePassword_"),
+    Host:     fmt.Sprintf("%s:%d", hostname, port),
+    RawQuery: query.Encode(),
+}
+
+db, err := sql.Open("sqlserver", u.String())
+if err != nil {
+    logrus.Fatalf("unable to open db: %v", err)
+}
+rows, err := db.Query("SELECT id, ssn FROM [dbo].[cid]")
+if err != nil {
+    logrus.Fatalf("unable to perform query: %v", err)
+}
+
+for ; rows.Next(); {
+    var dest struct {
+        Id int
+        SSN string
+    }
+    err = rows.Scan(&dest.Id, &dest.SSN)
+    if err != nil {
+        logrus.Fatalf("unable to scan into struct: %v", err)
+    }
+    fmt.Printf("%d, %s\n", dest.Id, dest.SSN)
+}
+```
+
+The code above, when used against an Always Encrypted column, returns
+the following:
+
+```
+1, 12345
+2, 00000
+```
+
+If `columnEncryption` is set to false, the result will be similar to the following:
+```
+1, B��v��3O뗇��a�R��o�l��U�
+�iE�#wOS�T횡5�R��1�i_n/Q��oLPBy��kL���8'/�
+2, �ކ��?�Y
+          Ѕ���i_n��-g|����v��2����x�Q)y�p�x��O��9������r��Bt�L�"N����.N]Rc
+```
+
 ## Executing Stored Procedures
 
 To run a stored procedure, set the query text to the procedure name:

always_encrypted_test.go

@@ -0,0 +1,34 @@
+package mssql
+
+import (
+	"fmt"
+	"testing"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAlwaysEncrypted(t *testing.T) {
+	conn := open(t)
+	defer conn.Close()
+	rows, err := conn.Query("SELECT id, ssn FROM [dbo].[cid]")
+	defer rows.Close()
+
+	if err != nil {
+		t.Fatalf("unable to query db: %s", err)
+	}
+
+	var dest struct {
+		Id int
+		SSN string
+	}
+
+	expectedValues := []string{"12345", "00000"}
+	expectedIdx := 0
+
+	for ; rows.Next() ; {
+		err = rows.Scan(&dest.Id, &dest.SSN)
+		assert.Equal(t, expectedValues[expectedIdx], dest.SSN)
+		expectedIdx++
+		assert.Nil(t, err)
+		fmt.Printf("col: %v\n", dest)
+	}
+}

buf.go

@@ -269,3 +269,41 @@ func (r *tdsBuffer) Read(buf []byte) (copied int, err error) {
 	r.rpos += copied
 	return
 }
+
+type sqlIdentifier struct {
+	serverName string
+	databaseName string
+	schemaName string
+	objectName string
+}
+
+func (r *tdsBuffer) sqlIdentifier() sqlIdentifier {
+	numParts := int(r.byte())
+	if numParts < 1 || numParts >= 5 {
+		panic("invalid sqlIdentifier: numparts is not between 1 and 4")
+	}
+
+	parts := make([]string, numParts)
+
+	for i := range parts {
+		parts[i] = r.UsVarChar()
+	}
+
+	sqlID := sqlIdentifier{
+		objectName: parts[0],
+	}
+
+	if numParts >= 2 {
+		sqlID.schemaName = parts[1]
+	}
+
+	if numParts >= 3{
+		sqlID.databaseName = parts[2]
+	}
+
+	if numParts == 4 {
+		sqlID.serverName = parts[3]
+	}
+
+	return sqlID
+}

cek.go

@@ -0,0 +1,29 @@
+package mssql
+
+type cekTable struct {
+	entries []cekTableEntry
+}
+
+type encryptionKeyInfo struct {
+	encryptedKey  []byte
+	databaseID    int
+	cekID         int
+	cekVersion    int
+	cekMdVersion  []byte
+	keyPath       string
+	keyStoreName  string
+	algorithmName string
+}
+
+type cekTableEntry struct {
+	databaseID int
+	keyId      int
+	keyVersion int
+	mdVersion  []byte
+	valueCount int
+	cekValues  []encryptionKeyInfo
+}
+
+func newCekTable(size uint16) cekTable {
+	return cekTable{entries: make([]cekTableEntry, size)}
+}

conn_str.go

@@ -39,11 +39,21 @@ type connectParams struct {
 	packetSize                uint16
 	fedAuthLibrary            int
 	fedAuthADALWorkflow       byte
+	columnEncryption          bool
+	keyStoreAuthentication    KeyStoreAuthentication
+	keyStoreLocation          string
+	keyStoreSecret            string
 }
 
 // default packet size for TDS buffer
 const defaultPacketSize = 4096
 
+type KeyStoreAuthentication string
+
+const (
+	PFXKeystoreAuth = "pfx"
+)
+
 func parseConnectParams(dsn string) (connectParams, error) {
 	p := connectParams{
 		fedAuthLibrary: fedAuthLibraryReserved,
@@ -169,6 +179,54 @@ func parseConnectParams(dsn string) (connectParams, error) {
 	} else {
 		p.trustServerCertificate = true
 	}
+
+	columnEncryption, ok := params["columnencryption"]
+	if ok {
+		if strings.EqualFold(columnEncryption, "true") {
+			p.columnEncryption = true
+		} else {
+			var err error
+			p.columnEncryption, err = strconv.ParseBool(columnEncryption)
+			if err != nil {
+				f := "invalid columnEncryption '%s': %s"
+				return p, fmt.Errorf(f, columnEncryption, err.Error())
+			}
+		}
+	} else {
+		p.columnEncryption = false
+	}
+
+	ksAuth, ok := params["keystoreauthentication"]
+	if ok {
+		var authMethod KeyStoreAuthentication
+		switch strings.ToLower(ksAuth) {
+		case "pfx":
+			authMethod = PFXKeystoreAuth
+		default:
+			return p, fmt.Errorf("invalid keystotreAuthentication '%s'", ksAuth)
+		}
+		p.keyStoreAuthentication = authMethod
+	}
+
+	ksLocation, ok := params["keystorelocation"]
+	if ok {
+		if ksLocation == "" {
+			return p, fmt.Errorf("invalid keystore location provided: '%s'", ksLocation)
+		}
+
+		_, err := os.Stat(ksLocation)
+		if err != nil {
+			return p, fmt.Errorf("unable to find keystore %s: %v", ksLocation, err)
+		}
+
+		p.keyStoreLocation = ksLocation
+	}
+
+	ksSecret, ok := params["keystoresecret"]
+	if ok {
+		p.keyStoreSecret = ksSecret
+	}
+
 	trust, ok := params["trustservercertificate"]
 	if ok {
 		var err error
@@ -248,6 +306,23 @@ func (p connectParams) toUrl() *url.URL {
 	if p.logFlags != 0 {
 		q.Add("log", strconv.FormatUint(p.logFlags, 10))
 	}
+
+	if p.columnEncryption {
+		q.Add("columnEncryption", "true")
+	}
+
+	if p.keyStoreAuthentication != "" {
+		q.Add("keyStoreAuthentication", string(p.keyStoreAuthentication))
+	}
+
+	if p.keyStoreLocation != "" {
+		q.Add("keyStoreLocation", p.keyStoreLocation)
+	}
+
+	if p.keyStoreSecret != "" {
+		q.Add("keyStoreSecret", p.keyStoreSecret)
+	}
+
 	res := url.URL{
 		Scheme: "sqlserver",
 		Host:   p.host,
@@ -256,6 +331,7 @@ func (p connectParams) toUrl() *url.URL {
 	if p.instance != "" {
 		res.Path = p.instance
 	}
+
 	if len(q) > 0 {
 		res.RawQuery = q.Encode()
 	}
@@ -274,7 +350,7 @@ func splitConnectionString(dsn string) (res map[string]string) {
 		if len(name) == 0 {
 			continue
 		}
-		var value string = ""
+		var value = ""
 		if len(lst) > 1 {
 			value = strings.TrimSpace(lst[1])
 		}

conn_str_test.go

@@ -2,6 +2,7 @@ package mssql
 
 import (
 	"bufio"
+	"github.com/stretchr/testify/assert"
 	"io"
 	"os"
 	"reflect"
@@ -186,10 +187,10 @@ func testConnParams(t testing.TB) connectParams {
 	}
 	if len(os.Getenv("HOST")) > 0 && len(os.Getenv("DATABASE")) > 0 {
 		return connectParams{
-			host: os.Getenv("HOST"),
+			host:     os.Getenv("HOST"),
 			instance: os.Getenv("INSTANCE"),
 			database: os.Getenv("DATABASE"),
-			user: os.Getenv("SQLUSER"),
+			user:     os.Getenv("SQLUSER"),
 			password: os.Getenv("SQLPASSWORD"),
 			logFlags: logFlags,
 		}
@@ -227,3 +228,16 @@ func TestConnParseRoundTripFixed(t *testing.T) {
 		t.Fatal("Parameters do not match after roundtrip", params, rtParams)
 	}
 }
+
+func TestConnParseAlwaysEncrypted(t *testing.T) {
+	connStr := "sqlserver://sa:sa@localhost/instance?database=master&columnEncryption=true&keyStoreAuthentication=pfx&keyStoreLocation=./resources/test/always-encrypted/ae-1.pfx&keyStoreSecret=password"
+	params, err := parseConnectParams(connStr)
+	if err != nil {
+		t.Fatal("Test URL is not valid", err)
+	}
+
+	assert.True(t, params.columnEncryption)
+	assert.Equal(t, KeyStoreAuthentication(PFXKeystoreAuth), params.keyStoreAuthentication)
+	assert.Equal(t, "./resources/test/always-encrypted/ae-1.pfx", params.keyStoreLocation)
+	assert.Equal(t, "password", params.keyStoreSecret)
+}

go.mod

@@ -3,6 +3,10 @@ module github.com/denisenkom/go-mssqldb
 go 1.11
 
 require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe
+	github.com/stretchr/testify v1.7.0
+	github.com/swisscom/mssql-always-encrypted v0.1.0
 	golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c
+	golang.org/x/text v0.3.5
 )

go.sum

@@ -1,5 +1,23 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/swisscom/mssql-always-encrypted v0.1.0 h1:bmYt1My3KgQsYkAJTDXkJt6b5wjRX3rSMrvyYHhK60Y=
+github.com/swisscom/mssql-always-encrypted v0.1.0/go.mod h1:FlEWLI3+svdMFq2w7GVMvk7iVhwBEBi7E7llAHb4B20=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0FNOmBrHfq7vN4btdGoDZgI=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=