Persistent Connections and Bitmap Codec Util (#55)

* Minor refactoring

* WIP persistent connections

* Final for ISO host persistent connections

* Final for ISO host persistent connections

README.md

@@ -3,6 +3,7 @@
 [![GoDev](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white&style=flat-square)](https://pkg.go.dev/github.com/rkbalgi/isosim?tab=doc)
 ![build](https://github.com/rkbalgi/isosim/workflows/build/badge.svg)
 ![GitHub release (latest by date including pre-releases)](https://img.shields.io/github/v/release/rkbalgi/isosim?include_prereleases&style=flat)
+![Docker Pulls](https://img.shields.io/docker/pulls/rkbalgi/isosim?color=%23FF6528&label=docker%20pulls)
 
 # ISO WebSim
 A very short screencast - [https://youtu.be/vSRZ_nzU-Jg](https://youtu.be/vSRZ_nzU-Jg)

cmd/isoserver/isoserver.go

@@ -7,7 +7,7 @@ import (
 	"io/ioutil"
 	"isosim/internal/iso"
 	"isosim/internal/iso/server"
-	"isosim/internal/services/v0/data"
+	"isosim/internal/services/data"
 	"os"
 	"sync"
 )

cmd/isosim/isosim.go

@@ -49,7 +49,7 @@ func main() {
 		log.SetLevel(log.ErrorLevel)
 	default:
 		log.Warn("Invalid log-level specified, will default to DEBUG")
-		log.SetLevel(log.InfoLevel)
+		log.SetLevel(log.DebugLevel)
 	}
 
 	log.SetFormatter(&log.TextFormatter{ForceColors: true, DisableColors: false})
@@ -81,7 +81,12 @@ func main() {
 			certFile := os.Getenv("TLS_CERT_FILE")
 			keyFile := os.Getenv("TLS_KEY_FILE")
 
-			log.Infof("Using certificate file - %s, key file: %s", certFile, keyFile)
+			log.Infof("TLS settings: Using Certificate file : %s, Key file: %s", certFile, keyFile)
+
+			if certFile == "" || keyFile == "" {
+				log.Fatalf("SSL enabled, but certificate/key file unspecified.")
+			}
+
 			log.Fatal(http.ListenAndServeTLS(":"+strconv.Itoa(*httpPort), certFile, keyFile, nil))
 		} else {
 			log.Fatal(http.ListenAndServe(":"+strconv.Itoa(*httpPort), nil))

cmd/isosim/version.go

@@ -1,4 +1,4 @@
 package main
 
-var version = "0.8.0"
-var build = "d01aad2c"
+var version = "0.10.0"
+var build = "087f55ec"

githooks/update_commit_id.sh

@@ -3,5 +3,5 @@
 commit_id=`git log -1 HEAD | head -1 | awk '{print substr($2,0,8)}'`
 echo "package main
 
-var version=\"0.8.0\"
+var version=\"0.10.0\"
 var build=\"$commit_id\"" > ../cmd/isosim/version.go

go.mod

@@ -5,7 +5,7 @@ go 1.14
 require (
 	github.com/go-kit/kit v0.10.0
 	github.com/google/uuid v1.0.0
-	github.com/rkbalgi/libiso v0.1.5-0.20200424111508-cb62515a9b4a
+	github.com/rkbalgi/libiso v0.1.5-0.20200513063448-de9fd4ba7bce
 	github.com/sirupsen/logrus v1.4.2
 	github.com/stretchr/testify v1.5.1
 	go.etcd.io/bbolt v1.3.3

go.sum

@@ -206,8 +206,8 @@ github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
-github.com/rkbalgi/libiso v0.1.5-0.20200424111508-cb62515a9b4a h1:GBq34G1CNMVLs1C2hOt/wTNJns2/h6NJrgSZ5xbJUk8=
-github.com/rkbalgi/libiso v0.1.5-0.20200424111508-cb62515a9b4a/go.mod h1:h4uRr0I72BG7sajrG5uljWWfQq3SPZTE21W7NOVadvY=
+github.com/rkbalgi/libiso v0.1.5-0.20200513063448-de9fd4ba7bce h1:ID0vpI8Ad2Eh0yeNnoRPnk0LrddQVpotCN5u5zlfUdA=
+github.com/rkbalgi/libiso v0.1.5-0.20200513063448-de9fd4ba7bce/go.mod h1:h4uRr0I72BG7sajrG5uljWWfQq3SPZTE21W7NOVadvY=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=

internal/db/data_set_manager.go

@@ -14,7 +14,7 @@ import (
 	"strings"
 	"sync"
 
-	"isosim/internal/services/v0/data"
+	"isosim/internal/services/data"
 )
 
 type dataSetManager struct{}

internal/db/db.go

@@ -9,11 +9,11 @@ import (
 	"github.com/google/uuid"
 	log "github.com/sirupsen/logrus"
 	bolt "go.etcd.io/bbolt"
-	"isosim/internal/services/v0/data"
+	"isosim/internal/services/data"
 	"time"
 )
 
-var timeFormat = "2006-01-02T15"
+const timeFormat = "2006-01-02T15"
 
 // DbMessage is an entry of a request/response that will be persisted to
 // storage
@@ -116,6 +116,7 @@ func ReadLast(specID int, msgID int, n int) ([]string, error) {
 				k, v := c.Last()
 
 				if k == nil || v == nil {
+					now = now.Add(-1 * time.Hour)
 					continue
 				}
 				for len(res) < n {

internal/db/db_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 )
 
-func Test_ReadWriteToBold(t *testing.T) {
+func Test_ReadWriteToBolt(t *testing.T) {
 
 	//t.SkipNow()
 	if err := Init("."); err != nil {

internal/iso/field.go

@@ -11,12 +11,18 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+const (
+	Fixed     = "fixed"
+	Variable  = "variable"
+	Bitmapped = "bitmap"
+)
+
 // NewField is a constructor for Field
 func NewField(info []string) (*Field, error) {
 
 	fieldInfo := &Field{}
 	switch info[0] {
-	case "fixed":
+	case Fixed:
 		{
 			fieldInfo.Type = FixedType
 			hasConstraints := false
@@ -54,7 +60,7 @@ func NewField(info []string) (*Field, error) {
 			}
 
 		}
-	case "bitmap":
+	case Bitmapped:
 		{
 			fieldInfo.Type = BitmappedType
 			if err := setEncoding(&(*fieldInfo).DataEncoding, info[1]); err != nil {
@@ -67,7 +73,7 @@ func NewField(info []string) (*Field, error) {
 			}
 
 		}
-	case "variable":
+	case Variable:
 		{
 			fieldInfo.Type = VariableType
 			hasConstraints := false

internal/iso/field_assembler.go

@@ -10,7 +10,7 @@ import (
 )
 
 // assemble assembles all the field into the dst Buffer buf
-func assemble(buf *bytes.Buffer, parsedMsg *ParsedMsg, fieldData *FieldData) error {
+func assemble(buf *bytes.Buffer, meta *MetaInfo, parsedMsg *ParsedMsg, fieldData *FieldData) error {
 
 	asmLog := log.WithFields(log.Fields{"component": "assembler"})
 
@@ -23,6 +23,9 @@ func assemble(buf *bytes.Buffer, parsedMsg *ParsedMsg, fieldData *FieldData) err
 		// from the children (nested fields) else we take it from the parent field
 		if !fieldData.Field.HasChildren() {
 			asmLog.Debugf("Field %s, Length: %d, Value: %s\n", field.Name, len(fieldData.Data), hex.EncodeToString(fieldData.Data))
+			if field.Key == true {
+				meta.MessageKey += fieldData.Value()
+			}
 			buf.Write(fieldData.Data)
 		}
 	case VariableType:
@@ -41,12 +44,15 @@ func assemble(buf *bytes.Buffer, parsedMsg *ParsedMsg, fieldData *FieldData) err
 				}
 
 				asmLog.Debugf("Field %s, LL (Variable): %s, Value: %s\n", field.Name, hex.EncodeToString(lenBuf.Bytes()), hex.EncodeToString(fieldData.Data))
+				if field.Key == true {
+					meta.MessageKey += fieldData.Value()
+				}
 				buf.Write(lenBuf.Bytes())
 				buf.Write(fieldData.Data)
 			}
 		}
 	case BitmappedType:
-		asmLog.Debugf("Field %s, Length (bitmapped): -, Value: %s\n", field.Name, hex.EncodeToString(fieldData.Bitmap.Bytes()))
+		asmLog.Debugf("Field %s, Length (Bitmapped): -, Value: %s\n", field.Name, hex.EncodeToString(fieldData.Bitmap.Bytes()))
 		buf.Write(fieldData.Bitmap.Bytes())
 
 	}
@@ -57,7 +63,7 @@ func assemble(buf *bytes.Buffer, parsedMsg *ParsedMsg, fieldData *FieldData) err
 			bmp := fieldData.Bitmap
 			for _, childField := range fieldData.Field.Children {
 				if bmp.IsOn(childField.Position) {
-					if err := assemble(buf, parsedMsg, parsedMsg.FieldDataMap[childField.ID]); err != nil {
+					if err := assemble(buf, meta, parsedMsg, parsedMsg.FieldDataMap[childField.ID]); err != nil {
 						return err
 					}
 				}
@@ -66,19 +72,22 @@ func assemble(buf *bytes.Buffer, parsedMsg *ParsedMsg, fieldData *FieldData) err
 			if field.Type == FixedType {
 				tempBuf := bytes.Buffer{}
 				for _, cf := range fieldData.Field.Children {
-					if err := assemble(&tempBuf, parsedMsg, parsedMsg.FieldDataMap[cf.ID]); err != nil {
+					if err := assemble(&tempBuf, meta, parsedMsg, parsedMsg.FieldDataMap[cf.ID]); err != nil {
 						return err
 					}
 				}
 				buf.Write(tempBuf.Bytes())
 				fieldData.Data = tempBuf.Bytes()
+				if field.Key == true {
+					meta.MessageKey += fieldData.Value()
+				}
 				asmLog.Debugf("Field %s, Length (Fixed): %d, Value: %s\n", field.Name, len(fieldData.Data), hex.EncodeToString(fieldData.Data))
 
 			} else if field.Type == VariableType {
 				//assemble all child fields and then construct the parent
 				tempBuf := bytes.Buffer{}
 				for _, cf := range fieldData.Field.Children {
-					if err := assemble(&tempBuf, parsedMsg, parsedMsg.FieldDataMap[cf.ID]); err != nil {
+					if err := assemble(&tempBuf, meta, parsedMsg, parsedMsg.FieldDataMap[cf.ID]); err != nil {
 						return err
 					}
 				}
@@ -96,6 +105,10 @@ func assemble(buf *bytes.Buffer, parsedMsg *ParsedMsg, fieldData *FieldData) err
 				fieldData.Data = tempBuf.Bytes()
 				asmLog.Debugf("Field %s, LL (Variable): %s, Value: %s\n", field.Name, hex.EncodeToString(lenBuf.Bytes()), hex.EncodeToString(fieldData.Data))
 
+				if field.Key == true {
+					meta.MessageKey += fieldData.Value()
+				}
+
 				buf.Write(lenBuf.Bytes())
 				buf.Write(tempBuf.Bytes())
 

internal/iso/field_parser.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"strconv"
 
-	"github.com/rkbalgi/libiso/encoding/ebcdic"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -25,8 +24,13 @@ var ErrInvalidEncoding = errors.New("isosim: Invalid encoding")
 type ParsedMsg struct {
 	IsRequest bool
 	Msg       *Message
+
 	//A map of Id to FieldData
 	FieldDataMap map[int]*FieldData
+
+	// MessageKey is a value that unique identifies a transaction
+	// (usually a combination of fields like STAN, PAN etc)
+	MessageKey string
 }
 
 // Get returns the field-data from the parsed message given its name
@@ -57,6 +61,7 @@ func (pMsg *ParsedMsg) Copy() *ParsedMsg {
 	}
 
 	newParsedMsg.Msg = pMsg.Msg
+	newParsedMsg.MessageKey = pMsg.MessageKey
 
 	return newParsedMsg
 
@@ -111,7 +116,9 @@ func parseFixed(buf *bytes.Buffer, parsedMsg *ParsedMsg, field *Field) error {
 	if fieldData.Data, err = NextBytes(buf, field.Size); err != nil {
 		return err
 	}
-
+	if field.Key {
+		parsedMsg.MessageKey += fieldData.Value()
+	}
 	log.WithFields(log.Fields{"component": "parser"}).Debugf("Field %s, Length: %d, Value: %s\n", field.Name, field.Size, hex.EncodeToString(fieldData.Data))
 
 	parsedMsg.FieldDataMap[field.ID] = fieldData
@@ -200,8 +207,7 @@ func parseVariable(buf *bytes.Buffer, parsedMsg *ParsedMsg, field *Field) error
 			return err
 		}
 	case EBCDIC:
-
-		if length, err = strconv.ParseUint(ebcdic.EncodeToString(lenData), 10, 64); err != nil {
+		if length, err = strconv.ParseUint(EBCDIC.EncodeToString(lenData), 10, 64); err != nil {
 			return err
 		}
 	default:
@@ -222,6 +228,9 @@ func parseVariable(buf *bytes.Buffer, parsedMsg *ParsedMsg, field *Field) error
 	if fieldData.Data, err = NextBytes(buf, int(length)); err != nil {
 		return err
 	}
+	if field.Key {
+		parsedMsg.MessageKey += fieldData.Value()
+	}
 
 	log.WithFields(log.Fields{"component": "parser"}).Debugf("Field %s, Length: %d, Value: %s\n", field.Name, length, hex.EncodeToString(fieldData.Data))