Add files for 5 lab

datastore/db.go

@@ -0,0 +1,131 @@
+package datastore
+
+import (
+	"bufio"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+const outFileName = "current-data"
+
+var ErrNotFound = fmt.Errorf("record does not exist")
+
+type hashIndex map[string]int64
+
+type Db struct {
+	out *os.File
+	outPath string
+	outOffset int64
+
+	index hashIndex
+}
+
+func NewDb(dir string) (*Db, error) {
+	outputPath := filepath.Join(dir, outFileName)
+	f, err := os.OpenFile(outputPath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o600)
+	if err != nil {
+		return nil, err
+	}
+	db := &Db{
+		outPath: outputPath,
+		out:     f,
+		index:   make(hashIndex),
+	}
+	err = db.recover()
+	if err != nil && err != io.EOF {
+		return nil, err
+	}
+	return db, nil
+}
+
+const bufSize = 8192
+
+func (db *Db) recover() error {
+	input, err := os.Open(db.outPath)
+	if err != nil {
+		return err
+	}
+	defer input.Close()
+
+	var buf [bufSize]byte
+	in := bufio.NewReaderSize(input, bufSize)
+	for err == nil {
+		var (
+			header, data []byte
+			n int
+		)
+		header, err = in.Peek(bufSize)
+		if err == io.EOF {
+			if len(header) == 0 {
+				return err
+			}
+		} else if err != nil {
+			return err
+		}
+		size := binary.LittleEndian.Uint32(header)
+
+		if size < bufSize {
+			data = buf[:size]
+		} else {
+			data = make([]byte, size)
+		}
+		n, err = in.Read(data)
+
+		if err == nil {
+			if n != int(size) {
+				return fmt.Errorf("corrupted file")
+			}
+
+			var e entry
+			e.Decode(data)
+			db.index[e.key] = db.outOffset
+			db.outOffset += int64(n)
+		}
+	}
+	return err
+}
+
+func (db *Db) Close() error {
+	return db.out.Close()
+}
+
+func (db *Db) Get(key string) (string, error) {
+	position, ok := db.index[key]
+	if !ok {
+		return "", ErrNotFound
+	}
+
+	file, err := os.Open(db.outPath)
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+
+	_, err = file.Seek(position, 0)
+	if err != nil {
+		return "", err
+	}
+
+	reader := bufio.NewReader(file)
+	value, err := readValue(reader)
+	if err != nil {
+		return "", err
+	}
+	return value, nil
+}
+
+func (db *Db) Put(key, value string) error {
+	e := entry{
+		key:   key,
+		value: value,
+	}
+	n, err := db.out.Write(e.Encode())
+	if err == nil {
+		db.index[key] = db.outOffset
+		db.outOffset += int64(n)
+	}
+	return err
+}

datastore/db_test.go

@@ -0,0 +1,92 @@
+package datastore
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestDb_Put(t *testing.T) {
+	dir, err := ioutil.TempDir("", "test-db")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	db, err := NewDb(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	pairs := [][]string {
+		{"key1", "value1"},
+		{"key2", "value2"},
+		{"key3", "value3"},
+	}
+
+	outFile, err := os.Open(filepath.Join(dir, outFileName))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("put/get", func(t *testing.T) {
+		for _, pair := range pairs {
+			err := db.Put(pair[0], pair[1])
+			if err != nil {
+				t.Errorf("Cannot put %s: %s", pairs[0], err)
+			}
+			value, err := db.Get(pair[0])
+			if err != nil {
+				t.Errorf("Cannot get %s: %s", pairs[0], err)
+			}
+			if value != pair[1] {
+				t.Errorf("Bad value returned expected %s, got %s", pair[1], value)
+			}
+		}
+	})
+
+	outInfo, err := outFile.Stat()
+	if err != nil {
+		t.Fatal(err)
+	}
+	size1 := outInfo.Size()
+
+	t.Run("file growth", func(t *testing.T) {
+		for _, pair := range pairs {
+			err := db.Put(pair[0], pair[1])
+			if err != nil {
+				t.Errorf("Cannot put %s: %s", pairs[0], err)
+			}
+		}
+		outInfo, err := outFile.Stat()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if size1 * 2 != outInfo.Size() {
+			t.Errorf("Unexpected size (%d vs %d)", size1, outInfo.Size())
+		}
+	})
+
+	t.Run("new db process", func(t *testing.T) {
+		if err := db.Close(); err != nil {
+			t.Fatal(err)
+		}
+		db, err = NewDb(dir)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		for _, pair := range pairs {
+			value, err := db.Get(pair[0])
+			if err != nil {
+				t.Errorf("Cannot put %s: %s", pairs[0], err)
+			}
+			if value != pair[1] {
+				t.Errorf("Bad value returned expected %s, got %s", pair[1], value)
+			}
+		}
+	})
+
+}

datastore/entry.go

@@ -0,0 +1,69 @@
+package datastore
+
+import (
+	"bufio"
+	"encoding/binary"
+	"fmt"
+)
+
+type entry struct {
+	key, value string
+}
+
+func (e *entry) Encode() []byte {
+	kl := len(e.key)
+	vl := len(e.value)
+	size := kl + vl + 12
+	res := make([]byte, size)
+	binary.LittleEndian.PutUint32(res, uint32(size))
+	binary.LittleEndian.PutUint32(res[4:], uint32(kl))
+	copy(res[8:], e.key)
+	binary.LittleEndian.PutUint32(res[kl+8:], uint32(vl))
+	copy(res[kl+12:], e.value)
+	return res
+}
+
+func (e *entry) Decode(input []byte) {
+	kl := binary.LittleEndian.Uint32(input[4:])
+	keyBuf := make([]byte, kl)
+	copy(keyBuf, input[8:kl+8])
+	e.key = string(keyBuf)
+
+	vl := binary.LittleEndian.Uint32(input[kl+8:])
+	valBuf := make([]byte, vl)
+	copy(valBuf, input[kl+12:kl+12+vl])
+	e.value = string(valBuf)
+}
+
+func readValue(in *bufio.Reader) (string, error) {
+	header, err := in.Peek(8)
+	if err != nil {
+		return "", err
+	}
+	keySize := int(binary.LittleEndian.Uint32(header[4:]))
+	_, err = in.Discard(keySize + 8)
+	if err != nil {
+		return "", err
+	}
+
+	header, err = in.Peek(4)
+	if err != nil {
+		return "", err
+	}
+	valSize := int(binary.LittleEndian.Uint32(header))
+	_, err = in.Discard(4)
+	if err != nil {
+		return "", err
+	}
+
+	data := make([]byte, valSize)
+	n, err := in.Read(data)
+	if err != nil {
+		return "", err
+	}
+	if n != valSize {
+		return "", fmt.Errorf("can't read value bytes (read %d, expected %d)", n, valSize)
+	}
+
+	return string(data), nil
+}

datastore/entry_test.go

@@ -0,0 +1,30 @@
+package datastore
+
+import (
+	"bufio"
+	"bytes"
+	"testing"
+)
+
+func TestEntry_Encode(t *testing.T) {
+	e := entry{"key", "value"}
+	e.Decode(e.Encode())
+	if e.key != "key" {
+		t.Error("incorrect key")
+	}
+	if e.value != "value" {
+		t.Error("incorrect value")
+	}
+}
+
+func TestReadValue(t *testing.T) {
+	e := entry{"key", "test-value"}
+	data := e.Encode()
+	v, err := readValue(bufio.NewReader(bytes.NewReader(data)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if v != "test-value" {
+		t.Errorf("Got bat value [%s]", v)
+	}
+}