Merge pull request #292 from noborus/add-text-format

Add Text Format.

Author: noborus

cmd/cmd.go

@@ -124,14 +124,15 @@ func (cli Cli) Run(args []string) int {
 	flags.IntVar(&inLimitRead, "ilr", 0, "limited number of rows to read.")
 	flags.StringVar(&inJQuery, "ijq", "", "jq expression string for input(JSON/JSONL only).")
 	flags.Var(&inNull, "inull", "value(string) to convert to null on input.")
+	flags.BoolVar(&inRowNumber, "inum", false, "add row number column.")
 
 	flags.BoolVar(&inFlag.CSV, "icsv", false, "CSV format for input.")
 	flags.BoolVar(&inFlag.LTSV, "iltsv", false, "LTSV format for input.")
 	flags.BoolVar(&inFlag.JSON, "ijson", false, "JSON format for input.")
 	flags.BoolVar(&inFlag.YAML, "iyaml", false, "YAML format for input.")
 	flags.BoolVar(&inFlag.TBLN, "itbln", false, "TBLN format for input.")
 	flags.BoolVar(&inFlag.WIDTH, "iwidth", false, "width specification format for input.")
-	flags.BoolVar(&inRowNumber, "inum", false, "add row number column.")
+	flags.BoolVar(&inFlag.TEXT, "itext", false, "text format for input.")
 
 	flags.StringVar(&outFile, "out", "", "output file name.")
 	flags.BoolVar(&outWithoutGuess, "out-without-guess", false, "output without guessing (when using -out).")
@@ -501,6 +502,7 @@ type inputFlag struct {
 	YAML  bool
 	TBLN  bool
 	WIDTH bool
+	TEXT  bool
 }
 
 // inputFormat returns format from flag.
@@ -518,14 +520,16 @@ func inputFormat(i inputFlag) trdsql.Format {
 		return trdsql.TBLN
 	case i.WIDTH:
 		return trdsql.WIDTH
+	case i.TEXT:
+		return trdsql.TEXT
 	default:
 		return trdsql.GUESS
 	}
 }
 
 func isInFormat(name string) bool {
 	switch name {
-	case "ig", "icsv", "iltsv", "ijson", "iyaml", "itbln", "iwidth":
+	case "ig", "icsv", "iltsv", "ijson", "iyaml", "itbln", "iwidth", "itext":
 		return true
 	}
 	return false

input_text.go

@@ -0,0 +1,65 @@
+package trdsql
+
+import (
+	"bufio"
+	"io"
+	"strings"
+)
+
+// TextReader provides a reader for text format.
+type TextReader struct {
+	reader *bufio.Reader
+	num    int
+	maxNum int
+}
+
+// NewTextReader returns a new TextReader.
+func NewTextReader(reader io.Reader, opts *ReadOpts) (*TextReader, error) {
+	r := &TextReader{
+		reader: bufio.NewReader(reader),
+	}
+
+	if opts.InSkip > 0 {
+		skipRead(r, opts.InSkip)
+	}
+
+	if opts.InLimitRead {
+		r.maxNum = opts.InPreRead
+	}
+	return r, nil
+}
+
+// Names returns column names.
+func (r *TextReader) Names() ([]string, error) {
+	return []string{"text"}, nil
+}
+
+// Types returns column types.
+func (r *TextReader) Types() ([]string, error) {
+	return []string{"text"}, nil
+}
+
+// PreReadRow returns pre-read rows.
+func (r *TextReader) PreReadRow() [][]any {
+	return nil
+}
+
+// ReadRow reads a row.
+func (r *TextReader) ReadRow([]any) ([]any, error) {
+	var builder strings.Builder
+	for {
+		if r.maxNum > 0 && r.num >= r.maxNum {
+			return []any{""}, io.EOF
+		}
+		line, isPrefix, err := r.reader.ReadLine()
+		if err != nil {
+			return []any{""}, err
+		}
+		builder.Write(line)
+		if isPrefix {
+			continue
+		}
+		r.num++
+		return []any{builder.String()}, nil
+	}
+}

input_text_test.go

@@ -0,0 +1,101 @@
+package trdsql
+
+import (
+	"io"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestNewTextReader(t *testing.T) {
+	type args struct {
+		reader io.Reader
+		opts   *ReadOpts
+	}
+	tests := []struct {
+		name string
+		args args
+	}{
+		{
+			name: "test1",
+			args: args{
+				reader: strings.NewReader("a\nb\nc\n"),
+				opts:   NewReadOpts(),
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := NewTextReader(tt.args.reader, tt.args.opts)
+			if err != nil {
+				t.Fatal(err)
+			}
+			names, err := got.Names()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !reflect.DeepEqual(names, []string{"text"}) {
+				t.Errorf("TextReader.Names() != text %v", names)
+			}
+			types, err := got.Types()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !reflect.DeepEqual(types, []string{"text"}) {
+				t.Errorf("TextReader.Types() != text %v", types)
+			}
+		})
+	}
+}
+
+func TestTextReaderFile(t *testing.T) {
+	tests := []struct {
+		name     string
+		fileName string
+		opts     *ReadOpts
+		want     []any
+		wantErr  bool
+	}{
+		{
+			name:     "test.csv",
+			fileName: "test.csv",
+			opts:     NewReadOpts(),
+			want:     []any{"1,Orange"},
+			wantErr:  false,
+		},
+		{
+			name:     "test.csv2",
+			fileName: "test.csv",
+			opts:     &ReadOpts{InSkip: 1},
+			want:     []any{"2,Melon"},
+			wantErr:  false,
+		},
+		{
+			name:     "test.csv3",
+			fileName: "test.csv",
+			opts:     &ReadOpts{InLimitRead: true, InPreRead: 1},
+			want:     []any{"1,Orange"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			file, err := singleFileOpen(filepath.Join(dataDir, tt.fileName))
+			if err != nil {
+				t.Error(err)
+			}
+			r, err := NewTextReader(file, tt.opts)
+			if err != nil {
+				t.Fatal(err)
+			}
+			got, err := r.ReadRow(nil)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("TextReader.ReadRow() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("TextReader.ReadRow() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

reader.go

@@ -49,6 +49,9 @@ var readerFuncs = map[Format]ReaderFunc{
 	WIDTH: func(reader io.Reader, opts *ReadOpts) (Reader, error) {
 		return NewGWReader(reader, opts)
 	},
+	TEXT: func(reader io.Reader, opts *ReadOpts) (Reader, error) {
+		return NewTextReader(reader, opts)
+	},
 }
 
 var (

trdsql.go

@@ -77,6 +77,9 @@ const (
 	// Format using guesswidth library.
 	WIDTH
 
+	// import
+	TEXT
+
 	// export
 	// Output as it is.
 	// Multiple characters can be selected as delimiter.

trdsql_test.go

@@ -554,6 +554,34 @@ func TestTBLNRun(t *testing.T) {
 	}
 }
 
+func TestTextRun(t *testing.T) {
+	testText := [][]string{
+		{"test.csv", `1,"1,Orange"
+2,"2,Melon"
+3,"3,Apple"
+`},
+		{"aiu.csv", "1,あ\n2,い\n3,う\n"},
+	}
+	outStream := new(bytes.Buffer)
+	importer := NewImporter(
+		InFormat(TEXT),
+		InRowNumber(true),
+	)
+	exporter := NewExporter(NewWriter(OutStream(outStream)))
+	trd := NewTRDSQL(importer, exporter)
+	for _, c := range testText {
+		sqlQuery := "SELECT * FROM " + filepath.Join(dataDir, c[0])
+		err := trd.Exec(sqlQuery)
+		if err != nil {
+			t.Errorf("trdsql error %s", err)
+		}
+		if outStream.String() != c[1] {
+			t.Fatalf("trdsql error %s:%s:%s", c[0], c[1], outStream)
+		}
+		outStream.Reset()
+	}
+}
+
 func setOutFormatTRDSQL(outFormat Format, outStream io.Writer) *TRDSQL {
 	importer := NewImporter(
 		InFormat(GUESS),