Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add handling for struct in named tuples #1500

Merged
merged 3 commits into from
Feb 27, 2025
+263 −0
Merged

Add handling for struct in named tuples #1500

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4a87f90
Add handling for struct in named tuples
Exca-DK Feb 15, 2025
594d2ab
Fix test
Exca-DK Feb 17, 2025
43f4092
Merge branch 'main' into Exca-DK/struct-named-tuple
SpencerTorres Feb 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 62 additions & 0 deletions lib/column/tuple.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -511,6 +511,68 @@ func (col *Tuple) AppendRow(v any) error {
value = value.Elem()
}
switch value.Kind() {
case reflect.Struct:
if valuer, ok := v.(driver.Valuer); ok {
val, err := valuer.Value()
if err != nil {
return &ColumnConverterError{
Op: "AppendRow",
To: string(col.chType),
From: fmt.Sprintf("%T", v),
Hint: "could not get driver.Valuer value",
}
}
return col.AppendRow(val)
}

if !col.isNamed {
return &Error{
ColumnType: string(col.chType),
Err: fmt.Errorf("converting from %T is not supported for unnamed tuples - use a slice", v),
}
}

valueType := value.Type()
fieldNames := make(map[string]struct{}, value.NumField())
for i := 0; i < value.NumField(); i++ {
if !value.Field(i).CanInterface() {
// can't interface - likely not exported so ignore the field
continue
}
name, omit := getStructFieldName(valueType.Field(i))
if omit {
continue
}
fieldNames[name] = struct{}{}
}

if len(fieldNames) != len(col.columns) {
return &Error{
ColumnType: string(col.chType),
Err: fmt.Errorf("invalid size. expected %d got %d", len(col.columns), len(fieldNames)),
}
}

for i := 0; i < value.NumField(); i++ {
if !value.Field(i).CanInterface() {
// can't interface - likely not exported so ignore the field
continue
}
name, omit := getStructFieldName(valueType.Field(i))
if omit {
continue
}
if _, ok := col.index[name]; !ok {
return &Error{
ColumnType: string(col.chType),
Err: fmt.Errorf("sub column '%s' does not exist in %s", name, col.Name()),
}
}
if err := col.columns[col.index[name]].AppendRow(value.Field(i).Interface()); err != nil {
return err
}
}
return nil
case reflect.Map:
if !col.isNamed {
return &Error{
Expand Down
201 changes: 201 additions & 0 deletions tests/tuple_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,174 @@ func TestNamedTupleWithTypedMap(t *testing.T) {
assert.Equal(t, col1Data, col1)
}

// named tuples work with typed structs
func TestNamedTupleWithStruct(t *testing.T) {
conn, err := GetNativeConnection(nil, nil, nil)
ctx := context.Background()
require.NoError(t, err)
// https://github.com/ClickHouse/ClickHouse/pull/36544
if !CheckMinServerServerVersion(conn, 22, 5, 0) {
t.Skip(fmt.Errorf("unsupported clickhouse version"))
return
}
const ddl = "CREATE TABLE test_tuple (Col1 Tuple(Id Int64, Code Int64)) Engine MergeTree() ORDER BY tuple()"

defer func() {
conn.Exec(ctx, "DROP TABLE IF EXISTS test_tuple")
}()
require.NoError(t, conn.Exec(ctx, ddl))
batch, err := conn.PrepareBatch(ctx, "INSERT INTO test_tuple")
require.NoError(t, err)
var (
col1Data = struct {
Code int64
Id int64
}{
Code: 1,
Id: 2,
}
)
require.NoError(t, batch.Append(col1Data))
require.Equal(t, 1, batch.Rows())
require.NoError(t, batch.Send())
var (
col1 struct {
Code int64
Id int64
}
)
require.NoError(t, conn.QueryRow(ctx, "SELECT * FROM test_tuple").Scan(&col1))
assert.Equal(t, col1Data, col1)
}

// named tuples work with typed structs tags
func TestNamedTupleWithStructTags(t *testing.T) {
conn, err := GetNativeConnection(nil, nil, nil)
ctx := context.Background()
require.NoError(t, err)
// https://github.com/ClickHouse/ClickHouse/pull/36544
if !CheckMinServerServerVersion(conn, 22, 5, 0) {
t.Skip(fmt.Errorf("unsupported clickhouse version"))
return
}
const ddl = "CREATE TABLE test_tuple (Col1 Tuple(id Int64, code Int64)) Engine MergeTree() ORDER BY tuple()"

defer func() {
conn.Exec(ctx, "DROP TABLE IF EXISTS test_tuple")
}()
require.NoError(t, conn.Exec(ctx, ddl))
batch, err := conn.PrepareBatch(ctx, "INSERT INTO test_tuple")
require.NoError(t, err)
var (
col1Data = struct {
Code int64 `ch:"code"`
Id int64 `ch:"id"`
}{
Code: 1,
Id: 2,
}
)
require.NoError(t, batch.Append(col1Data))
require.Equal(t, 1, batch.Rows())
require.NoError(t, batch.Send())
var (
col1 struct {
Code int64 `ch:"code"`
Id int64 `ch:"id"`
}
)
require.NoError(t, conn.QueryRow(ctx, "SELECT * FROM test_tuple").Scan(&col1))
assert.Equal(t, col1Data, col1)
}

// named tuples will not work with unexported fields
func TestNamedTupleWithUnexportedStructField(t *testing.T) {
conn, err := GetNativeConnection(nil, nil, nil)
ctx := context.Background()
require.NoError(t, err)
// https://github.com/ClickHouse/ClickHouse/pull/36544
if !CheckMinServerServerVersion(conn, 22, 5, 0) {
t.Skip(fmt.Errorf("unsupported clickhouse version"))
return
}
const ddl = "CREATE TABLE test_tuple (Col1 Tuple(id Int64, code Int64)) Engine MergeTree() ORDER BY tuple()"

defer func() {
conn.Exec(ctx, "DROP TABLE IF EXISTS test_tuple")
}()
require.NoError(t, conn.Exec(ctx, ddl))
batch, err := conn.PrepareBatch(ctx, "INSERT INTO test_tuple")
require.NoError(t, err)
var (
col1Data = struct {
foo int64 // unexported field shouldn't be counted.
Bar int64
}{}
)
err = batch.Append(col1Data)
require.Error(t, err)
require.Equal(t, "clickhouse [AppendRow]: (Col1 Tuple(id Int64, code Int64)) invalid size. expected 2 got 1", err.Error())
}

// named tuples will not work with too many fields
func TestNamedTupleWithTooManyFields(t *testing.T) {
conn, err := GetNativeConnection(nil, nil, nil)
ctx := context.Background()
require.NoError(t, err)
// https://github.com/ClickHouse/ClickHouse/pull/36544
if !CheckMinServerServerVersion(conn, 22, 5, 0) {
t.Skip(fmt.Errorf("unsupported clickhouse version"))
return
}
const ddl = "CREATE TABLE test_tuple (Col1 Tuple(id Int64, code Int64)) Engine MergeTree() ORDER BY tuple()"

defer func() {
conn.Exec(ctx, "DROP TABLE IF EXISTS test_tuple")
}()
require.NoError(t, conn.Exec(ctx, ddl))
batch, err := conn.PrepareBatch(ctx, "INSERT INTO test_tuple")
require.NoError(t, err)
var (
col1Data = struct {
Foo int64
Bar int64
Baz int64
}{}
)
err = batch.Append(col1Data)
require.Error(t, err)
require.Equal(t, "clickhouse [AppendRow]: (Col1 Tuple(id Int64, code Int64)) invalid size. expected 2 got 3", err.Error())
}

// named tuples will not work with invalid tags
func TestNamedTupleWithDuplicateTags(t *testing.T) {
conn, err := GetNativeConnection(nil, nil, nil)
ctx := context.Background()
require.NoError(t, err)
// https://github.com/ClickHouse/ClickHouse/pull/36544
if !CheckMinServerServerVersion(conn, 22, 5, 0) {
t.Skip(fmt.Errorf("unsupported clickhouse version"))
return
}
const ddl = "CREATE TABLE test_tuple (Col1 Tuple(id Int64, code Int64)) Engine MergeTree() ORDER BY tuple()"

defer func() {
conn.Exec(ctx, "DROP TABLE IF EXISTS test_tuple")
}()
require.NoError(t, conn.Exec(ctx, ddl))
batch, err := conn.PrepareBatch(ctx, "INSERT INTO test_tuple")
require.NoError(t, err)
var (
col1Data = struct {
Id int64 `ch:"id"`
Code int64 `ch:"id"` // duplicate tag, should be counted only once.
}{}
)
err = batch.Append(col1Data)
require.Error(t, err)
require.Equal(t, "clickhouse [AppendRow]: (Col1 Tuple(id Int64, code Int64)) invalid size. expected 2 got 1", err.Error())
}

// test column names which need escaping
func TestNamedTupleWithEscapedColumns(t *testing.T) {
conn, err := GetNativeConnection(nil, nil, nil)
Expand Down Expand Up @@ -332,6 +500,39 @@ func TestUnNamedTupleWithMap(t *testing.T) {
require.Equal(t, "clickhouse [ScanRow]: (Col1) converting Tuple(String, Int64) to map[string]interface {} is unsupported. cannot use maps for unnamed tuples, use slice", err.Error())
}

// unnamed tuples will not work with structs - keys cannot be attributed to fields
func TestUnNamedTupleWithStruct(t *testing.T) {
conn, err := GetNativeConnection(nil, nil, nil)
ctx := context.Background()
require.NoError(t, err)
// https://github.com/ClickHouse/ClickHouse/pull/36544
if !CheckMinServerServerVersion(conn, 22, 5, 0) {
t.Skip(fmt.Errorf("unsupported clickhouse version"))
return
}
const ddl = "CREATE TABLE test_tuple (Col1 Tuple(String, Int64)) Engine MergeTree() ORDER BY tuple()"

defer func() {
conn.Exec(ctx, "DROP TABLE IF EXISTS test_tuple")
}()
require.NoError(t, conn.Exec(ctx, ddl))
batch, err := conn.PrepareBatch(ctx, "INSERT INTO test_tuple")
require.NoError(t, err)
var (
col1Data = struct {
Name string
Id int64
}{
Name: "a",
Id: 1,
}
)
// this will fail - struct can't be used for unnamed tuples
err = batch.Append(col1Data)
require.Error(t, err)
require.Equal(t, "clickhouse [AppendRow]: (Col1 Tuple(String, Int64)) converting from struct { Name string; Id int64 } is not supported for unnamed tuples - use a slice", err.Error())
}

func TestColumnarTuple(t *testing.T) {
conn, err := GetNativeConnection(nil, nil, nil)
ctx := context.Background()
Expand Down
Loading