Skip to content

Commit 27f19ea

Browse files
ndyakovndyakov
authored
fix(aggregate, search): ft.aggregate bugfixes (redis#3263)
* fix: rearange args for ft.aggregate apply should be before any groupby or sortby * improve test * wip: add scorer and addscores * enable all tests * fix ftsearch with count test * make linter happy * Addscores is available in later redisearch releases. For safety state it is available in redis ce 8 * load an apply seem to break scorer and addscores
1 parent 84cb9d2 commit 27f19ea

File tree

2 files changed

+158
-25
lines changed

2 files changed

+158
-25
lines changed

Diff for: ‎search_commands.go

+58-23
Original file line numberDiff line numberDiff line change
@@ -240,13 +240,20 @@ type FTAggregateWithCursor struct {
240240
}
241241

242242
type FTAggregateOptions struct {
243-
Verbatim bool
244-
LoadAll bool
245-
Load []FTAggregateLoad
246-
Timeout int
247-
GroupBy []FTAggregateGroupBy
248-
SortBy []FTAggregateSortBy
249-
SortByMax int
243+
Verbatim bool
244+
LoadAll bool
245+
Load []FTAggregateLoad
246+
Timeout int
247+
GroupBy []FTAggregateGroupBy
248+
SortBy []FTAggregateSortBy
249+
SortByMax int
250+
// Scorer is used to set scoring function, if not set passed, a default will be used.
251+
// The default scorer depends on the Redis version:
252+
// - `BM25` for Redis >= 8
253+
// - `TFIDF` for Redis < 8
254+
Scorer string
255+
// AddScores is available in Redis CE 8
256+
AddScores bool
250257
Apply []FTAggregateApply
251258
LimitOffset int
252259
Limit int
@@ -490,6 +497,15 @@ func FTAggregateQuery(query string, options *FTAggregateOptions) AggregateQuery
490497
if options.Verbatim {
491498
queryArgs = append(queryArgs, "VERBATIM")
492499
}
500+
501+
if options.Scorer != "" {
502+
queryArgs = append(queryArgs, "SCORER", options.Scorer)
503+
}
504+
505+
if options.AddScores {
506+
queryArgs = append(queryArgs, "ADDSCORES")
507+
}
508+
493509
if options.LoadAll && options.Load != nil {
494510
panic("FT.AGGREGATE: LOADALL and LOAD are mutually exclusive")
495511
}
@@ -505,9 +521,18 @@ func FTAggregateQuery(query string, options *FTAggregateOptions) AggregateQuery
505521
}
506522
}
507523
}
524+
508525
if options.Timeout > 0 {
509526
queryArgs = append(queryArgs, "TIMEOUT", options.Timeout)
510527
}
528+
529+
for _, apply := range options.Apply {
530+
queryArgs = append(queryArgs, "APPLY", apply.Field)
531+
if apply.As != "" {
532+
queryArgs = append(queryArgs, "AS", apply.As)
533+
}
534+
}
535+
511536
if options.GroupBy != nil {
512537
for _, groupBy := range options.GroupBy {
513538
queryArgs = append(queryArgs, "GROUPBY", len(groupBy.Fields))
@@ -549,12 +574,6 @@ func FTAggregateQuery(query string, options *FTAggregateOptions) AggregateQuery
549574
if options.SortByMax > 0 {
550575
queryArgs = append(queryArgs, "MAX", options.SortByMax)
551576
}
552-
for _, apply := range options.Apply {
553-
queryArgs = append(queryArgs, "APPLY", apply.Field)
554-
if apply.As != "" {
555-
queryArgs = append(queryArgs, "AS", apply.As)
556-
}
557-
}
558577
if options.LimitOffset > 0 {
559578
queryArgs = append(queryArgs, "LIMIT", options.LimitOffset)
560579
}
@@ -581,6 +600,7 @@ func FTAggregateQuery(query string, options *FTAggregateOptions) AggregateQuery
581600
queryArgs = append(queryArgs, key, value)
582601
}
583602
}
603+
584604
if options.DialectVersion > 0 {
585605
queryArgs = append(queryArgs, "DIALECT", options.DialectVersion)
586606
}
@@ -661,11 +681,12 @@ func (cmd *AggregateCmd) readReply(rd *proto.Reader) (err error) {
661681
data, err := rd.ReadSlice()
662682
if err != nil {
663683
cmd.err = err
664-
return nil
684+
return err
665685
}
666686
cmd.val, err = ProcessAggregateResult(data)
667687
if err != nil {
668688
cmd.err = err
689+
return err
669690
}
670691
return nil
671692
}
@@ -681,6 +702,12 @@ func (c cmdable) FTAggregateWithArgs(ctx context.Context, index string, query st
681702
if options.Verbatim {
682703
args = append(args, "VERBATIM")
683704
}
705+
if options.Scorer != "" {
706+
args = append(args, "SCORER", options.Scorer)
707+
}
708+
if options.AddScores {
709+
args = append(args, "ADDSCORES")
710+
}
684711
if options.LoadAll && options.Load != nil {
685712
panic("FT.AGGREGATE: LOADALL and LOAD are mutually exclusive")
686713
}
@@ -699,6 +726,12 @@ func (c cmdable) FTAggregateWithArgs(ctx context.Context, index string, query st
699726
if options.Timeout > 0 {
700727
args = append(args, "TIMEOUT", options.Timeout)
701728
}
729+
for _, apply := range options.Apply {
730+
args = append(args, "APPLY", apply.Field)
731+
if apply.As != "" {
732+
args = append(args, "AS", apply.As)
733+
}
734+
}
702735
if options.GroupBy != nil {
703736
for _, groupBy := range options.GroupBy {
704737
args = append(args, "GROUPBY", len(groupBy.Fields))
@@ -740,12 +773,6 @@ func (c cmdable) FTAggregateWithArgs(ctx context.Context, index string, query st
740773
if options.SortByMax > 0 {
741774
args = append(args, "MAX", options.SortByMax)
742775
}
743-
for _, apply := range options.Apply {
744-
args = append(args, "APPLY", apply.Field)
745-
if apply.As != "" {
746-
args = append(args, "AS", apply.As)
747-
}
748-
}
749776
if options.LimitOffset > 0 {
750777
args = append(args, "LIMIT", options.LimitOffset)
751778
}
@@ -1693,7 +1720,8 @@ func (cmd *FTSearchCmd) readReply(rd *proto.Reader) (err error) {
16931720

16941721
// FTSearch - Executes a search query on an index.
16951722
// The 'index' parameter specifies the index to search, and the 'query' parameter specifies the search query.
1696-
// For more information, please refer to the Redis documentation:
1723+
// For more information, please refer to the Redis documentation about [FT.SEARCH].
1724+
//
16971725
// [FT.SEARCH]: (https://redis.io/commands/ft.search/)
16981726
func (c cmdable) FTSearch(ctx context.Context, index string, query string) *FTSearchCmd {
16991727
args := []interface{}{"FT.SEARCH", index, query}
@@ -1704,6 +1732,12 @@ func (c cmdable) FTSearch(ctx context.Context, index string, query string) *FTSe
17041732

17051733
type SearchQuery []interface{}
17061734

1735+
// FTSearchQuery - Executes a search query on an index with additional options.
1736+
// The 'index' parameter specifies the index to search, the 'query' parameter specifies the search query,
1737+
// and the 'options' parameter specifies additional options for the search.
1738+
// For more information, please refer to the Redis documentation about [FT.SEARCH].
1739+
//
1740+
// [FT.SEARCH]: (https://redis.io/commands/ft.search/)
17071741
func FTSearchQuery(query string, options *FTSearchOptions) SearchQuery {
17081742
queryArgs := []interface{}{query}
17091743
if options != nil {
@@ -1816,7 +1850,8 @@ func FTSearchQuery(query string, options *FTSearchOptions) SearchQuery {
18161850
// FTSearchWithArgs - Executes a search query on an index with additional options.
18171851
// The 'index' parameter specifies the index to search, the 'query' parameter specifies the search query,
18181852
// and the 'options' parameter specifies additional options for the search.
1819-
// For more information, please refer to the Redis documentation:
1853+
// For more information, please refer to the Redis documentation about [FT.SEARCH].
1854+
//
18201855
// [FT.SEARCH]: (https://redis.io/commands/ft.search/)
18211856
func (c cmdable) FTSearchWithArgs(ctx context.Context, index string, query string, options *FTSearchOptions) *FTSearchCmd {
18221857
args := []interface{}{"FT.SEARCH", index, query}
@@ -1908,7 +1943,7 @@ func (c cmdable) FTSearchWithArgs(ctx context.Context, index string, query strin
19081943
}
19091944
}
19101945
if options.SortByWithCount {
1911-
args = append(args, "WITHCOUT")
1946+
args = append(args, "WITHCOUNT")
19121947
}
19131948
}
19141949
if options.LimitOffset >= 0 && options.Limit > 0 {

Diff for: ‎search_test.go

+100-2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package redis_test
22

33
import (
44
"context"
5+
"fmt"
6+
"strconv"
57
"time"
68

79
. "github.com/bsm/ginkgo/v2"
@@ -127,8 +129,11 @@ var _ = Describe("RediSearch commands Resp 2", Label("search"), func() {
127129

128130
res3, err := client.FTSearchWithArgs(ctx, "num", "foo", &redis.FTSearchOptions{NoContent: true, SortBy: []redis.FTSearchSortBy{sortBy2}, SortByWithCount: true}).Result()
129131
Expect(err).NotTo(HaveOccurred())
130-
Expect(res3.Total).To(BeEquivalentTo(int64(0)))
132+
Expect(res3.Total).To(BeEquivalentTo(int64(3)))
131133

134+
res4, err := client.FTSearchWithArgs(ctx, "num", "notpresentf00", &redis.FTSearchOptions{NoContent: true, SortBy: []redis.FTSearchSortBy{sortBy2}, SortByWithCount: true}).Result()
135+
Expect(err).NotTo(HaveOccurred())
136+
Expect(res4.Total).To(BeEquivalentTo(int64(0)))
132137
})
133138

134139
It("should FTCreate and FTSearch example", Label("search", "ftcreate", "ftsearch"), func() {
@@ -640,6 +645,100 @@ var _ = Describe("RediSearch commands Resp 2", Label("search"), func() {
640645
Expect(res.Rows[0].Fields["t2"]).To(BeEquivalentTo("world"))
641646
})
642647

648+
It("should FTAggregate with scorer and addscores", Label("search", "ftaggregate", "NonRedisEnterprise"), func() {
649+
SkipBeforeRedisMajor(8, "ADDSCORES is available in Redis CE 8")
650+
title := &redis.FieldSchema{FieldName: "title", FieldType: redis.SearchFieldTypeText, Sortable: false}
651+
description := &redis.FieldSchema{FieldName: "description", FieldType: redis.SearchFieldTypeText, Sortable: false}
652+
val, err := client.FTCreate(ctx, "idx1", &redis.FTCreateOptions{OnHash: true, Prefix: []interface{}{"product:"}}, title, description).Result()
653+
Expect(err).NotTo(HaveOccurred())
654+
Expect(val).To(BeEquivalentTo("OK"))
655+
WaitForIndexing(client, "idx1")
656+
657+
client.HSet(ctx, "product:1", "title", "New Gaming Laptop", "description", "this is not a desktop")
658+
client.HSet(ctx, "product:2", "title", "Super Old Not Gaming Laptop", "description", "this laptop is not a new laptop but it is a laptop")
659+
client.HSet(ctx, "product:3", "title", "Office PC", "description", "office desktop pc")
660+
661+
options := &redis.FTAggregateOptions{
662+
AddScores: true,
663+
Scorer: "BM25",
664+
SortBy: []redis.FTAggregateSortBy{{
665+
FieldName: "@__score",
666+
Desc: true,
667+
}},
668+
}
669+
670+
res, err := client.FTAggregateWithArgs(ctx, "idx1", "laptop", options).Result()
671+
Expect(err).NotTo(HaveOccurred())
672+
Expect(res).ToNot(BeNil())
673+
Expect(len(res.Rows)).To(BeEquivalentTo(2))
674+
score1, err := strconv.ParseFloat(fmt.Sprintf("%s", res.Rows[0].Fields["__score"]), 64)
675+
Expect(err).NotTo(HaveOccurred())
676+
score2, err := strconv.ParseFloat(fmt.Sprintf("%s", res.Rows[1].Fields["__score"]), 64)
677+
Expect(err).NotTo(HaveOccurred())
678+
Expect(score1).To(BeNumerically(">", score2))
679+
680+
optionsDM := &redis.FTAggregateOptions{
681+
AddScores: true,
682+
Scorer: "DISMAX",
683+
SortBy: []redis.FTAggregateSortBy{{
684+
FieldName: "@__score",
685+
Desc: true,
686+
}},
687+
}
688+
689+
resDM, err := client.FTAggregateWithArgs(ctx, "idx1", "laptop", optionsDM).Result()
690+
Expect(err).NotTo(HaveOccurred())
691+
Expect(resDM).ToNot(BeNil())
692+
Expect(len(resDM.Rows)).To(BeEquivalentTo(2))
693+
score1DM, err := strconv.ParseFloat(fmt.Sprintf("%s", resDM.Rows[0].Fields["__score"]), 64)
694+
Expect(err).NotTo(HaveOccurred())
695+
score2DM, err := strconv.ParseFloat(fmt.Sprintf("%s", resDM.Rows[1].Fields["__score"]), 64)
696+
Expect(err).NotTo(HaveOccurred())
697+
Expect(score1DM).To(BeNumerically(">", score2DM))
698+
699+
Expect(score1DM).To(BeEquivalentTo(float64(4)))
700+
Expect(score2DM).To(BeEquivalentTo(float64(1)))
701+
Expect(score1).NotTo(BeEquivalentTo(score1DM))
702+
Expect(score2).NotTo(BeEquivalentTo(score2DM))
703+
})
704+
705+
It("should FTAggregate apply and groupby", Label("search", "ftaggregate"), func() {
706+
text1 := &redis.FieldSchema{FieldName: "PrimaryKey", FieldType: redis.SearchFieldTypeText, Sortable: true}
707+
num1 := &redis.FieldSchema{FieldName: "CreatedDateTimeUTC", FieldType: redis.SearchFieldTypeNumeric, Sortable: true}
708+
val, err := client.FTCreate(ctx, "idx1", &redis.FTCreateOptions{}, text1, num1).Result()
709+
Expect(err).NotTo(HaveOccurred())
710+
Expect(val).To(BeEquivalentTo("OK"))
711+
WaitForIndexing(client, "idx1")
712+
713+
// 6 feb
714+
client.HSet(ctx, "doc1", "PrimaryKey", "9::362330", "CreatedDateTimeUTC", "1738823999")
715+
716+
// 12 feb
717+
client.HSet(ctx, "doc2", "PrimaryKey", "9::362329", "CreatedDateTimeUTC", "1739342399")
718+
client.HSet(ctx, "doc3", "PrimaryKey", "9::362329", "CreatedDateTimeUTC", "1739353199")
719+
720+
reducer := redis.FTAggregateReducer{Reducer: redis.SearchCount, As: "perDay"}
721+
722+
options := &redis.FTAggregateOptions{
723+
Apply: []redis.FTAggregateApply{{Field: "floor(@CreatedDateTimeUTC /(60*60*24))", As: "TimestampAsDay"}},
724+
GroupBy: []redis.FTAggregateGroupBy{{
725+
Fields: []interface{}{"@TimestampAsDay"},
726+
Reduce: []redis.FTAggregateReducer{reducer},
727+
}},
728+
SortBy: []redis.FTAggregateSortBy{{
729+
FieldName: "@perDay",
730+
Desc: true,
731+
}},
732+
}
733+
734+
res, err := client.FTAggregateWithArgs(ctx, "idx1", "*", options).Result()
735+
Expect(err).NotTo(HaveOccurred())
736+
Expect(res).ToNot(BeNil())
737+
Expect(len(res.Rows)).To(BeEquivalentTo(2))
738+
Expect(res.Rows[0].Fields["perDay"]).To(BeEquivalentTo("2"))
739+
Expect(res.Rows[1].Fields["perDay"]).To(BeEquivalentTo("1"))
740+
})
741+
643742
It("should FTAggregate apply", Label("search", "ftaggregate"), func() {
644743
text1 := &redis.FieldSchema{FieldName: "PrimaryKey", FieldType: redis.SearchFieldTypeText, Sortable: true}
645744
num1 := &redis.FieldSchema{FieldName: "CreatedDateTimeUTC", FieldType: redis.SearchFieldTypeNumeric, Sortable: true}
@@ -684,7 +783,6 @@ var _ = Describe("RediSearch commands Resp 2", Label("search"), func() {
684783
Expect(res.Rows[0].Fields["age"]).To(BeEquivalentTo("19"))
685784
Expect(res.Rows[1].Fields["age"]).To(BeEquivalentTo("25"))
686785
}
687-
688786
})
689787

690788
It("should FTSearch SkipInitialScan", Label("search", "ftsearch"), func() {

0 commit comments

Comments
 (0)