Skip to content

Commit ec56d3d

Browse files
githubgxllgithubgxll
authored
[fix][dingo-executor] Fix document index scan filter issues (#1350)
1 parent 72c3e7b commit ec56d3d

File tree

11 files changed

+142
-113
lines changed

11 files changed

+142
-113
lines changed

‎dingo-calcite/src/main/java/io/dingodb/calcite/DingoSqlToRelConverter.java

+1
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,7 @@ protected void convertCollectionTable(Blackboard bb, SqlCall call) {
273273
null,
274274
null,
275275
new ArrayList<>(),
276+
null,
276277
false
277278
);
278279
}

‎dingo-calcite/src/main/java/io/dingodb/calcite/rel/DingoDocument.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,11 @@ public DingoDocument(
5353
TupleMapping selection,
5454
RexNode filter,
5555
List<RelHint> hints,
56+
String queryStr,
5657
boolean isDocumentScanFilter
5758
) {
58-
super(cluster, traitSet, call, table, operands, indexTableId, indexTable, selection, filter, hints, isDocumentScanFilter);
59+
super(cluster, traitSet, call, table, operands, indexTableId, indexTable, selection,
60+
filter, hints, queryStr, isDocumentScanFilter);
5961
}
6062

6163
@Override

‎dingo-calcite/src/main/java/io/dingodb/calcite/rel/LogicalDingoDocument.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ public LogicalDingoDocument(RelOptCluster cluster,
9898
TupleMapping selection,
9999
RexNode filter,
100100
List<RelHint> hints,
101+
String queryStr,
101102
boolean isDocumentScanFilter
102103
) {
103104
super(cluster, traitSet, Collections.emptyList(), call, null, call.type, null);
@@ -110,7 +111,8 @@ public LogicalDingoDocument(RelOptCluster cluster,
110111
this.rowType = null;
111112
this.realSelection = selection;
112113
this.hints = hints;
113-
this.queryStr = Objects.requireNonNull((SqlCharStringLiteral) operands.get(2)).getStringValue();
114+
this.queryStr = (queryStr == null ?
115+
Objects.requireNonNull((SqlCharStringLiteral) operands.get(2)).getStringValue(): queryStr);
114116
this.isDocumentScanFilter = isDocumentScanFilter;
115117
DingoTable dingoTable = table.unwrap(DingoTable.class);
116118
if (selection != null) {
@@ -156,7 +158,7 @@ public TableFunctionScan copy(RelTraitSet traitSet,
156158
return new LogicalDingoDocument(
157159
getCluster(),
158160
traitSet,
159-
call, table, operands, indexTableId, indexTable, selection, filter, hints, isDocumentScanFilter);
161+
call, table, operands, indexTableId, indexTable, selection, filter, hints, queryStr, isDocumentScanFilter);
160162
}
161163

162164
@Override

‎dingo-calcite/src/main/java/io/dingodb/calcite/rule/DingoDocumentFilterRule.java

+1
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ public void onMatch(RelOptRuleCall call) {
4747
document.getSelection(),
4848
filter.getCondition(),
4949
filter.getHints(),
50+
null,
5051
document.isDocumentScanFilter()
5152
)
5253
);

‎dingo-calcite/src/main/java/io/dingodb/calcite/rule/DingoDocumentIndexRule.java

+47-31
Original file line numberDiff line numberDiff line change
@@ -307,45 +307,45 @@ private static RelNode prePrimaryOrScalarPlan(
307307
match = false;
308308
break outer;
309309
}
310-
if (type instanceof StringType) {
311-
Properties properties = document.getIndexTable().getProperties();
312-
String json = (String) properties.get("text_fields");
313-
if (json == null) {
314-
match = false;
315-
break outer;
316-
}
317-
try {
318-
ObjectMapper JSON = new ObjectMapper();
319-
JsonNode jsonNode = JSON.readTree(json);
320-
Iterator<Map.Entry<String, JsonNode>> fields = jsonNode.fields();
321-
boolean flag = false;
322-
while (fields.hasNext()) {
323-
Map.Entry<String, JsonNode> next = fields.next();
324-
json = json.replace(next.getKey(), next.getKey().toUpperCase());
325-
JsonNode tokenizer = next.getValue().get("tokenizer");
326-
if (tokenizer == null) {
327-
match = false;
328-
break outer;
329-
}
330-
if (!next.getKey().equalsIgnoreCase(column.getName())) {
331-
continue;
332-
}
333-
flag = true;
310+
Properties properties = document.getIndexTable().getProperties();
311+
String json = (String) properties.get("text_fields");
312+
if (json == null) {
313+
match = false;
314+
break outer;
315+
}
316+
try {
317+
ObjectMapper JSON = new ObjectMapper();
318+
JsonNode jsonNode = JSON.readTree(json);
319+
Iterator<Map.Entry<String, JsonNode>> fields = jsonNode.fields();
320+
boolean flag = false;
321+
while (fields.hasNext()) {
322+
Map.Entry<String, JsonNode> next = fields.next();
323+
json = json.replace(next.getKey(), next.getKey().toUpperCase());
324+
JsonNode tokenizer = next.getValue().get("tokenizer");
325+
if (tokenizer == null) {
326+
match = false;
327+
break outer;
328+
}
329+
if (!next.getKey().equalsIgnoreCase(column.getName())) {
330+
continue;
331+
}
332+
flag = true;
333+
if (type instanceof StringType) {
334334
String tokenType = next.getValue().get("tokenizer").get("type").asText();
335335
if (!tokenType.equalsIgnoreCase("raw")) {
336336
match = false;
337337
break outer;
338338
}
339-
break ;
340-
}
341-
if (!flag) {
342-
match = false;
343-
break;
344339
}
345-
} catch (Exception e) {
340+
break ;
341+
}
342+
if (!flag) {
346343
match = false;
347-
break outer;
344+
break;
348345
}
346+
} catch (Exception e) {
347+
match = false;
348+
break outer;
349349
}
350350
}
351351
}
@@ -367,6 +367,22 @@ private static RelNode prePrimaryOrScalarPlan(
367367
if (flag) {
368368
document.setQueryStr(queryString);
369369
document.setDocumentScanFilter(true);
370+
DingoDocument dingoDocument = new DingoDocument(
371+
document.getCluster(),
372+
document.getTraitSet(),
373+
document.getCall(),
374+
document.getTable(),
375+
document.getOperands(),
376+
document.getIndexTableId(),
377+
document.getIndexTable(),
378+
document.getSelection(),
379+
document.getFilter(),
380+
document.getHints(),
381+
queryString,
382+
true
383+
);
384+
dingoDocument.setQueryStr(queryString);
385+
return dingoDocument;
370386
}
371387
}
372388
return null;

‎dingo-calcite/src/main/java/io/dingodb/calcite/rule/DingoDocumentProjectRule.java

+1
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ select id, distance from xx where not in (sub item size > 20) -> generate same
9898
TupleMapping.of(selectedColumns),
9999
filter,
100100
document.hints,
101+
document.getQueryStr(),
101102
document.isDocumentScanFilter()
102103
);
103104
final List<RexNode> newProjectRexNodes = RexUtil.apply(mapping, project.getProjects());

‎dingo-calcite/src/main/java/io/dingodb/calcite/rule/DingoFunctionScanRule.java

+1
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ public DingoFunctionScanRule(Config config) {
9393
document.getSelection(),
9494
document.getFilter(),
9595
document.hints,
96+
document.getQueryStr(),
9697
document.isDocumentScanFilter()
9798
);
9899
} else if (rel instanceof LogicalDingoHybridSearch && !(rel instanceof DingoHybridSearch)) {

‎dingo-calcite/src/main/java/io/dingodb/calcite/rule/DingoIndexScanMatchRule.java

+79-76
Original file line numberDiff line numberDiff line change
@@ -656,44 +656,45 @@ private static RelNode getIndexFullScanRelNode(LogicalProject project, LogicalDi
656656
match = false;
657657
break outer;
658658
}
659-
if (type instanceof StringType) {
660-
Properties properties = result.matchIndexTable.getProperties();
661-
String json = (String) properties.get("text_fields");
662-
if (json == null) {
663-
match = false;
664-
break outer;
665-
}
666-
try {
667-
ObjectMapper JSON = new ObjectMapper();
668-
JsonNode jsonNode = JSON.readTree(json);
669-
Iterator<Map.Entry<String, JsonNode>> fields = jsonNode.fields();
670-
boolean flag = false;
671-
while (fields.hasNext()) {
672-
Map.Entry<String, JsonNode> next = fields.next();
673-
json = json.replace(next.getKey(), next.getKey().toUpperCase());
674-
JsonNode tokenizer = next.getValue().get("tokenizer");
675-
if (tokenizer == null) {
676-
match = false;
677-
break outer;
678-
}
679-
if (!next.getKey().equalsIgnoreCase(column.getName())) {
680-
continue;
681-
}
682-
flag = true;
659+
Properties properties = result.matchIndexTable.getProperties();
660+
String json = (String) properties.get("text_fields");
661+
if (json == null) {
662+
match = false;
663+
break outer;
664+
}
665+
try {
666+
ObjectMapper JSON = new ObjectMapper();
667+
JsonNode jsonNode = JSON.readTree(json);
668+
Iterator<Map.Entry<String, JsonNode>> fields = jsonNode.fields();
669+
boolean flag = false;
670+
while (fields.hasNext()) {
671+
Map.Entry<String, JsonNode> next = fields.next();
672+
json = json.replace(next.getKey(), next.getKey().toUpperCase());
673+
JsonNode tokenizer = next.getValue().get("tokenizer");
674+
if (tokenizer == null) {
675+
match = false;
676+
break outer;
677+
}
678+
if (!next.getKey().equalsIgnoreCase(column.getName())) {
679+
continue;
680+
}
681+
flag = true;
682+
if (type instanceof StringType) {
683683
String tokenType = next.getValue().get("tokenizer").get("type").asText();
684684
if (!tokenType.equalsIgnoreCase("raw")) {
685685
match = false;
686686
break outer;
687687
}
688688
}
689-
if (!flag) {
690-
match = false;
691-
break;
692-
}
693-
} catch (Exception e) {
689+
break ;
690+
}
691+
if (!flag) {
694692
match = false;
695-
break outer;
693+
break;
696694
}
695+
} catch (Exception e) {
696+
match = false;
697+
break outer;
697698
}
698699
}
699700
}
@@ -714,8 +715,8 @@ private static RelNode getIndexFullScanRelNode(LogicalProject project, LogicalDi
714715
boolean needLookup = isNeedLookUp(finalSelection, tupleMapping, table.columns.size());
715716

716717
RelNode relNode;
717-
boolean flag = false;
718-
String queryString = "";
718+
// boolean flag = false;
719+
// String queryString = "";
719720
if (result.isDocumentIndex) {
720721
DocumentScanFilterVisitor documentScanFilterVisitor = new DocumentScanFilterVisitor(
721722
scan.getCluster().getRexBuilder(),
@@ -726,60 +727,62 @@ private static RelNode getIndexFullScanRelNode(LogicalProject project, LogicalDi
726727
.build()
727728
);
728729
DocumentScanFilterOb accept = rexNode.accept(documentScanFilterVisitor);
729-
flag = accept.isMatch();
730-
queryString = accept.getQueryStr();
731-
}
732-
if (result.isDocumentIndex && flag) {
733-
LogicalDocumentScanFilter indexScan = new LogicalDocumentScanFilter(
734-
scan.getCluster(),
735-
scan.getTraitSet(),
736-
scan.getHints(),
737-
scan.getTable(),
738-
scan.getFilter(),
739-
finalSelection,
740-
result.matchIndexTable,
741-
result.indexId,
742-
scan.isPushDown(),
743-
needLookup,
744-
0,
745-
queryString);
746-
relNode = new LogicalProject(
747-
project.getCluster(),
748-
project.getTraitSet(),
749-
project.getHints(),
750-
indexScan,
751-
newProjectRexNodes,
752-
project.getRowType(),
753-
project.getVariablesSet()
754-
);
755-
} else {
756-
LogicalIndexFullScan indexFullScan = new LogicalIndexFullScan(
757-
scan.getCluster(),
758-
scan.getTraitSet(),
759-
scan.getHints(),
760-
scan.getTable(),
761-
scan.getFilter(),
762-
finalSelection,
763-
result.matchIndexTable,
764-
result.indexId,
765-
result.selectionIxList,
766-
scan.isPushDown(),
767-
needLookup,
768-
0);
769-
if (RexUtil.isIdentity(newProjectRexNodes, scan.getSelectedType())) {
770-
relNode = indexFullScan;
771-
} else {
730+
boolean flag = accept.isMatch();
731+
String queryString = accept.getQueryStr();
732+
if (flag) {
733+
LogicalDocumentScanFilter indexScan = new LogicalDocumentScanFilter(
734+
scan.getCluster(),
735+
scan.getTraitSet(),
736+
scan.getHints(),
737+
scan.getTable(),
738+
scan.getFilter(),
739+
finalSelection,
740+
result.matchIndexTable,
741+
result.indexId,
742+
scan.isPushDown(),
743+
needLookup,
744+
0,
745+
queryString);
772746
relNode = new LogicalProject(
773747
project.getCluster(),
774748
project.getTraitSet(),
775749
project.getHints(),
776-
indexFullScan,
750+
indexScan,
777751
newProjectRexNodes,
778752
project.getRowType(),
779753
project.getVariablesSet()
780754
);
755+
return relNode;
756+
} else {
757+
return null;
781758
}
782759
}
760+
LogicalIndexFullScan indexFullScan = new LogicalIndexFullScan(
761+
scan.getCluster(),
762+
scan.getTraitSet(),
763+
scan.getHints(),
764+
scan.getTable(),
765+
scan.getFilter(),
766+
finalSelection,
767+
result.matchIndexTable,
768+
result.indexId,
769+
result.selectionIxList,
770+
scan.isPushDown(),
771+
needLookup,
772+
0);
773+
if (RexUtil.isIdentity(newProjectRexNodes, scan.getSelectedType())) {
774+
relNode = indexFullScan;
775+
} else {
776+
relNode = new LogicalProject(
777+
project.getCluster(),
778+
project.getTraitSet(),
779+
project.getHints(),
780+
indexFullScan,
781+
newProjectRexNodes,
782+
project.getRowType(),
783+
project.getVariablesSet()
784+
);
785+
}
783786
return relNode;
784787
}
785788

‎dingo-calcite/src/main/java/io/dingodb/calcite/visitor/function/DingoDocumentVisitFun.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ public static Collection<Vertex> visit(
117117
RexNode rexFilter = rel.getFilter();
118118

119119
String queryString;
120-
if (rel.isDocumentScanFilter()) {
120+
if (!rel.isDocumentScanFilter()) {
121121
queryString = Objects.requireNonNull((SqlCharStringLiteral) operandsList.get(2)).getStringValue();
122122
} else {
123123
queryString = rel.getQueryStr();

‎dingo-exec/src/main/java/io/dingodb/exec/transaction/util/TransactionCacheToMutation.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ public static Mutation cacheToMutation(@Nullable int op, @NonNull byte[] key,
162162
} else if (type instanceof TimestampType) {
163163
Timestamp data = (Timestamp) record[colNames.indexOf(columnDef.getName())];
164164
fieldType = DocumentValue.ScalarFieldType.DATETIME;
165-
scalarField = new ScalarField(TimeUtils.to(data));
165+
scalarField = new ScalarField(data != null ? TimeUtils.to(data) : null);
166166
} else if (type instanceof BooleanType) {
167167
Boolean data = (Boolean) record[colNames.indexOf(columnDef.getName())];
168168
fieldType = DocumentValue.ScalarFieldType.BOOL;

‎dingo-store-proxy/src/main/java/io/dingodb/store/proxy/mapper/TxnMapper.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,9 @@ default ScalarField scalarFieldTo(
206206
) {
207207
switch (type) {
208208
case BOOL:
209-
return ScalarField.builder().data(ScalarField.DataNest.BoolData.of((Boolean) field.getData())).build();
209+
return ScalarField.builder().data(field.getData() == null ?
210+
null : ScalarField.DataNest.BoolData.of((Boolean) field.getData()))
211+
.build();
210212
case INTEGER:
211213
return ScalarField.builder().data(ScalarField.DataNest.IntData.of((Integer) field.getData())).build();
212214
case LONG:

0 commit comments

Comments
 (0)