Skip to content

Commit 4ad4fac

Browse files
umeshmaumeshma
authored
knowPro query experiments (#618)
"Where" predicate experiments - action predicates - entity predicates - where expression
1 parent f4a10ce commit 4ad4fac

File tree

6 files changed

+214
-36
lines changed

6 files changed

+214
-36
lines changed

‎ts/examples/chat/src/memory/knowproMemory.ts

+1
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,7 @@ export async function createKnowproCommands(
196196
context.printer.writeLine("No matches");
197197
return;
198198
}
199+
context.printer.writeLine();
199200
context.printer.writeSearchResults(
200201
conversation,
201202
matches,

‎ts/packages/knowPro/src/conversationIndex.ts

+5-2
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,11 @@ function addFacet(
2626
) {
2727
if (facet !== undefined) {
2828
semanticRefIndex.addTerm(facet.name, refIndex);
29-
if (facet.value !== undefined && typeof facet.value === "string") {
30-
semanticRefIndex.addTerm(facet.value, refIndex);
29+
if (facet.value !== undefined) {
30+
semanticRefIndex.addTerm(
31+
conversation.knowledgeValueToString(facet.value),
32+
refIndex,
33+
);
3134
}
3235
}
3336
}

‎ts/packages/knowPro/src/dataFormat.ts

+8
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,14 @@ export type Term = {
111111
score?: number | undefined;
112112
};
113113

114+
export type QueryTerm = {
115+
term: Term;
116+
/**
117+
* These can be supplied from fuzzy synonym tables and so on
118+
*/
119+
relatedTerms?: Term[] | undefined;
120+
};
121+
114122
export interface ITermToRelatedTermsIndex {
115123
lookupTerm(term: string): Promise<Term[] | undefined>;
116124
}

‎ts/packages/knowPro/src/index.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,4 @@ export * from "./dataFormat.js";
66
export * from "./conversationIndex.js";
77
export * from "./termIndex.js";
88
export * from "./search.js";
9-
export * from "./query.js";
9+
//export * from "./query.js";

‎ts/packages/knowPro/src/query.ts

+186-29
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,13 @@ import {
77
ITermToRelatedTermsIndex,
88
ITermToSemanticRefIndex,
99
KnowledgeType,
10+
QueryTerm,
1011
ScoredSemanticRef,
1112
SemanticRef,
1213
SemanticRefIndex,
1314
Term,
1415
} from "./dataFormat.js";
16+
import * as knowLib from "knowledge-processor";
1517

1618
export function isConversationSearchable(conversation: IConversation): boolean {
1719
return (
@@ -26,14 +28,6 @@ export interface IQueryOpExpr<T> {
2628
eval(context: QueryEvalContext): Promise<T>;
2729
}
2830

29-
export type QueryTerm = {
30-
term: Term;
31-
/**
32-
* These can be supplied from fuzzy synonym tables and so on
33-
*/
34-
relatedTerms?: Term[] | undefined;
35-
};
36-
3731
export class QueryEvalContext {
3832
constructor(private conversation: IConversation) {
3933
if (!isConversationSearchable(conversation)) {
@@ -69,17 +63,14 @@ export class SelectTopNExpr<T extends MatchAccumulator>
6963
}
7064

7165
export class TermsMatchExpr implements IQueryOpExpr<SemanticRefAccumulator> {
72-
constructor(
73-
public terms: IQueryOpExpr<QueryTerm[]>,
74-
public semanticRefIndex?: ITermToSemanticRefIndex,
75-
) {}
66+
constructor(public terms: IQueryOpExpr<QueryTerm[]>) {}
7667

7768
public async eval(
7869
context: QueryEvalContext,
7970
): Promise<SemanticRefAccumulator> {
8071
const matchAccumulator: SemanticRefAccumulator =
8172
new SemanticRefAccumulator();
82-
const index = this.semanticRefIndex ?? context.semanticRefIndex;
73+
const index = context.semanticRefIndex;
8374
const terms = await this.terms.eval(context);
8475
for (const queryTerm of terms) {
8576
this.accumulateMatches(index, matchAccumulator, queryTerm);
@@ -102,6 +93,7 @@ export class TermsMatchExpr implements IQueryOpExpr<SemanticRefAccumulator> {
10293
// BUT are scored with the score of the related term
10394
matchAccumulator.addRelatedTermMatch(
10495
queryTerm.term,
96+
relatedTerm,
10597
index.lookupTerm(relatedTerm.text),
10698
relatedTerm.score,
10799
);
@@ -111,14 +103,11 @@ export class TermsMatchExpr implements IQueryOpExpr<SemanticRefAccumulator> {
111103
}
112104

113105
export class ResolveRelatedTermsExpr implements IQueryOpExpr<QueryTerm[]> {
114-
constructor(
115-
public terms: IQueryOpExpr<QueryTerm[]>,
116-
public index?: ITermToRelatedTermsIndex,
117-
) {}
106+
constructor(public terms: IQueryOpExpr<QueryTerm[]>) {}
118107

119108
public async eval(context: QueryEvalContext): Promise<QueryTerm[]> {
120109
const terms = await this.terms.eval(context);
121-
const index = this.index ?? context.relatedTermIndex;
110+
const index = context.relatedTermIndex;
122111
if (index !== undefined) {
123112
for (const queryTerm of terms) {
124113
if (
@@ -184,6 +173,113 @@ export class SelectTopNKnowledgeGroupExpr
184173
}
185174
}
186175

176+
export class WhereSemanticRefExpr
177+
implements IQueryOpExpr<SemanticRefAccumulator>
178+
{
179+
constructor(
180+
public sourceExpr: IQueryOpExpr<SemanticRefAccumulator>,
181+
public predicates: IQueryOpPredicate[],
182+
) {}
183+
184+
public async eval(
185+
context: QueryEvalContext,
186+
): Promise<SemanticRefAccumulator> {
187+
const accumulator = await this.sourceExpr.eval(context);
188+
const filtered = new SemanticRefAccumulator(
189+
accumulator.queryTermMatches,
190+
);
191+
const semanticRefs = context.semanticRefs;
192+
filtered.setMatches(
193+
accumulator.getMatchesWhere((match) =>
194+
this.testOr(semanticRefs, accumulator.queryTermMatches, match),
195+
),
196+
);
197+
return filtered;
198+
}
199+
200+
private testOr(
201+
semanticRefs: SemanticRef[],
202+
queryTermMatches: QueryTermAccumulator,
203+
match: Match<SemanticRefIndex>,
204+
) {
205+
for (let i = 0; i < this.predicates.length; ++i) {
206+
const semanticRef = semanticRefs[match.value];
207+
if (this.predicates[i].eval(queryTermMatches, semanticRef)) {
208+
return true;
209+
}
210+
}
211+
return false;
212+
}
213+
}
214+
215+
export interface IQueryOpPredicate {
216+
eval(termMatches: QueryTermAccumulator, semanticRef: SemanticRef): boolean;
217+
}
218+
219+
export class EntityPredicate implements IQueryOpPredicate {
220+
constructor(
221+
public type: string | undefined,
222+
public name: string | undefined,
223+
public facetName: string | undefined,
224+
) {}
225+
226+
public eval(
227+
termMatches: QueryTermAccumulator,
228+
semanticRef: SemanticRef,
229+
): boolean {
230+
if (semanticRef.knowledgeType !== "entity") {
231+
return false;
232+
}
233+
const entity =
234+
semanticRef.knowledge as knowLib.conversation.ConcreteEntity;
235+
return (
236+
termMatches.matched(entity.type, this.type) &&
237+
termMatches.matched(entity.name, this.name) &&
238+
this.matchFacet(termMatches, entity, this.facetName)
239+
);
240+
}
241+
242+
private matchFacet(
243+
termMatches: QueryTermAccumulator,
244+
entity: knowLib.conversation.ConcreteEntity,
245+
facetName?: string | undefined,
246+
): boolean {
247+
if (facetName === undefined || entity.facets === undefined) {
248+
return false;
249+
}
250+
for (const facet of entity.facets) {
251+
if (termMatches.matched(facet.name, facetName)) {
252+
return true;
253+
}
254+
}
255+
return false;
256+
}
257+
}
258+
259+
export class ActionPredicate implements IQueryOpPredicate {
260+
constructor(
261+
public subjectEntityName?: string | undefined,
262+
public objectEntityName?: string | undefined,
263+
) {}
264+
265+
public eval(
266+
termMatches: QueryTermAccumulator,
267+
semanticRef: SemanticRef,
268+
): boolean {
269+
if (semanticRef.knowledgeType !== "action") {
270+
return false;
271+
}
272+
const action = semanticRef.knowledge as knowLib.conversation.Action;
273+
return (
274+
termMatches.matched(
275+
action.subjectEntityName,
276+
this.subjectEntityName,
277+
) &&
278+
termMatches.matched(action.objectEntityName, this.objectEntityName)
279+
);
280+
}
281+
}
282+
187283
export interface Match<T = any> {
188284
value: T;
189285
score: number;
@@ -335,7 +431,7 @@ export class MatchAccumulator<T = any> {
335431
}
336432

337433
export class SemanticRefAccumulator extends MatchAccumulator<SemanticRefIndex> {
338-
constructor(public termMatches: Set<string> = new Set<string>()) {
434+
constructor(public queryTermMatches = new QueryTermAccumulator()) {
339435
super();
340436
}
341437

@@ -349,17 +445,20 @@ export class SemanticRefAccumulator extends MatchAccumulator<SemanticRefIndex> {
349445
for (const match of semanticRefs) {
350446
this.add(match.semanticRefIndex, match.score + scoreBoost);
351447
}
352-
this.recordTermMatch(term.text);
448+
this.queryTermMatches.add(term);
353449
}
354450
}
355451

356452
public addRelatedTermMatch(
357-
term: Term,
453+
primaryTerm: Term,
454+
relatedTerm: Term,
358455
semanticRefs: ScoredSemanticRef[] | undefined,
359456
scoreBoost?: number,
360457
) {
361458
if (semanticRefs) {
362-
scoreBoost ??= term.score ?? 0;
459+
// Related term matches count as matches for the queryTerm...
460+
// BUT are scored with the score of the related term
461+
scoreBoost ??= relatedTerm.score ?? 0;
363462
for (const semanticRef of semanticRefs) {
364463
let score = semanticRef.score + scoreBoost;
365464
let match = this.getMatch(semanticRef.semanticRefIndex);
@@ -376,14 +475,10 @@ export class SemanticRefAccumulator extends MatchAccumulator<SemanticRefIndex> {
376475
this.setMatch(match);
377476
}
378477
}
379-
this.recordTermMatch(term.text);
478+
this.queryTermMatches.add(primaryTerm, relatedTerm);
380479
}
381480
}
382481

383-
public recordTermMatch(term: string) {
384-
this.termMatches.add(term);
385-
}
386-
387482
public override getSortedByScore(
388483
minHitCount?: number,
389484
): Match<SemanticRefIndex>[] {
@@ -409,7 +504,7 @@ export class SemanticRefAccumulator extends MatchAccumulator<SemanticRefIndex> {
409504
let group = groups.get(semanticRef.knowledgeType);
410505
if (group === undefined) {
411506
group = new SemanticRefAccumulator();
412-
group.termMatches = this.termMatches;
507+
group.queryTermMatches = this.queryTermMatches;
413508
groups.set(semanticRef.knowledgeType, group);
414509
}
415510
group.setMatch(match);
@@ -427,6 +522,68 @@ export class SemanticRefAccumulator extends MatchAccumulator<SemanticRefIndex> {
427522
}
428523

429524
private getMinHitCount(minHitCount?: number): number {
430-
return minHitCount !== undefined ? minHitCount : this.termMatches.size;
525+
return minHitCount !== undefined
526+
? minHitCount
527+
: this.queryTermMatches.termMatches.size;
528+
}
529+
}
530+
531+
export class QueryTermAccumulator {
532+
constructor(
533+
public termMatches: Set<string> = new Set<string>(),
534+
public relatedTermToTerms: Map<string, Set<string>> = new Map<
535+
string,
536+
Set<string>
537+
>(),
538+
) {}
539+
540+
public add(term: Term, relatedTerm?: Term) {
541+
this.termMatches.add(term.text);
542+
if (relatedTerm !== undefined) {
543+
let relatedTermToTerms = this.relatedTermToTerms.get(
544+
relatedTerm.text,
545+
);
546+
if (relatedTermToTerms === undefined) {
547+
relatedTermToTerms = new Set<string>();
548+
this.relatedTermToTerms.set(
549+
relatedTerm.text,
550+
relatedTermToTerms,
551+
);
552+
}
553+
relatedTermToTerms.add(term.text);
554+
}
555+
}
556+
557+
public matched(
558+
testText: string | string[] | undefined,
559+
expectedText: string | undefined,
560+
): boolean {
561+
if (expectedText === undefined) {
562+
return true;
563+
}
564+
if (testText === undefined) {
565+
return false;
566+
}
567+
568+
if (Array.isArray(testText)) {
569+
for (const text of testText) {
570+
if (this.matched(text, expectedText)) {
571+
return true;
572+
}
573+
}
574+
return false;
575+
}
576+
577+
if (testText === expectedText) {
578+
return true;
579+
}
580+
581+
// Maybe the test text matched a related term.
582+
// If so, the matching related term should have matched on behalf of
583+
// of a term === expectedTerm
584+
const relatedTermToTerms = this.relatedTermToTerms.get(testText);
585+
return relatedTermToTerms !== undefined
586+
? relatedTermToTerms.has(expectedText)
587+
: false;
431588
}
432589
}

0 commit comments

Comments
 (0)