Fix doublefetching of query messages

Author: yshym

modules/web-discovery-project/sources/content-extractor.es

@@ -97,7 +97,7 @@ export class ContentExtractor {
     this.urlAnalyzer = new UrlAnalyzer(this.patterns);
   }
 
-  run(pageContent, url) {
+  run(pageContent, url, addStrictQuery) {
     function discard(reason = "") {
       logger.debug("No messages found for query:", query, "Reason:", reason);
       return {
@@ -109,6 +109,9 @@ export class ContentExtractor {
     const { found, type, query } = this.urlAnalyzer.parseLinks(url);
     if (!found) return discard("No content found.");
 
+    if (addStrictQuery)
+      this.wdp.addStrictQueries(url, query);
+
     const messages = this.extractMessages(pageContent, type, query, url);
     if (messages.length === 0) {
       return discard("No content found.");

modules/web-discovery-project/sources/url-analyzer.es

@@ -2,6 +2,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
+import { extractHostname } from "../core/tlds";
 import { parse } from "../core/url";
 import logger from "./logger";
 
@@ -135,4 +136,22 @@ export default class UrlAnalyzer {
     const parsedUrl = parse(url);
     return isBraveSearch && parsedUrl.searchParams.get("q");
   }
+
+  checkAnonSearchURL(url, query) {
+    const { found, type } = this.parseLinks(url);
+    if (!found) return { isSearchEngineUrl: false, queryUrl: null };
+    const isSearchEngineUrl = SEARCH_ENGINE_TYPES.has(type);
+    const urlPattern = URL_PATTERNS.find((p) => p.type == type);
+    const queryPrefix = urlPattern.prefix;
+    if (!queryPrefix) {
+      logger.debug(
+        `URL pattern with type '${urlPattern.type}' has no query prefix`
+      );
+      return { isSearchEngineUrl: false, queryUrl: null };
+    }
+    const encodedQuery = encodeURIComponent(query).replace(/%20/g, "+");
+    const hostname = extractHostname(url);
+    const queryUrl = `https://${hostname}/${queryPrefix}${encodedQuery}`;
+    return { isSearchEngineUrl, queryUrl };
+  }
 }

modules/web-discovery-project/sources/web-discovery-project.es

@@ -39,6 +39,12 @@ function _log(...msg) {
   }
 }
 
+function getRandomIntInclusive(min, max) {
+  const _min = Math.ceil(min);
+  const _max = Math.floor(max);
+  return Math.floor(random() * (_max - _min + 1)) + min;
+}
+
 function cleanFinalUrl(domain, href) {
   /*
     We need to get the final domain, there are 2 elements that we try to capture.
@@ -1597,7 +1603,7 @@ const WebDiscoveryProject = {
           WebDiscoveryProject.patterns.update(rules);
           WebDiscoveryProject._patternsLastUpdated = new Date();
           logger.info(
-            "WebDiscoveryProject patterns successfully updated at ${this._patternsLastUpdated}"
+            `WebDiscoveryProject patterns successfully updated at ${WebDiscoveryProject._patternsLastUpdated}`
           );
         } catch (e) {
           logger.warn("Failed to apply new WebDiscoveryProject patterns", e);
@@ -3382,7 +3388,7 @@ const WebDiscoveryProject = {
                 }
                 getContentDocument(originalURL)
                   .then((doc) => {
-                    WebDiscoveryProject.checkURL(doc, url);
+                    WebDiscoveryProject.checkURL(doc, url, true);
                   })
                   .catch((e) => {
                     logger.info(
@@ -3518,7 +3524,7 @@ const WebDiscoveryProject = {
                       )
                     ) {
                       try {
-                        WebDiscoveryProject.checkURL(cd, currURL);
+                        WebDiscoveryProject.checkURL(cd, currURL, false);
                       } catch (e) {}
                       //Check active usage...
                       // WebDiscoveryProject.activeUsage += 1;
@@ -3619,7 +3625,7 @@ const WebDiscoveryProject = {
               }
             })
             .catch((e) => {
-              _log("Error fetching fetching the currentURL: " + e);
+              _log("Error fetching the currentURL: " + e);
             });
 
           WebDiscoveryProject.counter += 4;
@@ -4701,10 +4707,11 @@ const WebDiscoveryProject = {
     else return null;
   },
 
-  checkURL(pageContent, url) {
+  checkURL(pageContent, url, addStrictQuery) {
     const { messages } = WebDiscoveryProject.contentExtractor.run(
       pageContent,
-      url
+      url,
+      addStrictQuery
     );
     for (const message of messages)
       WebDiscoveryProject.telemetry({
@@ -5043,7 +5050,7 @@ const WebDiscoveryProject = {
           e.qurl,
           function (url, page_data, ourl, x) {
             let cd = WebDiscoveryProject.docCache[url]["doc"];
-            WebDiscoveryProject.checkURL(cd, url);
+            WebDiscoveryProject.checkURL(cd, url, false);
           },
           function (a, b, c, d) {
             _log("Error aux>>>> " + d);
@@ -5714,6 +5721,38 @@ const WebDiscoveryProject = {
       }
     });
   },
+  addStrictQueries(url, query) {
+    // In some cases, we get query undefined.
+    if (!query) {
+      _log(">> Got an undefined query >>> " + url);
+      return;
+    }
+
+    if (WebDiscoveryProject.isSuspiciousQuery(query)) {
+      _log("Dropping suspicious query before double-fetch:", query);
+      return;
+    }
+
+    const { isSearchEngineUrl, queryUrl } =
+      WebDiscoveryProject.contentExtractor.urlAnalyzer.checkAnonSearchURL(
+        url,
+        query
+      );
+    if (isSearchEngineUrl) {
+      try {
+        const qObj = {
+          qurl: queryUrl,
+          ts: Date.now(),
+          tDiff: getRandomIntInclusive(1, 20),
+        };
+        logger.debug("PCN: pushed to strictQueries:", queryUrl);
+        WebDiscoveryProject.strictQueries.push(qObj);
+        WebDiscoveryProject.saveStrictQueries();
+      } catch (ee) {
+        logger.error("Failed to add query:", ee);
+      }
+    }
+  },
 };
 WebDiscoveryProject.contentExtractor = new ContentExtractor(
   WebDiscoveryProject.patterns,