feat(api): add JSON eXtract feature

Author: Дмитрий Бережнов

modules/router/methods.go

@@ -17,7 +17,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/davecgh/go-spew/spew"
 	"io/ioutil"
 	"log"
 	"net/http"
@@ -385,27 +384,37 @@ func (rt *Router) saveHintsToJsonFile(request apiRequest) error {
 	}
 	defer f.Close()
 
-	i := 0
+	scrollId := ""
 	for {
-		if i > 3 {
-			break
-		}
-		fmt.Println(">>>>", i, req)
-		sresponse, err := rt.doPost(host+request.Search.Index+"/_search?scroll=10m", req, "Search")
-		if err != nil {
-			return err
+		sresponse := []byte{}
+		if scrollId == "" {
+			r, err := rt.doPost(host+request.Search.Index+"/_search?scroll=10m", req, "Search")
+			if err != nil {
+				return err
+			}
+			sresponse = r
+		} else {
+			scroll := map[string]interface{}{"scroll": "10m", "scroll_id": scrollId}
+			r, err := rt.doPost(host+"_search/scroll", scroll, "Search")
+			if err != nil {
+				return err
+			}
+			sresponse = r
 		}
 
 		err = json.Unmarshal(sresponse, &scrollresponse)
 		if err != nil {
 			return err
 		}
 
-		spew.Println("Hints total", scrollresponse.HitsRoot.Total.Value)
-		spew.Println("Hints req len", len(scrollresponse.HitsRoot.Hits))
-		spew.Println("ScrollID", scrollresponse.ScrollID)
+		scrollId = scrollresponse.ScrollID
+
+		log.Println("Hints total", scrollresponse.HitsRoot.Total.Value)
+		log.Println("Hints req len", len(scrollresponse.HitsRoot.Hits))
+		log.Println("scrollId", scrollId)
 
 		if len(scrollresponse.HitsRoot.Hits) == 0 {
+			log.Println("Search is done!")
 			break
 		}
 
@@ -440,12 +449,6 @@ func (rt *Router) saveHintsToJsonFile(request apiRequest) error {
 			f.WriteString(string(jsonData) + "\n")
 
 		}
-
-		if scrollresponse.ScrollID != "" {
-			//req = map[string]interface{}{"scroll": "10m", "scroll_id": scrollresponse.ScrollID}
-		}
-
-		i = i + 1
 	}
 
 	return nil