Merge pull request #627 from arp242/merge

Merge pageviews

CHANGELOG.md

@@ -5,6 +5,10 @@ This list is not comprehensive, and only lists new features and major changes,
 but not every minor bugfix. The goatcounter.com service generally runs the
 latest master.
 
+unreleased
+----------
+- Can now merge paths instead of just deleting them.
+
 2022-10-17 v2.3.0
 -----------------
 - Expand campaigns: the `utm_campaign` or `campaign` parameter now is tracked

cmd/goatcounter/db.go

@@ -524,7 +524,7 @@ func cmdDBQuery(f zli.Flags, dbConnect, debug *string, createdb *bool) error {
 	}
 	defer db.Close()
 
-	q, err := zdb.Load(db, "db.query."+query[0]+".sql")
+	q, _, err := zdb.Load(db, "db.query."+query[0]+".sql")
 	if err != nil {
 		q = strings.Join(query, " ")
 	}

cmd/goatcounter/serve.go

@@ -19,13 +19,15 @@ import (
 
 	"github.com/go-chi/chi/v5"
 	"github.com/teamwork/reload"
+	"golang.org/x/text/language"
 	"zgo.at/blackmail"
 	"zgo.at/errors"
 	"zgo.at/goatcounter/v2"
 	"zgo.at/goatcounter/v2/acme"
 	"zgo.at/goatcounter/v2/bgrun"
 	"zgo.at/goatcounter/v2/cron"
 	"zgo.at/goatcounter/v2/handlers"
+	"zgo.at/z18n"
 	"zgo.at/zdb"
 	"zgo.at/zhttp"
 	"zgo.at/zli"
@@ -363,6 +365,8 @@ func setupServe(dbConnect, dbConn string, dev bool, flagTLS string, automigrate
 		return nil, nil, nil, nil, 0, err
 	}
 
+	ctx = z18n.With(ctx, z18n.NewBundle(language.English).Locale("en"))
+
 	if dev && (!zio.Exists("db/migrate") || !zio.Exists("tpl") || !zio.Exists("public")) {
 		return nil, nil, nil, nil, 0, errors.New("-dev flag was given but this doesn't seem like a GoatCounter source directory")
 	}

cron/language_stat.go

@@ -11,6 +11,7 @@ import (
 	"zgo.at/errors"
 	"zgo.at/goatcounter/v2"
 	"zgo.at/zdb"
+	"zgo.at/zstd/ztype"
 )
 
 func updateLanguageStats(ctx context.Context, hits []goatcounter.Hit) error {
@@ -29,11 +30,11 @@ func updateLanguageStats(ctx context.Context, hits []goatcounter.Hit) error {
 			}
 
 			day := h.CreatedAt.Format("2006-01-02")
-			k := day + h.Language + strconv.FormatInt(h.PathID, 10)
+			k := day + ztype.Deref(h.Language, "") + strconv.FormatInt(h.PathID, 10)
 			v := grouped[k]
 			if v.count == 0 {
 				v.day = day
-				v.language = h.Language
+				v.language = ztype.Deref(h.Language, "")
 				v.pathID = h.PathID
 			}
 

db/query/hit_list.ListPathsLike.gotxt

@@ -0,0 +1,20 @@
+with x as (
+	select path_id, path, title from paths
+	where site_id = :site and (
+		{{if .match_case}}
+			path like :search
+			{{if .match_title}}or title like :search{{end}}
+		{{else}}
+			lower(path) like lower(:search)
+			{{if .match_title}}or lower(title) like lower(:search){{end}}
+		{{end}}
+	)
+)
+select
+	path_id, path, title,
+	sum(total) as count
+from hit_counts
+join x using(path_id)
+where site_id = :site
+group by path_id, path, title
+order by count desc

go.mod

@@ -20,7 +20,7 @@ require (
 	golang.org/x/image v0.0.0-20220902085622-e7cb96979f69
 	golang.org/x/net v0.0.0-20221014081412-f15817d10f9b
 	golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0
-	golang.org/x/text v0.3.8
+	golang.org/x/text v0.4.0
 	zgo.at/blackmail v0.0.0-20211212060815-1f8e8a94692b
 	zgo.at/errors v1.1.0
 	zgo.at/follow v0.0.0-20211017230838-112008350298
@@ -30,23 +30,23 @@ require (
 	zgo.at/json v0.0.0-20211017213340-cc8bf51df08c
 	zgo.at/termtext v1.1.0
 	zgo.at/tz v0.0.0-20211017223207-006eae99adf6
-	zgo.at/z18n v0.0.0-20220606095325-513ddb98b28f
+	zgo.at/z18n v0.0.0-20221018165830-c235ed037573
 	zgo.at/zcache v1.2.0
 	zgo.at/zcache/v2 v2.1.0
-	zgo.at/zdb v0.0.0-20220822042559-7b1209555166
+	zgo.at/zdb v0.0.0-20221018164415-44f0726b1ff7
 	zgo.at/zhttp v0.0.0-20221001220656-a9e60fe5f8e3
 	zgo.at/zli v0.0.0-20221012220610-d6a5a841b943
 	zgo.at/zlog v0.0.0-20211008102840-46c1167bf2a9
 	zgo.at/zprof v0.0.0-20211217104121-c3c12596d8f0
 	zgo.at/zstd v0.0.0-20221013104704-16fa49fadc62
-	zgo.at/ztpl v0.0.0-20211128061406-6ff34b1256c4
+	zgo.at/ztpl v0.0.0-20221018165743-cbd2a654b6af
 	zgo.at/zvalidate v0.0.0-20211128195927-d13b18611e62
 )
 
 require (
 	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/lib/pq v1.10.6 // indirect
+	github.com/lib/pq v1.10.7 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/oschwald/maxminddb-golang v1.8.0 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect

handlers/backend_test.go

@@ -44,7 +44,7 @@ func TestBackendTpl(t *testing.T) {
 		{"/settings/users", "Access"},
 		{"/settings/users/add", "Password"},
 		{"/settings/users/1", "Password"},
-		{"/settings/purge", "Remove all pageviews for a page"},
+		{"/settings/purge", "Remove or merge pageviews"},
 		{"/settings/export", "includes all pageviews"},
 		{"/settings/delete-account", "The site will be marked as deleted"},
 		{"/settings/change-code", "Change your site code and login domain"},

handlers/count.go

@@ -68,7 +68,8 @@ func (h backend) count(w http.ResponseWriter, r *http.Request) error {
 		if len(tags) > 0 {
 			base, c := tags[0].Base()
 			if c == language.Exact || c == language.High {
-				hit.Language = base.ISO3()
+				l := base.ISO3()
+				hit.Language = &l
 			}
 		}
 	}

handlers/settings.go

@@ -72,11 +72,9 @@ func (h settings) mount(r chi.Router) {
 		set.Get("/settings/change-code", zhttp.Wrap(h.changeCode))
 		set.Post("/settings/change-code", zhttp.Wrap(h.changeCode))
 
-		set.Get("/settings/purge", zhttp.Wrap(func(w http.ResponseWriter, r *http.Request) error {
-			return h.purge(nil)(w, r)
-		}))
-		set.Get("/settings/purge/confirm", zhttp.Wrap(h.purgeConfirm))
+		set.Get("/settings/purge", zhttp.Wrap(h.purge))
 		set.Post("/settings/purge", zhttp.Wrap(h.purgeDo))
+		set.Post("/settings/merge", zhttp.Wrap(h.merge))
 
 		set.Get("/settings/export", zhttp.Wrap(func(w http.ResponseWriter, r *http.Request) error {
 			return h.export(nil)(w, r)
@@ -427,30 +425,35 @@ func (h settings) sitesCopySettings(w http.ResponseWriter, r *http.Request) erro
 	return zhttp.SeeOther(w, "/settings/sites")
 }
 
-func (h settings) purge(verr *zvalidate.Validator) zhttp.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) error {
-		return zhttp.Template(w, "settings_purge.gohtml", struct {
-			Globals
-			Validate *zvalidate.Validator
-		}{newGlobals(w, r), verr})
-	}
-}
+func (h settings) purge(w http.ResponseWriter, r *http.Request) error {
+	var (
+		path       = strings.TrimSpace(r.URL.Query().Get("path"))
+		matchTitle = r.URL.Query().Get("match-title") == "on"
+		matchCase  = r.URL.Query().Get("match-case") == "on"
+		list       goatcounter.HitLists
+		paths      goatcounter.Paths
+	)
 
-func (h settings) purgeConfirm(w http.ResponseWriter, r *http.Request) error {
-	path := strings.TrimSpace(r.URL.Query().Get("path"))
-	title := r.URL.Query().Get("match-title") == "on"
+	if path != "" {
+		err := list.ListPathsLike(r.Context(), path, matchTitle, matchCase)
+		if err != nil {
+			return err
+		}
 
-	var list goatcounter.HitLists
-	err := list.ListPathsLike(r.Context(), path, title)
-	if err != nil {
-		return err
+		err = paths.List(r.Context(), goatcounter.MustGetSite(r.Context()).ID)
+		if err != nil {
+			return err
+		}
 	}
 
-	return zhttp.Template(w, "settings_purge_confirm.gohtml", struct {
+	return zhttp.Template(w, "settings_purge.gohtml", struct {
 		Globals
-		PurgePath string
-		List      goatcounter.HitLists
-	}{newGlobals(w, r), path, list})
+		PurgePath  string
+		MatchTitle bool
+		MatchCase  bool
+		List       goatcounter.HitLists
+		AllPaths   goatcounter.Paths
+	}{newGlobals(w, r), path, matchTitle, matchCase, list, paths})
 }
 
 func (h settings) purgeDo(w http.ResponseWriter, r *http.Request) error {
@@ -472,6 +475,31 @@ func (h settings) purgeDo(w http.ResponseWriter, r *http.Request) error {
 	return zhttp.SeeOther(w, "/settings/purge")
 }
 
+func (h settings) merge(w http.ResponseWriter, r *http.Request) error {
+	paths, err := zint.Split(r.Form.Get("paths"), ",")
+	if err != nil {
+		return err
+	}
+
+	v := goatcounter.NewValidate(r.Context())
+	dst := v.Integer("merge_with", r.Form.Get("merge_with"))
+	if v.HasErrors() {
+		return v
+	}
+
+	ctx := goatcounter.CopyContextValues(r.Context())
+	bgrun.Run(fmt.Sprintf("merge:%d", Site(ctx).ID), func() {
+		var list goatcounter.Hits
+		err := list.Merge(ctx, dst, paths)
+		if err != nil {
+			zlog.Error(err)
+		}
+	})
+
+	zhttp.Flash(w, T(r.Context(), "notify/started-background-process|Started in the background; may take about 10-20 seconds to fully process."))
+	return zhttp.SeeOther(w, "/settings/purge")
+}
+
 func (h settings) export(verr *zvalidate.Validator) zhttp.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) error {
 		var exports goatcounter.Exports

handlers/settings_test.go

@@ -30,7 +30,7 @@ func TestSettingsTpl(t *testing.T) {
 				}...)
 			},
 			router:   newBackend,
-			path:     "/settings/purge/confirm?path=/asd",
+			path:     "/settings/purge?path=/asd",
 			auth:     true,
 			wantCode: 200,
 			wantBody: "<tr><td>2</td><td>/asd</td><td>AAA</td></tr>",

hit.go

@@ -45,7 +45,7 @@ type Hit struct {
 	RefScheme       *string    `db:"ref_scheme" json:"-"`
 	UserAgentHeader string     `db:"-" json:"-"`
 	Location        string     `db:"location" json:"-"`
-	Language        string     `db:"language" json:"-"`
+	Language        *string    `db:"language" json:"-"`
 	FirstVisit      zbool.Bool `db:"first_visit" json:"-"`
 	CreatedAt       time.Time  `db:"created_at" json:"-"`
 
@@ -57,6 +57,9 @@ type Hit struct {
 	UserSessionID string `db:"-" json:"-"`
 	BrowserID     int64  `db:"-" json:"-"`
 	SystemID      int64  `db:"-" json:"-"`
+
+	// Don't process in memstore; for merging paths.
+	noProcess bool `db:"-" json:"-"`
 }
 
 func (h *Hit) Ignore() bool {
@@ -378,3 +381,34 @@ func (h *Hits) Purge(ctx context.Context, pathIDs []int64) error {
 		return nil
 	})
 }
+
+// Merge the given paths.
+func (h *Hits) Merge(ctx context.Context, dst int64, pathIDs []int64) error {
+	site := MustGetSite(ctx).ID
+
+	err := (&Path{}).ByID(ctx, dst) // Ensure this site owns the path.
+	if err != nil {
+		return errors.Wrap(err, "Hits.Merge")
+	}
+
+	// Push back to lot to memstore to re-add it again, and then just call
+	// Purge() to delete the old ones.
+	err = zdb.Select(ctx, h, `select * from hits where site_id=? and path_id in (?)`, site, pathIDs)
+	if err != nil {
+		return errors.Wrap(err, "Hits.Merge")
+	}
+	hh := *h
+	for i := range hh {
+		hh[i].PathID = dst
+		hh[i].noProcess = true
+	}
+
+	err = errors.Wrap(h.Purge(ctx, pathIDs), "Hits.Merge")
+	if err != nil {
+		return errors.Wrap(err, "Hits.Merge")
+	}
+
+	// Only push back if delete worked.
+	Memstore.Append(hh...)
+	return nil
+}

hit_list.go

@@ -72,11 +72,12 @@ func (h *HitList) SiteTotalUTC(ctx context.Context, rng ztime.Range) error {
 type HitLists []HitList
 
 // ListPathsLike lists all paths matching the like pattern.
-func (h *HitLists) ListPathsLike(ctx context.Context, search string, matchTitle bool) error {
+func (h *HitLists) ListPathsLike(ctx context.Context, search string, matchTitle, matchCase bool) error {
 	err := zdb.Select(ctx, h, "load:hit_list.ListPathsLike", zdb.P{
 		"site":        MustGetSite(ctx).ID,
 		"search":      search,
 		"match_title": matchTitle,
+		"match_case":  matchCase,
 	})
 	return errors.Wrap(err, "Hits.ListPathsLike")
 }