diff --git a/CHANGELOG.md b/CHANGELOG.md index aa6b326fd..d9db7c082 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ This list is not comprehensive, and only lists new features and major changes, but not every minor bugfix. The goatcounter.com service generally runs the latest master. +unreleased +---------- +- Can now merge paths instead of just deleting them. + 2022-10-17 v2.3.0 ----------------- - Expand campaigns: the `utm_campaign` or `campaign` parameter now is tracked diff --git a/cmd/goatcounter/db.go b/cmd/goatcounter/db.go index 44a8a5f1e..145bc9586 100644 --- a/cmd/goatcounter/db.go +++ b/cmd/goatcounter/db.go @@ -524,7 +524,7 @@ func cmdDBQuery(f zli.Flags, dbConnect, debug *string, createdb *bool) error { } defer db.Close() - q, err := zdb.Load(db, "db.query."+query[0]+".sql") + q, _, err := zdb.Load(db, "db.query."+query[0]+".sql") if err != nil { q = strings.Join(query, " ") } diff --git a/cmd/goatcounter/serve.go b/cmd/goatcounter/serve.go index 774cdbd1a..c3d7457f4 100644 --- a/cmd/goatcounter/serve.go +++ b/cmd/goatcounter/serve.go @@ -19,6 +19,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/teamwork/reload" + "golang.org/x/text/language" "zgo.at/blackmail" "zgo.at/errors" "zgo.at/goatcounter/v2" @@ -26,6 +27,7 @@ import ( "zgo.at/goatcounter/v2/bgrun" "zgo.at/goatcounter/v2/cron" "zgo.at/goatcounter/v2/handlers" + "zgo.at/z18n" "zgo.at/zdb" "zgo.at/zhttp" "zgo.at/zli" @@ -363,6 +365,8 @@ func setupServe(dbConnect, dbConn string, dev bool, flagTLS string, automigrate return nil, nil, nil, nil, 0, err } + ctx = z18n.With(ctx, z18n.NewBundle(language.English).Locale("en")) + if dev && (!zio.Exists("db/migrate") || !zio.Exists("tpl") || !zio.Exists("public")) { return nil, nil, nil, nil, 0, errors.New("-dev flag was given but this doesn't seem like a GoatCounter source directory") } diff --git a/cron/language_stat.go b/cron/language_stat.go index ce0313051..dc5bd315a 100644 --- a/cron/language_stat.go +++ b/cron/language_stat.go @@ -11,6 +11,7 @@ import ( "zgo.at/errors" "zgo.at/goatcounter/v2" "zgo.at/zdb" + "zgo.at/zstd/ztype" ) func updateLanguageStats(ctx context.Context, hits []goatcounter.Hit) error { @@ -29,11 +30,11 @@ func updateLanguageStats(ctx context.Context, hits []goatcounter.Hit) error { } day := h.CreatedAt.Format("2006-01-02") - k := day + h.Language + strconv.FormatInt(h.PathID, 10) + k := day + ztype.Deref(h.Language, "") + strconv.FormatInt(h.PathID, 10) v := grouped[k] if v.count == 0 { v.day = day - v.language = h.Language + v.language = ztype.Deref(h.Language, "") v.pathID = h.PathID } diff --git a/db/query/hit_list.ListPathsLike.gotxt b/db/query/hit_list.ListPathsLike.gotxt new file mode 100644 index 000000000..e76c656f0 --- /dev/null +++ b/db/query/hit_list.ListPathsLike.gotxt @@ -0,0 +1,20 @@ +with x as ( + select path_id, path, title from paths + where site_id = :site and ( + {{if .match_case}} + path like :search + {{if .match_title}}or title like :search{{end}} + {{else}} + lower(path) like lower(:search) + {{if .match_title}}or lower(title) like lower(:search){{end}} + {{end}} + ) +) +select + path_id, path, title, + sum(total) as count +from hit_counts +join x using(path_id) +where site_id = :site +group by path_id, path, title +order by count desc diff --git a/db/query/hit_list.ListPathsLike.sql b/db/query/hit_list.ListPathsLike.sql deleted file mode 100644 index 640aadb60..000000000 --- a/db/query/hit_list.ListPathsLike.sql +++ /dev/null @@ -1,13 +0,0 @@ -with x as ( - select path_id, path, title from paths - where site_id = :site and - (lower(path) like lower(:search) {{:match_title or lower(title) like lower(:search)}}) -) -select - path_id, path, title, - sum(total) as count -from hit_counts -join x using(path_id) -where site_id = :site -group by path_id, path, title -order by count desc diff --git a/go.mod b/go.mod index 3a5f3b515..df1aaf9ea 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( golang.org/x/image v0.0.0-20220902085622-e7cb96979f69 golang.org/x/net v0.0.0-20221014081412-f15817d10f9b golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0 - golang.org/x/text v0.3.8 + golang.org/x/text v0.4.0 zgo.at/blackmail v0.0.0-20211212060815-1f8e8a94692b zgo.at/errors v1.1.0 zgo.at/follow v0.0.0-20211017230838-112008350298 @@ -30,23 +30,23 @@ require ( zgo.at/json v0.0.0-20211017213340-cc8bf51df08c zgo.at/termtext v1.1.0 zgo.at/tz v0.0.0-20211017223207-006eae99adf6 - zgo.at/z18n v0.0.0-20220606095325-513ddb98b28f + zgo.at/z18n v0.0.0-20221018165830-c235ed037573 zgo.at/zcache v1.2.0 zgo.at/zcache/v2 v2.1.0 - zgo.at/zdb v0.0.0-20220822042559-7b1209555166 + zgo.at/zdb v0.0.0-20221018164415-44f0726b1ff7 zgo.at/zhttp v0.0.0-20221001220656-a9e60fe5f8e3 zgo.at/zli v0.0.0-20221012220610-d6a5a841b943 zgo.at/zlog v0.0.0-20211008102840-46c1167bf2a9 zgo.at/zprof v0.0.0-20211217104121-c3c12596d8f0 zgo.at/zstd v0.0.0-20221013104704-16fa49fadc62 - zgo.at/ztpl v0.0.0-20211128061406-6ff34b1256c4 + zgo.at/ztpl v0.0.0-20221018165743-cbd2a654b6af zgo.at/zvalidate v0.0.0-20211128195927-d13b18611e62 ) require ( github.com/andybalholm/cascadia v1.3.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/lib/pq v1.10.6 // indirect + github.com/lib/pq v1.10.7 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/oschwald/maxminddb-golang v1.8.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect diff --git a/go.sum b/go.sum index 8bd228f87..8986146fb 100644 --- a/go.sum +++ b/go.sum @@ -22,13 +22,12 @@ github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbS github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8= github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= -github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= @@ -75,8 +74,8 @@ golang.org/x/term v0.0.0-20220411215600-e5f449aeb171/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= @@ -102,14 +101,14 @@ zgo.at/termtext v1.1.0 h1:fBwyR6BVw1DnOIGmRhlQwM3cePzoJtY9FHpc62TwCWU= zgo.at/termtext v1.1.0/go.mod h1:P0jvULwo8z82ad7+WR6vg5vTy86dqvbOmR3bT3NlYvc= zgo.at/tz v0.0.0-20211017223207-006eae99adf6 h1:5YGaOhnrchO+rebbvxJ4+ZOLYygCLh5JDLJg6kCilyw= zgo.at/tz v0.0.0-20211017223207-006eae99adf6/go.mod h1:A/XeaYjeMGoXptRB3EcR80tgir37tJnzCb6itDaHPxo= -zgo.at/z18n v0.0.0-20220606095325-513ddb98b28f h1:EFYfkKvD7X1Yka3NSL3vc+wooc3AWZIn5tlpxenVVS8= -zgo.at/z18n v0.0.0-20220606095325-513ddb98b28f/go.mod h1:7/jQw/L0ng1iFUikdkYC9meSdeUkhXYfbtdm5OB1KHA= +zgo.at/z18n v0.0.0-20221018165830-c235ed037573 h1:NxxdAQUWHPEMf5QH8i2r0nGDz4nmj7utakXxKqy+NSU= +zgo.at/z18n v0.0.0-20221018165830-c235ed037573/go.mod h1:VpKwLOg1qPGIVXdq8FRKPjKYzUYI5VfIbHrTh/GpdzI= zgo.at/zcache v1.2.0 h1:++0dNWOrmUBa10WSja+eHx5bEO2PzZLRY6MJlBD47yk= zgo.at/zcache v1.2.0/go.mod h1:xWQo2ha/bamTmx8CbfrZl9Nf8AoT5uNh2hWfbQi8TiE= zgo.at/zcache/v2 v2.1.0 h1:USo+ubK+R4vtjw4viGzTe/zjXyPw6R7SK/RL3epBBxs= zgo.at/zcache/v2 v2.1.0/go.mod h1:gyCeoLVo01QjDZynjime8xUGHHMbsLiPyUTBpDGd4Gk= -zgo.at/zdb v0.0.0-20220822042559-7b1209555166 h1:m+dUEQLpoVNzdamHFPwWFF0Bm9trm6/L8yVc2hKQ0xE= -zgo.at/zdb v0.0.0-20220822042559-7b1209555166/go.mod h1:JE0EY6ni7FDlOmQ4MLaqyxoyx84dDh381OAYLWXJN5c= +zgo.at/zdb v0.0.0-20221018164415-44f0726b1ff7 h1:TTqMMujW2PvqPjS6F0T1kMN3STObK0iLM6RG/Vy2f3I= +zgo.at/zdb v0.0.0-20221018164415-44f0726b1ff7/go.mod h1:OEKWLTSLiUI1xZV89s/oWMKuwmjBh9bz6Jr5twc964Q= zgo.at/zhttp v0.0.0-20221001220656-a9e60fe5f8e3 h1:9x0/pf1e0t3apoZeXeY2fbjQK8R4apGHZ7O6d2d6fXM= zgo.at/zhttp v0.0.0-20221001220656-a9e60fe5f8e3/go.mod h1:dcRzoKb63s2iqoEcouH8miO2aM80uEAI7/d5QmSpls0= zgo.at/zli v0.0.0-20221012220610-d6a5a841b943 h1:PeTHjJK0YmNdkwfglLXufJWEwjy2rGujj5DRWtDw0u8= @@ -120,11 +119,9 @@ zgo.at/zprof v0.0.0-20211217104121-c3c12596d8f0 h1:nUshmGDnI3+N1OeU265xaqR6weTN2 zgo.at/zprof v0.0.0-20211217104121-c3c12596d8f0/go.mod h1:JqClLxeT9Uui3apqyN3U6KryFanocqM7E3X4Gle2FAU= zgo.at/zstd v0.0.0-20210310054817-c39eb9b7df25/go.mod h1:bQqcOgaRPCBT7lZ52Ksg9GF1xwi0L97E3AyvW3oGJYE= zgo.at/zstd v0.0.0-20210320020631-01ce6df76a58/go.mod h1:sQqrTxBwKW0nlwcOg9RxXB8ikY+NBciJnJRPOq/gEuY= -zgo.at/zstd v0.0.0-20210512041107-8951517febd3/go.mod h1:sQqrTxBwKW0nlwcOg9RxXB8ikY+NBciJnJRPOq/gEuY= -zgo.at/zstd v0.0.0-20220622111728-4a78555db760/go.mod h1:KYkLDHZN6APaRYHlL+hpwBJW/g9w+MLmXi4MKE5YteY= zgo.at/zstd v0.0.0-20221013104704-16fa49fadc62 h1:2YmbyyIJEC9yIVDGfuG/M7qAmHh+tgRKn4Lu/Iot3AQ= zgo.at/zstd v0.0.0-20221013104704-16fa49fadc62/go.mod h1:o/Q8+EtSahHnfkbB3t8wXE0FnoDTmJ0sBDlzezv9XeM= -zgo.at/ztpl v0.0.0-20211128061406-6ff34b1256c4 h1:/GHTN42or9pD7HVnlQohlC+QegKpwBn9hhE0Fa8zJPc= -zgo.at/ztpl v0.0.0-20211128061406-6ff34b1256c4/go.mod h1:mhjSF7+pfwMjNhDgmw9Ywr1CixzeVWuvMjF7gLJT8Uw= +zgo.at/ztpl v0.0.0-20221018165743-cbd2a654b6af h1:R0y3IwJvX6+/V7+XNbUI5lRrTYCt52Yqivu+ylA8WQQ= +zgo.at/ztpl v0.0.0-20221018165743-cbd2a654b6af/go.mod h1:Mm3yhXeuH0S9qzQ4TS69OK5A4yP5R+PXrqWGj3OFsTc= zgo.at/zvalidate v0.0.0-20211128195927-d13b18611e62 h1:PUoQjNzZ5g4QDYrXNCCyTYMZnPWwOAmVVkSSxUtmDwI= zgo.at/zvalidate v0.0.0-20211128195927-d13b18611e62/go.mod h1:WiF1dChwUrGhTRTPCe96HbJSkj9ztxARSgP+tEWAdJ0= diff --git a/handlers/backend_test.go b/handlers/backend_test.go index 41c127971..babaeae33 100644 --- a/handlers/backend_test.go +++ b/handlers/backend_test.go @@ -44,7 +44,7 @@ func TestBackendTpl(t *testing.T) { {"/settings/users", "Access"}, {"/settings/users/add", "Password"}, {"/settings/users/1", "Password"}, - {"/settings/purge", "Remove all pageviews for a page"}, + {"/settings/purge", "Remove or merge pageviews"}, {"/settings/export", "includes all pageviews"}, {"/settings/delete-account", "The site will be marked as deleted"}, {"/settings/change-code", "Change your site code and login domain"}, diff --git a/handlers/count.go b/handlers/count.go index dcdac4d7f..2e22b5309 100644 --- a/handlers/count.go +++ b/handlers/count.go @@ -68,7 +68,8 @@ func (h backend) count(w http.ResponseWriter, r *http.Request) error { if len(tags) > 0 { base, c := tags[0].Base() if c == language.Exact || c == language.High { - hit.Language = base.ISO3() + l := base.ISO3() + hit.Language = &l } } } diff --git a/handlers/settings.go b/handlers/settings.go index 2c60144f6..308332a8e 100644 --- a/handlers/settings.go +++ b/handlers/settings.go @@ -72,11 +72,9 @@ func (h settings) mount(r chi.Router) { set.Get("/settings/change-code", zhttp.Wrap(h.changeCode)) set.Post("/settings/change-code", zhttp.Wrap(h.changeCode)) - set.Get("/settings/purge", zhttp.Wrap(func(w http.ResponseWriter, r *http.Request) error { - return h.purge(nil)(w, r) - })) - set.Get("/settings/purge/confirm", zhttp.Wrap(h.purgeConfirm)) + set.Get("/settings/purge", zhttp.Wrap(h.purge)) set.Post("/settings/purge", zhttp.Wrap(h.purgeDo)) + set.Post("/settings/merge", zhttp.Wrap(h.merge)) set.Get("/settings/export", zhttp.Wrap(func(w http.ResponseWriter, r *http.Request) error { return h.export(nil)(w, r) @@ -427,30 +425,35 @@ func (h settings) sitesCopySettings(w http.ResponseWriter, r *http.Request) erro return zhttp.SeeOther(w, "/settings/sites") } -func (h settings) purge(verr *zvalidate.Validator) zhttp.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) error { - return zhttp.Template(w, "settings_purge.gohtml", struct { - Globals - Validate *zvalidate.Validator - }{newGlobals(w, r), verr}) - } -} +func (h settings) purge(w http.ResponseWriter, r *http.Request) error { + var ( + path = strings.TrimSpace(r.URL.Query().Get("path")) + matchTitle = r.URL.Query().Get("match-title") == "on" + matchCase = r.URL.Query().Get("match-case") == "on" + list goatcounter.HitLists + paths goatcounter.Paths + ) -func (h settings) purgeConfirm(w http.ResponseWriter, r *http.Request) error { - path := strings.TrimSpace(r.URL.Query().Get("path")) - title := r.URL.Query().Get("match-title") == "on" + if path != "" { + err := list.ListPathsLike(r.Context(), path, matchTitle, matchCase) + if err != nil { + return err + } - var list goatcounter.HitLists - err := list.ListPathsLike(r.Context(), path, title) - if err != nil { - return err + err = paths.List(r.Context(), goatcounter.MustGetSite(r.Context()).ID) + if err != nil { + return err + } } - return zhttp.Template(w, "settings_purge_confirm.gohtml", struct { + return zhttp.Template(w, "settings_purge.gohtml", struct { Globals - PurgePath string - List goatcounter.HitLists - }{newGlobals(w, r), path, list}) + PurgePath string + MatchTitle bool + MatchCase bool + List goatcounter.HitLists + AllPaths goatcounter.Paths + }{newGlobals(w, r), path, matchTitle, matchCase, list, paths}) } func (h settings) purgeDo(w http.ResponseWriter, r *http.Request) error { @@ -472,6 +475,31 @@ func (h settings) purgeDo(w http.ResponseWriter, r *http.Request) error { return zhttp.SeeOther(w, "/settings/purge") } +func (h settings) merge(w http.ResponseWriter, r *http.Request) error { + paths, err := zint.Split(r.Form.Get("paths"), ",") + if err != nil { + return err + } + + v := goatcounter.NewValidate(r.Context()) + dst := v.Integer("merge_with", r.Form.Get("merge_with")) + if v.HasErrors() { + return v + } + + ctx := goatcounter.CopyContextValues(r.Context()) + bgrun.Run(fmt.Sprintf("merge:%d", Site(ctx).ID), func() { + var list goatcounter.Hits + err := list.Merge(ctx, dst, paths) + if err != nil { + zlog.Error(err) + } + }) + + zhttp.Flash(w, T(r.Context(), "notify/started-background-process|Started in the background; may take about 10-20 seconds to fully process.")) + return zhttp.SeeOther(w, "/settings/purge") +} + func (h settings) export(verr *zvalidate.Validator) zhttp.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) error { var exports goatcounter.Exports diff --git a/handlers/settings_test.go b/handlers/settings_test.go index bd6fa3708..9f5c39c7b 100644 --- a/handlers/settings_test.go +++ b/handlers/settings_test.go @@ -30,7 +30,7 @@ func TestSettingsTpl(t *testing.T) { }...) }, router: newBackend, - path: "/settings/purge/confirm?path=/asd", + path: "/settings/purge?path=/asd", auth: true, wantCode: 200, wantBody: "2/asdAAA", diff --git a/hit.go b/hit.go index 2662640eb..c5c4b2f3a 100644 --- a/hit.go +++ b/hit.go @@ -45,7 +45,7 @@ type Hit struct { RefScheme *string `db:"ref_scheme" json:"-"` UserAgentHeader string `db:"-" json:"-"` Location string `db:"location" json:"-"` - Language string `db:"language" json:"-"` + Language *string `db:"language" json:"-"` FirstVisit zbool.Bool `db:"first_visit" json:"-"` CreatedAt time.Time `db:"created_at" json:"-"` @@ -57,6 +57,9 @@ type Hit struct { UserSessionID string `db:"-" json:"-"` BrowserID int64 `db:"-" json:"-"` SystemID int64 `db:"-" json:"-"` + + // Don't process in memstore; for merging paths. + noProcess bool `db:"-" json:"-"` } func (h *Hit) Ignore() bool { @@ -378,3 +381,34 @@ func (h *Hits) Purge(ctx context.Context, pathIDs []int64) error { return nil }) } + +// Merge the given paths. +func (h *Hits) Merge(ctx context.Context, dst int64, pathIDs []int64) error { + site := MustGetSite(ctx).ID + + err := (&Path{}).ByID(ctx, dst) // Ensure this site owns the path. + if err != nil { + return errors.Wrap(err, "Hits.Merge") + } + + // Push back to lot to memstore to re-add it again, and then just call + // Purge() to delete the old ones. + err = zdb.Select(ctx, h, `select * from hits where site_id=? and path_id in (?)`, site, pathIDs) + if err != nil { + return errors.Wrap(err, "Hits.Merge") + } + hh := *h + for i := range hh { + hh[i].PathID = dst + hh[i].noProcess = true + } + + err = errors.Wrap(h.Purge(ctx, pathIDs), "Hits.Merge") + if err != nil { + return errors.Wrap(err, "Hits.Merge") + } + + // Only push back if delete worked. + Memstore.Append(hh...) + return nil +} diff --git a/hit_list.go b/hit_list.go index 99d995fa1..2e454a04a 100644 --- a/hit_list.go +++ b/hit_list.go @@ -72,11 +72,12 @@ func (h *HitList) SiteTotalUTC(ctx context.Context, rng ztime.Range) error { type HitLists []HitList // ListPathsLike lists all paths matching the like pattern. -func (h *HitLists) ListPathsLike(ctx context.Context, search string, matchTitle bool) error { +func (h *HitLists) ListPathsLike(ctx context.Context, search string, matchTitle, matchCase bool) error { err := zdb.Select(ctx, h, "load:hit_list.ListPathsLike", zdb.P{ "site": MustGetSite(ctx).ID, "search": search, "match_title": matchTitle, + "match_case": matchCase, }) return errors.Wrap(err, "Hits.ListPathsLike") } diff --git a/hit_list_test.go b/hit_list_test.go index 2647eb993..fa90f4770 100644 --- a/hit_list_test.go +++ b/hit_list_test.go @@ -169,9 +169,9 @@ func TestHitListsList(t *testing.T) { var stats HitLists totalDisplay, uniqueDisplay, more, err := stats.List(ctx, rng, pathsFilter, tt.inExclude, 2, false) - got := fmt.Sprintf("%d %d %t %v", totalDisplay, uniqueDisplay, more, err) - if got != tt.wantReturn { - t.Errorf("wrong return\nout: %s\nwant: %s\n", got, tt.wantReturn) + have := fmt.Sprintf("%d %d %t %v", totalDisplay, uniqueDisplay, more, err) + if have != tt.wantReturn { + t.Errorf("wrong return\nhave: %s\nwant: %s\n", have, tt.wantReturn) zdb.Dump(ctx, os.Stdout, "select * from paths") zdb.Dump(ctx, os.Stdout, "select * from hit_counts") zdb.Dump(ctx, os.Stdout, "select * from hit_stats") @@ -210,13 +210,13 @@ func TestGetMax(t *testing.T) { t.Run("hourly", func(t *testing.T) { want := []int{10, 10, 10, 10} for i, filter := range [][]int64{nil, []int64{1}, []int64{2}, []int64{1, 2}} { - got, err := GetMax(ctx, rng, filter, false) + have, err := GetMax(ctx, rng, filter, false) if err != nil { t.Fatal(err) } w := want[i] - if got != w { - t.Errorf("got %d; want %d (filter=%v)", got, w, filter) + if have != w { + t.Errorf("have %d; want %d (filter=%v)", have, w, filter) } } }) @@ -224,13 +224,13 @@ func TestGetMax(t *testing.T) { t.Run("daily", func(t *testing.T) { want := []int{10, 10, 10, 10} for i, filter := range [][]int64{nil, []int64{1}, []int64{2}, []int64{1, 2}} { - got, err := GetMax(ctx, rng, filter, true) + have, err := GetMax(ctx, rng, filter, true) if err != nil { t.Fatal(err) } w := want[i] - if got != w { - t.Errorf("got %d; want %d (filter=%v)", got, w, filter) + if have != w { + t.Errorf("have %d; want %d (filter=%v)", have, w, filter) } } }) @@ -255,9 +255,9 @@ func TestGetTotalCount(t *testing.T) { t.Fatal(err) } want := "{5 3 3 2 1}" - got := fmt.Sprintf("%v", tt) - if want != got { - t.Errorf("\nwant: %s\ngot: %s", want, got) + have := fmt.Sprintf("%v", tt) + if want != have { + t.Errorf("\nwant: %s\nhave: %s", want, have) } } } @@ -285,16 +285,16 @@ func TestHitListTotals(t *testing.T) { rng := ztime.NewRange(ztime.Now()).To(ztime.Now()) want := []string{ - `12 {"Count":12,"CountUnique":2,"PathID":0,"Path":"TOTAL ","Event":false,"Title":"","RefScheme":null,"Max":0,"Stats":[` + + `10 {"Count":12,"CountUnique":2,"PathID":0,"Path":"TOTAL ","Event":false,"Title":"","RefScheme":null,"Max":0,"Stats":[` + `{"Day":"2020-06-18","Hourly":[0,0,0,0,0,0,0,0,0,0,0,0,12,0,0,0,0,0,0,0,0,0,0,0],"HourlyUnique":[0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0],"Daily":0,"DailyUnique":0}]}`, - `11 {"Count":11,"CountUnique":1,"PathID":0,"Path":"TOTAL ","Event":false,"Title":"","RefScheme":null,"Max":0,"Stats":[` + + `10 {"Count":11,"CountUnique":1,"PathID":0,"Path":"TOTAL ","Event":false,"Title":"","RefScheme":null,"Max":0,"Stats":[` + `{"Day":"2020-06-18","Hourly":[0,0,0,0,0,0,0,0,0,0,0,0,11,0,0,0,0,0,0,0,0,0,0,0],"HourlyUnique":[0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0],"Daily":0,"DailyUnique":0}]}`, `10 {"Count":1,"CountUnique":1,"PathID":0,"Path":"TOTAL ","Event":false,"Title":"","RefScheme":null,"Max":0,"Stats":[` + `{"Day":"2020-06-18","Hourly":[0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0],"HourlyUnique":[0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0],"Daily":0,"DailyUnique":0}]}`, - `12 {"Count":12,"CountUnique":2,"PathID":0,"Path":"TOTAL ","Event":false,"Title":"","RefScheme":null,"Max":0,"Stats":[` + + `10 {"Count":12,"CountUnique":2,"PathID":0,"Path":"TOTAL ","Event":false,"Title":"","RefScheme":null,"Max":0,"Stats":[` + `{"Day":"2020-06-18","Hourly":[0,0,0,0,0,0,0,0,0,0,0,0,12,0,0,0,0,0,0,0,0,0,0,0],"HourlyUnique":[0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0],"Daily":0,"DailyUnique":0}]}`, } for i, filter := range [][]int64{nil, []int64{1}, []int64{2}, []int64{1, 2}} { @@ -304,10 +304,10 @@ func TestHitListTotals(t *testing.T) { t.Fatal(err) } - got := fmt.Sprintf("%d %s", count, zjson.MustMarshal(hs)) + have := fmt.Sprintf("%d %s", count, zjson.MustMarshal(hs)) w := want[i] - if got != w { - t.Errorf("\nwant: %s\ngot: %v\nfilter: %v", w, got, filter) + if have != w { + t.Errorf("\nwant: %s\nhave: %v\nfilter: %v", w, have, filter) } } }) @@ -336,10 +336,10 @@ func TestHitListTotals(t *testing.T) { t.Fatal(err) } - got := fmt.Sprintf("%d %s", count, zjson.MustMarshal(hs)) + have := fmt.Sprintf("%d %s", count, zjson.MustMarshal(hs)) w := want[i] - if got != w { - t.Errorf("\nwant: %s\ngot: %v\nfilter: %v", w, got, filter) + if have != w { + t.Errorf("\nwant: %s\nhave: %v\nfilter: %v", w, have, filter) } } }) diff --git a/memstore.go b/memstore.go index e48930733..aa3dd1284 100644 --- a/memstore.go +++ b/memstore.go @@ -269,6 +269,10 @@ func (m *ms) processHit(ctx context.Context, h *Hit) bool { l := zlog.Module("memstore") + if h.noProcess { + return true + } + // Ignore spammers. h.RefURL, _ = url.Parse(h.Ref) if h.RefURL != nil { @@ -281,14 +285,14 @@ func (m *ms) processHit(ctx context.Context, h *Hit) bool { var site Site err := site.ByID(ctx, h.Site) if err != nil { - l.Field("hit", h).Error(err) + l.Field("hit", fmt.Sprintf("%#v", h)).Error(err) return false } ctx = WithSite(ctx, &site) err = h.Defaults(ctx, false) if err != nil { - l.Field("hit", h).Error(err) + l.Field("hit", fmt.Sprintf("%#v", h)).Error(err) return false } @@ -314,7 +318,7 @@ func (m *ms) processHit(ctx context.Context, h *Hit) bool { h.UserAgentID = nil } if !site.Settings.Collect.Has(CollectLanguage) { - h.Language = "" + h.Language = nil } if !site.Settings.Collect.Has(CollectLocation) { h.Location = "" @@ -340,7 +344,7 @@ func (m *ms) processHit(ctx context.Context, h *Hit) bool { err = h.Validate(ctx, false) if err != nil { - l.Field("hit", h).Error(err) + l.Field("hit", fmt.Sprintf("%#v", h)).Error(err) return false } diff --git a/path.go b/path.go index fd6d9e6b7..0a6c194e5 100644 --- a/path.go +++ b/path.go @@ -37,6 +37,12 @@ func (p *Path) Validate(ctx context.Context) error { return v.ErrorOrNil() } +func (p *Path) ByID(ctx context.Context, id int64) error { + return errors.Wrapf(zdb.Get(ctx, p, + `/* Path.ByID */ select * from paths where path_id=? and site_id=?`, + id, MustGetSite(ctx).ID), "Path.ByID %d", id) +} + func (p *Path) GetOrInsert(ctx context.Context) error { site := MustGetSite(ctx) title := p.Title @@ -132,6 +138,14 @@ func (p Path) updateTitle(ctx context.Context, currentTitle, newTitle string) er return nil } +type Paths []Path + +// List all paths for a site. +func (u *Paths) List(ctx context.Context, siteID int64) error { + return errors.Wrap(zdb.Select(ctx, u, + `select * from paths where site_id=$1`, siteID), "Paths.List") +} + // PathFilter returns a list of IDs matching the path name. // // if matchTitle is true it will match the title as well. diff --git a/tpl/_settings_nav.gohtml b/tpl/_settings_nav.gohtml index b7e0a1145..34dddbd1e 100644 --- a/tpl/_settings_nav.gohtml +++ b/tpl/_settings_nav.gohtml @@ -1,6 +1,6 @@