Add limit to alert messages.

.gitignore

@@ -2,6 +2,8 @@
 /.vscode
 /.build
 /vendor
+build/
+test/
 
 *.tar.gz
 coverage.txt

Dockerfile_linux_amd64

@@ -0,0 +1,3 @@
+FROM alpine:3.16
+ADD build/dingtalk-webhook-linux-amd64 /dingtalk-webhook-linux-amd64
+ENTRYPOINT [ "/dingtalk-webhook-linux-amd64"]

Dockerfile_linux_arm64

@@ -0,0 +1,3 @@
+FROM alpine:3.16
+ADD build/dingtalk-webhook-linux-arm64 /dingtalk-webhook-linux-arm64
+ENTRYPOINT [ "/dingtalk-webhook-linux-arm64"]

Makefile

@@ -61,3 +61,23 @@ test: common-test
 .PHONY: clean
 clean:
 	- @rm -rf "$(REACT_APP_OUTPUT_DIR)"s
+
+go-build-linux-arm64:
+	GOOS=linux  GOARCH=arm64 \
+	go build -o build/dingtalk-webhook-linux-arm64 cmd/prometheus-webhook-dingtalk/main.go
+
+go-build-linux-amd64:
+	GOOS=linux  GOARCH=amd64 \
+	go build -o build/dingtalk-webhook-linux-amd64 cmd/prometheus-webhook-dingtalk/main.go
+
+docker-build-linux-arm64:
+	docker build -t dingtalk-webhook:v2.1.1 --platform=linux/arm64 -f Dockerfile_linux_arm64 .
+
+docker-build-linux-amd64:
+	docker build -t dingtalk-webhook:v2.1.1 --platform=linux/amd64 -f Dockerfile_linux_amd64 .
+
+docker-tag:
+	docker tag dingtalk-webhook:v2.1.1 docker.io/ahfuzhang/dingtalk-webhook:v2.1.1
+
+docker-push:
+	docker push docker.io/ahfuzhang/dingtalk-webhook:v2.1.1

README.md

@@ -58,6 +58,11 @@ Flags:
       --config.file=config.yml  Path to the configuration file.
       --log.level=info          Only log messages with the given severity or above. One of: [debug, info, warn, error]
       --log.format=logfmt       Output format of log messages. One of: [logfmt, json]
+      --pushmetrics.extraLabel  Extra label for push metrics. see: https://github.com/VictoriaMetrics/metrics
+      --pushmetrics.interval=15s
+                                Interval for push metrics.
+      --pushmetrics.url         Urls for push metrics.eg: http://host.docker.internal:8480/insert/0/prometheus/api/v1/import/prometheus.
+      --maxalertcount=30        Max alert count to send to ding talk.
       --version                 Show application version.
 ```
 

cmd/prometheus-webhook-dingtalk/main.go

@@ -16,6 +16,7 @@ import (
 	"github.com/prometheus/common/version"
 	"gopkg.in/alecthomas/kingpin.v2"
 
+	"github.com/VictoriaMetrics/metrics"
 	"github.com/timonwong/prometheus-webhook-dingtalk/config"
 	"github.com/timonwong/prometheus-webhook-dingtalk/template"
 	"github.com/timonwong/prometheus-webhook-dingtalk/web"
@@ -43,6 +44,24 @@ func run() int {
 			"config.file",
 			"Path to the configuration file.",
 		).Default("config.yml").ExistingFile()
+		// for push metrics
+		extraLabel = kingpin.Flag(
+			"pushmetrics.extraLabel",
+			"extraLabel for push metrics.",
+		).Default("").String()
+		intervalForPushMetrics = kingpin.Flag(
+			"pushmetrics.interval",
+			"interval for push metrics.",
+		).Default("15s").Duration()
+		urlForPushMetrics = kingpin.Flag(
+			"pushmetrics.url",
+			"urls for push metrics.",
+		).Default("").String()
+		// aviod too many alert
+		maxAlertCount = kingpin.Flag(
+			"maxalertcount",
+			"max alert count to send to ding talk.",
+		).Default("30").Uint16()
 	)
 
 	// DO NOT REMOVE. For compatibility purpose
@@ -59,6 +78,11 @@ func run() int {
 	level.Info(logger).Log("msg", "Starting prometheus-webhook-dingtalk", "version", version.Info())
 	level.Info(logger).Log("msg", "Build context", version.BuildContext())
 
+	if len(*urlForPushMetrics) > 0 {
+		metrics.InitPush(*urlForPushMetrics,
+			*intervalForPushMetrics, *extraLabel, true)
+	}
+
 	flagsMap := map[string]string{}
 	// Exclude kingpin default flags to expose only Prometheus ones.
 	boilerplateFlags := kingpin.New("", "").Version("")
@@ -87,7 +111,8 @@ func run() int {
 			BuildDate: version.BuildDate,
 			GoVersion: version.GoVersion,
 		},
-		Flags: flagsMap,
+		Flags:         flagsMap,
+		MaxAlertCount: *maxAlertCount,
 	})
 
 	configLogger := log.With(logger, "component", "configuration")

go.mod

@@ -16,6 +16,7 @@ require (
 require (
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.1.1 // indirect
+	github.com/VictoriaMetrics/metrics v1.25.3 // indirect
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -34,7 +35,9 @@ require (
 	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/spf13/cast v1.3.1 // indirect
+	github.com/valyala/fastrand v1.1.0 // indirect
+	github.com/valyala/histogram v1.2.0 // indirect
 	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect
-	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
+	golang.org/x/sys v0.15.0 // indirect
 	google.golang.org/protobuf v1.26.0 // indirect
 )

go.sum

@@ -39,6 +39,8 @@ github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030I
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8=
 github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk=
+github.com/VictoriaMetrics/metrics v1.25.3 h1:Zcxyj8JbAB6CQU51Er3D7RBRupcP55DevVQi9cFqo2Q=
+github.com/VictoriaMetrics/metrics v1.25.3/go.mod h1:ZKmlI+QN6b9LUC0OiHNp2LiGQGlBy4U1re6Slooln1o=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -232,6 +234,10 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8=
+github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
+github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ=
+github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -366,6 +372,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

web/dingtalk/dingtalk.go

@@ -2,6 +2,7 @@ package dingtalk
 
 import (
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"sync"
@@ -11,6 +12,7 @@ import (
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 
+	"github.com/VictoriaMetrics/metrics"
 	"github.com/timonwong/prometheus-webhook-dingtalk/config"
 	"github.com/timonwong/prometheus-webhook-dingtalk/notifier"
 	"github.com/timonwong/prometheus-webhook-dingtalk/pkg/chilog"
@@ -27,6 +29,8 @@ type API struct {
 	targets    map[string]config.Target
 	httpClient *http.Client
 	logger     log.Logger
+
+	MaxAlertCount uint16 // to avoid too many alert
 }
 
 func NewAPI(logger log.Logger) *API {
@@ -60,6 +64,7 @@ func (api *API) Routes() chi.Router {
 }
 
 func (api *API) serveSend(w http.ResponseWriter, r *http.Request) {
+	metrics.GetOrCreateCounter("http_requests_total{path=\"" + r.RequestURI + "\"}").Inc()
 	api.mtx.RLock()
 	targets := api.targets
 	conf := api.conf
@@ -72,36 +77,60 @@ func (api *API) serveSend(w http.ResponseWriter, r *http.Request) {
 
 	target, ok := targets[targetName]
 	if !ok {
-		level.Warn(logger).Log("msg", "target not found")
+		const reason = "target not found"
+		level.Warn(logger).Log("msg", reason)
 		http.NotFound(w, r)
+		metrics.GetOrCreateCounter(
+			fmt.Sprintf(`http_requests_error{path="%s",reason="%s"}`, r.RequestURI, reason)).Inc()
 		return
 	}
 
 	var promMessage models.WebhookMessage
 	if err := json.NewDecoder(r.Body).Decode(&promMessage); err != nil {
-		level.Error(logger).Log("msg", "Cannot decode prometheus webhook JSON request", "err", err)
+		const reason = "Cannot decode prometheus webhook JSON request"
+		level.Error(logger).Log("msg", reason, "err", err)
 		http.Error(w, "Bad Request", http.StatusBadRequest)
+		metrics.GetOrCreateCounter(
+			fmt.Sprintf(`http_requests_error{path="%s",reason="%s"}`, r.RequestURI, reason)).Inc()
 		return
 	}
 
+	metrics.GetOrCreateCounter(
+		fmt.Sprintf(`alert_count{path="%s"}`, r.RequestURI)).Add(len(promMessage.Alerts))
+	if api.MaxAlertCount > 0 && len(promMessage.Alerts) > int(api.MaxAlertCount) {
+		metrics.GetOrCreateCounter(
+			fmt.Sprintf(`alert_drop_count{path="%s"}`, r.RequestURI)).Add(len(promMessage.Alerts) - int(api.MaxAlertCount))
+		promMessage.Alerts = promMessage.Alerts[:api.MaxAlertCount]
+	}
+
 	builder := notifier.NewDingNotificationBuilder(tmpl, conf, &target)
 	notification, err := builder.Build(&promMessage)
 	if err != nil {
-		level.Error(logger).Log("msg", "Failed to build notification", "err", err)
+		const reason = "Failed to build notification"
+		level.Error(logger).Log("msg", reason, "err", err)
 		http.Error(w, "Bad Request", http.StatusBadRequest)
+		metrics.GetOrCreateCounter(
+			fmt.Sprintf(`http_requests_error{path="%s",reason="%s"}`, r.RequestURI, reason)).Inc()
 		return
 	}
 
 	robotResp, err := notifier.SendNotification(notification, httpClient, &target)
 	if err != nil {
-		level.Error(logger).Log("msg", "Failed to send notification", "err", err)
+		const reason = "Failed to send notification"
+		level.Error(logger).Log("msg", reason, "err", err)
 		http.Error(w, "Bad Request", http.StatusBadRequest)
+		metrics.GetOrCreateCounter(
+			fmt.Sprintf(`http_requests_error{path="%s",reason="%s"}`, r.RequestURI, reason)).Inc()
 		return
 	}
 
 	if robotResp.ErrorCode != 0 {
-		level.Error(logger).Log("msg", "Failed to send notification to DingTalk", "respCode", robotResp.ErrorCode, "respMsg", robotResp.ErrorMessage)
+		const reason = "Failed to send notification to DingTalk"
+		level.Error(logger).Log("msg", reason, "respCode", robotResp.ErrorCode, "respMsg", robotResp.ErrorMessage)
 		http.Error(w, "Unable to talk to DingTalk", http.StatusBadRequest)
+		metrics.GetOrCreateCounter(
+			fmt.Sprintf(`http_requests_error{path="%s",reason="%s",respCode="%d",respMsg="%s"}`,
+				r.RequestURI, reason, robotResp.ErrorCode, robotResp.ErrorMessage)).Inc()
 		return
 	}
 

web/web.go

@@ -34,6 +34,7 @@ import (
 	"github.com/prometheus/common/server"
 	"go.uber.org/atomic"
 
+	"github.com/VictoriaMetrics/metrics"
 	"github.com/timonwong/prometheus-webhook-dingtalk/config"
 	"github.com/timonwong/prometheus-webhook-dingtalk/template"
 	"github.com/timonwong/prometheus-webhook-dingtalk/web/apiv1"
@@ -57,6 +58,7 @@ type Options struct {
 	EnableLifecycle bool
 	Version         *VersionInfo
 	Flags           map[string]string
+	MaxAlertCount   uint16
 }
 
 type VersionInfo = apiv1.VersionInfo
@@ -120,6 +122,7 @@ func New(logger log.Logger, o *Options) *Handler {
 		h.runtimeInfo,
 	)
 	h.dingTalk = dingtalk.NewAPI(logger)
+	h.dingTalk.MaxAlertCount = o.MaxAlertCount
 
 	router.Mount("/dingtalk", h.dingTalk.Routes())
 
@@ -189,6 +192,9 @@ func New(logger log.Logger, o *Options) *Handler {
 		})
 	}
 
+	router.Get("/metrics", func(w http.ResponseWriter, r *http.Request) {
+		metrics.WritePrometheus(w, true)
+	})
 	return h
 }