Merge pull request #336 from actiontech/log_optimization

Log optimization

internal/apiserver/service/router.go

@@ -1,7 +1,10 @@
 package service
 
 import (
+	"bytes"
+	"compress/gzip"
 	"fmt"
+	"io"
 	"strings"
 
 	"github.com/actiontech/dms/internal/dms/biz"
@@ -190,24 +193,68 @@ func (s *APIServer) initRouter() error {
 	return nil
 }
 
+func SwaggerMiddleWare(next echo.HandlerFunc) echo.HandlerFunc {
+	return func(c echo.Context) error {
+		// swagger 请求分为两种,一种是swagger html页面请求,一种是swagger json请求. eg:
+		// swagger/index.html 获取html
+		// swagger/dms/doc.yaml 获取json
+		hasPkPrefix := strings.HasPrefix(c.Request().RequestURI, "/swagger/index.html?urls.primaryName=")
+		if hasPkPrefix {
+			// 为了避免404
+			c.Request().RequestURI = "/swagger/index.html"
+		}
+
+		return next(c)
+	}
+}
+
+// 检查 reply 是否是 Gzip 数据
+func isGzip(data []byte) bool {
+	return len(data) >= 2 && data[0] == 0x1f && data[1] == 0x8b
+}
+
+// 解码 Gzip 数据
+func decodeGzip(data []byte) string {
+	reader, err := gzip.NewReader(bytes.NewReader(data))
+	if err != nil {
+		return fmt.Sprintf("Gzip decode error: %v", err)
+	}
+	defer reader.Close()
+
+	decoded, err := io.ReadAll(reader)
+	if err != nil {
+		return fmt.Sprintf("Read Gzip data error: %v", err)
+	}
+	return string(decoded)
+}
+
 func (s *APIServer) installMiddleware() error {
 	// Middleware
-	s.echo.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{
-		Skipper: middleware.DefaultSkipper,
-		Format: `${time_custom} ECHO id:${id}, remote_ip:${remote_ip}, ` +
-			`host:${host}, method:${method}, uri:${uri}, user_agent:${user_agent}, ` +
-			`status:${status}, error:${error}, latency:${latency}, latency_human:${latency_human}` +
-			`, bytes_in:${bytes_in}, bytes_out:${bytes_out}` + "\n",
-		CustomTimeFormat: pkgLog.LogTimeLayout,
-	}))
 
 	s.echo.Use(middleware.BodyDumpWithConfig(middleware.BodyDumpConfig{
 		Skipper: func(c echo.Context) bool {
 			return !strings.HasPrefix(c.Request().RequestURI, dmsV1.GroupV1)
 		},
 		Handler: func(context echo.Context, req []byte, reply []byte) {
 			userUid, _ := jwt.GetUserUidStrFromContext(context)
-			commonLog.NewHelper(s.logger).Log(commonLog.LevelDebug, "middleware.uri", context.Request().RequestURI, "user_id", userUid, "req", string(req), "reply", string(reply))
+
+			// 将请求转为字符串
+			reqStr := string(req)
+			// 尝试解码 reply（gzip 格式）
+			var replyStr string
+			if isGzip(reply) {
+				replyStr = decodeGzip(reply)
+			} else {
+				replyStr = string(reply)
+			}
+
+			commonLog.NewHelper(s.logger).Log(
+				commonLog.LevelDebug,
+				"middleware.uri", context.Request().RequestURI,
+				"user_id", userUid,
+				"req", reqStr, // 输出处理后的请求数据
+				"reply", replyStr, // 输出处理后的响应数据
+			)
 		},
 	}))
 

internal/dms/biz/db_service.go

@@ -218,11 +218,7 @@ type ListDBServicesOption struct {
 }
 
 func (d *DBServiceUsecase) ListDBService(ctx context.Context, option *ListDBServicesOption, projectUid, currentUserUid string) (dbServices []*DBService, total int64, err error) {
-	// 只允许系统用户查询所有数据源,同步数据到其他服务(provision)
-	// 检查项目是否归档/删除
-	if projectUid == "" && currentUserUid != pkgConst.UIDOfUserSys {
-		return nil, 0, fmt.Errorf("list db service error: project is empty")
-	}
+
 	services, total, err := d.repo.ListDBServices(ctx, option)
 	if err != nil {
 		return nil, 0, fmt.Errorf("list db services failed: %w", err)

internal/dms/biz/plugin.go

@@ -34,8 +34,8 @@ type Plugin struct {
 }
 
 func (p *Plugin) String() string {
-	return fmt.Sprintf("name=%v,addDBServicePreCheckUrl=%v,delDBServicePreCheckUrl=%v,delUserPreCheckUrl=%v,delUserGroupPreCheckUrl=%v,OperateDataHandleUrl=%v",
-		p.Name, p.AddDBServicePreCheckUrl, p.DelDBServicePreCheckUrl, p.DelUserPreCheckUrl, p.DelUserGroupPreCheckUrl, p.OperateDataResourceHandleUrl)
+	return fmt.Sprintf("name=%v,OperateDataHandleUrl=%v",
+		p.Name, p.OperateDataResourceHandleUrl)
 }
 
 func NewDMSPluginUsecase(logger utilLog.Logger, repo DMSPluginRepo) (*PluginUsecase, error) {

internal/dms/pkg/errors/error.go

@@ -30,6 +30,6 @@ func WrapStorageErr(log *utilLog.Helper, originalErr error) error {
 
 func WrapErrStorageNoData(log *utilLog.Helper, originalErr error) error {
 	err := fmt.Errorf("%w:%v", ErrStorageNoData, originalErr)
-	log.Errorf(err.Error())
+	// log.Errorf(err.Error())
 	return err
 }

internal/dms/service/db_service.go

@@ -230,11 +230,6 @@ func (d *DMSService) convertPeriodToMaintenanceTime(p periods.Periods) []*dmsCom
 }
 
 func (d *DMSService) ListDBServices(ctx context.Context, req *dmsCommonV1.ListDBServiceReq, currentUserUid string) (reply *dmsCommonV1.ListDBServiceReply, err error) {
-	d.log.Infof("ListDBServices.req=%v", req)
-	defer func() {
-		d.log.Infof("ListDBServices.req=%v;reply=%v;error=%v", req, reply, err)
-	}()
-
 	var orderBy biz.DBServiceField
 	switch req.OrderBy {
 	case dmsCommonV1.DBServiceOrderByName:

internal/dms/service/op_permission.go

@@ -9,10 +9,6 @@ import (
 )
 
 func (d *DMSService) ListOpPermissions(ctx context.Context, req *dmsV1.ListOpPermissionReq) (reply *dmsV1.ListOpPermissionReply, err error) {
-	d.log.Infof("ListOpPermissions.req=%v", req)
-	defer func() {
-		d.log.Infof("ListOpPermissions.req=%v;reply=%v;error=%v", req, reply, err)
-	}()
 
 	var orderBy biz.OpPermissionField
 	switch req.OrderBy {

internal/dms/service/proxy.go

@@ -10,9 +10,9 @@ import (
 )
 
 func (d *DMSService) RegisterDMSProxyTarget(ctx context.Context, currentUserUid string, req *dmsV1.RegisterDMSProxyTargetReq) (err error) {
-	d.log.Infof("RegisterDMSProxyTarget.req=%v", req)
+	d.log.Infof("RegisterDMSProxyTarget.req=%v", *req)
 	defer func() {
-		d.log.Infof("RegisterDMSProxyTarget.req=%v;error=%v", req, err)
+		d.log.Infof("RegisterDMSProxyTarget.req=%v;error=%v", *req, err)
 	}()
 	scenairo, err := convertProxyScenario(req.DMSProxyTarget.Scenario)
 	if err != nil {

internal/dms/service/user.go

@@ -52,10 +52,6 @@ func (d *DMSService) AfterUserLogin(ctx context.Context, req *dmsV1.AfterUserLog
 }
 
 func (d *DMSService) GetCurrentUser(ctx context.Context, req *dmsV1.GetUserBySessionReq) (reply *dmsV1.GetUserBySessionReply, err error) {
-	d.log.Infof("GetCurrentUser,req=%v", req)
-	defer func() {
-		d.log.Infof("GetCurrentUser.req=%v,reply=%v;error=%v", req, reply, err)
-	}()
 
 	user, err := d.UserUsecase.GetUser(ctx, req.UserUid)
 	if nil != err {
@@ -139,10 +135,6 @@ func (d *DMSService) DelUser(ctx context.Context, currentUserUid string, req *dm
 }
 
 func (d *DMSService) ListUsers(ctx context.Context, req *dmsCommonV1.ListUserReq) (reply *dmsCommonV1.ListUserReply, err error) {
-	d.log.Infof("ListUsers.req=%v", req)
-	defer func() {
-		d.log.Infof("ListUsers.req=%v;reply=%v;error=%v", req, reply, err)
-	}()
 
 	var orderBy biz.UserField
 	switch req.OrderBy {
@@ -305,10 +297,6 @@ func (d *DMSService) DelUserGroup(ctx context.Context, currentUserUid string, re
 }
 
 func (d *DMSService) ListUserGroups(ctx context.Context, req *dmsV1.ListUserGroupReq) (reply *dmsV1.ListUserGroupReply, err error) {
-	d.log.Infof("ListUserGroups.req=%v", req)
-	defer func() {
-		d.log.Infof("ListUserGroups.req=%v;reply=%v;error=%v", req, reply, err)
-	}()
 
 	var orderBy biz.UserGroupField
 	switch req.OrderBy {
@@ -419,16 +407,10 @@ func (d *DMSService) GetUserOpPermission(ctx context.Context, req *dmsCommonV1.G
 		}{IsAdmin: isAdmin, OpPermissionList: replyOpPermission},
 	}
 
-	d.log.Infof("GetUserOpPermission.resp=%v", reply)
 	return reply, nil
 }
 
 func (d *DMSService) GetUser(ctx context.Context, req *dmsCommonV1.GetUserReq) (reply *dmsCommonV1.GetUserReply, err error) {
-	d.log.Infof("GetUser.req=%v", req)
-	defer func() {
-		d.log.Infof("GetUser.req=%v;error=%v", req, err)
-	}()
-
 	u, err := d.UserUsecase.GetUser(ctx, req.UserUid)
 	if err != nil {
 		return nil, fmt.Errorf("get user error: %v", err)
@@ -526,7 +508,6 @@ func (d *DMSService) GetUser(ctx context.Context, req *dmsCommonV1.GetUserReq) (
 		Data: dmsCommonUser,
 	}
 
-	d.log.Infof("GetUser.resp=%v", reply)
 	return reply, nil
 }
 

pkg/dms-common/api/dms/v1/proxy.go

@@ -1,6 +1,10 @@
 package v1
 
-import base "github.com/actiontech/dms/pkg/dms-common/api/base/v1"
+import (
+	"fmt"
+
+	base "github.com/actiontech/dms/pkg/dms-common/api/base/v1"
+)
 
 // swagger:parameters RegisterDMSProxyTarget
 type RegisterDMSProxyTargetReq struct {
@@ -27,6 +31,10 @@ type DMSProxyTarget struct {
 	Scenario ProxyScenario `json:"scenario"`
 }
 
+func (s *DMSProxyTarget) String() string {
+	return fmt.Sprintf("{name: %v, addr: %v, version: %v, Scenario %v}", s.Name, s.Addr, s.Version, s.Scenario)
+}
+
 // swagger:enum ProxyScenario
 type ProxyScenario string