Skip to content

Commit

Permalink
Add Put/Delete/Head metrics and average datain/dataout
Browse files Browse the repository at this point in the history
  • Loading branch information
@vkuznet
vkuznet committed Jun 30, 2024
1 parent 5062c5a commit 5a02123
Show file tree
Hide file tree
Showing 4 changed files with 141 additions and 35 deletions.
45 changes: 27 additions & 18 deletions data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,24 +111,33 @@ type Mem struct {

// Metrics provide various metrics about our server
type Metrics struct {
CPU []float64 `json:"cpu"` // cpu metrics from gopsutils
Connections []net.ConnectionStat `json:"conenctions"` // connections metrics from gopsutils
DataIn float64 `json:"data_in"` // data into APS (in bytes)
DataOut float64 `json:"data_out"` // data out of APS (in bytes)
Load load.AvgStat `json:"load"` // load metrics from gopsutils
Memory Mem `json:"memory"` // memory metrics from gopsutils
OpenFiles []process.OpenFilesStat `json:"openFiles"` // open files metrics from gopsutils
GoRoutines uint64 `json:"goroutines"` // total number of go routines at run-time
Uptime float64 `json:"uptime"` // uptime of the server
GetX509Requests uint64 `json:"x509GetRequests"` // total number of get x509 requests
PostX509Requests uint64 `json:"x509PostRequests"` // total number of post X509 requests
GetOAuthRequests uint64 `json:"oAuthGetRequests"` // total number of get requests form OAuth server
PostOAuthRequests uint64 `json:"oAuthPostRequests"` // total number of post requests from OAuth server
GetRequests uint64 `json:"getRequests"` // total number of get requests across all services
PostRequests uint64 `json:"postRequests"` // total number of post requests across all services
RPS float64 `json:"rps"` // throughput req/sec
RPSPhysical float64 `json:"rpsPhysical"` // throughput req/sec using physical cpu
RPSLogical float64 `json:"rpsLogical"` // throughput req/sec using logical cpu
CPU []float64 `json:"cpu"` // cpu metrics from gopsutils
Connections []net.ConnectionStat `json:"conenctions"` // connections metrics from gopsutils
DataIn float64 `json:"data_in"` // data into APS (in bytes)
DataOut float64 `json:"data_out"` // data out of APS (in bytes)
Load load.AvgStat `json:"load"` // load metrics from gopsutils
Memory Mem `json:"memory"` // memory metrics from gopsutils
OpenFiles []process.OpenFilesStat `json:"openFiles"` // open files metrics from gopsutils
GoRoutines uint64 `json:"goroutines"` // total number of go routines at run-time
Uptime float64 `json:"uptime"` // uptime of the server
GetX509Requests uint64 `json:"x509GetRequests"` // total number of get x509 requests
PostX509Requests uint64 `json:"x509PostRequests"` // total number of post X509 requests
PutX509Requests uint64 `json:"x509PutRequests"` // total number of put X509 requests
DeleteX509Requests uint64 `json:"x509DeleteRequests"` // total number of delete X509 requests
HeadX509Requests uint64 `json:"x509HeadRequests"` // total number of head X509 requests
GetOAuthRequests uint64 `json:"oAuthGetRequests"` // total number of get requests form OAuth server
PostOAuthRequests uint64 `json:"oAuthPostRequests"` // total number of post requests from OAuth server
PutOAuthRequests uint64 `json:"oAuthPutRequests"` // total number of put requests from OAuth server
DeleteOAuthRequests uint64 `json:"oAuthDeleteRequests"` // total number of delete requests from OAuth server
HeadOAuthRequests uint64 `json:"oAuthHeadRequests"` // total number of head requests from OAuth server
GetRequests uint64 `json:"getRequests"` // total number of get requests across all services
PostRequests uint64 `json:"postRequests"` // total number of post requests across all services
PutRequests uint64 `json:"putRequests"` // total number of put requests across all services
DeleteRequests uint64 `json:"deleteRequests"` // total number of delete requests across all services
HeadRequests uint64 `json:"headRequests"` // total number of head requests across all services
RPS float64 `json:"rps"` // throughput req/sec
RPSPhysical float64 `json:"rpsPhysical"` // throughput req/sec using physical cpu
RPSLogical float64 `json:"rpsLogical"` // throughput req/sec using logical cpu
}

// ScitokensConfig represents configuration of scitokens service
Expand Down
77 changes: 69 additions & 8 deletions metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,15 @@ var TotalGetRequests uint64
// TotalPostRequests counts total number of POST requests received by the server
var TotalPostRequests uint64

// TotalPutRequests counts total number of PUT requests received by the server
var TotalPutRequests uint64

// TotalDeleteRequests counts total number of DELETE requests received by the server
var TotalDeleteRequests uint64

// TotalHeadRequests counts total number of HEAD requests received by the server
var TotalHeadRequests uint64

// MetricsLastUpdateTime keeps track of last update time of the metrics
var MetricsLastUpdateTime time.Time

Expand Down Expand Up @@ -68,20 +77,33 @@ func metrics() Metrics {
}
}
metrics.Uptime = time.Since(StartTime).Seconds()

metrics.GetX509Requests = TotalX509GetRequests
metrics.PostX509Requests = TotalX509PostRequests
metrics.PutX509Requests = TotalX509PutRequests
metrics.DeleteX509Requests = TotalX509DeleteRequests
metrics.HeadX509Requests = TotalX509HeadRequests

metrics.GetOAuthRequests = TotalOAuthGetRequests
metrics.PostOAuthRequests = TotalOAuthPostRequests
metrics.PutOAuthRequests = TotalOAuthPutRequests
metrics.DeleteOAuthRequests = TotalOAuthDeleteRequests
metrics.HeadOAuthRequests = TotalOAuthHeadRequests

metrics.GetRequests = metrics.GetX509Requests + metrics.GetOAuthRequests
metrics.PostRequests = metrics.PostX509Requests + metrics.PostOAuthRequests
if (metrics.GetRequests + metrics.PostRequests) > 0 {
metrics.RPS = RPS / float64(metrics.GetRequests+metrics.PostRequests)
metrics.PutRequests = metrics.PutX509Requests + metrics.PutOAuthRequests
metrics.DeleteRequests = metrics.DeleteX509Requests + metrics.DeleteOAuthRequests
metrics.HeadRequests = metrics.HeadX509Requests + metrics.HeadOAuthRequests

if (metrics.GetRequests + metrics.PostRequests + metrics.PutRequests + metrics.DeleteRequests + metrics.HeadRequests) > 0 {
metrics.RPS = RPS / float64(metrics.GetRequests+metrics.PostRequests+metrics.PutRequests+metrics.DeleteRequests+metrics.HeadRequests)
}
if metrics.GetRequests+metrics.PostRequests > 0 {
metrics.RPSPhysical = RPSPhysical / float64(metrics.GetRequests+metrics.PostRequests)
if (metrics.GetRequests + metrics.PostRequests + metrics.PutRequests + metrics.DeleteRequests + metrics.HeadRequests) > 0 {
metrics.RPSPhysical = RPSPhysical / float64(metrics.GetRequests+metrics.PostRequests+metrics.PutRequests+metrics.DeleteRequests+metrics.HeadRequests)
}
if metrics.GetRequests+metrics.PostRequests > 0 {
metrics.RPSLogical = RPSLogical / float64(metrics.GetRequests+metrics.PostRequests)
if (metrics.GetRequests + metrics.PostRequests + metrics.PutRequests + metrics.DeleteRequests + metrics.HeadRequests) > 0 {
metrics.RPSLogical = RPSLogical / float64(metrics.GetRequests+metrics.PostRequests+metrics.PutRequests+metrics.DeleteRequests+metrics.HeadRequests)
}
metrics.DataIn = DataIn
metrics.DataOut = DataOut
Expand Down Expand Up @@ -185,31 +207,70 @@ func promMetrics() string {
out += fmt.Sprintf("# HELP %s_get_x509_requests reports total number of X509 HTTP GET requests\n", prefix)
out += fmt.Sprintf("# TYPE %s_get_x509_requests counter\n", prefix)
out += fmt.Sprintf("%s_get_x509_requests %v\n", prefix, data.GetX509Requests)

out += fmt.Sprintf("# HELP %s_post_x509_requests reports total number of X509 HTTP POST requests\n", prefix)
out += fmt.Sprintf("# TYPE %s_post_x509_requests counter\n", prefix)
out += fmt.Sprintf("%s_post_x509_requests %v\n", prefix, data.PostX509Requests)

out += fmt.Sprintf("# HELP %s_put_x509_requests reports total number of X509 HTTP PUT requests\n", prefix)
out += fmt.Sprintf("# TYPE %s_put_x509_requests counter\n", prefix)
out += fmt.Sprintf("%s_put_x509_requests %v\n", prefix, data.PutX509Requests)

out += fmt.Sprintf("# HELP %s_delete_x509_requests reports total number of X509 HTTP DELETE requests\n", prefix)
out += fmt.Sprintf("# TYPE %s_delete_x509_requests counter\n", prefix)
out += fmt.Sprintf("%s_delete_x509_requests %v\n", prefix, data.DeleteX509Requests)

out += fmt.Sprintf("# HELP %s_head_x509_requests reports total number of X509 HTTP HEAD requests\n", prefix)
out += fmt.Sprintf("# TYPE %s_head_x509_requests counter\n", prefix)
out += fmt.Sprintf("%s_head_x509_requests %v\n", prefix, data.HeadX509Requests)

// oauth requests
out += fmt.Sprintf("# HELP %s_get_oauth_requests reports total number of OAuth HTTP GET requests\n", prefix)
out += fmt.Sprintf("# TYPE %s_get_oauth_requests counter\n", prefix)
out += fmt.Sprintf("%s_get_oauth_requests %v\n", prefix, data.GetOAuthRequests)

out += fmt.Sprintf("# HELP %s_post_oauth_requests reports total number of OAuth HTTP POST requests\n", prefix)
out += fmt.Sprintf("# TYPE %s_post_oauth_requests counter\n", prefix)
out += fmt.Sprintf("%s_post_oauth_requests %v\n", prefix, data.PostOAuthRequests)

out += fmt.Sprintf("# HELP %s_put_oauth_requests reports total number of OAuth HTTP PUT requests\n", prefix)
out += fmt.Sprintf("# TYPE %s_put_oauth_requests counter\n", prefix)
out += fmt.Sprintf("%s_put_oauth_requests %v\n", prefix, data.PutOAuthRequests)

out += fmt.Sprintf("# HELP %s_delete_oauth_requests reports total number of OAuth HTTP DELETE requests\n", prefix)
out += fmt.Sprintf("# TYPE %s_delete_oauth_requests counter\n", prefix)
out += fmt.Sprintf("%s_delete_oauth_requests %v\n", prefix, data.DeleteOAuthRequests)

out += fmt.Sprintf("# HELP %s_head_oauth_requests reports total number of OAuth HTTP HEAD requests\n", prefix)
out += fmt.Sprintf("# TYPE %s_head_oauth_requests counter\n", prefix)
out += fmt.Sprintf("%s_head_oauth_requests %v\n", prefix, data.HeadOAuthRequests)

// total requests
out += fmt.Sprintf("# HELP %s_get_requests reports total number of HTTP GET requests\n", prefix)
out += fmt.Sprintf("# TYPE %s_get_requests counter\n", prefix)
out += fmt.Sprintf("%s_get_requests %v\n", prefix, data.GetRequests)

out += fmt.Sprintf("# HELP %s_post_requests reports total number of HTTP POST requests\n", prefix)
out += fmt.Sprintf("# TYPE %s_post_requests counter\n", prefix)
out += fmt.Sprintf("%s_post_requests %v\n", prefix, data.PostRequests)

out += fmt.Sprintf("# HELP %s_put_requests reports total number of HTTP PUT requests\n", prefix)
out += fmt.Sprintf("# TYPE %s_put_requests counter\n", prefix)
out += fmt.Sprintf("%s_put_requests %v\n", prefix, data.PutRequests)

out += fmt.Sprintf("# HELP %s_delete_requests reports total number of HTTP DELETE requests\n", prefix)
out += fmt.Sprintf("# TYPE %s_delete_requests counter\n", prefix)
out += fmt.Sprintf("%s_delete_requests %v\n", prefix, data.DeleteRequests)

out += fmt.Sprintf("# HELP %s_head_requests reports total number of HTTP HEAD requests\n", prefix)
out += fmt.Sprintf("# TYPE %s_head_requests counter\n", prefix)
out += fmt.Sprintf("%s_head_requests %v\n", prefix, data.HeadRequests)

// data in/data out numbers
out += fmt.Sprintf("# HELP %s_data_in reports total number of bytes going into HTTP server\n", prefix)
out += fmt.Sprintf("# HELP %s_data_in reports average number of bytes going into HTTP server\n", prefix)
out += fmt.Sprintf("# TYPE %s_data_in counter\n", prefix)
out += fmt.Sprintf("%s_data_in %v\n", prefix, data.DataIn)
out += fmt.Sprintf("# HELP %s_data_out reports total number of bytes going out of HTTP server\n", prefix)
out += fmt.Sprintf("# HELP %s_data_out reports average number of bytes going out of HTTP server\n", prefix)
out += fmt.Sprintf("# TYPE %s_data_out counter\n", prefix)
out += fmt.Sprintf("%s_data_out %v\n", prefix, data.DataOut)

Expand Down
26 changes: 22 additions & 4 deletions oauth.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,18 @@ var TotalOAuthGetRequests uint64
// TotalOAuthPostRequests counts total number of POST requests received by the server
var TotalOAuthPostRequests uint64

// TotalOAuthPutRequests counts total number of PUT requests received by the server
var TotalOAuthPutRequests uint64

// TotalOAuthDeleteRequests counts total number of DELETE requests received by the server
var TotalOAuthDeleteRequests uint64

// TotalOAuthHeadRequests counts total number of HEAD requests received by the server
var TotalOAuthHeadRequests uint64

// TotalOAuthRequests counts total number of all requests received by the server
var TotalOAuthRequests uint64

// AuthTokenURL holds url for token authentication
var AuthTokenURL string

Expand Down Expand Up @@ -386,10 +398,16 @@ func oauthRequestHandler(w http.ResponseWriter, r *http.Request) {
// increment GET/POST counters
if r.Method == "GET" {
atomic.AddUint64(&TotalOAuthGetRequests, 1)
}
if r.Method == "POST" {
} else if r.Method == "POST" {
atomic.AddUint64(&TotalOAuthPostRequests, 1)
} else if r.Method == "PUT" {
atomic.AddUint64(&TotalOAuthPostRequests, 1)
} else if r.Method == "DELETE" {
atomic.AddUint64(&TotalOAuthDeleteRequests, 1)
} else if r.Method == "HEAD" {
atomic.AddUint64(&TotalOAuthHeadRequests, 1)
}
atomic.AddUint64(&TotalOAuthRequests, 1)
defer getRPS(start)

status := http.StatusOK
Expand Down Expand Up @@ -421,8 +439,8 @@ func oauthRequestHandler(w http.ResponseWriter, r *http.Request) {
crw := &logging.CustomResponseWriter{ResponseWriter: w}
// collect how much bytes we consume and write out with every HTTP request
defer func() {
DataIn += float64(r.ContentLength)
DataOut += float64(crw.BytesWritten)
DataIn += float64(r.ContentLength) / float64(TotalOAuthRequests)
DataOut += float64(crw.BytesWritten) / float64(TotalOAuthRequests)
}()

attrs, err := checkAccessToken(r)
Expand Down
28 changes: 23 additions & 5 deletions x509.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,34 @@ var TotalX509GetRequests uint64
// TotalX509PostRequests counts total number of POST requests received by the server
var TotalX509PostRequests uint64

// TotalX509PutRequests counts total number of PUT requests received by the server
var TotalX509PutRequests uint64

// TotalX509HeadRequests counts total number of HEAD requests received by the server
var TotalX509HeadRequests uint64

// TotalX509DeleteRequests counts total number of DELETE requests received by the server
var TotalX509DeleteRequests uint64

// TotalX509Requests counts total number of all requests received by the server
var TotalX509Requests uint64

// x509RequestHandler handle requests for x509 clients
func x509RequestHandler(w http.ResponseWriter, r *http.Request) {
start := time.Now()
// increment GET/POST counters
// increment requests counters
if r.Method == "GET" {
atomic.AddUint64(&TotalX509GetRequests, 1)
}
if r.Method == "POST" {
} else if r.Method == "POST" {
atomic.AddUint64(&TotalX509PostRequests, 1)
} else if r.Method == "PUT" {
atomic.AddUint64(&TotalX509PutRequests, 1)
} else if r.Method == "DELETE" {
atomic.AddUint64(&TotalX509DeleteRequests, 1)
} else if r.Method == "HEAD" {
atomic.AddUint64(&TotalX509HeadRequests, 1)
}
atomic.AddUint64(&TotalX509Requests, 1)
defer getRPS(start)

// check if user provides valid credentials
Expand Down Expand Up @@ -63,8 +81,8 @@ func x509RequestHandler(w http.ResponseWriter, r *http.Request) {
crw := &logging.CustomResponseWriter{ResponseWriter: w}
// collect how much bytes we consume and write out with every HTTP request
defer func() {
DataIn += float64(r.ContentLength)
DataOut += float64(crw.BytesWritten)
DataIn += float64(r.ContentLength) / float64(TotalX509Requests)
DataOut += float64(crw.BytesWritten) / float64(TotalX509Requests)
}()

// add LogRequest after we set cms headers in HTTP request
Expand Down

0 comments on commit 5a02123

Please sign in to comment.