Skip to content

Commit

Permalink
feat: add-termination-metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
@danielstokes
danielstokes committed Feb 26, 2025
1 parent 56d591b commit 6d9b760
Show file tree
Hide file tree
Showing 5 changed files with 89 additions and 53 deletions.
26 changes: 22 additions & 4 deletions src/kubelet/metric/pods.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@ import (
"strings"
"time"

log "github.com/sirupsen/logrus"
v1 "k8s.io/api/core/v1"

"github.com/newrelic/nri-kubernetes/v3/src/client"
"github.com/newrelic/nri-kubernetes/v3/src/definition"

log "github.com/sirupsen/logrus"
v1 "k8s.io/api/core/v1"
)

// KubeletPodsPath is the path where kubelet serves information about pods.
Expand Down Expand Up @@ -182,6 +182,16 @@ func fillContainerStatuses(pod *v1.Pod, dest map[string]definition.RawMetrics) {
name := c.Name
id := containerID(pod, name)

// Set the ExitCode. Zero if no terminated Exit Code.
var lastTerminatedExitCode int32 = 0

Check failure on line 186 in src/kubelet/metric/pods.go

View workflow job for this annotation

GitHub Actions / Static analysis and linting 

var-declaration: should drop = 0 from declaration of var lastTerminatedExitCode; it is the zero value (revive)
var lastTerminatedExitReason string = "None"

Check failure on line 187 in src/kubelet/metric/pods.go

View workflow job for this annotation

GitHub Actions / Static analysis and linting 

var-declaration: should omit type string from declaration of var lastTerminatedExitReason; it will be inferred from the right-hand side (revive)
var lastTerminatedFinishedAt string
if c.LastTerminationState.Terminated != nil {
lastTerminatedExitCode = c.LastTerminationState.Terminated.ExitCode
lastTerminatedExitReason = c.LastTerminationState.Terminated.Reason
lastTerminatedFinishedAt = c.LastTerminationState.Terminated.FinishedAt.Time.In(time.UTC).Format(time.RFC3339)
}

dest[id] = make(definition.RawMetrics)

switch {
Expand All @@ -190,15 +200,23 @@ func fillContainerStatuses(pod *v1.Pod, dest map[string]definition.RawMetrics) {
dest[id]["startedAt"] = c.State.Running.StartedAt.Time.In(time.UTC) // TODO WE DO NOT REPORT THAT METRIC
dest[id]["restartCount"] = c.RestartCount
dest[id]["isReady"] = c.Ready
dest[id]["lastTerminatedExitCode"] = lastTerminatedExitCode
dest[id]["lastTerminatedExitReason"] = lastTerminatedExitReason
dest[id]["lastTerminatedTimestamp"] = lastTerminatedFinishedAt
case c.State.Waiting != nil:
dest[id]["status"] = "Waiting"
dest[id]["reason"] = c.State.Waiting.Reason
dest[id]["restartCount"] = c.RestartCount
dest[id]["lastTerminatedExitCode"] = lastTerminatedExitCode
dest[id]["lastTerminatedExitReason"] = lastTerminatedExitReason
dest[id]["lastTerminatedTimestamp"] = lastTerminatedFinishedAt
case c.State.Terminated != nil:
dest[id]["status"] = "Terminated"
dest[id]["reason"] = c.State.Terminated.Reason
dest[id]["restartCount"] = c.RestartCount
dest[id]["startedAt"] = c.State.Terminated.StartedAt.Time.In(time.UTC) // TODO WE DO NOT REPORT THAT METRIC
dest[id]["lastTerminatedExitCode"] = lastTerminatedExitCode
dest[id]["lastTerminatedExitReason"] = lastTerminatedExitReason
dest[id]["lastTerminatedTimestamp"] = lastTerminatedFinishedAt
default:
dest[id]["status"] = "Unknown"
}
Expand Down
78 changes: 42 additions & 36 deletions src/kubelet/metric/testdata/kubelet_group_expected.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,31 +128,34 @@ var ExpectedGroupData = definition.RawGroups{
},
"container": {
"kube-system_newrelic-infra-rz225_newrelic-infra": {
"containerName": "newrelic-infra",
"containerID": "69d7203a8f2d2d027ffa51d61002eac63357f22a17403363ef79e66d1c3146b2",
"containerImage": "newrelic/ohaik:1.0.0-beta3",
"containerImageID": "sha256:1a95d0df2997f93741fbe2a15d2c31a394e752fd942ec29bf16a44163342f6a1",
"namespace": "kube-system",
"podName": "newrelic-infra-rz225",
"daemonsetName": "newrelic-infra",
"nodeName": "minikube",
"nodeIP": "192.168.99.100",
"restartCount": int32(6),
"isReady": true,
"status": "Running",
"startedAt": parseTime("2018-02-27T15:21:16Z"),
"cpuRequestedCores": int64(100),
"memoryRequestedBytes": int64(104857600),
"memoryLimitBytes": int64(104857600),
"usageBytes": uint64(18083840),
"workingSetBytes": uint64(17113088),
"usageNanoCores": uint64(17428240),
"fsAvailableBytes": uint64(14924988416),
"fsUsedBytes": uint64(126976),
"fsCapacityBytes": uint64(17293533184),
"fsInodesFree": uint64(9713372),
"fsInodes": uint64(9732096),
"fsInodesUsed": uint64(36),
"containerName": "newrelic-infra",
"containerID": "69d7203a8f2d2d027ffa51d61002eac63357f22a17403363ef79e66d1c3146b2",
"containerImage": "newrelic/ohaik:1.0.0-beta3",
"containerImageID": "sha256:1a95d0df2997f93741fbe2a15d2c31a394e752fd942ec29bf16a44163342f6a1",
"namespace": "kube-system",
"podName": "newrelic-infra-rz225",
"daemonsetName": "newrelic-infra",
"nodeName": "minikube",
"nodeIP": "192.168.99.100",
"restartCount": int32(6),
"isReady": true,
"status": "Running",
"startedAt": parseTime("2018-02-27T15:21:16Z"),
"lastTerminatedExitCode": int32(0),
"lastTerminatedExitReason": "Completed",
"lastTerminatedTimestamp": "2018-02-27T15:21:10Z",
"cpuRequestedCores": int64(100),
"memoryRequestedBytes": int64(104857600),
"memoryLimitBytes": int64(104857600),
"usageBytes": uint64(18083840),
"workingSetBytes": uint64(17113088),
"usageNanoCores": uint64(17428240),
"fsAvailableBytes": uint64(14924988416),
"fsUsedBytes": uint64(126976),
"fsCapacityBytes": uint64(17293533184),
"fsInodesFree": uint64(9713372),
"fsInodes": uint64(9732096),
"fsInodesUsed": uint64(36),
"labels": map[string]string{
"controller-revision-hash": "3887482659",
"name": "newrelic-infra",
Expand Down Expand Up @@ -240,14 +243,17 @@ var ExpectedGroupData = definition.RawGroups{
},
},
"kube-system_kube-controller-manager-minikube_kube-controller-manager": {
"containerName": "kube-controller-manager",
"containerImage": "k8s.gcr.io/kube-controller-manager:v1.16.0",
"nodeIP": "192.168.99.100",
"cpuRequestedCores": int64(200),
"status": "Running",
"startedAt": parseTime("2019-10-23T17:10:49Z"),
"restartCount": int32(1),
"isReady": bool(true),
"containerName": "kube-controller-manager",
"containerImage": "k8s.gcr.io/kube-controller-manager:v1.16.0",
"nodeIP": "192.168.99.100",
"cpuRequestedCores": int64(200),
"status": "Running",
"lastTerminatedExitCode": int32(255),
"lastTerminatedExitReason": "Error",
"lastTerminatedTimestamp": "2019-10-23T17:10:25Z",
"startedAt": parseTime("2019-10-23T17:10:49Z"),
"restartCount": int32(1),
"isReady": bool(true),
"labels": map[string]string{
"k8s-app": "kube-controller-manager",
"component": "kube-controller-manager",
Expand Down Expand Up @@ -287,9 +293,9 @@ var ExpectedGroupData = definition.RawGroups{
"usageCoreNanoSeconds": uint64(22332102208229),
"usageNanoCores": uint64(228759290),
"labels": map[string]string{
"kubernetes.io/arch": "amd64",
"kubernetes.io/hostname": "minikube",
"kubernetes.io/os": "linux",
"kubernetes.io/arch": "amd64",
"kubernetes.io/hostname": "minikube",
"kubernetes.io/os": "linux",
"node-role.kubernetes.io/control-plane": "",
},
"cpuRequestedCores": int64(501),
Expand Down
32 changes: 19 additions & 13 deletions src/kubelet/metric/testdata/kubelet_pods_expected.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,10 +99,13 @@ var ExpectedRawData = definition.RawGroups{
"isReady": true,
"status": "Running",
// "reason": "", // TODO
"startedAt": parseTime("2018-02-27T15:21:16Z"),
"cpuRequestedCores": int64(100),
"memoryRequestedBytes": int64(104857600),
"memoryLimitBytes": int64(104857600),
"startedAt": parseTime("2018-02-27T15:21:16Z"),
"lastTerminatedExitCode": int32(0),
"lastTerminatedExitReason": "Completed",
"lastTerminatedTimestamp": "2018-02-27T15:21:10Z",
"cpuRequestedCores": int64(100),
"memoryRequestedBytes": int64(104857600),
"memoryLimitBytes": int64(104857600),
"labels": map[string]string{
"controller-revision-hash": "3887482659",
"name": "newrelic-infra",
Expand Down Expand Up @@ -177,15 +180,18 @@ var ExpectedRawData = definition.RawGroups{
"k8s-app": "kube-controller-manager",
"component": "kube-controller-manager",
},
"podName": "kube-controller-manager-minikube",
"containerImage": "k8s.gcr.io/kube-controller-manager:v1.16.0",
"namespace": "kube-system",
"nodeIP": "192.168.99.100",
"cpuRequestedCores": int64(200),
"status": "Running",
"startedAt": parseTime("2019-10-23T17:10:49Z"),
"restartCount": int32(1),
"containerName": "kube-controller-manager",
"podName": "kube-controller-manager-minikube",
"containerImage": "k8s.gcr.io/kube-controller-manager:v1.16.0",
"namespace": "kube-system",
"nodeIP": "192.168.99.100",
"cpuRequestedCores": int64(200),
"status": "Running",
"lastTerminatedExitCode": int32(255),
"lastTerminatedExitReason": "Error",
"lastTerminatedTimestamp": "2019-10-23T17:10:25Z",
"startedAt": parseTime("2019-10-23T17:10:49Z"),
"restartCount": int32(1),
"containerName": "kube-controller-manager",
},
},
}
Expand Down
3 changes: 3 additions & 0 deletions src/metric/definition.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1312,6 +1312,9 @@ var KubeletSpecs = definition.SpecGroups{
{Name: "status", ValueFunc: definition.FromRaw("status"), Type: sdkMetric.ATTRIBUTE},
{Name: "isReady", ValueFunc: definition.Transform(definition.FromRaw("isReady"), toNumericBoolean), Type: sdkMetric.GAUGE},
{Name: "reason", ValueFunc: definition.FromRaw("reason"), Type: sdkMetric.ATTRIBUTE, Optional: true}, // Previously called statusWaitingReason
{Name: "lastTerminatedExitCode", ValueFunc: definition.FromRaw("lastTerminatedExitCode"), Type: sdkMetric.GAUGE, Optional: true},
{Name: "lastTerminatedExitReason", ValueFunc: definition.FromRaw("lastTerminatedExitReason"), Type: sdkMetric.ATTRIBUTE, Optional: true},
{Name: "lastTerminatedTimestamp", ValueFunc: definition.FromRaw("lastTerminatedTimestamp"), Type: sdkMetric.ATTRIBUTE, Optional: true},

// Inherit from pod
{Name: "label.*", ValueFunc: definition.Transform(definition.FromRaw("labels"), kubeletMetric.OneMetricPerLabel), Type: sdkMetric.ATTRIBUTE},
Expand Down
3 changes: 3 additions & 0 deletions src/scrape/scrape_job_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,9 @@ var expectedEntities = []*integration.Entity{
"status": "Running",
"isReady": float64(1),
//"reason": "", // TODO ?
"lastTerminatedExitCode": float64(0),
"lastTerminatedExitReason": "Completed",
"lastTerminatedTimestamp": "2018-02-27T15:21:10Z",
"displayName": "newrelic-infra", // From entity attributes
"clusterName": "test-cluster", // From entity attributes
"label.controller-revision-hash": "3887482659",
Expand Down

0 comments on commit 6d9b760

Please sign in to comment.