From 3d1ba667afed4f177835f9411dfcde7a49e3a2b0 Mon Sep 17 00:00:00 2001 From: dongjiang Date: Thu, 6 Mar 2025 20:32:57 +0800 Subject: [PATCH] add build info and go runtime metrics Signed-off-by: dongjiang add build info and go runtime metrics Co-Authored-By: Hongcai Ren --- go.mod | 2 +- go.sum | 4 +- hack/util.sh | 2 + operator/cmd/operator/app/operator.go | 7 ++ pkg/metrics/version.go | 51 ++++++++ pkg/version/base.go | 7 +- pkg/version/version.go | 30 ++--- pkg/version/version_test.go | 17 +-- vendor/modules.txt | 2 +- .../controller-runtime/pkg/cache/cache.go | 43 ++++++- .../pkg/cache/internal/informers.go | 16 ++- .../pkg/certwatcher/certwatcher.go | 110 ++++++++++++------ .../pkg/certwatcher/metrics/metrics.go | 1 + .../pkg/internal/controller/controller.go | 2 +- .../internal/controller/metrics/metrics.go | 4 +- .../pkg/leaderelection/leader_election.go | 25 +++- .../controller-runtime/pkg/manager/manager.go | 1 + 17 files changed, 249 insertions(+), 75 deletions(-) create mode 100644 pkg/metrics/version.go diff --git a/go.mod b/go.mod index f7128c17c233..5ba24f5e7d50 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,7 @@ require ( k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf sigs.k8s.io/cluster-api v1.7.1 - sigs.k8s.io/controller-runtime v0.19.1 + sigs.k8s.io/controller-runtime v0.19.6 sigs.k8s.io/custom-metrics-apiserver v1.30.1-0.20241105195130-84dc8cfe2555 sigs.k8s.io/kind v0.25.0 sigs.k8s.io/mcs-api v0.1.0 diff --git a/go.sum b/go.sum index ca30be2327b2..44ebe8ae96e4 100644 --- a/go.sum +++ b/go.sum @@ -1480,8 +1480,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1 sigs.k8s.io/cluster-api v1.7.1 h1:JkMAbAMzBM+WBHxXLTJXTiCisv1PAaHRzld/3qrmLYY= sigs.k8s.io/cluster-api v1.7.1/go.mod h1:V9ZhKLvQtsDODwjXOKgbitjyCmC71yMBwDcMyNNIov0= sigs.k8s.io/controller-runtime v0.6.1/go.mod h1:XRYBPdbf5XJu9kpS84VJiZ7h/u1hF3gEORz0efEja7A= -sigs.k8s.io/controller-runtime v0.19.1 h1:Son+Q40+Be3QWb+niBXAg2vFiYWolDjjRfO8hn/cxOk= -sigs.k8s.io/controller-runtime v0.19.1/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= +sigs.k8s.io/controller-runtime v0.19.6 h1:fuq53qTLQ7aJTA7aNsklNnu7eQtSFqJUomOyM+phPLk= +sigs.k8s.io/controller-runtime v0.19.6/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= sigs.k8s.io/controller-tools v0.3.0/go.mod h1:enhtKGfxZD1GFEoMgP8Fdbu+uKQ/cq1/WGJhdVChfvI= sigs.k8s.io/custom-metrics-apiserver v1.30.1-0.20241105195130-84dc8cfe2555 h1:GYU1Vmegcr1cs7+D06pa6+saS2DDu31JIHvDIbvWtcE= sigs.k8s.io/custom-metrics-apiserver v1.30.1-0.20241105195130-84dc8cfe2555/go.mod h1:JL2q3g2QCWnIDvo73jpkksZOVd3ee3FWzZs4EHvx5NE= diff --git a/hack/util.sh b/hack/util.sh index a2d84af25940..1c2717be4550 100755 --- a/hack/util.sh +++ b/hack/util.sh @@ -710,6 +710,7 @@ function util::get_version() { function util::version_ldflags() { # Git information GIT_VERSION=$(util::get_version) + GIT_SHORT_COMMIT=$(git rev-parse --short HEAD) GIT_COMMIT_HASH=$(git rev-parse HEAD) if git_status=$(git status --porcelain 2>/dev/null) && [[ -z ${git_status} ]]; then GIT_TREESTATE="clean" @@ -720,6 +721,7 @@ function util::version_ldflags() { LDFLAGS="-X github.com/karmada-io/karmada/pkg/version.gitVersion=${GIT_VERSION} \ -X github.com/karmada-io/karmada/pkg/version.gitCommit=${GIT_COMMIT_HASH} \ -X github.com/karmada-io/karmada/pkg/version.gitTreeState=${GIT_TREESTATE} \ + -X github.com/karmada-io/karmada/pkg/version.gitShortCommit=${GIT_SHORT_COMMIT} \ -X github.com/karmada-io/karmada/pkg/version.buildDate=${BUILDDATE}" echo $LDFLAGS } diff --git a/operator/cmd/operator/app/operator.go b/operator/cmd/operator/app/operator.go index 871a043927a4..e4b12e08a7ed 100644 --- a/operator/cmd/operator/app/operator.go +++ b/operator/cmd/operator/app/operator.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/healthz" + ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "github.com/karmada-io/karmada/operator/cmd/operator/app/options" @@ -39,6 +40,7 @@ import ( ctrlctx "github.com/karmada-io/karmada/operator/pkg/controller/context" "github.com/karmada-io/karmada/operator/pkg/controller/karmada" "github.com/karmada-io/karmada/operator/pkg/scheme" + versionmetrics "github.com/karmada-io/karmada/pkg/metrics" "github.com/karmada-io/karmada/pkg/sharedcli" "github.com/karmada-io/karmada/pkg/sharedcli/klogflag" "github.com/karmada-io/karmada/pkg/version" @@ -110,6 +112,11 @@ func Run(ctx context.Context, o *options.Options) error { return err } + // `karmada_operator_build_info` metrics for operator version upgrade + ctrlmetrics.Registry.MustRegister( + versionmetrics.NewCollector("karmada_operator"), + ) + controllerCtx := ctrlctx.Context{ Controllers: o.Controllers, Manager: manager, diff --git a/pkg/metrics/version.go b/pkg/metrics/version.go new file mode 100644 index 000000000000..3a4987caddf8 --- /dev/null +++ b/pkg/metrics/version.go @@ -0,0 +1,51 @@ +/* +Copyright 2025 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "fmt" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/karmada-io/karmada/pkg/version" +) + +// NewCollector returns a collector that exports metrics about current version +// information. +func NewCollector(program string) prometheus.Collector { + return prometheus.NewGaugeFunc( + prometheus.GaugeOpts{ + Namespace: program, + Name: "build_info", + Help: fmt.Sprintf( + "A metric with a constant '1' value labeled by version, revision, branch, goversion from which %s was built, and the goos and goarch for the build.", + program, + ), + ConstLabels: prometheus.Labels{ + "git_version": version.Get().GitVersion, + "git_commit": version.Get().GitCommit, + "git_short_commit": version.Get().GitShortCommit, + "git_tree_state": version.Get().GitTreeState, + "build_date": version.Get().BuildDate, + "go_version": version.Get().GoVersion, + "compiler": version.Get().Compiler, + "platform": version.Get().Platform, + }, + }, + func() float64 { return 1 }, + ) +} diff --git a/pkg/version/base.go b/pkg/version/base.go index 3b4b83efd830..cc0c704927bf 100644 --- a/pkg/version/base.go +++ b/pkg/version/base.go @@ -23,9 +23,10 @@ package version // version for ad-hoc builds (e.g. `go build`) that cannot get the version // information from git. var ( - gitVersion = "v0.0.0-master" - gitCommit = "unknown" // sha1 from git, output of $(git rev-parse HEAD) - gitTreeState = "unknown" // state of git tree, either "clean" or "dirty" + gitVersion = "v0.0.0-master" + gitCommit = "unknown" // sha1 from git, output of $(git rev-parse HEAD) + gitTreeState = "unknown" // state of git tree, either "clean" or "dirty" + gitShortCommit = "unknown" // short sha1 from git, output of $(git rev-parse --short HEAD) buildDate = "unknown" // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ') ) diff --git a/pkg/version/version.go b/pkg/version/version.go index a50d74f95390..c9d99d4959d1 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -23,13 +23,14 @@ import ( // Info contains versioning information. type Info struct { - GitVersion string `json:"gitVersion"` - GitCommit string `json:"gitCommit"` - GitTreeState string `json:"gitTreeState"` - BuildDate string `json:"buildDate"` - GoVersion string `json:"goVersion"` - Compiler string `json:"compiler"` - Platform string `json:"platform"` + GitVersion string `json:"gitVersion"` + GitCommit string `json:"gitCommit"` + GitShortCommit string `json:"gitShortCommit"` + GitTreeState string `json:"gitTreeState"` + BuildDate string `json:"buildDate"` + GoVersion string `json:"goVersion"` + Compiler string `json:"compiler"` + Platform string `json:"platform"` } // String returns a Go-syntax representation of the Info. @@ -41,12 +42,13 @@ func (info Info) String() string { // what code a binary was built from. func Get() Info { return Info{ - GitVersion: gitVersion, - GitCommit: gitCommit, - GitTreeState: gitTreeState, - BuildDate: buildDate, - GoVersion: runtime.Version(), - Compiler: runtime.Compiler, - Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), + GitVersion: gitVersion, + GitShortCommit: gitShortCommit, + GitCommit: gitCommit, + GitTreeState: gitTreeState, + BuildDate: buildDate, + GoVersion: runtime.Version(), + Compiler: runtime.Compiler, + Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), } } diff --git a/pkg/version/version_test.go b/pkg/version/version_test.go index 3bef19c9dd42..c6e750e2ed93 100644 --- a/pkg/version/version_test.go +++ b/pkg/version/version_test.go @@ -29,15 +29,16 @@ func TestInfo_String(t *testing.T) { { name: "test1", info: Info{ - GitVersion: "1.3.0", - GitCommit: "da070e68f3318410c8c70ed8186a2bc4736dacbd", - GitTreeState: "clean", - BuildDate: "2022-08-31T13:09:22Z", - GoVersion: "go1.18.3", - Compiler: "gc", - Platform: "linux/amd64", + GitVersion: "1.3.0", + GitCommit: "da070e68f3318410c8c70ed8186a2bc4736dacbd", + GitTreeState: "clean", + GitShortCommit: "851c78564", + BuildDate: "2022-08-31T13:09:22Z", + GoVersion: "go1.18.3", + Compiler: "gc", + Platform: "linux/amd64", }, - want: `version.Info{GitVersion:"1.3.0", GitCommit:"da070e68f3318410c8c70ed8186a2bc4736dacbd", GitTreeState:"clean", BuildDate:"2022-08-31T13:09:22Z", GoVersion:"go1.18.3", Compiler:"gc", Platform:"linux/amd64"}`, + want: `version.Info{GitVersion:"1.3.0", GitCommit:"da070e68f3318410c8c70ed8186a2bc4736dacbd", GitShortCommit:"851c78564", GitTreeState:"clean", BuildDate:"2022-08-31T13:09:22Z", GoVersion:"go1.18.3", Compiler:"gc", Platform:"linux/amd64"}`, }, } for _, tt := range tests { diff --git a/vendor/modules.txt b/vendor/modules.txt index 14d437e279c3..718dc913bdcc 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1771,7 +1771,7 @@ sigs.k8s.io/cluster-api/errors sigs.k8s.io/cluster-api/feature sigs.k8s.io/cluster-api/util/certs sigs.k8s.io/cluster-api/util/secret -# sigs.k8s.io/controller-runtime v0.19.1 +# sigs.k8s.io/controller-runtime v0.19.6 ## explicit; go 1.22.0 sigs.k8s.io/controller-runtime sigs.k8s.io/controller-runtime/pkg/builder diff --git a/vendor/sigs.k8s.io/controller-runtime/pkg/cache/cache.go b/vendor/sigs.k8s.io/controller-runtime/pkg/cache/cache.go index 706f9c6cdd49..9c7bf2325542 100644 --- a/vendor/sigs.k8s.io/controller-runtime/pkg/cache/cache.go +++ b/vendor/sigs.k8s.io/controller-runtime/pkg/cache/cache.go @@ -222,6 +222,18 @@ type Options struct { // DefaultNamespaces. DefaultUnsafeDisableDeepCopy *bool + // DefaultEnableWatchBookmarks requests watch events with type "BOOKMARK". + // Servers that do not implement bookmarks may ignore this flag and + // bookmarks are sent at the server's discretion. Clients should not + // assume bookmarks are returned at any specific interval, nor may they + // assume the server will send any BOOKMARK event during a session. + // + // This will be used for all object types, unless it is set in ByObject or + // DefaultNamespaces. + // + // Defaults to false. + DefaultEnableWatchBookmarks *bool + // ByObject restricts the cache's ListWatch to the desired fields per GVK at the specified object. // If unset, this will fall through to the Default* settings. ByObject map[client.Object]ByObject @@ -272,6 +284,15 @@ type ByObject struct { // Be very careful with this, when enabled you must DeepCopy any object before mutating it, // otherwise you will mutate the object in the cache. UnsafeDisableDeepCopy *bool + + // EnableWatchBookmarks requests watch events with type "BOOKMARK". + // Servers that do not implement bookmarks may ignore this flag and + // bookmarks are sent at the server's discretion. Clients should not + // assume bookmarks are returned at any specific interval, nor may they + // assume the server will send any BOOKMARK event during a session. + // + // Defaults to false. + EnableWatchBookmarks *bool } // Config describes all potential options for a given watch. @@ -298,6 +319,15 @@ type Config struct { // UnsafeDisableDeepCopy specifies if List and Get requests against the // cache should not DeepCopy. A nil value allows to default this. UnsafeDisableDeepCopy *bool + + // EnableWatchBookmarks requests watch events with type "BOOKMARK". + // Servers that do not implement bookmarks may ignore this flag and + // bookmarks are sent at the server's discretion. Clients should not + // assume bookmarks are returned at any specific interval, nor may they + // assume the server will send any BOOKMARK event during a session. + // + // Defaults to false. + EnableWatchBookmarks *bool } // NewCacheFunc - Function for creating a new cache from the options and a rest config. @@ -367,6 +397,7 @@ func optionDefaultsToConfig(opts *Options) Config { FieldSelector: opts.DefaultFieldSelector, Transform: opts.DefaultTransform, UnsafeDisableDeepCopy: opts.DefaultUnsafeDisableDeepCopy, + EnableWatchBookmarks: opts.DefaultEnableWatchBookmarks, } } @@ -376,6 +407,7 @@ func byObjectToConfig(byObject ByObject) Config { FieldSelector: byObject.Field, Transform: byObject.Transform, UnsafeDisableDeepCopy: byObject.UnsafeDisableDeepCopy, + EnableWatchBookmarks: byObject.EnableWatchBookmarks, } } @@ -398,6 +430,7 @@ func newCache(restConfig *rest.Config, opts Options) newCacheFunc { Transform: config.Transform, WatchErrorHandler: opts.DefaultWatchErrorHandler, UnsafeDisableDeepCopy: ptr.Deref(config.UnsafeDisableDeepCopy, false), + EnableWatchBookmarks: ptr.Deref(config.EnableWatchBookmarks, false), NewInformer: opts.newInformer, }), readerFailOnMissingInformer: opts.ReaderFailOnMissingInformer, @@ -434,6 +467,8 @@ func defaultOpts(config *rest.Config, opts Options) (Options, error) { } } + opts.ByObject = maps.Clone(opts.ByObject) + opts.DefaultNamespaces = maps.Clone(opts.DefaultNamespaces) for obj, byObject := range opts.ByObject { isNamespaced, err := apiutil.IsObjectNamespaced(obj, opts.Scheme, opts.Mapper) if err != nil { @@ -445,6 +480,8 @@ func defaultOpts(config *rest.Config, opts Options) (Options, error) { if isNamespaced && byObject.Namespaces == nil { byObject.Namespaces = maps.Clone(opts.DefaultNamespaces) + } else { + byObject.Namespaces = maps.Clone(byObject.Namespaces) } // Default the namespace-level configs first, because they need to use the undefaulted type-level config @@ -452,7 +489,6 @@ func defaultOpts(config *rest.Config, opts Options) (Options, error) { for namespace, config := range byObject.Namespaces { // 1. Default from the undefaulted type-level config config = defaultConfig(config, byObjectToConfig(byObject)) - // 2. Default from the namespace-level config. This was defaulted from the global default config earlier, but // might not have an entry for the current namespace. if defaultNamespaceSettings, hasDefaultNamespace := opts.DefaultNamespaces[namespace]; hasDefaultNamespace { @@ -482,6 +518,7 @@ func defaultOpts(config *rest.Config, opts Options) (Options, error) { byObject.Field = defaultedConfig.FieldSelector byObject.Transform = defaultedConfig.Transform byObject.UnsafeDisableDeepCopy = defaultedConfig.UnsafeDisableDeepCopy + byObject.EnableWatchBookmarks = defaultedConfig.EnableWatchBookmarks } opts.ByObject[obj] = byObject @@ -523,7 +560,9 @@ func defaultConfig(toDefault, defaultFrom Config) Config { if toDefault.UnsafeDisableDeepCopy == nil { toDefault.UnsafeDisableDeepCopy = defaultFrom.UnsafeDisableDeepCopy } - + if toDefault.EnableWatchBookmarks == nil { + toDefault.EnableWatchBookmarks = defaultFrom.EnableWatchBookmarks + } return toDefault } diff --git a/vendor/sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go b/vendor/sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go index cd8c6774cacb..a40382d6f378 100644 --- a/vendor/sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go +++ b/vendor/sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go @@ -51,6 +51,7 @@ type InformersOpts struct { Selector Selector Transform cache.TransformFunc UnsafeDisableDeepCopy bool + EnableWatchBookmarks bool WatchErrorHandler cache.WatchErrorHandler } @@ -78,6 +79,7 @@ func NewInformers(config *rest.Config, options *InformersOpts) *Informers { selector: options.Selector, transform: options.Transform, unsafeDisableDeepCopy: options.UnsafeDisableDeepCopy, + enableWatchBookmarks: options.EnableWatchBookmarks, newInformer: newInformer, watchErrorHandler: options.WatchErrorHandler, } @@ -174,6 +176,7 @@ type Informers struct { selector Selector transform cache.TransformFunc unsafeDisableDeepCopy bool + enableWatchBookmarks bool // NewInformer allows overriding of the shared index informer constructor for testing. newInformer func(cache.ListerWatcher, runtime.Object, time.Duration, cache.Indexers) cache.SharedIndexInformer @@ -361,8 +364,10 @@ func (ip *Informers) addInformerToMap(gvk schema.GroupVersionKind, obj runtime.O return listWatcher.ListFunc(opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - ip.selector.ApplyToList(&opts) opts.Watch = true // Watch needs to be set to true separately + opts.AllowWatchBookmarks = ip.enableWatchBookmarks + + ip.selector.ApplyToList(&opts) return listWatcher.WatchFunc(opts) }, }, obj, calculateResyncPeriod(ip.resync), cache.Indexers{ @@ -444,6 +449,9 @@ func (ip *Informers) makeListWatcher(gvk schema.GroupVersionKind, obj runtime.Ob }, // Setup the watch function WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + opts.Watch = true // Watch needs to be set to true separately + opts.AllowWatchBookmarks = ip.enableWatchBookmarks + if namespace != "" { return resources.Namespace(namespace).Watch(ip.ctx, opts) } @@ -486,6 +494,9 @@ func (ip *Informers) makeListWatcher(gvk schema.GroupVersionKind, obj runtime.Ob }, // Setup the watch function WatchFunc: func(opts metav1.ListOptions) (watcher watch.Interface, err error) { + opts.Watch = true // Watch needs to be set to true separately + opts.AllowWatchBookmarks = ip.enableWatchBookmarks + if namespace != "" { watcher, err = resources.Namespace(namespace).Watch(ip.ctx, opts) } else { @@ -527,6 +538,9 @@ func (ip *Informers) makeListWatcher(gvk schema.GroupVersionKind, obj runtime.Ob }, // Setup the watch function WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + opts.Watch = true // Watch needs to be set to true separately + opts.AllowWatchBookmarks = ip.enableWatchBookmarks + // Build the request. req := client.Get().Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec) if namespace != "" { diff --git a/vendor/sigs.k8s.io/controller-runtime/pkg/certwatcher/certwatcher.go b/vendor/sigs.k8s.io/controller-runtime/pkg/certwatcher/certwatcher.go index fe15fc0dd71a..c32324098234 100644 --- a/vendor/sigs.k8s.io/controller-runtime/pkg/certwatcher/certwatcher.go +++ b/vendor/sigs.k8s.io/controller-runtime/pkg/certwatcher/certwatcher.go @@ -17,9 +17,11 @@ limitations under the License. package certwatcher import ( + "bytes" "context" "crypto/tls" "fmt" + "os" "sync" "time" @@ -33,18 +35,24 @@ import ( var log = logf.RuntimeLog.WithName("certwatcher") -// CertWatcher watches certificate and key files for changes. When either file -// changes, it reads and parses both and calls an optional callback with the new -// certificate. +const defaultWatchInterval = 10 * time.Second + +// CertWatcher watches certificate and key files for changes. +// It always returns the cached version, +// but periodically reads and parses certificate and key for changes +// and calls an optional callback with the new certificate. type CertWatcher struct { sync.RWMutex currentCert *tls.Certificate watcher *fsnotify.Watcher + interval time.Duration certPath string keyPath string + cachedKeyPEMBlock []byte + // callback is a function to be invoked when the certificate changes. callback func(tls.Certificate) } @@ -56,6 +64,7 @@ func New(certPath, keyPath string) (*CertWatcher, error) { cw := &CertWatcher{ certPath: certPath, keyPath: keyPath, + interval: defaultWatchInterval, } // Initial read of certificate and key. @@ -71,6 +80,12 @@ func New(certPath, keyPath string) (*CertWatcher, error) { return cw, nil } +// WithWatchInterval sets the watch interval and returns the CertWatcher pointer +func (cw *CertWatcher) WithWatchInterval(interval time.Duration) *CertWatcher { + cw.interval = interval + return cw +} + // RegisterCallback registers a callback to be invoked when the certificate changes. func (cw *CertWatcher) RegisterCallback(callback func(tls.Certificate)) { cw.Lock() @@ -112,12 +127,20 @@ func (cw *CertWatcher) Start(ctx context.Context) error { go cw.Watch() - log.Info("Starting certificate watcher") - - // Block until the context is done. - <-ctx.Done() + ticker := time.NewTicker(cw.interval) + defer ticker.Stop() - return cw.watcher.Close() + log.Info("Starting certificate poll+watcher", "interval", cw.interval) + for { + select { + case <-ctx.Done(): + return cw.watcher.Close() + case <-ticker.C: + if err := cw.ReadCertificate(); err != nil { + log.Error(err, "failed read certificate") + } + } + } } // Watch reads events from the watcher's channel and reacts to changes. @@ -131,7 +154,6 @@ func (cw *CertWatcher) Watch() { } cw.handleEvent(event) - case err, ok := <-cw.watcher.Errors: // Channel is closed. if !ok { @@ -143,20 +165,48 @@ func (cw *CertWatcher) Watch() { } } +// updateCachedCertificate checks if the new certificate differs from the cache, +// updates it and returns the result if it was updated or not +func (cw *CertWatcher) updateCachedCertificate(cert *tls.Certificate, keyPEMBlock []byte) bool { + cw.Lock() + defer cw.Unlock() + + if cw.currentCert != nil && + bytes.Equal(cw.currentCert.Certificate[0], cert.Certificate[0]) && + bytes.Equal(cw.cachedKeyPEMBlock, keyPEMBlock) { + log.V(7).Info("certificate already cached") + return false + } + cw.currentCert = cert + cw.cachedKeyPEMBlock = keyPEMBlock + return true +} + // ReadCertificate reads the certificate and key files from disk, parses them, -// and updates the current certificate on the watcher. If a callback is set, it +// and updates the current certificate on the watcher if updated. If a callback is set, it // is invoked with the new certificate. func (cw *CertWatcher) ReadCertificate() error { metrics.ReadCertificateTotal.Inc() - cert, err := tls.LoadX509KeyPair(cw.certPath, cw.keyPath) + certPEMBlock, err := os.ReadFile(cw.certPath) + if err != nil { + metrics.ReadCertificateErrors.Inc() + return err + } + keyPEMBlock, err := os.ReadFile(cw.keyPath) if err != nil { metrics.ReadCertificateErrors.Inc() return err } - cw.Lock() - cw.currentCert = &cert - cw.Unlock() + cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock) + if err != nil { + metrics.ReadCertificateErrors.Inc() + return err + } + + if !cw.updateCachedCertificate(&cert, keyPEMBlock) { + return nil + } log.Info("Updated current TLS certificate") @@ -173,36 +223,20 @@ func (cw *CertWatcher) ReadCertificate() error { func (cw *CertWatcher) handleEvent(event fsnotify.Event) { // Only care about events which may modify the contents of the file. - if !(isWrite(event) || isRemove(event) || isCreate(event) || isChmod(event)) { - return - } - - log.V(1).Info("certificate event", "event", event) - - // If the file was removed or renamed, re-add the watch to the previous name - if isRemove(event) || isChmod(event) { + switch { + case event.Op.Has(fsnotify.Write): + case event.Op.Has(fsnotify.Create): + case event.Op.Has(fsnotify.Chmod), event.Op.Has(fsnotify.Remove): + // If the file was removed or renamed, re-add the watch to the previous name if err := cw.watcher.Add(event.Name); err != nil { log.Error(err, "error re-watching file") } + default: + return } + log.V(1).Info("certificate event", "event", event) if err := cw.ReadCertificate(); err != nil { log.Error(err, "error re-reading certificate") } } - -func isWrite(event fsnotify.Event) bool { - return event.Op.Has(fsnotify.Write) -} - -func isCreate(event fsnotify.Event) bool { - return event.Op.Has(fsnotify.Create) -} - -func isRemove(event fsnotify.Event) bool { - return event.Op.Has(fsnotify.Remove) -} - -func isChmod(event fsnotify.Event) bool { - return event.Op.Has(fsnotify.Chmod) -} diff --git a/vendor/sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics/metrics.go b/vendor/sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics/metrics.go index 05869eff0309..f128abbcf0e4 100644 --- a/vendor/sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics/metrics.go +++ b/vendor/sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics/metrics.go @@ -18,6 +18,7 @@ package metrics import ( "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" ) diff --git a/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go b/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go index dfe407f3b8e5..1f7752ba6e6b 100644 --- a/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go +++ b/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go @@ -183,7 +183,7 @@ func (c *Controller[request]) Start(ctx context.Context) error { c.LogConstructor(nil).Info("Starting Controller") for _, watch := range c.startWatches { - syncingSource, ok := watch.(source.SyncingSource) + syncingSource, ok := watch.(source.TypedSyncingSource[request]) if !ok { continue } diff --git a/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics/metrics.go b/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics/metrics.go index fbf15669d5dc..6d562efb9371 100644 --- a/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics/metrics.go +++ b/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics/metrics.go @@ -88,7 +88,7 @@ func init() { ActiveWorkers, // expose process metrics like CPU, Memory, file descriptor usage etc. collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), - // expose Go runtime metrics like GC stats, memory stats etc. - collectors.NewGoCollector(), + // expose all Go runtime metrics like GC stats, memory stats etc. + collectors.NewGoCollector(collectors.WithGoCollectorRuntimeMetrics(collectors.MetricsAll)), ) } diff --git a/vendor/sigs.k8s.io/controller-runtime/pkg/leaderelection/leader_election.go b/vendor/sigs.k8s.io/controller-runtime/pkg/leaderelection/leader_election.go index ee4fcf4cbe01..5cc253917a3b 100644 --- a/vendor/sigs.k8s.io/controller-runtime/pkg/leaderelection/leader_election.go +++ b/vendor/sigs.k8s.io/controller-runtime/pkg/leaderelection/leader_election.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "os" + "time" "k8s.io/apimachinery/pkg/util/uuid" coordinationv1client "k8s.io/client-go/kubernetes/typed/coordination/v1" @@ -49,6 +50,12 @@ type Options struct { // LeaderElectionID determines the name of the resource that leader election // will use for holding the leader lock. LeaderElectionID string + + // RenewDeadline is the renew deadline for this leader election client. + // Must be set to ensure the resource lock has an appropriate client timeout. + // Without that, a single slow response from the API server can result + // in losing leadership. + RenewDeadline time.Duration } // NewResourceLock creates a new resource lock for use in a leader election loop. @@ -88,6 +95,20 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op // Construct clients for leader election rest.AddUserAgent(config, "leader-election") + + if options.RenewDeadline != 0 { + return resourcelock.NewFromKubeconfig(options.LeaderElectionResourceLock, + options.LeaderElectionNamespace, + options.LeaderElectionID, + resourcelock.ResourceLockConfig{ + Identity: id, + EventRecorder: recorderProvider.GetEventRecorderFor(id), + }, + config, + options.RenewDeadline, + ) + } + corev1Client, err := corev1client.NewForConfig(config) if err != nil { return nil, err @@ -97,7 +118,6 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op if err != nil { return nil, err } - return resourcelock.New(options.LeaderElectionResourceLock, options.LeaderElectionNamespace, options.LeaderElectionID, @@ -106,7 +126,8 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op resourcelock.ResourceLockConfig{ Identity: id, EventRecorder: recorderProvider.GetEventRecorderFor(id), - }) + }, + ) } func getInClusterNamespace() (string, error) { diff --git a/vendor/sigs.k8s.io/controller-runtime/pkg/manager/manager.go b/vendor/sigs.k8s.io/controller-runtime/pkg/manager/manager.go index 3166f4818fbe..92906fe6ca77 100644 --- a/vendor/sigs.k8s.io/controller-runtime/pkg/manager/manager.go +++ b/vendor/sigs.k8s.io/controller-runtime/pkg/manager/manager.go @@ -389,6 +389,7 @@ func New(config *rest.Config, options Options) (Manager, error) { LeaderElectionResourceLock: options.LeaderElectionResourceLock, LeaderElectionID: options.LeaderElectionID, LeaderElectionNamespace: options.LeaderElectionNamespace, + RenewDeadline: *options.RenewDeadline, }) if err != nil { return nil, err