feat: add helm configuration for reporting in different rules (kyvern…

…o#11376) (kyverno#11388) * feat: add helm configuration for reporting in different rules (forgot signoff) * fix: linter and tests * feat: rename reporting.imageVerification --------- Signed-off-by: Vishal Choudhary <[email protected]> Signed-off-by: ShutingZhao <[email protected]> Co-authored-by: Vishal Choudhary <[email protected]> Co-authored-by: shuting <[email protected]>
realshuting · Oct 11, 2024 · c2d6840 · c2d6840
1 parent 7f4ca6e
commit c2d6840
Show file tree

Hide file tree

Showing 27 changed files with 268 additions and 106 deletions.
diff --git a/charts/kyverno/README.md b/charts/kyverno/README.md
@@ -327,7 +327,11 @@ The chart values are organised per component.
 | features.aggregateReports.enabled | bool | `true` | Enables the feature |
 | features.policyReports.enabled | bool | `true` | Enables the feature |
 | features.validatingAdmissionPolicyReports.enabled | bool | `false` | Enables the feature |
-| features.backgroundReports.enabled | bool | `true` | Enables the feature |
+| features.reporting.validate | bool | `true` | Enables the feature |
+| features.reporting.mutate | bool | `true` | Enables the feature |
+| features.reporting.mutateExisting | bool | `true` | Enables the feature |
+| features.reporting.imageVerify | bool | `true` | Enables the feature |
+| features.reporting.generate | bool | `true` | Enables the feature |
 | features.autoUpdateWebhooks.enabled | bool | `true` | Enables the feature |
 | features.backgroundScan.enabled | bool | `true` | Enables the feature |
 | features.backgroundScan.backgroundScanWorkers | int | `2` | Number of background scan workers |

diff --git a/charts/kyverno/templates/_helpers.tpl b/charts/kyverno/templates/_helpers.tpl
@@ -25,9 +25,6 @@
 {{- with .validatingAdmissionPolicyReports -}}
   {{- $flags = append $flags (print "--validatingAdmissionPolicyReports=" .enabled) -}}
 {{- end -}}
-{{- with .backgroundReports -}}
-  {{- $flags = append $flags (print "--backgroundReports=" .enabled) -}}
-{{- end -}}
 {{- with .autoUpdateWebhooks -}}
   {{- $flags = append $flags (print "--autoUpdateWebhooks=" .enabled) -}}
 {{- end -}}
@@ -97,6 +94,25 @@
     {{- $flags = append $flags (print "--tufRootRaw=" .) -}}
   {{- end -}}
 {{- end -}}
+{{- with .reporting -}}
+  {{- $reportingConfig := list -}}
+  {{- with .validate -}}
+    {{- $reportingConfig = append $reportingConfig "validate" -}}
+  {{- end -}}
+  {{- with .mutate -}}
+    {{- $reportingConfig = append $reportingConfig "mutate" -}}
+  {{- end -}}
+  {{- with .mutateExisting -}}
+    {{- $reportingConfig = append $reportingConfig "mutateExisting" -}}
+  {{- end -}}
+  {{- with .imageVerify -}}
+    {{- $reportingConfig = append $reportingConfig "imageVerify" -}}
+  {{- end -}}
+  {{- with .generate -}}
+    {{- $reportingConfig = append $reportingConfig "generate" -}}
+  {{- end -}}
+  {{- $flags = append $flags (print "--enableReporting=" (join "," $reportingConfig)) -}}
+{{- end -}}
 {{- with $flags -}}
   {{- toYaml . -}}
 {{- end -}}

diff --git a/charts/kyverno/templates/admission-controller/deployment.yaml b/charts/kyverno/templates/admission-controller/deployment.yaml
@@ -174,6 +174,7 @@ spec:
             - --imagePullSecrets={{- join "," (concat (keys .Values.imagePullSecrets) .Values.existingImagePullSecrets) }}
             {{- end }}
             {{- include "kyverno.features.flags" (pick (mergeOverwrite .Values.features .Values.admissionController.featuresOverride)
+              "reporting"
               "admissionReports"
               "autoUpdateWebhooks"
               "configMapCaching"

diff --git a/charts/kyverno/templates/background-controller/deployment.yaml b/charts/kyverno/templates/background-controller/deployment.yaml
@@ -119,7 +119,7 @@ spec:
             - --imagePullSecrets={{- join "," (concat (keys .Values.imagePullSecrets) .Values.existingImagePullSecrets) }}
             {{- end }}
             {{- include "kyverno.features.flags" (pick (mergeOverwrite .Values.features .Values.backgroundController.featuresOverride)
-              "backgroundReports"
+              "reporting"
               "configMapCaching"
               "deferredLoading"
               "globalContext"

diff --git a/charts/kyverno/templates/reports-controller/deployment.yaml b/charts/kyverno/templates/reports-controller/deployment.yaml
@@ -119,6 +119,7 @@ spec:
             - --imagePullSecrets={{- join "," (concat (keys .Values.imagePullSecrets) .Values.existingImagePullSecrets) }}
             {{- end }}
             {{- include "kyverno.features.flags" (pick (mergeOverwrite .Values.features .Values.reportsController.featuresOverride)
+              "reporting"
               "admissionReports"
               "aggregateReports"
               "policyReports"

diff --git a/charts/kyverno/values.yaml b/charts/kyverno/values.yaml
@@ -636,9 +636,17 @@ features:
   validatingAdmissionPolicyReports:
     # -- Enables the feature
     enabled: false
-  backgroundReports:
+  reporting:
     # -- Enables the feature
-    enabled: true
+    validate: true
+    # -- Enables the feature
+    mutate: true
+    # -- Enables the feature
+    mutateExisting: true
+    # -- Enables the feature
+    imageVerify: true
+    # -- Enables the feature
+    generate: true
   autoUpdateWebhooks:
     # -- Enables the feature
     enabled: true

diff --git a/cmd/background-controller/main.go b/cmd/background-controller/main.go
@@ -29,6 +29,7 @@ import (
 	"github.com/kyverno/kyverno/pkg/policy"
 	"github.com/kyverno/kyverno/pkg/utils/generator"
 	kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
+	reportutils "github.com/kyverno/kyverno/pkg/utils/report"
 	apiserver "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	kubeinformers "k8s.io/client-go/informers"
 	kyamlopenapi "sigs.k8s.io/kustomize/kyaml/openapi"
@@ -55,7 +56,7 @@ func createrLeaderControllers(
 	jp jmespath.Interface,
 	backgroundScanInterval time.Duration,
 	urGenerator generator.UpdateRequestGenerator,
-	backgroundReports bool,
+	reportsConfig reportutils.ReportingConfiguration,
 	reportsBreaker breaker.Breaker,
 ) ([]internal.Controller, error) {
 	policyCtrl, err := policy.NewPolicyController(
@@ -88,7 +89,7 @@ func createrLeaderControllers(
 		eventGenerator,
 		configuration,
 		jp,
-		backgroundReports,
+		reportsConfig,
 		reportsBreaker,
 	)
 	return []internal.Controller{
@@ -103,7 +104,6 @@ func main() {
 		maxQueuedEvents          int
 		omitEvents               string
 		maxAPICallResponseLength int64
-		backgroundReports        bool
 		maxBackgroundReports     int
 	)
 	flagset := flag.NewFlagSet("updaterequest-controller", flag.ExitOnError)
@@ -112,7 +112,6 @@ func main() {
 	flagset.StringVar(&omitEvents, "omitEvents", "", "Set this flag to a comma sperated list of PolicyViolation, PolicyApplied, PolicyError, PolicySkipped to disable events, e.g. --omitEvents=PolicyApplied,PolicyViolation")
 	flagset.Int64Var(&maxAPICallResponseLength, "maxAPICallResponseLength", 2*1000*1000, "Maximum allowed response size from API Calls. A value of 0 bypasses checks (not recommended).")
 	flagset.IntVar(&maxBackgroundReports, "maxBackgroundReports", 10000, "Maximum number of ephemeralreports created for the background policies.")
-	flagset.BoolVar(&backgroundReports, "backgroundReports", true, "Enables or disables reports for mutate existing and generate rules.")
 
 	// config
 	appConfig := internal.NewConfiguration(
@@ -132,6 +131,7 @@ func main() {
 		internal.WithApiServerClient(),
 		internal.WithMetadataClient(),
 		internal.WithFlagSets(flagset),
+		internal.WithReporting(),
 	)
 	// parse flags
 	internal.ParseFlags(appConfig)
@@ -252,7 +252,7 @@ func main() {
 					setup.Jp,
 					bgscanInterval,
 					urGenerator,
-					backgroundReports,
+					setup.ReportingConfiguration,
 					reportsBreaker,
 				)
 				if err != nil {

diff --git a/cmd/internal/config.go b/cmd/internal/config.go
@@ -22,6 +22,7 @@ type Configuration interface {
 	UsesMetadataClient() bool
 	UsesKyvernoDynamicClient() bool
 	UsesEventsClient() bool
+	UsesReporting() bool
 	FlagSets() []*flag.FlagSet
 }
 
@@ -145,6 +146,12 @@ func WithFlagSets(flagsets ...*flag.FlagSet) ConfigurationOption {
 	}
 }
 
+func WithReporting() ConfigurationOption {
+	return func(c *configuration) {
+		c.usesReporting = true
+	}
+}
+
 type configuration struct {
 	usesMetrics              bool
 	usesTracing              bool
@@ -163,6 +170,7 @@ type configuration struct {
 	usesMetadataClient       bool
 	usesKyvernoDynamicClient bool
 	usesEventsClient         bool
+	usesReporting            bool
 	flagSets                 []*flag.FlagSet
 }
 
@@ -234,6 +242,10 @@ func (c *configuration) UsesEventsClient() bool {
 	return c.usesEventsClient
 }
 
+func (c *configuration) UsesReporting() bool {
+	return c.usesReporting
+}
+
 func (c *configuration) FlagSets() []*flag.FlagSet {
 	return c.flagSets
 }
diff --git a/cmd/internal/flag.go b/cmd/internal/flag.go
@@ -59,6 +59,8 @@ var (
 	imageVerifyCacheMaxSize     int64
 	// global context
 	enableGlobalContext bool
+	// reporting
+	enableReporting string
 )
 
 func initLoggingFlags() {
@@ -137,6 +139,10 @@ func initCleanupFlags() {
 	flag.StringVar(&cleanupServerPort, "cleanupServerPort", "9443", "kyverno cleanup server port, defaults to '9443'.")
 }
 
+func initReportingFlags() {
+	flag.StringVar(&enableReporting, "enableReporting", "validate,mutate,mutateExisting,generate,imageVerify", "Comma separated list to enables reporting for different rule types. (validate,mutate,mutateExisting,generate,imageVerify)")
+}
+
 type options struct {
 	clientRateLimitQPS   float64
 	clientRateLimitBurst int
@@ -220,6 +226,11 @@ func initFlags(config Configuration, opts ...Option) {
 	if config.UsesLeaderElection() {
 		initLeaderElectionFlags()
 	}
+	// reporting
+	if config.UsesReporting() {
+		initReportingFlags()
+	}
+
 	initCleanupFlags()
 	for _, flagset := range config.FlagSets() {
 		flagset.VisitAll(func(f *flag.Flag) {

diff --git a/cmd/internal/reporting.go b/cmd/internal/reporting.go
@@ -0,0 +1,15 @@
+package internal
+
+import (
+	"strings"
+
+	"github.com/go-logr/logr"
+	reportutils "github.com/kyverno/kyverno/pkg/utils/report"
+)
+
+func setupReporting(logger logr.Logger) reportutils.ReportingConfiguration {
+	logger = logger.WithName("setup-reporting").WithValues("enableReporting", enableReporting)
+	cfg := reportutils.NewReportingConfig(strings.Split(enableReporting, ",")...)
+	logger.Info("setting up reporting...", "validate", cfg.ValidateReportsEnabled(), "mutate", cfg.MutateReportsEnabled(), "mutateExisiting", cfg.MutateExistingReportsEnabled(), "imageVerify", cfg.ImageVerificationReportsEnabled(), "generate", cfg.GenerateReportsEnabled())
+	return cfg
+}
diff --git a/cmd/internal/setup.go b/cmd/internal/setup.go
@@ -16,6 +16,7 @@ import (
 	"github.com/kyverno/kyverno/pkg/imageverifycache"
 	"github.com/kyverno/kyverno/pkg/metrics"
 	"github.com/kyverno/kyverno/pkg/registryclient"
+	reportutils "github.com/kyverno/kyverno/pkg/utils/report"
 	eventsv1 "k8s.io/client-go/kubernetes/typed/events/v1"
 	corev1listers "k8s.io/client-go/listers/core/v1"
 )
@@ -48,6 +49,7 @@ type SetupResult struct {
 	MetadataClient         metadataclient.UpstreamInterface
 	KyvernoDynamicClient   dclient.Interface
 	EventsClient           eventsv1.EventsV1Interface
+	ReportingConfiguration reportutils.ReportingConfiguration
 }
 
 func Setup(config Configuration, name string, skipResourceFilters bool) (context.Context, SetupResult, context.CancelFunc) {
@@ -105,6 +107,10 @@ func Setup(config Configuration, name string, skipResourceFilters bool) (context
 	if config.UsesMetadataClient() {
 		metadataClient = createMetadataClient(logger, metadataclient.WithMetrics(metricsManager, metrics.MetadataClient), metadataclient.WithTracing())
 	}
+	var reportingConfig reportutils.ReportingConfiguration
+	if config.UsesReporting() {
+		reportingConfig = setupReporting(logger)
+	}
 	return ctx,
 		SetupResult{
 			Logger:                 logger,
@@ -123,6 +129,7 @@ func Setup(config Configuration, name string, skipResourceFilters bool) (context
 			MetadataClient:         metadataClient,
 			KyvernoDynamicClient:   dClient,
 			EventsClient:           eventsClient,
+			ReportingConfiguration: reportingConfig,
 		},
 		shutdown(logger.WithName("shutdown"), sdownMaxProcs, sdownMetrics, sdownTracing, sdownSignals)
 }
diff --git a/cmd/kyverno/main.go b/cmd/kyverno/main.go
@@ -320,6 +320,7 @@ func main() {
 		internal.WithApiServerClient(),
 		internal.WithMetadataClient(),
 		internal.WithFlagSets(flagset),
+		internal.WithReporting(),
 	)
 	// parse flags
 	internal.ParseFlags(appConfig)
@@ -578,6 +579,7 @@ func main() {
 			setup.Jp,
 			maxAuditWorkers,
 			maxAuditCapacity,
+			setup.ReportingConfiguration,
 			reportsBreaker,
 		)
 		exceptionHandlers := webhooksexception.NewHandlers(exception.ValidationOptions{

diff --git a/cmd/reports-controller/main.go b/cmd/reports-controller/main.go
@@ -27,6 +27,7 @@ import (
 	"github.com/kyverno/kyverno/pkg/leaderelection"
 	"github.com/kyverno/kyverno/pkg/logging"
 	kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
+	reportutils "github.com/kyverno/kyverno/pkg/utils/report"
 	"github.com/kyverno/kyverno/pkg/validatingadmissionpolicy"
 	apiserver "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	kubeinformers "k8s.io/client-go/informers"
@@ -66,6 +67,7 @@ func createReportControllers(
 	configuration config.Configuration,
 	jp jmespath.Interface,
 	eventGenerator event.Interface,
+	reportsConfig reportutils.ReportingConfiguration,
 	reportsBreaker breaker.Breaker,
 ) ([]internal.Controller, func(context.Context) error) {
 	var ctrls []internal.Controller
@@ -126,6 +128,7 @@ func createReportControllers(
 				jp,
 				eventGenerator,
 				policyReports,
+				reportsConfig,
 				reportsBreaker,
 			)
 			ctrls = append(ctrls, internal.NewController(
@@ -149,6 +152,7 @@ func createrLeaderControllers(
 	eng engineapi.Engine,
 	backgroundScan bool,
 	admissionReports bool,
+	reportsConfig reportutils.ReportingConfiguration,
 	aggregateReports bool,
 	policyReports bool,
 	validatingAdmissionPolicyReports bool,
@@ -183,6 +187,7 @@ func createrLeaderControllers(
 		configuration,
 		jp,
 		eventGenerator,
+		reportsConfig,
 		reportsBreaker,
 	)
 	return reportControllers, warmup, nil
@@ -238,6 +243,7 @@ func main() {
 		internal.WithEventsClient(),
 		internal.WithApiServerClient(),
 		internal.WithFlagSets(flagset),
+		internal.WithReporting(),
 	)
 	// parse flags
 	internal.ParseFlags(
@@ -350,6 +356,7 @@ func main() {
 					engine,
 					backgroundScan,
 					admissionReports,
+					setup.ReportingConfiguration,
 					aggregateReports,
 					policyReports,
 					validatingAdmissionPolicyReports,

diff --git a/config/install-latest-testing.yaml b/config/install-latest-testing.yaml
@@ -50737,6 +50737,7 @@ spec:
             - --protectManagedResources=false
             - --allowInsecureRegistry=false
             - --registryCredentialHelpers=default,google,amazon,azure,github
+            - --enableReporting=validate,mutate,mutateExisting,imageVerify,generate
 
           resources:
             limits:
@@ -50881,14 +50882,14 @@ spec:
             - --disableMetrics=false
             - --otelConfig=prometheus
             - --metricsPort=8000
-            - --backgroundReports=true
             - --enableConfigMapCaching=true
             - --enableDeferredLoading=true
             - --maxAPICallResponseLength=2000000
             - --loggingFormat=text
             - --v=2
             - --omitEvents=PolicyApplied,PolicySkipped
             - --enablePolicyException=true
+            - --enableReporting=validate,mutate,mutateExisting,imageVerify,generate
 
           env:
           - name: KYVERNO_SERVICEACCOUNT_NAME
@@ -51142,6 +51143,7 @@ spec:
             - --enablePolicyException=true
             - --allowInsecureRegistry=false
             - --registryCredentialHelpers=default,google,amazon,azure,github
+            - --enableReporting=validate,mutate,mutateExisting,imageVerify,generate
 
           env:
           - name: KYVERNO_SERVICEACCOUNT_NAME