From 4176e6c6cb40f3c52eed7d6134c602554549be3e Mon Sep 17 00:00:00 2001 From: Gang Liu Date: Thu, 4 May 2023 13:50:03 +0800 Subject: [PATCH] set pod annotations for contour Signed-off-by: Gang Liu --- .../v1alpha1/contourdeployment.go | 4 ++ .../v1alpha1/zz_generated.deepcopy.go | 7 +++ examples/contour/01-crds.yaml | 6 +++ examples/render/contour-deployment.yaml | 6 +++ .../render/contour-gateway-provisioner.yaml | 6 +++ examples/render/contour-gateway.yaml | 6 +++ examples/render/contour.yaml | 6 +++ internal/provisioner/controller/gateway.go | 4 ++ .../provisioner/controller/gateway_test.go | 46 +++++++++++++++++++ internal/provisioner/model/model.go | 8 +++- .../objects/deployment/deployment.go | 36 +++++++++++++-- .../objects/deployment/deployment_test.go | 17 +++++++ .../docs/main/config/api-reference.html | 13 ++++++ 13 files changed, 158 insertions(+), 7 deletions(-) diff --git a/apis/projectcontour/v1alpha1/contourdeployment.go b/apis/projectcontour/v1alpha1/contourdeployment.go index 272b026c223..dfb4ae87747 100644 --- a/apis/projectcontour/v1alpha1/contourdeployment.go +++ b/apis/projectcontour/v1alpha1/contourdeployment.go @@ -114,6 +114,10 @@ type ContourSettings struct { // Deployment describes the settings for running contour as a `Deployment`. // +optional Deployment *DeploymentSettings `json:"deployment,omitempty"` + + // PodAnnotations defines annotations to add to the Contour pods. + // +optional + PodAnnotations map[string]string `json:"podAnnotations,omitempty"` } // DeploymentSettings contains settings for Deployment resources. diff --git a/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go b/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go index 63ad9b433a0..cbe409f2ad0 100644 --- a/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go +++ b/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go @@ -356,6 +356,13 @@ func (in *ContourSettings) DeepCopyInto(out *ContourSettings) { *out = new(DeploymentSettings) (*in).DeepCopyInto(*out) } + if in.PodAnnotations != nil { + in, out := &in.PodAnnotations, &out.PodAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContourSettings. diff --git a/examples/contour/01-crds.yaml b/examples/contour/01-crds.yaml index 1e76311a976..2bf9721ca2b 100644 --- a/examples/contour/01-crds.yaml +++ b/examples/contour/01-crds.yaml @@ -1146,6 +1146,12 @@ spec: type: object type: array type: object + podAnnotations: + additionalProperties: + type: string + description: PodAnnotations defines annotations to add to the + Contour pods. + type: object replicas: description: "Deprecated: Use `DeploymentSettings.Replicas` instead. \n Replicas is the desired number of Contour replicas. If if diff --git a/examples/render/contour-deployment.yaml b/examples/render/contour-deployment.yaml index 54c3e47cfad..eff9b1daec6 100644 --- a/examples/render/contour-deployment.yaml +++ b/examples/render/contour-deployment.yaml @@ -1359,6 +1359,12 @@ spec: type: object type: array type: object + podAnnotations: + additionalProperties: + type: string + description: PodAnnotations defines annotations to add to the + Contour pods. + type: object replicas: description: "Deprecated: Use `DeploymentSettings.Replicas` instead. \n Replicas is the desired number of Contour replicas. If if diff --git a/examples/render/contour-gateway-provisioner.yaml b/examples/render/contour-gateway-provisioner.yaml index 1068485fdd0..483620dd67e 100644 --- a/examples/render/contour-gateway-provisioner.yaml +++ b/examples/render/contour-gateway-provisioner.yaml @@ -1160,6 +1160,12 @@ spec: type: object type: array type: object + podAnnotations: + additionalProperties: + type: string + description: PodAnnotations defines annotations to add to the + Contour pods. + type: object replicas: description: "Deprecated: Use `DeploymentSettings.Replicas` instead. \n Replicas is the desired number of Contour replicas. If if diff --git a/examples/render/contour-gateway.yaml b/examples/render/contour-gateway.yaml index 1ec4d2d45cc..6506a628a71 100644 --- a/examples/render/contour-gateway.yaml +++ b/examples/render/contour-gateway.yaml @@ -1365,6 +1365,12 @@ spec: type: object type: array type: object + podAnnotations: + additionalProperties: + type: string + description: PodAnnotations defines annotations to add to the + Contour pods. + type: object replicas: description: "Deprecated: Use `DeploymentSettings.Replicas` instead. \n Replicas is the desired number of Contour replicas. If if diff --git a/examples/render/contour.yaml b/examples/render/contour.yaml index 00c9850fe1e..3d88ebca02c 100644 --- a/examples/render/contour.yaml +++ b/examples/render/contour.yaml @@ -1359,6 +1359,12 @@ spec: type: object type: array type: object + podAnnotations: + additionalProperties: + type: string + description: PodAnnotations defines annotations to add to the + Contour pods. + type: object replicas: description: "Deprecated: Use `DeploymentSettings.Replicas` instead. \n Replicas is the desired number of Contour replicas. If if diff --git a/internal/provisioner/controller/gateway.go b/internal/provisioner/controller/gateway.go index cdf3d579bfc..02a18bc7e93 100644 --- a/internal/provisioner/controller/gateway.go +++ b/internal/provisioner/controller/gateway.go @@ -275,6 +275,10 @@ func (r *gatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct contourParams.Deployment.Strategy != nil { contourModel.Spec.ContourDeploymentStrategy = *contourParams.Deployment.Strategy } + + for k, v := range contourParams.PodAnnotations { + contourModel.Spec.ContourPodAnnotations[k] = v + } } if gatewayClassParams.Spec.Envoy != nil { diff --git a/internal/provisioner/controller/gateway_test.go b/internal/provisioner/controller/gateway_test.go index 18a5a612592..519f1077837 100644 --- a/internal/provisioner/controller/gateway_test.go +++ b/internal/provisioner/controller/gateway_test.go @@ -1259,6 +1259,52 @@ func TestGatewayReconcile(t *testing.T) { }, }, + "If ContourDeployment.Spec.Contour.PodAnnotations is specified, the Contour pods' have annotations for prometheus & user-defined": { + + gatewayClass: reconcilableGatewayClassWithParams("gatewayclass-1", controller), + gatewayClassParams: &contourv1alpha1.ContourDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "projectcontour", + Name: "gatewayclass-1-params", + }, + Spec: contourv1alpha1.ContourDeploymentSpec{ + Contour: &contourv1alpha1.ContourSettings{ + PodAnnotations: map[string]string{ + "key": "val", + }, + }, + }, + }, + gateway: &gatewayv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "gateway-1", + Name: "gateway-1", + }, + Spec: gatewayv1beta1.GatewaySpec{ + GatewayClassName: gatewayv1beta1.ObjectName("gatewayclass-1"), + }, + }, + assertions: func(t *testing.T, r *gatewayReconciler, gw *gatewayv1beta1.Gateway, reconcileErr error) { + require.NoError(t, reconcileErr) + + // Verify the Gateway has a "Accepted: true" condition + require.NoError(t, r.client.Get(context.Background(), keyFor(gw), gw)) + require.Len(t, gw.Status.Conditions, 1) + assert.Equal(t, string(gatewayv1beta1.GatewayConditionAccepted), gw.Status.Conditions[0].Type) + assert.Equal(t, metav1.ConditionTrue, gw.Status.Conditions[0].Status) + + // Verify the deployment has been created + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "gateway-1", + Name: "contour-gateway-1", + }, + } + require.NoError(t, r.client.Get(context.Background(), keyFor(deploy), deploy)) + assert.Contains(t, deploy.Spec.Template.ObjectMeta.Annotations, "key") + }, + }, + "If ContourDeployment.Spec.Envoy.WorkloadType is set to DaemonSet," + "an Envoy daemonset is provisioned with the strategy that come from DaemonsetSettings": { gatewayClass: reconcilableGatewayClassWithParams("gatewayclass-1", controller), diff --git a/internal/provisioner/model/model.go b/internal/provisioner/model/model.go index ef7aeda5758..96cb572c5b4 100644 --- a/internal/provisioner/model/model.go +++ b/internal/provisioner/model/model.go @@ -67,8 +67,9 @@ func Default(namespace, name string) *Contour { MaxUnavailable: ref.To(intstr.FromString("25%")), }, }, - ResourceLabels: map[string]string{}, - EnvoyPodAnnotations: map[string]string{}, + ResourceLabels: map[string]string{}, + EnvoyPodAnnotations: map[string]string{}, + ContourPodAnnotations: map[string]string{}, }, } } @@ -208,6 +209,9 @@ type ContourSpec struct { // EnvoyPodAnnotations holds the annotations that will be add to the envoy‘s pod. EnvoyPodAnnotations map[string]string + // ContourPodAnnotations holds the annotations that will be add to the contour's pod. + ContourPodAnnotations map[string]string + // Compute Resources required by envoy container. EnvoyResources corev1.ResourceRequirements diff --git a/internal/provisioner/objects/deployment/deployment.go b/internal/provisioner/objects/deployment/deployment.go index 95822733760..9dd20097810 100644 --- a/internal/provisioner/objects/deployment/deployment.go +++ b/internal/provisioner/objects/deployment/deployment.go @@ -210,11 +210,8 @@ func DesiredDeployment(contour *model.Contour, image string) *appsv1.Deployment ObjectMeta: metav1.ObjectMeta{ // TODO [danehans]: Remove the prometheus annotations when Contour is updated to // show how the Prometheus Operator is used to scrape Contour/Envoy metrics. - Annotations: map[string]string{ - "prometheus.io/scrape": "true", - "prometheus.io/port": fmt.Sprintf("%d", metricsPort), - }, - Labels: contourPodLabels(contour), + Annotations: contourPodAnnotations(contour), + Labels: contourPodLabels(contour), }, Spec: corev1.PodSpec{ // TODO [danehans]: Readdress anti-affinity when https://github.com/projectcontour/contour/issues/2997 @@ -301,3 +298,32 @@ func contourPodLabels(contour *model.Contour) map[string]string { } return labels } + +// contourPodAnnotations returns the annotations for contour's pods +func contourPodAnnotations(contour *model.Contour) map[string]string { + annotations := map[string]string{} + for k, v := range contour.Spec.ContourPodAnnotations { + annotations[k] = v + } + + metricsPort := metricsPort + if contour.Spec.RuntimeSettings != nil && + contour.Spec.RuntimeSettings.Metrics != nil && + contour.Spec.RuntimeSettings.Metrics.Port > 0 { + metricsPort = contour.Spec.RuntimeSettings.Metrics.Port + } + + annotations["prometheus.io/scrape"] = "true" + annotations["prometheus.io/port"] = fmt.Sprint(metricsPort) + + return annotations +} + +/* + Annotations: map[string]string{ + "prometheus.io/scrape": "true", + "prometheus.io/port": fmt.Sprintf("%d", metricsPort), + }, +*/ + +// Annotations: envoyPodAnnotations(contour), diff --git a/internal/provisioner/objects/deployment/deployment_test.go b/internal/provisioner/objects/deployment/deployment_test.go index 6307859dfa8..9aedd831756 100644 --- a/internal/provisioner/objects/deployment/deployment_test.go +++ b/internal/provisioner/objects/deployment/deployment_test.go @@ -68,6 +68,17 @@ func checkDeploymentHasLabels(t *testing.T, deploy *appsv1.Deployment, expected t.Errorf("deployment has unexpected %q labels", deploy.Labels) } +func checkPodHasAnnotations(t *testing.T, tmpl *corev1.PodTemplateSpec, annotations map[string]string) { + t.Helper() + + for k, v := range annotations { + if val, ok := tmpl.Annotations[k]; !ok || val != v { + t.Errorf("pod template has unexpected %q annotations", tmpl.Annotations) + } + } + +} + func checkContainerHasArg(t *testing.T, container *corev1.Container, arg string) { t.Helper() @@ -140,6 +151,10 @@ func TestDesiredDeployment(t *testing.T) { corev1.ResourceMemory: resource.MustParse("25Mi"), }, } + + annotations := map[string]string{ + "key": "value", + } cntr.Spec.ContourResources = resQutoa // Change the Kubernetes log level to test --kubernetes-debug. @@ -151,6 +166,7 @@ func TestDesiredDeployment(t *testing.T) { cntr.Spec.ResourceLabels = map[string]string{ "key": "value", } + cntr.Spec.ContourPodAnnotations = annotations // Use non-default container ports to test that --envoy-service-http(s)-port // flags are added. @@ -167,6 +183,7 @@ func TestDesiredDeployment(t *testing.T) { checkDeploymentHasEnvVar(t, deploy, contourNsEnvVar) checkDeploymentHasEnvVar(t, deploy, contourPodEnvVar) checkDeploymentHasLabels(t, deploy, cntr.AppLabels()) + checkPodHasAnnotations(t, &deploy.Spec.Template, annotations) for _, port := range cntr.Spec.NetworkPublishing.Envoy.Ports { switch port.Name { diff --git a/site/content/docs/main/config/api-reference.html b/site/content/docs/main/config/api-reference.html index 44f3c2c4e5d..39a5801aada 100644 --- a/site/content/docs/main/config/api-reference.html +++ b/site/content/docs/main/config/api-reference.html @@ -5938,6 +5938,19 @@

ContourSettings

Deployment describes the settings for running contour as a Deployment.

+ + +podAnnotations +
+ +map[string]string + + + +(Optional) +

PodAnnotations defines annotations to add to the Contour pods.

+ +

CustomTag