[node-labeler] Refactor node labeling to use taints instead of labels (…

…#20652) * [node-labeler] Refactor node labeling to use taints instead of labels * [agent-smith] Add toleration to daemonset * Add workspace component tolerations to various Gitpod components if it running in Full installation * Apply suggestions from code review Co-authored-by: Kyle Brennan <[email protected]> * Update components/node-labeler/cmd/run.go Co-authored-by: Kyle Brennan <[email protected]> --------- Co-authored-by: Kyle Brennan <[email protected]>
gitpod-io · Mar 11, 2025 · 52a7727 · 52a7727
1 parent 1becba7
commit 52a7727
Show file tree

Hide file tree

Showing 26 changed files with 383 additions and 71 deletions.
diff --git a/components/node-labeler/cmd/run.go b/components/node-labeler/cmd/run.go
diff --git a/install/installer/pkg/common/toleration.go b/install/installer/pkg/common/toleration.go
@@ -0,0 +1,33 @@
+// Copyright (c) 2025 Gitpod GmbH. All rights reserved.
+// Licensed under the GNU Affero General Public License (AGPL).
+// See License.AGPL.txt in the project root for license information.
+
+package common
+
+import (
+	configv1 "github.com/gitpod-io/gitpod/installer/pkg/config/v1"
+	corev1 "k8s.io/api/core/v1"
+)
+
+const (
+	RegistryFacadeTaintKey = "gitpod.io/registry-facade-not-ready"
+	WsDaemonTaintKey       = "gitpod.io/ws-daemon-not-ready"
+)
+
+func WithTolerationWorkspaceComponentNotReady(ctx *RenderContext) []corev1.Toleration {
+	if ctx.Config.Kind != configv1.InstallationFull {
+		return []corev1.Toleration{}
+	}
+	return []corev1.Toleration{
+		{
+			Key:      RegistryFacadeTaintKey,
+			Operator: corev1.TolerationOpExists,
+			Effect:   corev1.TaintEffectNoSchedule,
+		},
+		{
+			Key:      WsDaemonTaintKey,
+			Operator: corev1.TolerationOpExists,
+			Effect:   corev1.TaintEffectNoSchedule,
+		},
+	}
+}
diff --git a/install/installer/pkg/components/agent-smith/daemonset.go b/install/installer/pkg/components/agent-smith/daemonset.go
@@ -105,6 +105,12 @@ func daemonset(ctx *common.RenderContext) ([]runtime.Object, error) {
 						},
 						common.CAVolume(),
 					},
+					Tolerations: []corev1.Toleration{
+						{
+							Effect:   corev1.TaintEffectNoSchedule,
+							Operator: corev1.TolerationOpExists,
+						},
+					},
 				},
 			},
 			UpdateStrategy: common.DaemonSetRolloutStrategy(),

diff --git a/install/installer/pkg/components/blobserve/deployment.go b/install/installer/pkg/components/blobserve/deployment.go
@@ -168,6 +168,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 								FailureThreshold:    3,
 							},
 						}, *common.KubeRBACProxyContainer(ctx)},
+						Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 					},
 				},
 			},

diff --git a/install/installer/pkg/components/content-service/deployment.go b/install/installer/pkg/components/content-service/deployment.go
@@ -88,6 +88,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 			},
 		}, *common.KubeRBACProxyContainer(ctx),
 		},
+		Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 	}
 
 	err = common.AddStorageMounts(ctx, &podSpec, Component)

diff --git a/install/installer/pkg/components/dashboard/deployment.go b/install/installer/pkg/components/dashboard/deployment.go
@@ -86,6 +86,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 								TimeoutSeconds:   1,
 							},
 						}},
+						Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 					},
 				},
 			},

diff --git a/install/installer/pkg/components/database/incluster/helm.go b/install/installer/pkg/components/database/incluster/helm.go
@@ -25,6 +25,15 @@ var Helm = common.CompositeHelmFunc(
 			return nil, err
 		}
 
+		tolerations, err := helm.WithTolerationWorkspaceComponentNotReadyYaml(cfg)
+		if err != nil {
+			return nil, err
+		}
+		tolerationsTemplate, err := helm.KeyFileValue("mysql.primary.tolerations", tolerations)
+		if err != nil {
+			return nil, err
+		}
+
 		imageRegistry := common.ThirdPartyContainerRepo(cfg.Config.Repository, common.DockerRegistryURL)
 
 		type EnvVar struct {
@@ -73,6 +82,7 @@ var Helm = common.CompositeHelmFunc(
 				FileValues: []string{
 					primaryAffinityTemplate,
 					extraEnvVarsTemplate,
+					tolerationsTemplate,
 				},
 			},
 		}, nil

diff --git a/install/installer/pkg/components/docker-registry/helm.go b/install/installer/pkg/components/docker-registry/helm.go
@@ -65,12 +65,25 @@ var Helm = common.CompositeHelmFunc(
 			)
 		}
 
+		tolerations, err := helm.WithTolerationWorkspaceComponentNotReadyYaml(cfg)
+		if err != nil {
+			return nil, err
+		}
+		tolerationsTemplate, err := helm.KeyFileValue("docker-registry.tolerations", tolerations)
+		if err != nil {
+			return nil, err
+		}
+
 		registryValues = append(registryValues, helm.KeyValue("docker-registry.persistence.enabled", enablePersistence))
 
 		return &common.HelmConfig{
 			Enabled: inCluster,
 			Values: &values.Options{
 				Values: registryValues,
+				// This is too complex to be sent as a string
+				FileValues: []string{
+					tolerationsTemplate,
+				},
 			},
 		}, nil
 	}),

diff --git a/install/installer/pkg/components/ide-metrics/deployment.go b/install/installer/pkg/components/ide-metrics/deployment.go
@@ -120,7 +120,8 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 						},
 							*common.KubeRBACProxyContainerWithConfig(ctx),
 						},
-						Volumes: volumes,
+						Volumes:     volumes,
+						Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 					},
 				},
 			},

diff --git a/install/installer/pkg/components/ide-proxy/deployment.go b/install/installer/pkg/components/ide-proxy/deployment.go
@@ -82,6 +82,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 								TimeoutSeconds:   1,
 							},
 						}},
+						Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 					},
 				},
 			},

diff --git a/install/installer/pkg/components/ide-service/deployment.go b/install/installer/pkg/components/ide-service/deployment.go
@@ -153,6 +153,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 								},
 							},
 						},
+						Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 					},
 				},
 			},

diff --git a/install/installer/pkg/components/image-builder-mk3/deployment.go b/install/installer/pkg/components/image-builder-mk3/deployment.go
@@ -188,6 +188,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 					},
 						*common.KubeRBACProxyContainer(ctx),
 					},
+					Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 				},
 			},
 		},

diff --git a/install/installer/pkg/components/migrations/job.go b/install/installer/pkg/components/migrations/job.go
@@ -56,6 +56,7 @@ func job(ctx *common.RenderContext) ([]runtime.Object, error) {
 							"cd /app/node_modules/@gitpod/gitpod-db && yarn run wait-for-db && yarn run typeorm migration:show || true && yarn run typeorm migration:run",
 						},
 					}},
+					Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 				},
 			},
 		},

diff --git a/install/installer/pkg/components/minio/incluster/minio.go b/install/installer/pkg/components/minio/incluster/minio.go
@@ -6,6 +6,7 @@ package incluster
 
 import (
 	"fmt"
+
 	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	"github.com/gitpod-io/gitpod/installer/pkg/helm"
@@ -26,6 +27,15 @@ var Helm = func(apiPort int32, consolePort int32, commonHelmValues []string) com
 				return nil, err
 			}
 
+			tolerations, err := helm.WithTolerationWorkspaceComponentNotReadyYaml(cfg)
+			if err != nil {
+				return nil, err
+			}
+			tolerationsTemplate, err := helm.KeyFileValue("minio.tolerations", tolerations)
+			if err != nil {
+				return nil, err
+			}
+
 			return &common.HelmConfig{
 				Enabled: true,
 				Values: &values.Options{
@@ -41,6 +51,7 @@ var Helm = func(apiPort int32, consolePort int32, commonHelmValues []string) com
 					// This is too complex to be sent as a string
 					FileValues: []string{
 						affinityTemplate,
+						tolerationsTemplate,
 					},
 				},
 			}, nil

diff --git a/install/installer/pkg/components/node-labeler/deployment.go b/install/installer/pkg/components/node-labeler/deployment.go
@@ -71,6 +71,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 			},
 			*common.KubeRBACProxyContainerWithConfig(ctx),
 		},
+		Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 	}
 
 	return []runtime.Object{

diff --git a/install/installer/pkg/components/openvsx-proxy/statefulset.go b/install/installer/pkg/components/openvsx-proxy/statefulset.go
@@ -195,6 +195,7 @@ func statefulset(ctx *common.RenderContext) ([]runtime.Object, error) {
 						}},
 					}, *common.KubeRBACProxyContainer(ctx),
 					},
+					Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 				},
 			},
 			VolumeClaimTemplates: volumeClaimTemplates,

diff --git a/install/installer/pkg/components/proxy/deployment.go b/install/installer/pkg/components/proxy/deployment.go
@@ -286,7 +286,9 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 									Value: segmentEndpoint,
 								}},
 							)),
-						}},
+						},
+						},
+						Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 					},
 				},
 			},

diff --git a/install/installer/pkg/components/public-api-server/deployment.go b/install/installer/pkg/components/public-api-server/deployment.go
@@ -187,7 +187,8 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 							},
 							*common.KubeRBACProxyContainer(ctx),
 						},
-						Volumes: volumes,
+						Volumes:     volumes,
+						Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 					},
 				},
 			},

diff --git a/install/installer/pkg/components/redis/deployment.go b/install/installer/pkg/components/redis/deployment.go
@@ -143,6 +143,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 							},
 							*common.KubeRBACProxyContainer(ctx),
 						},
+						Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 					},
 				},
 			},

diff --git a/install/installer/pkg/components/server/deployment.go b/install/installer/pkg/components/server/deployment.go
@@ -416,6 +416,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 								volumeMounts...,
 							),
 						}, *common.KubeRBACProxyContainer(ctx)},
+						Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 					},
 				},
 			},

diff --git a/install/installer/pkg/components/spicedb/deployment.go b/install/installer/pkg/components/spicedb/deployment.go
@@ -189,6 +189,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 							},
 							*common.KubeRBACProxyContainer(ctx),
 						},
+						Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 						Volumes: []v1.Volume{
 							bootstrapVolume,
 						},

diff --git a/install/installer/pkg/components/usage/deployment.go b/install/installer/pkg/components/usage/deployment.go
@@ -152,6 +152,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 						},
 							*common.KubeRBACProxyContainerWithConfig(ctx),
 						},
+						Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 					},
 				},
 			},

diff --git a/install/installer/pkg/components/ws-manager-bridge/deployment.go b/install/installer/pkg/components/ws-manager-bridge/deployment.go
@@ -209,6 +209,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 								PeriodSeconds:       20,
 							},
 						}, *common.KubeRBACProxyContainer(ctx)},
+						Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 					},
 				},
 			},

diff --git a/install/installer/pkg/components/ws-manager-mk2/deployment.go b/install/installer/pkg/components/ws-manager-mk2/deployment.go
@@ -146,6 +146,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 		},
 			*common.KubeRBACProxyContainer(ctx),
 		},
+		Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 		Volumes: append([]corev1.Volume{
 			{
 				Name: VolumeConfig,

diff --git a/install/installer/pkg/components/ws-proxy/deployment.go b/install/installer/pkg/components/ws-proxy/deployment.go
@@ -184,6 +184,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 		},
 			*common.KubeRBACProxyContainer(ctx),
 		},
+		Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 	}
 
 	return []runtime.Object{

diff --git a/install/installer/pkg/helm/helm.go b/install/installer/pkg/helm/helm.go
@@ -125,6 +125,17 @@ func AffinityYaml(orLabels ...string) ([]byte, error) {
 	return marshal, nil
 }
 
+func WithTolerationWorkspaceComponentNotReadyYaml(ctx *common.RenderContext) ([]byte, error) {
+	tolerations := common.WithTolerationWorkspaceComponentNotReady(ctx)
+
+	marshal, err := yaml.Marshal(tolerations)
+	if err != nil {
+		return nil, err
+	}
+
+	return marshal, nil
+}
+
 func nodeAffinity(orLabels ...string) *corev1.Affinity {
 	var terms []corev1.NodeSelectorTerm
 	for _, lbl := range orLabels {
-Original file line number
+Diff line change
@@ Expand Up @@
     								FailureThreshold:    3,
     							},
     						}, *common.KubeRBACProxyContainer(ctx)},
+    						Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
     					},
     				},
     			},
@@ Expand Down @@