test: NodeClaim CEL validation tests (#635)

kubernetes-sigs · Oct 24, 2023 · 7806fd3 · 7806fd3
1 parent 223dcd0
commit 7806fd3
Show file tree

Hide file tree

Showing 2 changed files with 372 additions and 0 deletions.
diff --git a/pkg/apis/v1beta1/nodeclaim_validation_cel_test.go b/pkg/apis/v1beta1/nodeclaim_validation_cel_test.go
@@ -0,0 +1,372 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1_test
+
+import (
+	"strings"
+	"time"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+
+	"github.com/Pallinder/go-randomdata"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"knative.dev/pkg/ptr"
+
+	. "github.com/aws/karpenter-core/pkg/apis/v1beta1"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+var _ = Describe("Validation", func() {
+	var nodeClaim *NodeClaim
+
+	BeforeEach(func() {
+		if env.Version.Minor() < 25 {
+			Skip("CEL Validation is for 1.25>")
+		}
+		nodeClaim = &NodeClaim{
+			ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())},
+			Spec: NodeClaimSpec{
+				NodeClassRef: &NodeClassReference{
+					Kind: "NodeClaim",
+					Name: "default",
+				},
+				Requirements: []v1.NodeSelectorRequirement{
+					{
+						Key:      CapacityTypeLabelKey,
+						Operator: v1.NodeSelectorOpExists,
+					},
+				},
+			},
+		}
+	})
+
+	Context("Taints", func() {
+		It("should succeed for valid taints", func() {
+			nodeClaim.Spec.Taints = []v1.Taint{
+				{Key: "a", Value: "b", Effect: v1.TaintEffectNoSchedule},
+				{Key: "c", Value: "d", Effect: v1.TaintEffectNoExecute},
+				{Key: "e", Value: "f", Effect: v1.TaintEffectPreferNoSchedule},
+				{Key: "key-only", Effect: v1.TaintEffectNoExecute},
+			}
+			Expect(env.Client.Create(ctx, nodeClaim)).To(Succeed())
+		})
+		It("should fail for invalid taint keys", func() {
+			nodeClaim.Spec.Taints = []v1.Taint{{Key: "???"}}
+			Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+		})
+		It("should fail for missing taint key", func() {
+			nodeClaim.Spec.Taints = []v1.Taint{{Effect: v1.TaintEffectNoSchedule}}
+			Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+		})
+		It("should fail for invalid taint value", func() {
+			nodeClaim.Spec.Taints = []v1.Taint{{Key: "invalid-value", Effect: v1.TaintEffectNoSchedule, Value: "???"}}
+			Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+		})
+		It("should fail for invalid taint effect", func() {
+			nodeClaim.Spec.Taints = []v1.Taint{{Key: "invalid-effect", Effect: "???"}}
+			Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+		})
+		It("should not fail for same key with different effects", func() {
+			nodeClaim.Spec.Taints = []v1.Taint{
+				{Key: "a", Effect: v1.TaintEffectNoSchedule},
+				{Key: "a", Effect: v1.TaintEffectNoExecute},
+			}
+			Expect(env.Client.Create(ctx, nodeClaim)).To(Succeed())
+		})
+	})
+	Context("Requirements", func() {
+		It("should allow supported ops", func() {
+			nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{
+				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}},
+				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{"1"}},
+				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{"1"}},
+				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn},
+				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpExists},
+			}
+			Expect(env.Client.Create(ctx, nodeClaim)).To(Succeed())
+		})
+		It("should fail for unsupported ops", func() {
+			for _, op := range []v1.NodeSelectorOperator{"unknown"} {
+				nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{
+					{Key: v1.LabelTopologyZone, Operator: op, Values: []string{"test"}},
+				}
+				Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+			}
+		})
+		It("should fail for restricted domains", func() {
+			for label := range RestrictedLabelDomains {
+				nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{
+					{Key: label + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}},
+				}
+				Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+			}
+		})
+		It("should allow restricted domains exceptions", func() {
+			oldNodeClaim := nodeClaim.DeepCopy()
+			for label := range LabelDomainExceptions {
+				nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{
+					{Key: label + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}},
+				}
+				Expect(env.Client.Create(ctx, nodeClaim)).To(Succeed())
+				Expect(env.Client.Delete(ctx, nodeClaim)).To(Succeed())
+				nodeClaim = oldNodeClaim.DeepCopy()
+			}
+		})
+		It("should allow well known label exceptions", func() {
+			oldNodeClaim := nodeClaim.DeepCopy()
+			for label := range WellKnownLabels.Difference(sets.New(NodePoolLabelKey)) {
+				nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{
+					{Key: label, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}},
+				}
+				Expect(env.Client.Create(ctx, nodeClaim)).To(Succeed())
+				Expect(env.Client.Delete(ctx, nodeClaim)).To(Succeed())
+				nodeClaim = oldNodeClaim.DeepCopy()
+			}
+		})
+		It("should allow non-empty set after removing overlapped value", func() {
+			nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{
+				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test", "foo"}},
+				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test", "bar"}},
+			}
+			Expect(env.Client.Create(ctx, nodeClaim)).To(Succeed())
+		})
+		It("should allow empty requirements", func() {
+			nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{}
+			Expect(env.Client.Create(ctx, nodeClaim)).To(Succeed())
+		})
+		It("should fail with invalid GT or LT values", func() {
+			for _, requirement := range []v1.NodeSelectorRequirement{
+				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{}},
+				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{"1", "2"}},
+				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{"a"}},
+				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{"-1"}},
+				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{}},
+				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{"1", "2"}},
+				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{"a"}},
+				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{"-1"}},
+			} {
+				nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{requirement}
+				Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+			}
+		})
+	})
+	Context("Kubelet", func() {
+		It("should fail on kubeReserved with invalid keys", func() {
+			nodeClaim.Spec.Kubelet = &KubeletConfiguration{
+				KubeReserved: v1.ResourceList{
+					v1.ResourcePods: resource.MustParse("2"),
+				},
+			}
+			Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+		})
+		It("should fail on systemReserved with invalid keys", func() {
+			nodeClaim.Spec.Kubelet = &KubeletConfiguration{
+				SystemReserved: v1.ResourceList{
+					v1.ResourcePods: resource.MustParse("2"),
+				},
+			}
+			Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+		})
+		Context("Eviction Signals", func() {
+			Context("Eviction Hard", func() {
+				It("should succeed on evictionHard with valid keys", func() {
+					nodeClaim.Spec.Kubelet = &KubeletConfiguration{
+						EvictionHard: map[string]string{
+							"memory.available":   "5%",
+							"nodefs.available":   "10%",
+							"nodefs.inodesFree":  "15%",
+							"imagefs.available":  "5%",
+							"imagefs.inodesFree": "5%",
+							"pid.available":      "5%",
+						},
+					}
+					Expect(env.Client.Create(ctx, nodeClaim)).To(Succeed())
+				})
+				It("should fail on evictionHard with invalid keys", func() {
+					nodeClaim.Spec.Kubelet = &KubeletConfiguration{
+						EvictionHard: map[string]string{
+							"memory": "5%",
+						},
+					}
+					Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+				})
+				It("should fail on invalid formatted percentage value in evictionHard", func() {
+					nodeClaim.Spec.Kubelet = &KubeletConfiguration{
+						EvictionHard: map[string]string{
+							"memory.available": "5%3",
+						},
+					}
+					Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+				})
+				It("should fail on invalid percentage value (too large) in evictionHard", func() {
+					nodeClaim.Spec.Kubelet = &KubeletConfiguration{
+						EvictionHard: map[string]string{
+							"memory.available": "110%",
+						},
+					}
+					Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+				})
+				It("should fail on invalid quantity value in evictionHard", func() {
+					nodeClaim.Spec.Kubelet = &KubeletConfiguration{
+						EvictionHard: map[string]string{
+							"memory.available": "110GB",
+						},
+					}
+					Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+				})
+			})
+		})
+		Context("Eviction Soft", func() {
+			It("should succeed on evictionSoft with valid keys", func() {
+				nodeClaim.Spec.Kubelet = &KubeletConfiguration{
+					EvictionSoft: map[string]string{
+						"memory.available":   "5%",
+						"nodefs.available":   "10%",
+						"nodefs.inodesFree":  "15%",
+						"imagefs.available":  "5%",
+						"imagefs.inodesFree": "5%",
+						"pid.available":      "5%",
+					},
+					EvictionSoftGracePeriod: map[string]metav1.Duration{
+						"memory.available":   {Duration: time.Minute},
+						"nodefs.available":   {Duration: time.Second * 90},
+						"nodefs.inodesFree":  {Duration: time.Minute * 5},
+						"imagefs.available":  {Duration: time.Hour},
+						"imagefs.inodesFree": {Duration: time.Hour * 24},
+						"pid.available":      {Duration: time.Minute},
+					},
+				}
+				Expect(env.Client.Create(ctx, nodeClaim)).To(Succeed())
+			})
+			It("should fail on evictionSoft with invalid keys", func() {
+				nodeClaim.Spec.Kubelet = &KubeletConfiguration{
+					EvictionSoft: map[string]string{
+						"memory": "5%",
+					},
+					EvictionSoftGracePeriod: map[string]metav1.Duration{
+						"memory": {Duration: time.Minute},
+					},
+				}
+				Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+			})
+			It("should fail on invalid formatted percentage value in evictionSoft", func() {
+				nodeClaim.Spec.Kubelet = &KubeletConfiguration{
+					EvictionSoft: map[string]string{
+						"memory.available": "5%3",
+					},
+					EvictionSoftGracePeriod: map[string]metav1.Duration{
+						"memory.available": {Duration: time.Minute},
+					},
+				}
+				Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+			})
+			It("should fail on invalid percentage value (too large) in evictionSoft", func() {
+				nodeClaim.Spec.Kubelet = &KubeletConfiguration{
+					EvictionSoft: map[string]string{
+						"memory.available": "110%",
+					},
+					EvictionSoftGracePeriod: map[string]metav1.Duration{
+						"memory.available": {Duration: time.Minute},
+					},
+				}
+				Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+			})
+			It("should fail on invalid quantity value in evictionSoft", func() {
+				nodeClaim.Spec.Kubelet = &KubeletConfiguration{
+					EvictionSoft: map[string]string{
+						"memory.available": "110GB",
+					},
+					EvictionSoftGracePeriod: map[string]metav1.Duration{
+						"memory.available": {Duration: time.Minute},
+					},
+				}
+				Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+			})
+			It("should fail when eviction soft doesn't have matching grace period", func() {
+				nodeClaim.Spec.Kubelet = &KubeletConfiguration{
+					EvictionSoft: map[string]string{
+						"memory.available": "200Mi",
+					},
+				}
+				Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+			})
+		})
+		Context("GCThresholdPercent", func() {
+			It("should succeed on a valid imageGCHighThresholdPercent", func() {
+				nodeClaim.Spec.Kubelet = &KubeletConfiguration{
+					ImageGCHighThresholdPercent: ptr.Int32(10),
+				}
+				Expect(env.Client.Create(ctx, nodeClaim)).To(Succeed())
+			})
+			It("should fail when imageGCHighThresholdPercent is less than imageGCLowThresholdPercent", func() {
+				nodeClaim.Spec.Kubelet = &KubeletConfiguration{
+					ImageGCHighThresholdPercent: ptr.Int32(50),
+					ImageGCLowThresholdPercent:  ptr.Int32(60),
+				}
+				Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+			})
+			It("should fail when imageGCLowThresholdPercent is greather than imageGCHighThresheldPercent", func() {
+				nodeClaim.Spec.Kubelet = &KubeletConfiguration{
+					ImageGCHighThresholdPercent: ptr.Int32(50),
+					ImageGCLowThresholdPercent:  ptr.Int32(60),
+				}
+				Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+			})
+		})
+		Context("Eviction Soft Grace Period", func() {
+			It("should succeed on evictionSoftGracePeriod with valid keys", func() {
+				nodeClaim.Spec.Kubelet = &KubeletConfiguration{
+					EvictionSoft: map[string]string{
+						"memory.available":   "5%",
+						"nodefs.available":   "10%",
+						"nodefs.inodesFree":  "15%",
+						"imagefs.available":  "5%",
+						"imagefs.inodesFree": "5%",
+						"pid.available":      "5%",
+					},
+					EvictionSoftGracePeriod: map[string]metav1.Duration{
+						"memory.available":   {Duration: time.Minute},
+						"nodefs.available":   {Duration: time.Second * 90},
+						"nodefs.inodesFree":  {Duration: time.Minute * 5},
+						"imagefs.available":  {Duration: time.Hour},
+						"imagefs.inodesFree": {Duration: time.Hour * 24},
+						"pid.available":      {Duration: time.Minute},
+					},
+				}
+				Expect(env.Client.Create(ctx, nodeClaim)).To(Succeed())
+			})
+			It("should fail on evictionSoftGracePeriod with invalid keys", func() {
+				nodeClaim.Spec.Kubelet = &KubeletConfiguration{
+					EvictionSoftGracePeriod: map[string]metav1.Duration{
+						"memory": {Duration: time.Minute},
+					},
+				}
+				Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+			})
+			It("should fail when eviction soft grace period doesn't have matching threshold", func() {
+				nodeClaim.Spec.Kubelet = &KubeletConfiguration{
+					EvictionSoftGracePeriod: map[string]metav1.Duration{
+						"memory.available": {Duration: time.Minute},
+					},
+				}
+				Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+			})
+		})
+	})
+})
diff --git a/...apis/v1beta1/nodeclaim_validation_test.go → ...eta1/nodeclaim_validation_webhook_test.go b/...apis/v1beta1/nodeclaim_validation_test.go → ...eta1/nodeclaim_validation_webhook_test.go