fix: update retina operator to take namespace helm value (microsoft#792)

# Description

- Replaced the hardcoded `kube-system` namespace with the helm value for
the retina operator and other places where it made sense
- Updated the E2E test to accept a namespace parameter to verify if the
retina-agents and retina-operator were deployed correctly
- Manually tested the changes in a local cluster for further
verification, as well as for my learning

## Related Issue
microsoft#731 

## Checklist

- [x] I have read the [contributing
documentation](https://retina.sh/docs/contributing).
- [x] I signed and signed-off the commits (`git commit -S -s ...`). See
[this
documentation](https://docs.github.com/en/authentication/managing-commit-signature-verification/about-commit-signature-verification)
on signing commits.
- [x] I have correctly attributed the author(s) of the code.
- [x] I have tested the changes locally.
- [x] I have followed the project's style guidelines.
- [x] I have updated the documentation, if necessary.
- [x] I have added tests, if applicable.

## Screenshots (if applicable) or Testing Completed
I've attached the E2E test output with the recent microsoft retina
manifest, i.e. 74b6ec2,

[updated_e2e_test_result.txt](https://github.com/user-attachments/files/17400534/updated_e2e_test_result.txt)


## Additional Notes
Add any additional notes or context about the pull request here.

---

Please refer to the [CONTRIBUTING.md](../CONTRIBUTING.md) file for more
information on how to contribute to this project.

test/e2e/common/common.go

@@ -7,5 +7,7 @@ package common
 const (
 	RetinaPort int = 10093
 	// netObsRGtag is used to tag resources created by this test suite
-	NetObsRGtag = "-e2e-netobs-"
+	NetObsRGtag         = "-e2e-netobs-"
+	KubeSystemNamespace = "kube-system"
+	TestPodNamespace    = "kube-system-test"
 )

test/e2e/framework/kubernetes/create-namespace.go

@@ -0,0 +1,40 @@
+package kubernetes
+
+import (
+	"context"
+	"fmt"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+func CreateNamespace(kubeconfigpath, namespace string) error {
+	config, err := clientcmd.BuildConfigFromFlags("", kubeconfigpath)
+	if err != nil {
+		return fmt.Errorf("error building kubeconfig: %w", err)
+	}
+
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return fmt.Errorf("error creating Kubernetes client: %w", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	_, err = clientset.CoreV1().Namespaces().Create(ctx, &v1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: namespace,
+		},
+	}, metav1.CreateOptions{})
+
+	if err != nil && !errors.IsAlreadyExists(err) {
+		return fmt.Errorf("error creating namespace: %w", err)
+	} else {
+		fmt.Printf("Namespace '%s' created successfully.\n", namespace)
+		return nil
+	}
+}

test/e2e/framework/kubernetes/install-retina-helm.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/microsoft/retina/test/e2e/common"
 	generic "github.com/microsoft/retina/test/e2e/framework/generic"
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/chart/loader"
@@ -46,6 +47,12 @@ func (i *InstallHelmChart) Run() error {
 		return fmt.Errorf("failed to initialize helm action config: %w", err)
 	}
 
+	// Creating extra namespace to deploy test pods
+	err = CreateNamespace(i.KubeConfigFilePath, common.TestPodNamespace)
+	if err != nil {
+		return fmt.Errorf("failed to create namespace %s: %w", i.Namespace, err)
+	}
+
 	tag := os.Getenv(generic.DefaultTagEnv)
 	if tag == "" {
 		return fmt.Errorf("tag is not set: %w", errEmpty)

test/e2e/framework/kubernetes/port-forward.go

@@ -138,12 +138,12 @@ func (p *PortForward) findPodsWithAffinity(ctx context.Context, clientset *kuber
 	}
 
 	// get all pods with optional label affinity
-	affinityPods, errAffinity := clientset.CoreV1().Pods(p.Namespace).List(ctx, metav1.ListOptions{
+	affinityPods, errAffinity := clientset.CoreV1().Pods(metav1.NamespaceAll).List(ctx, metav1.ListOptions{
 		LabelSelector: p.OptionalLabelAffinity,
 		FieldSelector: "status.phase=Running",
 	})
 	if errAffinity != nil {
-		return "", fmt.Errorf("could not list affinity pods in %q with label %q: %w", p.Namespace, p.OptionalLabelAffinity, errAffinity)
+		return "", fmt.Errorf("could not list affinity pods across all namespaces with label %q: %w", p.OptionalLabelAffinity, errAffinity)
 	}
 
 	// keep track of where the affinity pods are scheduled

test/e2e/jobs/jobs.go

@@ -1,6 +1,7 @@
 package retina
 
 import (
+	"github.com/microsoft/retina/test/e2e/common"
 	"github.com/microsoft/retina/test/e2e/framework/azure"
 	"github.com/microsoft/retina/test/e2e/framework/generic"
 	"github.com/microsoft/retina/test/e2e/framework/kubernetes"
@@ -74,20 +75,20 @@ func DeleteTestInfra(subID, clusterName, location string) *types.Job {
 	return job
 }
 
-func InstallAndTestRetinaBasicMetrics(kubeConfigFilePath, chartPath string) *types.Job {
+func InstallAndTestRetinaBasicMetrics(kubeConfigFilePath, chartPath string, testPodNamespace string) *types.Job {
 	job := types.NewJob("Install and test Retina with basic metrics")
 
 	job.AddStep(&kubernetes.InstallHelmChart{
-		Namespace:          "kube-system",
+		Namespace:          common.KubeSystemNamespace,
 		ReleaseName:        "retina",
 		KubeConfigFilePath: kubeConfigFilePath,
 		ChartPath:          chartPath,
 		TagEnv:             generic.DefaultTagEnv,
 	}, nil)
 
-	job.AddScenario(drop.ValidateDropMetric())
+	job.AddScenario(drop.ValidateDropMetric(testPodNamespace))
 
-	job.AddScenario(tcp.ValidateTCPMetrics())
+	job.AddScenario(tcp.ValidateTCPMetrics(testPodNamespace))
 
 	job.AddScenario(windows.ValidateWindowsBasicMetric())
 
@@ -133,22 +134,22 @@ func InstallAndTestRetinaBasicMetrics(kubeConfigFilePath, chartPath string) *typ
 	}
 
 	for _, scenario := range dnsScenarios {
-		job.AddScenario(dns.ValidateBasicDNSMetrics(scenario.name, scenario.req, scenario.resp))
+		job.AddScenario(dns.ValidateBasicDNSMetrics(scenario.name, scenario.req, scenario.resp, testPodNamespace))
 	}
 
 	job.AddStep(&kubernetes.EnsureStableCluster{
-		PodNamespace:  "kube-system",
+		PodNamespace:  common.KubeSystemNamespace,
 		LabelSelector: "k8s-app=retina",
 	}, nil)
 
 	return job
 }
 
-func UpgradeAndTestRetinaAdvancedMetrics(kubeConfigFilePath, chartPath, valuesFilePath string) *types.Job {
+func UpgradeAndTestRetinaAdvancedMetrics(kubeConfigFilePath, chartPath, valuesFilePath string, testPodNamespace string) *types.Job {
 	job := types.NewJob("Upgrade and test Retina with advanced metrics")
 	// enable advanced metrics
 	job.AddStep(&kubernetes.UpgradeRetinaHelmChart{
-		Namespace:          "kube-system",
+		Namespace:          common.KubeSystemNamespace,
 		ReleaseName:        "retina",
 		KubeConfigFilePath: kubeConfigFilePath,
 		ChartPath:          chartPath,
@@ -198,13 +199,13 @@ func UpgradeAndTestRetinaAdvancedMetrics(kubeConfigFilePath, chartPath, valuesFi
 	}
 
 	for _, scenario := range dnsScenarios {
-		job.AddScenario(dns.ValidateAdvancedDNSMetrics(scenario.name, scenario.req, scenario.resp, kubeConfigFilePath))
+		job.AddScenario(dns.ValidateAdvancedDNSMetrics(scenario.name, scenario.req, scenario.resp, kubeConfigFilePath, testPodNamespace))
 	}
 
-	job.AddScenario(latency.ValidateLatencyMetric())
+	job.AddScenario(latency.ValidateLatencyMetric(testPodNamespace))
 
 	job.AddStep(&kubernetes.EnsureStableCluster{
-		PodNamespace:  "kube-system",
+		PodNamespace:  common.KubeSystemNamespace,
 		LabelSelector: "k8s-app=retina",
 	}, nil)
 

test/e2e/retina_e2e_test.go

@@ -73,10 +73,10 @@ func TestE2ERetina(t *testing.T) {
 	}()
 
 	// Install and test Retina basic metrics
-	basicMetricsE2E := types.NewRunner(t, jobs.InstallAndTestRetinaBasicMetrics(kubeConfigFilePath, chartPath))
+	basicMetricsE2E := types.NewRunner(t, jobs.InstallAndTestRetinaBasicMetrics(kubeConfigFilePath, chartPath, common.TestPodNamespace))
 	basicMetricsE2E.Run()
 
 	// Upgrade and test Retina with advanced metrics
-	advanceMetricsE2E := types.NewRunner(t, jobs.UpgradeAndTestRetinaAdvancedMetrics(kubeConfigFilePath, chartPath, profilePath))
+	advanceMetricsE2E := types.NewRunner(t, jobs.UpgradeAndTestRetinaAdvancedMetrics(kubeConfigFilePath, chartPath, profilePath, common.TestPodNamespace))
 	advanceMetricsE2E.Run()
 }

test/e2e/scenarios/dns/scenarios.go

@@ -36,7 +36,7 @@ type ResponseValidationParams struct {
 }
 
 // ValidateBasicDNSMetrics validates basic DNS metrics present in the metrics endpoint
-func ValidateBasicDNSMetrics(scenarioName string, req *RequestValidationParams, resp *ResponseValidationParams) *types.Scenario {
+func ValidateBasicDNSMetrics(scenarioName string, req *RequestValidationParams, resp *ResponseValidationParams, namespace string) *types.Scenario {
 	// generate a random ID using rand
 	id := fmt.Sprintf("basic-dns-port-forward-%d", rand.Int()) // nolint:gosec // fine to use math/rand here
 	agnhostName := "agnhost-" + id
@@ -45,13 +45,13 @@ func ValidateBasicDNSMetrics(scenarioName string, req *RequestValidationParams,
 		{
 			Step: &kubernetes.CreateAgnhostStatefulSet{
 				AgnhostName:      agnhostName,
-				AgnhostNamespace: "kube-system",
+				AgnhostNamespace: namespace,
 			},
 		},
 		{
 			Step: &kubernetes.ExecInPod{
 				PodName:      podName,
-				PodNamespace: "kube-system",
+				PodNamespace: namespace,
 				Command:      req.Command,
 			},
 			Opts: &types.StepOptions{
@@ -68,7 +68,7 @@ func ValidateBasicDNSMetrics(scenarioName string, req *RequestValidationParams,
 		{
 			Step: &kubernetes.ExecInPod{
 				PodName:      podName,
-				PodNamespace: "kube-system",
+				PodNamespace: namespace,
 				Command:      req.Command,
 			},
 			Opts: &types.StepOptions{
@@ -83,7 +83,7 @@ func ValidateBasicDNSMetrics(scenarioName string, req *RequestValidationParams,
 		},
 		{
 			Step: &kubernetes.PortForward{
-				Namespace:             "kube-system",
+				Namespace:             common.KubeSystemNamespace,
 				LabelSelector:         "k8s-app=retina",
 				LocalPort:             strconv.Itoa(common.RetinaPort),
 				RemotePort:            strconv.Itoa(common.RetinaPort),
@@ -125,7 +125,7 @@ func ValidateBasicDNSMetrics(scenarioName string, req *RequestValidationParams,
 			Step: &kubernetes.DeleteKubernetesResource{
 				ResourceType:      kubernetes.TypeString(kubernetes.StatefulSet),
 				ResourceName:      agnhostName,
-				ResourceNamespace: "kube-system",
+				ResourceNamespace: namespace,
 			}, Opts: &types.StepOptions{
 				SkipSavingParametersToJob: true,
 			},
@@ -140,7 +140,7 @@ func ValidateBasicDNSMetrics(scenarioName string, req *RequestValidationParams,
 }
 
 // ValidateAdvancedDNSMetrics validates the advanced DNS metrics present in the metrics endpoint
-func ValidateAdvancedDNSMetrics(scenarioName string, req *RequestValidationParams, resp *ResponseValidationParams, kubeConfigFilePath string) *types.Scenario {
+func ValidateAdvancedDNSMetrics(scenarioName string, req *RequestValidationParams, resp *ResponseValidationParams, kubeConfigFilePath string, namespace string) *types.Scenario {
 	// random ID
 	id := fmt.Sprintf("adv-dns-port-forward-%d", rand.Int()) // nolint:gosec // fine to use math/rand here
 	agnhostName := "agnhost-" + id
@@ -149,13 +149,13 @@ func ValidateAdvancedDNSMetrics(scenarioName string, req *RequestValidationParam
 		{
 			Step: &kubernetes.CreateAgnhostStatefulSet{
 				AgnhostName:      agnhostName,
-				AgnhostNamespace: "kube-system",
+				AgnhostNamespace: namespace,
 			},
 		},
 		{
 			Step: &kubernetes.ExecInPod{
 				PodName:      podName,
-				PodNamespace: "kube-system",
+				PodNamespace: namespace,
 				Command:      req.Command,
 			},
 			Opts: &types.StepOptions{
@@ -172,7 +172,7 @@ func ValidateAdvancedDNSMetrics(scenarioName string, req *RequestValidationParam
 		{
 			Step: &kubernetes.ExecInPod{
 				PodName:      podName,
-				PodNamespace: "kube-system",
+				PodNamespace: namespace,
 				Command:      req.Command,
 			},
 			Opts: &types.StepOptions{
@@ -187,7 +187,7 @@ func ValidateAdvancedDNSMetrics(scenarioName string, req *RequestValidationParam
 		},
 		{
 			Step: &kubernetes.PortForward{
-				Namespace:             "kube-system",
+				Namespace:             common.KubeSystemNamespace,
 				LabelSelector:         "k8s-app=retina",
 				LocalPort:             strconv.Itoa(common.RetinaPort),
 				RemotePort:            strconv.Itoa(common.RetinaPort),
@@ -201,7 +201,7 @@ func ValidateAdvancedDNSMetrics(scenarioName string, req *RequestValidationParam
 		},
 		{
 			Step: &ValidateAdvancedDNSRequestMetrics{
-				Namespace:          "kube-system",
+				PodNamespace:       namespace,
 				PodName:            podName,
 				Query:              req.Query,
 				QueryType:          req.QueryType,
@@ -215,7 +215,7 @@ func ValidateAdvancedDNSMetrics(scenarioName string, req *RequestValidationParam
 		},
 		{
 			Step: &ValidateAdvanceDNSResponseMetrics{
-				Namespace:          "kube-system",
+				PodNamespace:       namespace,
 				NumResponse:        resp.NumResponse,
 				PodName:            podName,
 				Query:              resp.Query,
@@ -239,7 +239,7 @@ func ValidateAdvancedDNSMetrics(scenarioName string, req *RequestValidationParam
 			Step: &kubernetes.DeleteKubernetesResource{
 				ResourceType:      kubernetes.TypeString(kubernetes.StatefulSet),
 				ResourceName:      agnhostName,
-				ResourceNamespace: "kube-system",
+				ResourceNamespace: namespace,
 			}, Opts: &types.StepOptions{
 				SkipSavingParametersToJob: true,
 			},

test/e2e/scenarios/dns/validate-advanced-dns-metric.go

@@ -18,7 +18,7 @@ var (
 )
 
 type ValidateAdvancedDNSRequestMetrics struct {
-	Namespace    string
+	PodNamespace string
 	PodName      string
 	Query        string
 	QueryType    string
@@ -31,14 +31,14 @@ type ValidateAdvancedDNSRequestMetrics struct {
 func (v *ValidateAdvancedDNSRequestMetrics) Run() error {
 	metricsEndpoint := fmt.Sprintf("http://localhost:%d/metrics", common.RetinaPort)
 	// Get Pod IP address
-	podIP, err := kubernetes.GetPodIP(v.KubeConfigFilePath, v.Namespace, v.PodName)
+	podIP, err := kubernetes.GetPodIP(v.KubeConfigFilePath, v.PodNamespace, v.PodName)
 	if err != nil {
 		return errors.Wrapf(err, "failed to get pod IP address")
 	}
 
 	validateAdvancedDNSRequestMetrics := map[string]string{
 		"ip":            podIP,
-		"namespace":     v.Namespace,
+		"namespace":     v.PodNamespace,
 		"podname":       v.PodName,
 		"query":         v.Query,
 		"query_type":    v.QueryType,
@@ -64,7 +64,7 @@ func (v *ValidateAdvancedDNSRequestMetrics) Stop() error {
 }
 
 type ValidateAdvanceDNSResponseMetrics struct {
-	Namespace    string
+	PodNamespace string
 	NumResponse  string
 	PodName      string
 	Query        string
@@ -80,7 +80,7 @@ type ValidateAdvanceDNSResponseMetrics struct {
 func (v *ValidateAdvanceDNSResponseMetrics) Run() error {
 	metricsEndpoint := fmt.Sprintf("http://localhost:%d/metrics", common.RetinaPort)
 	// Get Pod IP address
-	podIP, err := kubernetes.GetPodIP(v.KubeConfigFilePath, v.Namespace, v.PodName)
+	podIP, err := kubernetes.GetPodIP(v.KubeConfigFilePath, v.PodNamespace, v.PodName)
 	if err != nil {
 		return errors.Wrapf(err, "failed to get pod IP address")
 	}
@@ -91,7 +91,7 @@ func (v *ValidateAdvanceDNSResponseMetrics) Run() error {
 
 	validateAdvanceDNSResponseMetrics := map[string]string{
 		"ip":            podIP,
-		"namespace":     v.Namespace,
+		"namespace":     v.PodNamespace,
 		"num_response":  v.NumResponse,
 		"podname":       v.PodName,
 		"query":         v.Query,