Skip to content

Commit 1566b98

Browse files
jimassajimassa
authored
test: Run e2e tests in both arm64 and amd64 arch. (#1129)
# Description This PR allows us to run plugin e2e in both arm64 and amd64 arch. It also replaces agnhost image with the standard one hosted in `registry.k8s.io/e2e-test-images`instead of the fork in `acnpublic`, as it is multi-arch and good practice. The test sets up agnhost in different args, and check plugin metrics on retina-agent on both amd64 and arm64. ![image](https://github.com/user-attachments/assets/065ee0f4-a3db-4c07-9c61-1526fc832e1f) ![image](https://github.com/user-attachments/assets/24176641-2e1b-4000-a46f-ef50cc3ace61) ## Related Issue If this pull request is related to any issue, please mention it here. Additionally, make sure that the issue is assigned to you before submitting this pull request. [#450](#450) ## Checklist - [x] I have read the [contributing documentation](https://retina.sh/docs/contributing). - [x] I signed and signed-off the commits (`git commit -S -s ...`). See [this documentation](https://docs.github.com/en/authentication/managing-commit-signature-verification/about-commit-signature-verification) on signing commits. - [x] I have correctly attributed the author(s) of the code. - [x] I have tested the changes locally. - [x] I have followed the project's style guidelines. - [x] I have updated the documentation, if necessary. - [x] I have added tests, if applicable. ## Screenshots (if applicable) or Testing Completed Please add any relevant screenshots or GIFs to showcase the changes made. ## Additional Notes Add any additional notes or context about the pull request here. --- Please refer to the [CONTRIBUTING.md](../CONTRIBUTING.md) file for more information on how to contribute to this project.
1 parent eefcb42 commit 1566b98

File tree

7 files changed

+68
-94
lines changed

7 files changed

+68
-94
lines changed

‎test/e2e/common/common.go

+1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ const (
2626

2727
var (
2828
AzureLocations = []string{"eastus2", "northeurope", "uksouth", "centralindia", "westus2"}
29+
Architectures = []string{"amd64", "arm64"}
2930
CreateInfra = flag.Bool("create-infra", true, "create a Resource group, vNET and AKS cluster for testing")
3031
DeleteInfra = flag.Bool("delete-infra", true, "delete a Resource group, vNET and AKS cluster for testing")
3132
)

‎test/e2e/framework/kubernetes/create-agnhost-statefulset.go

+17-9
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,18 @@ import (
1717
var ErrLabelMissingFromPod = fmt.Errorf("label missing from pod")
1818

1919
const (
20-
AgnhostHTTPPort = 80
21-
AgnhostReplicas = 1
20+
AgnhostHTTPPort = 80
21+
AgnhostReplicas = 1
22+
AgnhostArchAmd64 = "amd64"
23+
AgnhostArchArm64 = "arm64"
2224
)
2325

2426
type CreateAgnhostStatefulSet struct {
2527
AgnhostName string
2628
AgnhostNamespace string
2729
ScheduleOnSameNode bool
2830
KubeConfigFilePath string
31+
AgnhostArch string
2932
}
3033

3134
func (c *CreateAgnhostStatefulSet) Run() error {
@@ -42,14 +45,19 @@ func (c *CreateAgnhostStatefulSet) Run() error {
4245
ctx, cancel := context.WithTimeout(context.Background(), defaultTimeoutSeconds*time.Second)
4346
defer cancel()
4447

45-
agnhostStatefulest := c.getAgnhostDeployment()
48+
// set default arch to amd64
49+
if c.AgnhostArch == "" {
50+
c.AgnhostArch = AgnhostArchAmd64
51+
}
52+
53+
agnhostStatefulSet := c.getAgnhostDeployment(c.AgnhostArch)
4654

47-
err = CreateResource(ctx, agnhostStatefulest, clientset)
55+
err = CreateResource(ctx, agnhostStatefulSet, clientset)
4856
if err != nil {
4957
return fmt.Errorf("error agnhost component: %w", err)
5058
}
5159

52-
selector, exists := agnhostStatefulest.Spec.Selector.MatchLabels["app"]
60+
selector, exists := agnhostStatefulSet.Spec.Selector.MatchLabels["app"]
5361
if !exists {
5462
return fmt.Errorf("missing label \"app=%s\" from agnhost statefulset: %w", c.AgnhostName, ErrLabelMissingFromPod)
5563
}
@@ -71,7 +79,7 @@ func (c *CreateAgnhostStatefulSet) Stop() error {
7179
return nil
7280
}
7381

74-
func (c *CreateAgnhostStatefulSet) getAgnhostDeployment() *appsv1.StatefulSet {
82+
func (c *CreateAgnhostStatefulSet) getAgnhostDeployment(arch string) *appsv1.StatefulSet {
7583
reps := int32(AgnhostReplicas)
7684

7785
var affinity *v1.Affinity
@@ -90,7 +98,6 @@ func (c *CreateAgnhostStatefulSet) getAgnhostDeployment() *appsv1.StatefulSet {
9098
},
9199
},
92100
}
93-
94101
} else {
95102
affinity = &v1.Affinity{
96103
PodAntiAffinity: &v1.PodAntiAffinity{
@@ -141,12 +148,13 @@ func (c *CreateAgnhostStatefulSet) getAgnhostDeployment() *appsv1.StatefulSet {
141148
Spec: v1.PodSpec{
142149
Affinity: affinity,
143150
NodeSelector: map[string]string{
144-
"kubernetes.io/os": "linux",
151+
"kubernetes.io/os": "linux",
152+
"kubernetes.io/arch": arch,
145153
},
146154
Containers: []v1.Container{
147155
{
148156
Name: c.AgnhostName,
149-
Image: "acnpublic.azurecr.io/agnhost:2.40",
157+
Image: "registry.k8s.io/e2e-test-images/agnhost:2.40",
150158
Resources: v1.ResourceRequirements{
151159
Requests: v1.ResourceList{
152160
"memory": resource.MustParse("20Mi"),

‎test/e2e/framework/kubernetes/create-network-policy.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ func (c *CreateDenyAllNetworkPolicy) Run() error {
3636
ctx, cancel := context.WithCancel(context.Background())
3737
defer cancel()
3838

39-
agnhostStatefulest := getNetworkPolicy(c.NetworkPolicyNamespace, c.DenyAllLabelSelector)
40-
err = CreateResource(ctx, agnhostStatefulest, clientset)
39+
agnhostStatefulSet := getNetworkPolicy(c.NetworkPolicyNamespace, c.DenyAllLabelSelector)
40+
err = CreateResource(ctx, agnhostStatefulSet, clientset)
4141
if err != nil {
4242
return fmt.Errorf("error creating simple deny-all network policy: %w", err)
4343
}
@@ -96,8 +96,8 @@ func (d *DeleteDenyAllNetworkPolicy) Run() error {
9696
ctx, cancel := context.WithCancel(context.Background())
9797
defer cancel()
9898

99-
agnhostStatefulest := getNetworkPolicy(d.NetworkPolicyNamespace, d.DenyAllLabelSelector)
100-
err = DeleteResource(ctx, agnhostStatefulest, clientset)
99+
agnhostStatefulSet := getNetworkPolicy(d.NetworkPolicyNamespace, d.DenyAllLabelSelector)
100+
err = DeleteResource(ctx, agnhostStatefulSet, clientset)
101101
if err != nil {
102102
return fmt.Errorf("error creating simple deny-all network policy: %w", err)
103103
}

‎test/e2e/jobs/jobs.go

+15-60
Original file line numberDiff line numberDiff line change
@@ -117,12 +117,6 @@ func InstallAndTestRetinaBasicMetrics(kubeConfigFilePath, chartPath string, test
117117
TagEnv: generic.DefaultTagEnv,
118118
}, nil)
119119

120-
job.AddScenario(drop.ValidateDropMetric(testPodNamespace))
121-
122-
job.AddScenario(tcp.ValidateTCPMetrics(testPodNamespace))
123-
124-
job.AddScenario(windows.ValidateWindowsBasicMetric())
125-
126120
dnsScenarios := []struct {
127121
name string
128122
req *dns.RequestValidationParams
@@ -164,8 +158,16 @@ func InstallAndTestRetinaBasicMetrics(kubeConfigFilePath, chartPath string, test
164158
},
165159
}
166160

167-
for _, scenario := range dnsScenarios {
168-
job.AddScenario(dns.ValidateBasicDNSMetrics(scenario.name, scenario.req, scenario.resp, testPodNamespace))
161+
for _, arch := range common.Architectures {
162+
job.AddScenario(drop.ValidateDropMetric(testPodNamespace, arch))
163+
job.AddScenario(tcp.ValidateTCPMetrics(testPodNamespace, arch))
164+
165+
for _, scenario := range dnsScenarios {
166+
name := scenario.name + " - Arch: " + arch
167+
job.AddScenario(dns.ValidateBasicDNSMetrics(name, scenario.req, scenario.resp, testPodNamespace, arch))
168+
}
169+
170+
job.AddScenario(windows.ValidateWindowsBasicMetric())
169171
}
170172

171173
job.AddStep(&kubernetes.EnsureStableComponent{
@@ -230,8 +232,11 @@ func UpgradeAndTestRetinaAdvancedMetrics(kubeConfigFilePath, chartPath, valuesFi
230232
},
231233
}
232234

233-
for _, scenario := range dnsScenarios {
234-
job.AddScenario(dns.ValidateAdvancedDNSMetrics(scenario.name, scenario.req, scenario.resp, kubeConfigFilePath, testPodNamespace))
235+
for _, arch := range common.Architectures {
236+
for _, scenario := range dnsScenarios {
237+
name := scenario.name + " - Arch: " + arch
238+
job.AddScenario(dns.ValidateAdvancedDNSMetrics(name, scenario.req, scenario.resp, kubeConfigFilePath, testPodNamespace, arch))
239+
}
235240
}
236241

237242
job.AddScenario(latency.ValidateLatencyMetric(testPodNamespace))
@@ -260,55 +265,6 @@ func ValidateHubble(kubeConfigFilePath, chartPath string, testPodNamespace strin
260265

261266
job.AddScenario(hubble.ValidateHubbleUIService(kubeConfigFilePath))
262267

263-
job.AddScenario(drop.ValidateDropMetric(testPodNamespace))
264-
265-
job.AddScenario(tcp.ValidateTCPMetrics(testPodNamespace))
266-
267-
dnsScenarios := []struct {
268-
name string
269-
req *dns.RequestValidationParams
270-
resp *dns.ResponseValidationParams
271-
}{
272-
{
273-
name: "Validate basic DNS request and response metrics for a valid domain",
274-
req: &dns.RequestValidationParams{
275-
NumResponse: "0",
276-
Query: "kubernetes.default.svc.cluster.local.",
277-
QueryType: "A",
278-
Command: "nslookup kubernetes.default",
279-
ExpectError: false,
280-
},
281-
resp: &dns.ResponseValidationParams{
282-
NumResponse: "1",
283-
Query: "kubernetes.default.svc.cluster.local.",
284-
QueryType: "A",
285-
ReturnCode: "No Error",
286-
Response: "10.0.0.1",
287-
},
288-
},
289-
{
290-
name: "Validate basic DNS request and response metrics for a non-existent domain",
291-
req: &dns.RequestValidationParams{
292-
NumResponse: "0",
293-
Query: "some.non.existent.domain.",
294-
QueryType: "A",
295-
Command: "nslookup some.non.existent.domain",
296-
ExpectError: true,
297-
},
298-
resp: &dns.ResponseValidationParams{
299-
NumResponse: "0",
300-
Query: "some.non.existent.domain.",
301-
QueryType: "A",
302-
Response: dns.EmptyResponse, // hacky way to bypass the framework for now
303-
ReturnCode: "Non-Existent Domain",
304-
},
305-
},
306-
}
307-
308-
for _, scenario := range dnsScenarios {
309-
job.AddScenario(dns.ValidateBasicDNSMetrics(scenario.name, scenario.req, scenario.resp, testPodNamespace))
310-
}
311-
312268
job.AddStep(&kubernetes.EnsureStableComponent{
313269
PodNamespace: common.KubeSystemNamespace,
314270
LabelSelector: "k8s-app=retina",
@@ -365,4 +321,3 @@ func RunPerfTest(kubeConfigFilePath string, chartPath string) *types.Job {
365321

366322
return job
367323
}
368-

‎test/e2e/scenarios/dns/scenarios.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ type ResponseValidationParams struct {
3636
}
3737

3838
// ValidateBasicDNSMetrics validates basic DNS metrics present in the metrics endpoint
39-
func ValidateBasicDNSMetrics(scenarioName string, req *RequestValidationParams, resp *ResponseValidationParams, namespace string) *types.Scenario {
39+
func ValidateBasicDNSMetrics(scenarioName string, req *RequestValidationParams, resp *ResponseValidationParams, namespace, arch string) *types.Scenario {
4040
// generate a random ID using rand
4141
id := fmt.Sprintf("basic-dns-port-forward-%d", rand.Int()) // nolint:gosec // fine to use math/rand here
4242
agnhostName := "agnhost-" + id
@@ -46,6 +46,7 @@ func ValidateBasicDNSMetrics(scenarioName string, req *RequestValidationParams,
4646
Step: &kubernetes.CreateAgnhostStatefulSet{
4747
AgnhostName: agnhostName,
4848
AgnhostNamespace: namespace,
49+
AgnhostArch: arch,
4950
},
5051
},
5152
// Need this delay to guarantee that the pods will have bpf program attached
@@ -149,7 +150,7 @@ func ValidateBasicDNSMetrics(scenarioName string, req *RequestValidationParams,
149150
}
150151

151152
// ValidateAdvancedDNSMetrics validates the advanced DNS metrics present in the metrics endpoint
152-
func ValidateAdvancedDNSMetrics(scenarioName string, req *RequestValidationParams, resp *ResponseValidationParams, kubeConfigFilePath string, namespace string) *types.Scenario {
153+
func ValidateAdvancedDNSMetrics(scenarioName string, req *RequestValidationParams, resp *ResponseValidationParams, kubeConfigFilePath, namespace, arch string) *types.Scenario {
153154
// random ID
154155
id := fmt.Sprintf("adv-dns-port-forward-%d", rand.Int()) // nolint:gosec // fine to use math/rand here
155156
agnhostName := "agnhost-" + id
@@ -159,6 +160,7 @@ func ValidateAdvancedDNSMetrics(scenarioName string, req *RequestValidationParam
159160
Step: &kubernetes.CreateAgnhostStatefulSet{
160161
AgnhostName: agnhostName,
161162
AgnhostNamespace: namespace,
163+
AgnhostArch: arch,
162164
},
163165
},
164166
// Need this delay to guarantee that the pods will have bpf program attached

‎test/e2e/scenarios/drop/scenario.go

+14-10
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,11 @@ const (
1616
IPTableRuleDrop = "IPTABLE_RULE_DROP"
1717
)
1818

19-
func ValidateDropMetric(namespace string) *types.Scenario {
20-
name := "Drop Metrics"
19+
func ValidateDropMetric(namespace, arch string) *types.Scenario {
20+
id := "drop-port-forward-" + arch
21+
agnhostName := "agnhost-drop"
22+
podName := agnhostName + "-0"
23+
name := "Drop Metrics - Arch: " + arch
2124
steps := []*types.StepWrapper{
2225
{
2326
Step: &kubernetes.CreateDenyAllNetworkPolicy{
@@ -28,7 +31,8 @@ func ValidateDropMetric(namespace string) *types.Scenario {
2831
{
2932
Step: &kubernetes.CreateAgnhostStatefulSet{
3033
AgnhostNamespace: namespace,
31-
AgnhostName: "agnhost-drop",
34+
AgnhostName: agnhostName,
35+
AgnhostArch: arch,
3236
},
3337
},
3438
// Need this delay to guarantee that the pods will have bpf program attached
@@ -43,7 +47,7 @@ func ValidateDropMetric(namespace string) *types.Scenario {
4347
{
4448
Step: &kubernetes.ExecInPod{
4549
PodNamespace: namespace,
46-
PodName: "agnhost-drop-0",
50+
PodName: podName,
4751
Command: "curl -s -m 5 bing.com",
4852
},
4953
Opts: &types.StepOptions{
@@ -59,7 +63,7 @@ func ValidateDropMetric(namespace string) *types.Scenario {
5963
{
6064
Step: &kubernetes.ExecInPod{
6165
PodNamespace: namespace,
62-
PodName: "agnhost-drop-0",
66+
PodName: podName,
6367
Command: "curl -s -m 5 bing.com",
6468
},
6569
Opts: &types.StepOptions{
@@ -74,24 +78,24 @@ func ValidateDropMetric(namespace string) *types.Scenario {
7478
LocalPort: "10093",
7579
RemotePort: "10093",
7680
Endpoint: "metrics",
77-
OptionalLabelAffinity: "app=agnhost-drop", // port forward to a pod on a node that also has this pod with this label, assuming same namespace
81+
OptionalLabelAffinity: "app=" + agnhostName, // port forward to a pod on a node that also has this pod with this label, assuming same namespace
7882
},
7983
Opts: &types.StepOptions{
80-
RunInBackgroundWithID: "drop-port-forward",
84+
RunInBackgroundWithID: id,
8185
},
8286
},
8387
{
8488
Step: &ValidateRetinaDropMetric{
8589
PortForwardedRetinaPort: "10093",
86-
Source: "agnhost-drop",
90+
Source: agnhostName,
8791
Reason: IPTableRuleDrop,
8892
Direction: "unknown",
8993
Protocol: UDP,
9094
},
9195
},
9296
{
9397
Step: &types.Stop{
94-
BackgroundID: "drop-port-forward",
98+
BackgroundID: id,
9599
},
96100
},
97101

@@ -108,7 +112,7 @@ func ValidateDropMetric(namespace string) *types.Scenario {
108112
Step: &kubernetes.DeleteKubernetesResource{
109113
ResourceType: kubernetes.TypeString(kubernetes.StatefulSet),
110114
ResourceNamespace: namespace,
111-
ResourceName: "agnhost-drop",
115+
ResourceName: agnhostName,
112116
}, Opts: &types.StepOptions{
113117
SkipSavingParametersToJob: true,
114118
},

‎test/e2e/scenarios/tcp/scenario.go

+13-9
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,11 @@ const (
1616
IPTableRuleDrop = "IPTABLE_RULE_DROP"
1717
)
1818

19-
func ValidateTCPMetrics(namespace string) *types.Scenario {
20-
Name := "Flow Metrics"
19+
func ValidateTCPMetrics(namespace, arch string) *types.Scenario {
20+
id := "flow-port-forward-" + arch
21+
agnhostName := "agnhost-tcp"
22+
podName := agnhostName + "-0"
23+
Name := "Flow Metrics - Arch: " + arch
2124
Steps := []*types.StepWrapper{
2225
{
2326
Step: &kubernetes.CreateKapingerDeployment{
@@ -27,13 +30,14 @@ func ValidateTCPMetrics(namespace string) *types.Scenario {
2730
},
2831
{
2932
Step: &kubernetes.CreateAgnhostStatefulSet{
30-
AgnhostName: "agnhost-a",
33+
AgnhostName: agnhostName,
3134
AgnhostNamespace: namespace,
35+
AgnhostArch: arch,
3236
},
3337
},
3438
{
3539
Step: &kubernetes.ExecInPod{
36-
PodName: "agnhost-a-0",
40+
PodName: podName,
3741
PodNamespace: namespace,
3842
Command: "curl -s -m 5 bing.com",
3943
}, Opts: &types.StepOptions{
@@ -47,7 +51,7 @@ func ValidateTCPMetrics(namespace string) *types.Scenario {
4751
},
4852
{
4953
Step: &kubernetes.ExecInPod{
50-
PodName: "agnhost-a-0",
54+
PodName: podName,
5155
PodNamespace: namespace,
5256
Command: "curl -s -m 5 bing.com",
5357
}, Opts: &types.StepOptions{
@@ -61,11 +65,11 @@ func ValidateTCPMetrics(namespace string) *types.Scenario {
6165
LocalPort: "10093",
6266
RemotePort: "10093",
6367
Endpoint: "metrics",
64-
OptionalLabelAffinity: "app=agnhost-a", // port forward to a pod on a node that also has this pod with this label, assuming same namespace
68+
OptionalLabelAffinity: "app=" + agnhostName, // port forward to a pod on a node that also has this pod with this label, assuming same namespace
6569
},
6670
Opts: &types.StepOptions{
6771
SkipSavingParametersToJob: true,
68-
RunInBackgroundWithID: "drop-flow-forward",
72+
RunInBackgroundWithID: id,
6973
},
7074
},
7175
{
@@ -84,13 +88,13 @@ func ValidateTCPMetrics(namespace string) *types.Scenario {
8488
},
8589
{
8690
Step: &types.Stop{
87-
BackgroundID: "drop-flow-forward",
91+
BackgroundID: id,
8892
},
8993
},
9094
{
9195
Step: &kubernetes.DeleteKubernetesResource{
9296
ResourceType: kubernetes.TypeString(kubernetes.StatefulSet),
93-
ResourceName: "agnhost-a",
97+
ResourceName: agnhostName,
9498
ResourceNamespace: namespace,
9599
}, Opts: &types.StepOptions{
96100
SkipSavingParametersToJob: true,

0 commit comments

Comments
 (0)