Skip to content

Commit d776355

Browse files
ritwikranjanritwikranjan
authored
feat: add retina mode support for perf-test and abstract common e2e setups (#1244)
# Description This pull request includes several changes to the end-to-end (E2E) testing framework for the retina project. The main update is addition of running perf tests in advanced mode for Retina. The other updates involve the addition of new utility functions, refactoring of existing code for better modularity, and improvements to the performance testing workflow. ### New Utility Functions: * Added `RetinaChartPath`, `RetinaAdvancedProfilePath`, and `KubeConfigFilePath` functions to dynamically generate file paths. (`test/e2e/common/common.go`) ### Refactoring: * Moved the `CreateAzureTempK8sInfra` function to a new file `azure_temp_infra_setup.go` to modularize infrastructure setup. (`test/e2e/infra/azure_temp_infra_setup.go`) * Removed redundant code and simplified the `TestE2ERetina` and `TestE2EPerfRetina` functions by using new utility functions and the `CreateAzureTempK8sInfra` function. (`test/e2e/retina_e2e_test.go`, `test/e2e/retina_perf_test.go`) [[1]](diffhunk://#diff-66cf931d4cbc2a8832e1fcb70af2838ef6913eedb0294d0666fec1aa6787999bL24-R46) [[2]](diffhunk://#diff-4c195fe24e6f8e7cfb68381dd2d389ca275dc11282f0dfe785526a503dae509eL6-R62) ### Performance Testing: * Moved the performance test logic to a new file `perf.go` and added support for running tests in different retina modes (basic, advanced). (`test/e2e/jobs/perf.go`) * Updated the `PublishPerfResults` function to include the retina mode in telemetry data. (`test/e2e/scenarios/perf/publish-perf-results.go`) ### Code Cleanup: * Removed unused imports and redundant code blocks from various files to improve readability and maintainability. (`test/e2e/jobs/jobs.go`, `test/e2e/scale_test.go`) [[1]](diffhunk://#diff-96a5238dd492978468cfd659ceca6877583744425053d5f2ebbf9a62ae47e535L16) [[2]](diffhunk://#diff-cb7aaaca572345df6e4fbb0f6d5e54f8f392870192227f9746a244e095dcdaa8L52-R56) These changes collectively enhance the modularity, readability, and functionality of the E2E testing framework, making it easier to maintain and extend in the future. ## Checklist - [X] I have read the [contributing documentation](https://retina.sh/docs/contributing). - [X] I signed and signed-off the commits (`git commit -S -s ...`). See [this documentation](https://docs.github.com/en/authentication/managing-commit-signature-verification/about-commit-signature-verification) on signing commits. - [X] I have correctly attributed the author(s) of the code. - [X] I have tested the changes locally. - [X] I have followed the project's style guidelines. - [] I have updated the documentation, if necessary. - [] I have added tests, if applicable. ## Screenshots (if applicable) or Testing Completed I have tested the relevant changes locally and have verified that it works as nintended. ## Additional Notes Add any additional notes or context about the pull request here. --- Please refer to the [CONTRIBUTING.md](../CONTRIBUTING.md) file for more information on how to contribute to this project.
1 parent 8928514 commit d776355

File tree

9 files changed

+217
-143
lines changed

9 files changed

+217
-143
lines changed

‎test/e2e/common/common.go

+17
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"flag"
99
"os"
1010
"os/user"
11+
"path/filepath"
1112
"strconv"
1213
"testing"
1314
"time"
@@ -30,6 +31,22 @@ var (
3031
Architectures = []string{"amd64", "arm64"}
3132
CreateInfra = flag.Bool("create-infra", true, "create a Resource group, vNET and AKS cluster for testing")
3233
DeleteInfra = flag.Bool("delete-infra", true, "delete a Resource group, vNET and AKS cluster for testing")
34+
35+
// kubeconfig: path to kubeconfig file, in not provided,
36+
// a new k8s cluster will be created
37+
KubeConfig = flag.String("kubeConfig", "", "Path to kubeconfig file")
38+
)
39+
40+
var (
41+
RetinaChartPath = func(rootDir string) string {
42+
return filepath.Join(rootDir, "deploy", "legacy", "manifests", "controller", "helm", "retina")
43+
}
44+
RetinaAdvancedProfilePath = func(rootDir string) string {
45+
return filepath.Join(rootDir, "test", "profiles", "advanced", "values.yaml")
46+
}
47+
KubeConfigFilePath = func(rootDir string) string {
48+
return filepath.Join(rootDir, "test", "e2e", "test.pem")
49+
}
3350
)
3451

3552
func ClusterNameForE2ETest(t *testing.T) string {

‎test/e2e/framework/kubernetes/validate-service.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ func (v *ValidateResource) Run() error {
6262
return nil
6363
}
6464

65-
func serviceExists(ctx context.Context, clientset *kubernetes.Clientset, namespace, serviceName, labels string) (bool, error) {
65+
func serviceExists(ctx context.Context, clientset *kubernetes.Clientset, namespace, _, labels string) (bool, error) {
6666
var serviceList *corev1.ServiceList
6767
serviceList, err := clientset.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{LabelSelector: labels})
6868
if err != nil {

‎test/e2e/infra/azure_temp_infra_setup.go

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package infra
2+
3+
import (
4+
"context"
5+
"crypto/rand"
6+
"math/big"
7+
"os"
8+
"testing"
9+
10+
"github.com/microsoft/retina/test/e2e/common"
11+
"github.com/microsoft/retina/test/e2e/framework/types"
12+
jobs "github.com/microsoft/retina/test/e2e/jobs"
13+
"github.com/stretchr/testify/require"
14+
)
15+
16+
func CreateAzureTempK8sInfra(ctx context.Context, t *testing.T, rootDir string) string {
17+
kubeConfigFilePath := common.KubeConfigFilePath(rootDir)
18+
clusterName := common.ClusterNameForE2ETest(t)
19+
20+
subID := os.Getenv("AZURE_SUBSCRIPTION_ID")
21+
require.NotEmpty(t, subID, "AZURE_SUBSCRIPTION_ID environment variable must be set")
22+
23+
location := os.Getenv("AZURE_LOCATION")
24+
if location == "" {
25+
nBig, err := rand.Int(rand.Reader, big.NewInt(int64(len(common.AzureLocations))))
26+
if err != nil {
27+
t.Fatal("Failed to generate a secure random index", err)
28+
}
29+
location = common.AzureLocations[nBig.Int64()]
30+
}
31+
32+
rg := os.Getenv("AZURE_RESOURCE_GROUP")
33+
if rg == "" {
34+
// Use the cluster name as the resource group name by default.
35+
rg = clusterName
36+
}
37+
38+
// CreateTestInfra
39+
createTestInfra := types.NewRunner(t, jobs.CreateTestInfra(subID, rg, clusterName, location, kubeConfigFilePath, *common.CreateInfra))
40+
createTestInfra.Run(ctx)
41+
42+
t.Cleanup(func() {
43+
err := jobs.DeleteTestInfra(subID, rg, location, *common.DeleteInfra).Run()
44+
if err != nil {
45+
t.Logf("Failed to delete test infrastructure: %v", err)
46+
}
47+
})
48+
return kubeConfigFilePath
49+
}

‎test/e2e/jobs/jobs.go

+14-58
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
package retina
22

33
import (
4-
"fmt"
5-
"time"
6-
74
"github.com/microsoft/retina/test/e2e/common"
85
"github.com/microsoft/retina/test/e2e/framework/azure"
96
"github.com/microsoft/retina/test/e2e/framework/generic"
@@ -13,7 +10,6 @@ import (
1310
"github.com/microsoft/retina/test/e2e/scenarios/dns"
1411
"github.com/microsoft/retina/test/e2e/scenarios/drop"
1512
"github.com/microsoft/retina/test/e2e/scenarios/latency"
16-
"github.com/microsoft/retina/test/e2e/scenarios/perf"
1713
tcp "github.com/microsoft/retina/test/e2e/scenarios/tcp"
1814
"github.com/microsoft/retina/test/e2e/scenarios/windows"
1915
)
@@ -59,23 +55,19 @@ func CreateTestInfra(subID, rg, clusterName, location, kubeConfigFilePath string
5955
}, nil)
6056
}
6157

62-
job.AddStep(&generic.LoadFlags{
63-
TagEnv: generic.DefaultTagEnv,
64-
ImageNamespaceEnv: generic.DefaultImageNamespace,
65-
ImageRegistryEnv: generic.DefaultImageRegistry,
66-
}, nil)
67-
6858
return job
6959
}
7060

71-
func DeleteTestInfra(subID, rg, clusterName, location string) *types.Job {
61+
func DeleteTestInfra(subID, rg, location string, deleteInfra bool) *types.Job {
7262
job := types.NewJob("Delete e2e test infrastructure")
7363

74-
job.AddStep(&azure.DeleteResourceGroup{
75-
SubscriptionID: subID,
76-
ResourceGroupName: rg,
77-
Location: location,
78-
}, nil)
64+
if deleteInfra {
65+
job.AddStep(&azure.DeleteResourceGroup{
66+
SubscriptionID: subID,
67+
ResourceGroupName: rg,
68+
Location: location,
69+
}, nil)
70+
}
7971

8072
return job
8173
}
@@ -274,50 +266,14 @@ func ValidateHubble(kubeConfigFilePath, chartPath string, testPodNamespace strin
274266
return job
275267
}
276268

277-
func RunPerfTest(kubeConfigFilePath string, chartPath string) *types.Job {
278-
job := types.NewJob("Run performance tests")
279-
280-
benchmarkFile := fmt.Sprintf("netperf-benchmark-%s.json", time.Now().Format("20060102150405"))
281-
resultFile := fmt.Sprintf("netperf-result-%s.json", time.Now().Format("20060102150405"))
282-
regressionFile := fmt.Sprintf("netperf-regression-%s.json", time.Now().Format("20060102150405"))
269+
func LoadGenericFlags() *types.Job {
270+
job := types.NewJob("Loading Generic Flags to env")
283271

284-
job.AddStep(&perf.GetNetworkPerformanceMeasures{
285-
KubeConfigFilePath: kubeConfigFilePath,
286-
ResultTag: "no-retina",
287-
JsonOutputFile: benchmarkFile,
288-
}, &types.StepOptions{
289-
SkipSavingParametersToJob: true,
290-
})
291-
292-
job.AddStep(&kubernetes.InstallHelmChart{
293-
Namespace: "kube-system",
294-
ReleaseName: "retina",
295-
KubeConfigFilePath: kubeConfigFilePath,
296-
ChartPath: chartPath,
297-
TagEnv: generic.DefaultTagEnv,
272+
job.AddStep(&generic.LoadFlags{
273+
TagEnv: generic.DefaultTagEnv,
274+
ImageNamespaceEnv: generic.DefaultImageNamespace,
275+
ImageRegistryEnv: generic.DefaultImageRegistry,
298276
}, nil)
299277

300-
job.AddStep(&perf.GetNetworkPerformanceMeasures{
301-
KubeConfigFilePath: kubeConfigFilePath,
302-
ResultTag: "retina",
303-
JsonOutputFile: resultFile,
304-
}, &types.StepOptions{
305-
SkipSavingParametersToJob: true,
306-
})
307-
308-
job.AddStep(&perf.GetNetworkRegressionResults{
309-
BaseResultsFile: benchmarkFile,
310-
NewResultsFile: resultFile,
311-
RegressionResultsFile: regressionFile,
312-
}, &types.StepOptions{
313-
SkipSavingParametersToJob: true,
314-
})
315-
316-
job.AddStep(&perf.PublishPerfResults{
317-
ResultsFile: regressionFile,
318-
}, &types.StepOptions{
319-
SkipSavingParametersToJob: true,
320-
})
321-
322278
return job
323279
}

‎test/e2e/jobs/perf.go

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
package retina
2+
3+
import (
4+
"fmt"
5+
"time"
6+
7+
"github.com/microsoft/retina/test/e2e/framework/generic"
8+
"github.com/microsoft/retina/test/e2e/framework/kubernetes"
9+
"github.com/microsoft/retina/test/e2e/framework/types"
10+
"github.com/microsoft/retina/test/e2e/scenarios/perf"
11+
)
12+
13+
func RunPerfTest(kubeConfigFilePath, chartPath, advancedValuePath, retinaMode string) *types.Job {
14+
job := types.NewJob("Run performance tests")
15+
16+
benchmarkFile := fmt.Sprintf("netperf-benchmark-%s.json", time.Now().Format("20060102150405"))
17+
resultFile := fmt.Sprintf("netperf-result-%s.json", time.Now().Format("20060102150405"))
18+
regressionFile := fmt.Sprintf("netperf-regression-%s.json", time.Now().Format("20060102150405"))
19+
20+
job.AddStep(&perf.GetNetworkPerformanceMeasures{
21+
KubeConfigFilePath: kubeConfigFilePath,
22+
ResultTag: "no-retina",
23+
JsonOutputFile: benchmarkFile,
24+
}, &types.StepOptions{
25+
SkipSavingParametersToJob: true,
26+
})
27+
28+
job.AddStep(&kubernetes.InstallHelmChart{
29+
Namespace: "kube-system",
30+
ReleaseName: "retina",
31+
KubeConfigFilePath: kubeConfigFilePath,
32+
ChartPath: chartPath,
33+
TagEnv: generic.DefaultTagEnv,
34+
}, nil)
35+
36+
if retinaMode == "advanced" {
37+
job.AddStep(&kubernetes.UpgradeRetinaHelmChart{
38+
Namespace: "kube-system",
39+
ReleaseName: "retina",
40+
KubeConfigFilePath: kubeConfigFilePath,
41+
ChartPath: chartPath,
42+
ValuesFile: advancedValuePath,
43+
TagEnv: generic.DefaultTagEnv,
44+
}, &types.StepOptions{
45+
SkipSavingParametersToJob: true,
46+
})
47+
}
48+
49+
job.AddStep(&perf.GetNetworkPerformanceMeasures{
50+
KubeConfigFilePath: kubeConfigFilePath,
51+
ResultTag: "retina",
52+
JsonOutputFile: resultFile,
53+
}, &types.StepOptions{
54+
SkipSavingParametersToJob: true,
55+
})
56+
57+
job.AddStep(&perf.GetNetworkRegressionResults{
58+
BaseResultsFile: benchmarkFile,
59+
NewResultsFile: resultFile,
60+
RegressionResultsFile: regressionFile,
61+
}, &types.StepOptions{
62+
SkipSavingParametersToJob: true,
63+
})
64+
65+
job.AddStep(&perf.PublishPerfResults{
66+
ResultsFile: regressionFile,
67+
RetinaMode: retinaMode,
68+
}, &types.StepOptions{
69+
SkipSavingParametersToJob: true,
70+
})
71+
72+
return job
73+
}

‎test/e2e/retina_e2e_test.go

+25-37
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,14 @@
33
package retina
44

55
import (
6-
"crypto/rand"
7-
"math/big"
86
"os"
97
"path/filepath"
108
"testing"
119

1210
"github.com/microsoft/retina/test/e2e/common"
1311
"github.com/microsoft/retina/test/e2e/framework/helpers"
1412
"github.com/microsoft/retina/test/e2e/framework/types"
13+
"github.com/microsoft/retina/test/e2e/infra"
1514
jobs "github.com/microsoft/retina/test/e2e/jobs"
1615
"github.com/stretchr/testify/require"
1716
)
@@ -21,57 +20,46 @@ func TestE2ERetina(t *testing.T) {
2120
ctx, cancel := helpers.Context(t)
2221
defer cancel()
2322

24-
// Truncate the username to 8 characters
25-
clusterName := common.ClusterNameForE2ETest(t)
26-
27-
subID := os.Getenv("AZURE_SUBSCRIPTION_ID")
28-
require.NotEmpty(t, subID)
29-
30-
location := os.Getenv("AZURE_LOCATION")
31-
if location == "" {
32-
nBig, err := rand.Int(rand.Reader, big.NewInt(int64(len(common.AzureLocations))))
33-
if err != nil {
34-
t.Fatalf("Failed to generate a secure random index: %v", err)
35-
}
36-
location = common.AzureLocations[nBig.Int64()]
37-
}
38-
39-
rg := os.Getenv("AZURE_RESOURCE_GROUP")
40-
if rg == "" {
41-
// Use the cluster name as the resource group name by default.
42-
rg = clusterName
43-
}
44-
4523
cwd, err := os.Getwd()
4624
require.NoError(t, err)
4725

4826
// Get to root of the repo by going up two directories
4927
rootDir := filepath.Dir(filepath.Dir(cwd))
5028

51-
chartPath := filepath.Join(rootDir, "deploy", "legacy", "manifests", "controller", "helm", "retina")
5229
hubblechartPath := filepath.Join(rootDir, "deploy", "hubble", "manifests", "controller", "helm", "retina")
53-
profilePath := filepath.Join(rootDir, "test", "profiles", "advanced", "values.yaml")
54-
kubeConfigFilePath := filepath.Join(rootDir, "test", "e2e", "test.pem")
5530

56-
// CreateTestInfra
57-
createTestInfra := types.NewRunner(t, jobs.CreateTestInfra(subID, rg, clusterName, location, kubeConfigFilePath, *common.CreateInfra))
58-
createTestInfra.Run(ctx)
31+
err = jobs.LoadGenericFlags().Run()
32+
require.NoError(t, err, "failed to load generic flags")
5933

60-
t.Cleanup(func() {
61-
if *common.DeleteInfra {
62-
_ = jobs.DeleteTestInfra(subID, rg, clusterName, location).Run()
63-
}
64-
})
34+
if *common.KubeConfig == "" {
35+
*common.KubeConfig = infra.CreateAzureTempK8sInfra(ctx, t, rootDir)
36+
}
6537

6638
// Install and test Retina basic metrics
67-
basicMetricsE2E := types.NewRunner(t, jobs.InstallAndTestRetinaBasicMetrics(kubeConfigFilePath, chartPath, common.TestPodNamespace))
39+
basicMetricsE2E := types.NewRunner(t,
40+
jobs.InstallAndTestRetinaBasicMetrics(
41+
common.KubeConfigFilePath(rootDir),
42+
common.RetinaChartPath(rootDir),
43+
common.TestPodNamespace),
44+
)
6845
basicMetricsE2E.Run(ctx)
6946

7047
// Upgrade and test Retina with advanced metrics
71-
advanceMetricsE2E := types.NewRunner(t, jobs.UpgradeAndTestRetinaAdvancedMetrics(kubeConfigFilePath, chartPath, profilePath, common.TestPodNamespace))
48+
advanceMetricsE2E := types.NewRunner(t,
49+
jobs.UpgradeAndTestRetinaAdvancedMetrics(
50+
common.KubeConfigFilePath(rootDir),
51+
common.RetinaChartPath(rootDir),
52+
common.RetinaAdvancedProfilePath(rootDir),
53+
common.TestPodNamespace),
54+
)
7255
advanceMetricsE2E.Run(ctx)
7356

7457
// Install and test Hubble basic metrics
75-
validatehubble := types.NewRunner(t, jobs.ValidateHubble(kubeConfigFilePath, hubblechartPath, common.TestPodNamespace))
58+
validatehubble := types.NewRunner(t,
59+
jobs.ValidateHubble(
60+
common.KubeConfigFilePath(rootDir),
61+
hubblechartPath,
62+
common.TestPodNamespace),
63+
)
7664
validatehubble.Run(ctx)
7765
}

0 commit comments

Comments
 (0)