diff --git a/.github/workflows/basic-ci-amd64.yaml b/.github/workflows/basic-ci-amd64.yaml index 119891853098..714f27f7f0bc 100644 --- a/.github/workflows/basic-ci-amd64.yaml +++ b/.github/workflows/basic-ci-amd64.yaml @@ -22,7 +22,7 @@ jobs: fail-fast: false matrix: containerd_version: ['lts', 'active'] - vmm: ['clh', 'qemu'] + vmm: ['clh', 'qemu', 'stratovirt'] runs-on: garm-ubuntu-2204-smaller env: CONTAINERD_VERSION: ${{ matrix.containerd_version }} @@ -60,7 +60,7 @@ jobs: fail-fast: false matrix: containerd_version: ['lts', 'active'] - vmm: ['clh', 'qemu'] + vmm: ['clh', 'qemu', 'stratovirt'] runs-on: garm-ubuntu-2204-smaller env: CONTAINERD_VERSION: ${{ matrix.containerd_version }} @@ -101,7 +101,7 @@ jobs: fail-fast: false matrix: containerd_version: ['lts', 'active'] - vmm: ['clh', 'qemu', 'dragonball'] + vmm: ['clh', 'qemu', 'dragonball', 'stratovirt'] runs-on: garm-ubuntu-2204-smaller env: CONTAINERD_VERSION: ${{ matrix.containerd_version }} diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index 2fadc761ca8c..aea0a0240627 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -33,6 +33,7 @@ jobs: - clh - dragonball - qemu + - stratovirt instance-type: - small - normal diff --git a/.github/workflows/run-metrics.yaml b/.github/workflows/run-metrics.yaml index 53deeb7864f2..7628a41369a5 100644 --- a/.github/workflows/run-metrics.yaml +++ b/.github/workflows/run-metrics.yaml @@ -48,7 +48,7 @@ jobs: # all the tests due to a single flaky instance. fail-fast: false matrix: - vmm: ['clh', 'qemu'] + vmm: ['clh', 'qemu', 'stratovirt'] max-parallel: 1 runs-on: metrics env: diff --git a/src/runtime/cmd/kata-runtime/kata-check_amd64.go b/src/runtime/cmd/kata-runtime/kata-check_amd64.go index fcdb047fbef1..c40f5e9dcb1d 100644 --- a/src/runtime/cmd/kata-runtime/kata-check_amd64.go +++ b/src/runtime/cmd/kata-runtime/kata-check_amd64.go @@ -115,6 +115,8 @@ func setCPUtype(hypervisorType vc.HypervisorType) error { } switch hypervisorType { + case vc.StratovirtHypervisor: + fallthrough case vc.FirecrackerHypervisor: fallthrough case vc.ClhHypervisor: @@ -315,6 +317,8 @@ func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error { fallthrough case vc.ClhHypervisor: fallthrough + case vc.StratovirtHypervisor: + fallthrough case vc.FirecrackerHypervisor: return kvmIsUsable() case vc.AcrnHypervisor: diff --git a/src/runtime/virtcontainers/stratovirt_test.go b/src/runtime/virtcontainers/stratovirt_test.go new file mode 100644 index 000000000000..3b41eaf18ac3 --- /dev/null +++ b/src/runtime/virtcontainers/stratovirt_test.go @@ -0,0 +1,432 @@ +//go:build linux + +// Copyright (c) 2023 Huawei Technologies Co.,Ltd. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist" + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +func newStratovirtConfig() (HypervisorConfig, error) { + + setupStratovirt() + + if testStratovirtPath == "" { + return HypervisorConfig{}, errors.New("hypervisor fake path is empty") + } + + if testVirtiofsdPath == "" { + return HypervisorConfig{}, errors.New("virtiofsd fake path is empty") + } + + if _, err := os.Stat(testStratovirtPath); os.IsNotExist(err) { + return HypervisorConfig{}, err + } + + if _, err := os.Stat(testVirtiofsdPath); os.IsNotExist(err) { + return HypervisorConfig{}, err + } + + return HypervisorConfig{ + HypervisorPath: testStratovirtPath, + KernelPath: testStratovirtKernelPath, + InitrdPath: testStratovirtInitrdPath, + RootfsType: string(EXT4), + NumVCPUsF: defaultVCPUs, + BlockDeviceDriver: config.VirtioBlock, + MemorySize: defaultMemSzMiB, + DefaultMaxVCPUs: uint32(64), + SharedFS: config.VirtioFS, + VirtioFSCache: typeVirtioFSCacheModeAlways, + VirtioFSDaemon: testVirtiofsdPath, + }, nil +} + +func TestStratovirtCreateVM(t *testing.T) { + assert := assert.New(t) + + store, err := persist.GetDriver() + assert.NoError(err) + + network, err := NewNetwork() + assert.NoError(err) + + sv := stratovirt{ + config: HypervisorConfig{ + VMStorePath: store.RunVMStoragePath(), + RunStorePath: store.RunStoragePath(), + }, + } + + config0, err := newStratovirtConfig() + assert.NoError(err) + + config1, err := newStratovirtConfig() + assert.NoError(err) + config1.ImagePath = testStratovirtImagePath + config1.InitrdPath = "" + + config2, err := newStratovirtConfig() + assert.NoError(err) + config2.Debug = true + + config3, err := newStratovirtConfig() + assert.NoError(err) + config3.SharedFS = config.VirtioFS + + config4, err := newStratovirtConfig() + assert.NoError(err) + config4.SharedFS = config.VirtioFSNydus + + type testData struct { + config HypervisorConfig + expectError bool + configMatch bool + } + + data := []testData{ + {config0, false, true}, + {config1, false, true}, + {config2, false, true}, + {config3, false, true}, + {config4, false, true}, + } + + for i, d := range data { + msg := fmt.Sprintf("test[%d]", i) + + err = sv.CreateVM(context.Background(), "testSandbox", network, &d.config) + + if d.expectError { + assert.Error(err, msg) + continue + } + + assert.NoError(err, msg) + + if d.configMatch { + assert.Exactly(d.config, sv.config, msg) + } + } +} + +func TestStratovirtStartSandbox(t *testing.T) { + assert := assert.New(t) + sConfig, err := newStratovirtConfig() + assert.NoError(err) + sConfig.Debug = true + + network, err := NewNetwork() + assert.NoError(err) + + store, err := persist.GetDriver() + assert.NoError(err) + + sConfig.VMStorePath = store.RunVMStoragePath() + sConfig.RunStorePath = store.RunStoragePath() + + sv := &stratovirt{ + config: sConfig, + virtiofsDaemon: &virtiofsdMock{}, + } + + assert.Exactly(sv.stopped.Load(), false) + + err = sv.CreateVM(context.Background(), "testSandbox", network, &sConfig) + assert.NoError(err) + + mem := sv.GetTotalMemoryMB(context.Background()) + assert.True(mem > 0) + + err = sv.StartVM(context.Background(), 10) + assert.Error(err) +} + +func TestStratovirtCleanupVM(t *testing.T) { + assert := assert.New(t) + store, err := persist.GetDriver() + assert.NoError(err, "persist.GetDriver() unexpected error") + + sv := &stratovirt{ + id: "cleanVM", + config: HypervisorConfig{ + VMStorePath: store.RunVMStoragePath(), + RunStorePath: store.RunStoragePath(), + }, + } + sv.svConfig.vmPath = filepath.Join(sv.config.VMStorePath, sv.id) + sv.config.VMid = "cleanVM" + + err = sv.cleanupVM(true) + assert.NoError(err, "persist.GetDriver() unexpected error") + + dir := filepath.Join(store.RunVMStoragePath(), sv.id) + os.MkdirAll(dir, os.ModePerm) + + err = sv.cleanupVM(false) + assert.NoError(err, "persist.GetDriver() unexpected error") + + _, err = os.Stat(dir) + assert.Error(err, "dir should not exist %s", dir) + + assert.True(os.IsNotExist(err), "persist.GetDriver() unexpected error") +} + +func TestStratovirtAddFsDevice(t *testing.T) { + assert := assert.New(t) + sConfig, err := newStratovirtConfig() + assert.NoError(err) + sConfig.SharedFS = config.VirtioFS + mountTag := "testMountTag" + + sv := &stratovirt{ + ctx: context.Background(), + config: sConfig, + } + volume := types.Volume{ + MountTag: mountTag, + } + expected := []VirtioDev{ + virtioFs{ + backend: "socket", + charID: "virtio_fs", + charDev: "virtio_fs", + tag: volume.MountTag, + deviceID: "virtio-fs0", + driver: mmioBus, + }, + } + + err = sv.AddDevice(context.Background(), volume, FsDev) + assert.NoError(err) + assert.Exactly(sv.svConfig.devices, expected) +} + +func TestStratovirtAddBlockDevice(t *testing.T) { + assert := assert.New(t) + sConfig, err := newStratovirtConfig() + assert.NoError(err) + + sv := &stratovirt{ + ctx: context.Background(), + config: sConfig, + } + blockDrive := config.BlockDrive{} + expected := []VirtioDev{ + blkDevice{ + id: "rootfs", + filePath: sv.svConfig.rootfsPath, + deviceID: "virtio-blk0", + driver: mmioBus, + }, + } + + err = sv.AddDevice(context.Background(), blockDrive, BlockDev) + assert.NoError(err) + assert.Exactly(sv.svConfig.devices, expected) +} + +func TestStratovirtAddVsockDevice(t *testing.T) { + assert := assert.New(t) + sConfig, err := newStratovirtConfig() + assert.NoError(err) + + dir := t.TempDir() + vsockFilename := filepath.Join(dir, "vsock") + contextID := uint64(3) + port := uint32(1024) + vsockFile, fileErr := os.Create(vsockFilename) + assert.NoError(fileErr) + defer vsockFile.Close() + + sv := &stratovirt{ + ctx: context.Background(), + config: sConfig, + } + vsock := types.VSock{ + ContextID: contextID, + Port: port, + VhostFd: vsockFile, + } + expected := []VirtioDev{ + vhostVsock{ + id: "vsock-id", + guestID: fmt.Sprintf("%d", contextID), + VHostFD: vsockFile, + driver: mmioBus, + }, + } + + err = sv.AddDevice(context.Background(), vsock, VSockPCIDev) + assert.NoError(err) + assert.Exactly(sv.svConfig.devices, expected) +} + +func TestStratovirtAddConsole(t *testing.T) { + assert := assert.New(t) + sConfig, err := newStratovirtConfig() + assert.NoError(err) + + sv := &stratovirt{ + ctx: context.Background(), + config: sConfig, + } + sock := types.Socket{} + expected := []VirtioDev{ + consoleDevice{ + id: "virtio-serial0", + backend: "socket", + charID: "charconsole0", + devType: "virtconsole", + charDev: "charconsole0", + deviceID: "virtio-console0", + driver: mmioBus, + }, + } + + err = sv.AddDevice(context.Background(), sock, SerialPortDev) + assert.NoError(err) + assert.Exactly(sv.svConfig.devices, expected) +} + +func TestStratovirtGetSandboxConsole(t *testing.T) { + assert := assert.New(t) + store, err := persist.GetDriver() + assert.NoError(err) + + sandboxID := "testSandboxID" + sv := &stratovirt{ + id: sandboxID, + ctx: context.Background(), + config: HypervisorConfig{ + VMStorePath: store.RunVMStoragePath(), + RunStorePath: store.RunStoragePath(), + }, + } + expected := filepath.Join(store.RunVMStoragePath(), sandboxID, debugSocket) + + proto, result, err := sv.GetVMConsole(sv.ctx, sandboxID) + assert.NoError(err) + assert.Equal(result, expected) + assert.Equal(proto, consoleProtoUnix) +} + +func TestStratovirtCapabilities(t *testing.T) { + assert := assert.New(t) + + sConfig, err := newStratovirtConfig() + assert.NoError(err) + + sv := stratovirt{} + assert.Equal(sv.config, HypervisorConfig{}) + + sConfig.SharedFS = config.VirtioFS + + err = sv.setConfig(&sConfig) + assert.NoError(err) + + var ctx context.Context + c := sv.Capabilities(ctx) + assert.True(c.IsFsSharingSupported()) + + sConfig.SharedFS = config.NoSharedFS + + err = sv.setConfig(&sConfig) + assert.NoError(err) + + c = sv.Capabilities(ctx) + assert.False(c.IsFsSharingSupported()) +} + +func TestStratovirtSetConfig(t *testing.T) { + assert := assert.New(t) + + config, err := newStratovirtConfig() + assert.NoError(err) + + sv := stratovirt{} + assert.Equal(sv.config, HypervisorConfig{}) + + err = sv.setConfig(&config) + assert.NoError(err) + + assert.Equal(sv.config, config) +} + +func TestStratovirtCleanup(t *testing.T) { + assert := assert.New(t) + sConfig, err := newStratovirtConfig() + assert.NoError(err) + + sv := &stratovirt{ + ctx: context.Background(), + config: sConfig, + } + + err = sv.Cleanup(sv.ctx) + assert.Nil(err) +} + +func TestStratovirtGetpids(t *testing.T) { + assert := assert.New(t) + + sv := &stratovirt{} + pids := sv.GetPids() + assert.NotNil(pids) + assert.True(len(pids) == 1) + assert.True(pids[0] == 0) +} + +func TestStratovirtBinPath(t *testing.T) { + assert := assert.New(t) + + f, err := os.CreateTemp("", "stratovirt") + assert.NoError(err) + defer func() { _ = f.Close() }() + defer func() { _ = os.Remove(f.Name()) }() + + expectedPath := f.Name() + sConfig, err := newStratovirtConfig() + assert.NoError(err) + + sConfig.HypervisorPath = expectedPath + sv := &stratovirt{ + config: sConfig, + } + + // get config hypervisor path + path, err := sv.binPath() + assert.NoError(err) + assert.Equal(path, expectedPath) + + // config hypervisor path does not exist + sv.config.HypervisorPath = "/abc/xyz/123" + path, err = sv.binPath() + assert.Error(err) + assert.Equal(path, "") + + // get default stratovirt hypervisor path + sv.config.HypervisorPath = "" + path, err = sv.binPath() + if _, errStat := os.Stat(path); os.IsNotExist(errStat) { + assert.Error(err) + assert.Equal(path, "") + } else { + assert.NoError(err) + assert.Equal(path, defaultStratoVirt) + } +} diff --git a/src/runtime/virtcontainers/virtcontainers_test.go b/src/runtime/virtcontainers/virtcontainers_test.go index 0a269b8b9315..f366558d5009 100644 --- a/src/runtime/virtcontainers/virtcontainers_test.go +++ b/src/runtime/virtcontainers/virtcontainers_test.go @@ -48,6 +48,10 @@ var testAcrnKernelPath = "" var testAcrnImagePath = "" var testAcrnPath = "" var testAcrnCtlPath = "" +var testStratovirtKernelPath = "" +var testStratovirtImagePath = "" +var testStratovirtInitrdPath = "" +var testStratovirtPath = "" var testVirtiofsdPath = "" var testHyperstartCtlSocket = "" @@ -89,6 +93,18 @@ func setupClh() { } } +func setupStratovirt() { + os.Mkdir(filepath.Join(testDir, testBundle), DirMode) + + for _, filename := range []string{testStratovirtKernelPath, testStratovirtInitrdPath, testStratovirtPath, testVirtiofsdPath} { + _, err := os.Create(filename) + if err != nil { + fmt.Printf("Could not recreate %s:%v", filename, err) + os.Exit(1) + } + } +} + // TestMain is the common main function used by ALL the test functions // for this package. func TestMain(m *testing.M) { @@ -149,6 +165,13 @@ func TestMain(m *testing.M) { setupClh() + testStratovirtKernelPath = filepath.Join(testDir, testBundle, testKernel) + testStratovirtImagePath = filepath.Join(testDir, testBundle, testInitrd) + testStratovirtInitrdPath = filepath.Join(testDir, testBundle, testInitrd) + testStratovirtPath = filepath.Join(testDir, testBundle, testHypervisor) + + setupStratovirt() + // set now that configStoragePath has been overridden. sandboxDirState = filepath.Join(fs.MockRunStoragePath(), testSandboxID) diff --git a/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-stratovirt-kata-metric8.toml b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-stratovirt-kata-metric8.toml new file mode 100644 index 000000000000..b5ad92ca1935 --- /dev/null +++ b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-stratovirt-kata-metric8.toml @@ -0,0 +1,162 @@ +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# SPDX-License-Identifier: Apache-2.0 +# +# This file contains baseline expectations +# for checked results by checkmetrics tool. + +[[metric]] +name = "boot-times" +type = "json" +description = "measure container lifecycle timings" +# Min and Max values to set a 'range' that +# the median of the CSV Results data must fall +# within (inclusive) +checkvar = ".\"boot-times\".Results | .[] | .\"to-workload\".Result" +checktype = "mean" +midval = 0.62 +minpercent = 40.0 +maxpercent = 40.0 + +[[metric]] +name = "memory-footprint" +type = "json" +description = "measure memory usage" +# Min and Max values to set a 'range' that +# the median of the CSV Results data must fall +# within (inclusive) +checkvar = ".\"memory-footprint\".Results | .[] | .average.Result" +checktype = "mean" +midval = 129842.10 +minpercent = 30.0 +maxpercent = 30.0 + +[[metric]] +name = "memory-footprint-inside-container" +type = "json" +description = "measure memory inside the container" +# Min and Max values to set a 'range' that +# the median of the CSV Results data must fall +# within (inclusive) +checkvar = ".\"memory-footprint-inside-container\".Results | .[] | .memtotal.Result" +checktype = "mean" +midval = 2040568.0 +minpercent = 30.0 +maxpercent = 30.0 + +[[metric]] +name = "blogbench" +type = "json" +description = "measure container average of blogbench write" +# Min and Max values to set a 'range' that +# the median of the CSV Results data must fall +# within (inclusive) +checkvar = ".\"blogbench\".Results | .[] | .write.Result" +checktype = "mean" +midval = 603.0 +minpercent = 30.0 +maxpercent = 30.0 + +[[metric]] +name = "blogbench" +type = "json" +description = "measure container average of blogbench read" +# Min and Max values to set a 'range' that +# the median of the CSV Results data must fall +# within (inclusive) +checkvar = ".\"blogbench\".Results | .[] | .read.Result" +checktype = "mean" +midval = 37669.0 +minpercent = 30.0 +maxpercent = 30.0 + +[[metric]] +name = "tensorflow_nhwc" +type = "json" +description = "tensorflow resnet model" +# Min and Max values to set a 'range' that +# the median of the CSV Results data must fall +# within (inclusive) +checkvar = ".\"tensorflow_nhwc\".Results | .[] | .resnet.Result" +checktype = "mean" +midval = 2025.0 +minpercent = 30.0 +maxpercent = 30.0 + +[[metric]] +name = "tensorflow_nhwc" +type = "json" +description = "tensorflow alexnet model" +# Min and Max values to set a 'range' that +# the median of the CSV Results data must fall +# within (inclusive) +checkvar = ".\"tensorflow_nhwc\".Results | .[] | .alexnet.Result" +checktype = "mean" +midval = 75.0 +minpercent = 30.0 +maxpercent = 30.0 + +[[metric]] +name = "latency" +type = "json" +description = "measure container latency" +# Min and Max values to set a 'range' that +# the median of the CSV Results data must fall +# within (inclusive) +checkvar = ".\"latency\".Results | .[] | .latency.Result" +checktype = "mean" +midval = 0.78 +minpercent = 30.0 +maxpercent = 30.0 + +[[metric]] +name = "network-iperf3" +type = "json" +description = "measure container cpu utilization using iperf3" +# Min and Max values to set a 'range' that +# the median of the CSV Results data must fall +# within (inclusive) +checkvar = ".\"network-iperf3\".Results | .[] | .cpu.Result" +checktype = "mean" +midval = 60.10 +minpercent = 30.0 +maxpercent = 30.0 + +[[metric]] +name = "network-iperf3" +type = "json" +description = "measure container bandwidth using iperf3" +# Min and Max values to set a 'range' that +# the median of the CSV Results data must fall +# within (inclusive) +checkvar = ".\"network-iperf3\".Results | .[] | .bandwidth.Result" +checktype = "mean" +midval = 19959440840.94 +minpercent = 30.0 +maxpercent = 30.0 + +[[metric]] +name = "network-iperf3" +type = "json" +description = "measure container parallel bandwidth using iperf3" +# Min and Max values to set a 'range' that +# the median of the CSV Results data must fall +# within (inclusive) +checkvar = ".\"network-iperf3\".Results | .[] | .parallel.Result" +checktype = "mean" +midval = 25487333685.04 +minpercent = 30.0 +maxpercent = 30.0 + +[[metric]] +name = "network-iperf3" +type = "json" +description = "iperf" +# Min and Max values to set a 'range' that +# the median of the CSV Results data must fall +# within (inclusive) +checkvar = ".\"network-iperf3\".Results | .[] | .jitter.Result" +checktype = "mean" +midval = 0.038 +minpercent = 40.0 +maxpercent = 40.0