From 57d4dd8e57bfcab68fdc4baaa37163900909fd54 Mon Sep 17 00:00:00 2001 From: Yohei Ueda Date: Thu, 17 Feb 2022 11:04:38 +0900 Subject: [PATCH] runtime: Support the remote hypervisor type This patch adds the support of the remote hypervisor type. Shim opens a Unix domain socket specified in the config file, and sends TTPRC requests to a external process to control sandbox VMs. Fixes #4482 Co-authored-by: Pradipta Banerjee Co-authored-by: stevenhorsman Signed-off-by: Yohei Ueda (based on commit f9278f22c353e2ac253fa2eb2f129685af544bd0) --- .../cmd/kata-runtime/kata-check_amd64.go | 2 + .../cmd/kata-runtime/kata-check_arm64.go | 3 + .../cmd/kata-runtime/kata-check_ppc64le.go | 3 + .../cmd/kata-runtime/kata-check_s390x.go | 3 + src/runtime/pkg/katautils/config.go | 19 ++ src/runtime/pkg/katautils/config_test.go | 11 + .../protocols/hypervisor/hypervisor.proto | 3 + src/runtime/virtcontainers/hypervisor.go | 20 ++ .../hypervisor_config_darwin.go | 4 + .../virtcontainers/hypervisor_config_linux.go | 4 + .../virtcontainers/hypervisor_config_test.go | 9 + .../virtcontainers/hypervisor_linux.go | 2 + src/runtime/virtcontainers/hypervisor_test.go | 20 +- src/runtime/virtcontainers/kata_agent.go | 66 ++-- .../pkg/agent/protocols/client/client.go | 33 ++ src/runtime/virtcontainers/remote.go | 288 ++++++++++++++++++ src/runtime/virtcontainers/remote_test.go | 45 +++ src/runtime/virtcontainers/sandbox.go | 21 ++ src/runtime/virtcontainers/types/sandbox.go | 12 + .../virtcontainers/types/sandbox_test.go | 74 +++++ 20 files changed, 616 insertions(+), 26 deletions(-) create mode 100644 src/runtime/virtcontainers/remote.go create mode 100644 src/runtime/virtcontainers/remote_test.go diff --git a/src/runtime/cmd/kata-runtime/kata-check_amd64.go b/src/runtime/cmd/kata-runtime/kata-check_amd64.go index c40f5e9dcb1d..e2c2c26df284 100644 --- a/src/runtime/cmd/kata-runtime/kata-check_amd64.go +++ b/src/runtime/cmd/kata-runtime/kata-check_amd64.go @@ -323,6 +323,8 @@ func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error { return kvmIsUsable() case vc.AcrnHypervisor: return acrnIsUsable() + case vc.RemoteHypervisor: + return nil case vc.MockHypervisor: return nil default: diff --git a/src/runtime/cmd/kata-runtime/kata-check_arm64.go b/src/runtime/cmd/kata-runtime/kata-check_arm64.go index 66d81c71c3d1..933c9776095a 100644 --- a/src/runtime/cmd/kata-runtime/kata-check_arm64.go +++ b/src/runtime/cmd/kata-runtime/kata-check_arm64.go @@ -86,6 +86,9 @@ func checkKVMExtensions() error { } func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error { + if hypervisorType == "remote" { + return nil + } if err := kvmIsUsable(); err != nil { return err } diff --git a/src/runtime/cmd/kata-runtime/kata-check_ppc64le.go b/src/runtime/cmd/kata-runtime/kata-check_ppc64le.go index 7c5e7453ac63..de34f9614f74 100644 --- a/src/runtime/cmd/kata-runtime/kata-check_ppc64le.go +++ b/src/runtime/cmd/kata-runtime/kata-check_ppc64le.go @@ -61,6 +61,9 @@ func setCPUtype(hypervisorType vc.HypervisorType) error { } func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error { + if hypervisorType == "remote" { + return nil + } return kvmIsUsable() } diff --git a/src/runtime/cmd/kata-runtime/kata-check_s390x.go b/src/runtime/cmd/kata-runtime/kata-check_s390x.go index c9b1578b43f7..8ee51b7e5c32 100644 --- a/src/runtime/cmd/kata-runtime/kata-check_s390x.go +++ b/src/runtime/cmd/kata-runtime/kata-check_s390x.go @@ -55,6 +55,9 @@ func kvmIsUsable() error { } func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error { + if hypervisorType == "remote" { + return nil + } return kvmIsUsable() } diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 7aa09b995fee..1ca926964449 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -53,6 +53,7 @@ const ( acrnHypervisorTableType = "acrn" dragonballHypervisorTableType = "dragonball" stratovirtHypervisorTableType = "stratovirt" + remoteHypervisorTableType = "remote" // the maximum amount of PCI bridges that can be cold plugged in a VM maxPCIBridges uint32 = 5 @@ -105,6 +106,7 @@ type hypervisor struct { GuestMemoryDumpPath string `toml:"guest_memory_dump_path"` SeccompSandbox string `toml:"seccompsandbox"` BlockDeviceAIO string `toml:"block_device_aio"` + RemoteHypervisorSocket string `toml:"remote_hypervisor_socket"` HypervisorPathList []string `toml:"valid_hypervisor_paths"` JailerPathList []string `toml:"valid_jailer_paths"` CtlPathList []string `toml:"valid_ctlpaths"` @@ -134,6 +136,7 @@ type hypervisor struct { MemSlots uint32 `toml:"memory_slots"` DefaultBridges uint32 `toml:"default_bridges"` Msize9p uint32 `toml:"msize_9p"` + RemoteHypervisorTimeout uint32 `toml:"remote_hypervisor_timeout"` NumVCPUs float32 `toml:"default_vcpus"` BlockDeviceCacheSet bool `toml:"block_device_cache_set"` BlockDeviceCacheDirect bool `toml:"block_device_cache_direct"` @@ -1242,6 +1245,14 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { }, nil } +func newRemoteHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { + + return vc.HypervisorConfig{ + RemoteHypervisorSocket: h.RemoteHypervisorSocket, + RemoteHypervisorTimeout: h.RemoteHypervisorTimeout, + }, nil +} + func newFactoryConfig(f factory) (oci.FactoryConfig, error) { if f.TemplatePath == "" { f.TemplatePath = defaultTemplatePath @@ -1281,6 +1292,9 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi case stratovirtHypervisorTableType: config.HypervisorType = vc.StratovirtHypervisor hConfig, err = newStratovirtHypervisorConfig(hypervisor) + case remoteHypervisorTableType: + config.HypervisorType = vc.RemoteHypervisor + hConfig, err = newRemoteHypervisorConfig(hypervisor) default: err = fmt.Errorf("%s: %+q", errInvalidHypervisorPrefix, k) } @@ -1882,6 +1896,11 @@ func checkFactoryConfig(config oci.RuntimeConfig) error { // checkHypervisorConfig performs basic "sanity checks" on the hypervisor // config. func checkHypervisorConfig(config vc.HypervisorConfig) error { + + if config.RemoteHypervisorSocket != "" { + return nil + } + type image struct { path string initrd bool diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go index 39ccb49f2b74..3ecb1517e5c5 100644 --- a/src/runtime/pkg/katautils/config_test.go +++ b/src/runtime/pkg/katautils/config_test.go @@ -1564,6 +1564,17 @@ func TestCheckHypervisorConfig(t *testing.T) { // reset logger kataUtilsLogger.Logger.Out = savedOut } + + // Check remote hypervisor doesn't error with missing unnescessary config + remoteConfig := vc.HypervisorConfig{ + RemoteHypervisorSocket: "dummy_socket", + ImagePath: "", + InitrdPath: "", + MemorySize: 0, + } + + err := checkHypervisorConfig(remoteConfig) + assert.NoError(err, "remote hypervisor config") } func TestCheckNetNsConfig(t *testing.T) { diff --git a/src/runtime/protocols/hypervisor/hypervisor.proto b/src/runtime/protocols/hypervisor/hypervisor.proto index 4840be284e9a..bc8a87b2ef2f 100644 --- a/src/runtime/protocols/hypervisor/hypervisor.proto +++ b/src/runtime/protocols/hypervisor/hypervisor.proto @@ -1,3 +1,6 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + syntax = "proto3"; option go_package = "./"; diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 43ec8f91f363..7c16064e1ff3 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -57,6 +57,9 @@ const ( // VirtFrameworkHypervisor is the Darwin Virtualization.framework hypervisor VirtframeworkHypervisor HypervisorType = "virtframework" + // RemoteHypervisor is the Remote hypervisor. + RemoteHypervisor HypervisorType = "remote" + // MockHypervisor is a mock hypervisor for testing purposes MockHypervisor HypervisorType = "mock" @@ -240,6 +243,9 @@ func (hType *HypervisorType) Set(value string) error { case "virtframework": *hType = VirtframeworkHypervisor return nil + case "remote": + *hType = RemoteHypervisor + return nil case "mock": *hType = MockHypervisor return nil @@ -261,6 +267,8 @@ func (hType *HypervisorType) String() string { return string(ClhHypervisor) case StratovirtHypervisor: return string(StratovirtHypervisor) + case RemoteHypervisor: + return string(RemoteHypervisor) case MockHypervisor: return string(MockHypervisor) default: @@ -455,6 +463,15 @@ type HypervisorConfig struct { // BlockiDeviceAIO specifies the I/O API to be used. BlockDeviceAIO string + // The socket to connect to the remote hypervisor implementation on + RemoteHypervisorSocket string + + // The name of the sandbox (pod) + SandboxName string + + // The name of the namespace of the sandbox (pod) + SandboxNamespace string + // The user maps to the uid. User string @@ -563,6 +580,9 @@ type HypervisorConfig struct { // Group ID. Gid uint32 + // Timeout for actions e.g. startVM for the remote hypervisor + RemoteHypervisorTimeout uint32 + // BlockDeviceCacheSet specifies cache-related options will be set to block devices or not. BlockDeviceCacheSet bool diff --git a/src/runtime/virtcontainers/hypervisor_config_darwin.go b/src/runtime/virtcontainers/hypervisor_config_darwin.go index a949adf3a73c..1225271a2a4c 100644 --- a/src/runtime/virtcontainers/hypervisor_config_darwin.go +++ b/src/runtime/virtcontainers/hypervisor_config_darwin.go @@ -11,6 +11,10 @@ import ( func validateHypervisorConfig(conf *HypervisorConfig) error { + if conf.RemoteHypervisorSocket != "" { + return nil + } + if conf.KernelPath == "" { return fmt.Errorf("Missing kernel path") } diff --git a/src/runtime/virtcontainers/hypervisor_config_linux.go b/src/runtime/virtcontainers/hypervisor_config_linux.go index 8e34f98b5bc6..1bcd47218c3c 100644 --- a/src/runtime/virtcontainers/hypervisor_config_linux.go +++ b/src/runtime/virtcontainers/hypervisor_config_linux.go @@ -13,6 +13,10 @@ import ( func validateHypervisorConfig(conf *HypervisorConfig) error { + if conf.RemoteHypervisorSocket != "" { + return nil + } + if conf.KernelPath == "" { return fmt.Errorf("Missing kernel path") } diff --git a/src/runtime/virtcontainers/hypervisor_config_test.go b/src/runtime/virtcontainers/hypervisor_config_test.go index 49558f6a97a9..e51773eaafee 100644 --- a/src/runtime/virtcontainers/hypervisor_config_test.go +++ b/src/runtime/virtcontainers/hypervisor_config_test.go @@ -28,3 +28,12 @@ func TestHypervisorConfigNoKernelPath(t *testing.T) { testHypervisorConfigValid(t, hypervisorConfig, false) } + +func TestRemoteHypervisorConfigNoKernelPath(t *testing.T) { + hypervisorConfig := &HypervisorConfig{ + RemoteHypervisorSocket: "dummy_socket", + KernelPath: "", + } + + testHypervisorConfigValid(t, hypervisorConfig, true) +} diff --git a/src/runtime/virtcontainers/hypervisor_linux.go b/src/runtime/virtcontainers/hypervisor_linux.go index ed73d97bc05e..ba5c38881116 100644 --- a/src/runtime/virtcontainers/hypervisor_linux.go +++ b/src/runtime/virtcontainers/hypervisor_linux.go @@ -40,6 +40,8 @@ func NewHypervisor(hType HypervisorType) (Hypervisor, error) { return &stratovirt{}, nil case DragonballHypervisor: return &mockHypervisor{}, nil + case RemoteHypervisor: + return &remoteHypervisor{}, nil case MockHypervisor: return &mockHypervisor{}, nil default: diff --git a/src/runtime/virtcontainers/hypervisor_test.go b/src/runtime/virtcontainers/hypervisor_test.go index 19bfdf4773b1..64794249f8df 100644 --- a/src/runtime/virtcontainers/hypervisor_test.go +++ b/src/runtime/virtcontainers/hypervisor_test.go @@ -7,11 +7,12 @@ package virtcontainers import ( "fmt" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" - "github.com/stretchr/testify/assert" "os" "strings" "testing" + + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" + "github.com/stretchr/testify/assert" ) func TestGetKernelRootParams(t *testing.T) { @@ -186,6 +187,10 @@ func TestSetMockHypervisorType(t *testing.T) { testSetHypervisorType(t, "mock", MockHypervisor) } +func TestSetRemoteHypervisorType(t *testing.T) { + testSetHypervisorType(t, "remote", RemoteHypervisor) +} + func TestSetUnknownHypervisorType(t *testing.T) { var hypervisorType HypervisorType assert := assert.New(t) @@ -207,6 +212,11 @@ func TestStringFromQemuHypervisorType(t *testing.T) { testStringFromHypervisorType(t, hypervisorType, "qemu") } +func TestStringFromRemoteHypervisorType(t *testing.T) { + hypervisorType := RemoteHypervisor + testStringFromHypervisorType(t, hypervisorType, "remote") +} + func TestStringFromMockHypervisorType(t *testing.T) { hypervisorType := MockHypervisor testStringFromHypervisorType(t, hypervisorType, "mock") @@ -224,6 +234,12 @@ func testNewHypervisorFromHypervisorType(t *testing.T, hypervisorType Hypervisor assert.Exactly(hy, expected) } +func TestNewHypervisorFromRemoteHypervisorType(t *testing.T) { + hypervisorType := RemoteHypervisor + expectedHypervisor := &remoteHypervisor{} + testNewHypervisorFromHypervisorType(t, hypervisorType, expectedHypervisor) +} + func TestNewHypervisorFromMockHypervisorType(t *testing.T) { hypervisorType := MockHypervisor expectedHypervisor := &mockHypervisor{} diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 9bb9f138fa9d..e9f7a61edf44 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -78,9 +78,13 @@ const ( defaultSeLinuxContainerType = "container_t" ) +type customRequestTimeoutKeyType struct{} + var ( checkRequestTimeout = 30 * time.Second defaultRequestTimeout = 60 * time.Second + remoteRequestTimeout = 300 * time.Second + customRequestTimeoutKey = customRequestTimeoutKeyType(struct{}{}) errorMissingOCISpec = errors.New("Missing OCI specification") defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/" defaultKataGuestSharedDir = "/run/kata-containers/shared/containers/" @@ -376,6 +380,8 @@ func (k *kataAgent) agentURL() (string, error) { return s.String(), nil case types.HybridVSock: return s.String(), nil + case types.RemoteSock: + return s.String(), nil case types.MockHybridVSock: return s.String(), nil default: @@ -426,6 +432,7 @@ func (k *kataAgent) configure(ctx context.Context, h Hypervisor, id, sharePath s if err != nil { return err } + case types.RemoteSock: case types.MockHybridVSock: default: return types.ErrInvalidConfigType @@ -745,37 +752,43 @@ func (k *kataAgent) startSandbox(ctx context.Context, sandbox *Sandbox) error { return err } - // Check grpc server is serving - if err = k.check(ctx); err != nil { - return err - } + var kmodules []*grpc.KernelModule - // If a Policy has been specified, send it to the agent. - if len(sandbox.config.AgentConfig.Policy) > 0 { - if err := sandbox.agent.setPolicy(ctx, sandbox.config.AgentConfig.Policy); err != nil { + if sandbox.config.HypervisorType == RemoteHypervisor { + ctx = context.WithValue(ctx, customRequestTimeoutKey, remoteRequestTimeout) + } else { + // Check grpc server is serving + if err = k.check(ctx); err != nil { return err } - } - // Setup network interfaces and routes - interfaces, routes, neighs, err := generateVCNetworkStructures(ctx, sandbox.network) - if err != nil { - return err - } - if err = k.updateInterfaces(ctx, interfaces); err != nil { - return err - } - if _, err = k.updateRoutes(ctx, routes); err != nil { - return err - } - if err = k.addARPNeighbors(ctx, neighs); err != nil { - return err + // If a Policy has been specified, send it to the agent. + if len(sandbox.config.AgentConfig.Policy) > 0 { + if err := sandbox.agent.setPolicy(ctx, sandbox.config.AgentConfig.Policy); err != nil { + return err + } + } + + // Setup network interfaces and routes + interfaces, routes, neighs, err := generateVCNetworkStructures(ctx, sandbox.network) + if err != nil { + return err + } + if err = k.updateInterfaces(ctx, interfaces); err != nil { + return err + } + if _, err = k.updateRoutes(ctx, routes); err != nil { + return err + } + if err = k.addARPNeighbors(ctx, neighs); err != nil { + return err + } + + kmodules = setupKernelModules(k.kmodules) } storages := setupStorages(ctx, sandbox) - kmodules := setupKernelModules(k.kmodules) - req := &grpc.CreateSandboxRequest{ Hostname: hostname, Dns: dns, @@ -2104,7 +2117,12 @@ func (k *kataAgent) getReqContext(ctx context.Context, reqName string) (newCtx c case grpcCheckRequest: newCtx, cancel = context.WithTimeout(ctx, checkRequestTimeout) default: - newCtx, cancel = context.WithTimeout(ctx, defaultRequestTimeout) + var requestTimeout = defaultRequestTimeout + + if timeout, ok := ctx.Value(customRequestTimeoutKey).(time.Duration); ok { + requestTimeout = timeout + } + newCtx, cancel = context.WithTimeout(ctx, requestTimeout) } return newCtx, cancel diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/client/client.go b/src/runtime/virtcontainers/pkg/agent/protocols/client/client.go index b31c86ad84b4..b44ee0d34ef5 100644 --- a/src/runtime/virtcontainers/pkg/agent/protocols/client/client.go +++ b/src/runtime/virtcontainers/pkg/agent/protocols/client/client.go @@ -34,6 +34,7 @@ import ( const ( VSockSocketScheme = "vsock" HybridVSockScheme = "hvsock" + RemoteSockScheme = "remote" MockHybridVSockScheme = "mock" ) @@ -235,6 +236,11 @@ func parse(sock string) (string, *url.URL, error) { } hybridVSockPort = uint32(port) grpcAddr = HybridVSockScheme + ":" + hvsocket[0] + case RemoteSockScheme: + if addr.Host != "" { + return "", nil, grpcStatus.Errorf(codes.InvalidArgument, "Invalid remote sock scheme: host address must be empty: %s", sock) + } + grpcAddr = RemoteSockScheme + ":" + addr.Path // just for tests use. case MockHybridVSockScheme: if addr.Path == "" { @@ -255,6 +261,8 @@ func agentDialer(addr *url.URL) dialer { return VsockDialer case HybridVSockScheme: return HybridVSockDialer + case RemoteSockScheme: + return RemoteSockDialer case MockHybridVSockScheme: return MockHybridVSockDialer default: @@ -435,6 +443,31 @@ func HybridVSockDialer(sock string, timeout time.Duration) (net.Conn, error) { return commonDialer(timeout, dialFunc, timeoutErr) } +// RemoteSockDialer dials to an agent in a remote hypervisor sandbox +func RemoteSockDialer(sock string, timeout time.Duration) (net.Conn, error) { + + s := strings.Split(sock, ":") + if !(len(s) == 2 && s[0] == RemoteSockScheme) { + return nil, fmt.Errorf("failed to parse remote sock: %q", sock) + } + socketPath := s[1] + + logrus.Printf("Dialing remote sock: %q %q", socketPath, sock) + + dialFunc := func() (net.Conn, error) { + conn, err := net.Dial("unix", socketPath) + if err != nil { + logrus.Errorf("failed to dial remote sock %q: %v", socketPath, err) + return nil, err + } + return conn, nil + } + + timeoutErr := grpcStatus.Errorf(codes.DeadlineExceeded, "timed out connecting to remote sock: %s", socketPath) + + return commonDialer(timeout, dialFunc, timeoutErr) +} + // just for tests use. func MockHybridVSockDialer(sock string, timeout time.Duration) (net.Conn, error) { sock = strings.TrimPrefix(sock, "mock:") diff --git a/src/runtime/virtcontainers/remote.go b/src/runtime/virtcontainers/remote.go new file mode 100644 index 000000000000..f79a21f3ff09 --- /dev/null +++ b/src/runtime/virtcontainers/remote.go @@ -0,0 +1,288 @@ +// Copyright (c) 2022 IBM Corporation +// SPDX-License-Identifier: Apache-2.0 + +package virtcontainers + +import ( + "context" + "fmt" + "os" + "time" + + cri "github.com/containerd/containerd/pkg/cri/annotations" + persistapi "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" + pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor" + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +const defaultMinTimeout = 60 + +type remoteHypervisor struct { + sandboxID remoteHypervisorSandboxID + agentSocketPath string + config HypervisorConfig +} + +type remoteHypervisorSandboxID string + +type remoteService struct { + conn *grpc.ClientConn + client pb.HypervisorClient +} + +func openRemoteService(socketPath string) (*remoteService, error) { + + conn, err := grpc.Dial(fmt.Sprintf("unix://%s", socketPath), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, fmt.Errorf("failed to connect to remote hypervisor socket: %w", err) + } + defer conn.Close() + client := pb.NewHypervisorClient(conn) + + s := &remoteService{ + conn: conn, + client: client, + } + + return s, nil +} + +func (s *remoteService) Close() error { + return s.conn.Close() +} + +func (rh *remoteHypervisor) CreateVM(ctx context.Context, id string, network Network, hypervisorConfig *HypervisorConfig) error { + + rh.sandboxID = remoteHypervisorSandboxID(id) + + if err := rh.setConfig(hypervisorConfig); err != nil { + return err + } + + s, err := openRemoteService(hypervisorConfig.RemoteHypervisorSocket) + if err != nil { + return err + } + defer s.Close() + + annotations := map[string]string{} + annotations[cri.SandboxName] = hypervisorConfig.SandboxName + annotations[cri.SandboxNamespace] = hypervisorConfig.SandboxNamespace + + req := &pb.CreateVMRequest{ + Id: id, + Annotations: annotations, + NetworkNamespacePath: network.NetworkID(), + } + + res, err := s.client.CreateVM(ctx, req) + if err != nil { + return fmt.Errorf("remote hypervisor call failed: %w", err) + } + + if res.AgentSocketPath == "" { + return errors.New("remote hypervisor does not return tunnel socket path") + } + + rh.agentSocketPath = res.AgentSocketPath + + return nil +} + +func (rh *remoteHypervisor) StartVM(ctx context.Context, timeout int) error { + + minTimeout := defaultMinTimeout + if rh.config.RemoteHypervisorTimeout > 0 { + minTimeout = int(rh.config.RemoteHypervisorTimeout) + } + + if timeout < minTimeout { + timeout = minTimeout + } + + s, err := openRemoteService(rh.config.RemoteHypervisorSocket) + if err != nil { + return err + } + defer s.Close() + + req := &pb.StartVMRequest{ + Id: string(rh.sandboxID), + } + + ctx2, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) + defer cancel() + + logrus.Printf("calling remote hypervisor StartVM (timeout: %d)", timeout) + + if _, err := s.client.StartVM(ctx2, req); err != nil { + return fmt.Errorf("remote hypervisor call failed: %w", err) + } + + return nil +} + +func (rh *remoteHypervisor) AttestVM(ctx context.Context) error { + return nil +} + +func (rh *remoteHypervisor) StopVM(ctx context.Context, waitOnly bool) error { + + s, err := openRemoteService(rh.config.RemoteHypervisorSocket) + if err != nil { + return err + } + defer s.Close() + + req := &pb.StopVMRequest{ + Id: string(rh.sandboxID), + } + + if _, err := s.client.StopVM(ctx, req); err != nil { + return fmt.Errorf("remote hypervisor call failed: %w", err) + } + + return nil +} + +func (rh *remoteHypervisor) GenerateSocket(id string) (interface{}, error) { + + socketPath := rh.agentSocketPath + if len(socketPath) == 0 { + return nil, errors.New("failed to generate remote sock: TunnelSocketPath is not set") + } + + remoteSock := types.RemoteSock{ + SandboxID: id, + TunnelSocketPath: socketPath, + } + + return remoteSock, nil +} + +func notImplemented(name string) error { + + err := errors.Errorf("%s: not implemented", name) + + logrus.Errorf(err.Error()) + + if tracer, ok := err.(interface{ StackTrace() errors.StackTrace }); ok { + for _, f := range tracer.StackTrace() { + logrus.Errorf("%+s:%d\n", f, f) + } + } + + return err +} + +func (rh *remoteHypervisor) PauseVM(ctx context.Context) error { + return notImplemented("PauseVM") +} + +func (rh *remoteHypervisor) SaveVM() error { + return notImplemented("SaveVM") +} + +func (rh *remoteHypervisor) ResumeVM(ctx context.Context) error { + return notImplemented("ResumeVM") +} + +func (rh *remoteHypervisor) AddDevice(ctx context.Context, devInfo interface{}, devType DeviceType) error { + // TODO should we return notImplemented("AddDevice"), rather than nil and ignoring it? + logrus.Printf("addDevice: deviceType=%v devInfo=%#v", devType, devInfo) + return nil +} + +func (rh *remoteHypervisor) HotplugAddDevice(ctx context.Context, devInfo interface{}, devType DeviceType) (interface{}, error) { + return nil, notImplemented("HotplugAddDevice") +} + +func (rh *remoteHypervisor) HotplugRemoveDevice(ctx context.Context, devInfo interface{}, devType DeviceType) (interface{}, error) { + return nil, notImplemented("HotplugRemoveDevice") +} + +func (rh *remoteHypervisor) ResizeMemory(ctx context.Context, memMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, MemoryDevice, error) { + return memMB, MemoryDevice{}, nil +} + +func (rh *remoteHypervisor) GetTotalMemoryMB(ctx context.Context) uint32 { + //The remote hypervisor uses the peer pod config to determine the memory of the VM, so we need to use static resource management + logrus.Error("GetTotalMemoryMB - remote hypervisor cannot update resources") + return 0 +} + +func (rh *remoteHypervisor) ResizeVCPUs(ctx context.Context, vcpus uint32) (uint32, uint32, error) { + return vcpus, vcpus, nil +} + +func (rh *remoteHypervisor) GetVMConsole(ctx context.Context, sandboxID string) (string, string, error) { + return "", "", notImplemented("GetVMConsole") +} + +func (rh *remoteHypervisor) Disconnect(ctx context.Context) { + notImplemented("Disconnect") +} + +func (rh *remoteHypervisor) Capabilities(ctx context.Context) types.Capabilities { + var caps types.Capabilities + caps.SetBlockDeviceHotplugSupport() + return caps +} + +func (rh *remoteHypervisor) HypervisorConfig() HypervisorConfig { + return rh.config +} + +func (rh *remoteHypervisor) GetThreadIDs(ctx context.Context) (VcpuThreadIDs, error) { + // Not supported. return success + // Just allocating an empty map + return VcpuThreadIDs{}, nil +} + +func (rh *remoteHypervisor) Cleanup(ctx context.Context) error { + return nil +} + +func (rh *remoteHypervisor) setConfig(config *HypervisorConfig) error { + // Create a Validator specific for remote hypervisor + rh.config = *config + + return nil +} + +func (rh *remoteHypervisor) GetPids() []int { + // let's use shim pid as it used by crio to fetch start time + return []int{os.Getpid()} +} + +func (rh *remoteHypervisor) GetVirtioFsPid() *int { + panic(notImplemented("GetVirtioFsPid")) +} + +func (rh *remoteHypervisor) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, j []byte) error { + panic(notImplemented("fromGrpc")) +} + +func (rh *remoteHypervisor) toGrpc(ctx context.Context) ([]byte, error) { + panic(notImplemented("toGrpc")) +} + +func (rh *remoteHypervisor) Check() error { + return nil +} + +func (rh *remoteHypervisor) Save() persistapi.HypervisorState { + return persistapi.HypervisorState{} +} + +func (rh *remoteHypervisor) Load(persistapi.HypervisorState) { + notImplemented("Load") +} + +func (rh *remoteHypervisor) IsRateLimiterBuiltin() bool { + return false +} diff --git a/src/runtime/virtcontainers/remote_test.go b/src/runtime/virtcontainers/remote_test.go new file mode 100644 index 000000000000..36d52b7aca41 --- /dev/null +++ b/src/runtime/virtcontainers/remote_test.go @@ -0,0 +1,45 @@ +// Copyright (c) 2023 IBM Corporation +// SPDX-License-Identifier: Apache-2.0 + +package virtcontainers + +import ( + "testing" + + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" + "github.com/stretchr/testify/assert" +) + +func newRemoteConfig() HypervisorConfig { + return HypervisorConfig{ + RemoteHypervisorSocket: "/run/peerpod/hypervisor.sock", + RemoteHypervisorTimeout: 600, + DisableGuestSeLinux: true, + EnableAnnotations: []string{}, + } +} + +func TestRemoteHypervisorGenerateSocket(t *testing.T) { + assert := assert.New(t) + + remoteHypervisor := remoteHypervisor{ + config: newRemoteConfig(), + } + id := "sandboxId" + + // No socketPath should error + _, err := remoteHypervisor.GenerateSocket(id) + assert.Error(err) + + socketPath := "socketPath" + remoteHypervisor.agentSocketPath = socketPath + + result, err := remoteHypervisor.GenerateSocket(id) + assert.NoError(err) + + expected := types.RemoteSock{ + SandboxID: id, + TunnelSocketPath: socketPath, + } + assert.Equal(result, expected) +} diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 9762467411ae..d921988292da 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -28,6 +28,8 @@ import ( "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" + cri "github.com/containerd/containerd/pkg/cri/annotations" + crio "github.com/containers/podman/v4/pkg/annotations" "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers" @@ -635,6 +637,8 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor } + setHypervisorConfigAnnotations(&sandboxConfig) + coldPlugVFIO, err := s.coldOrHotPlugVFIO(&sandboxConfig) if err != nil { return nil, err @@ -722,6 +726,23 @@ func (s *Sandbox) coldOrHotPlugVFIO(sandboxConfig *SandboxConfig) (bool, error) return coldPlugVFIO, nil } +func setHypervisorConfigAnnotations(sandboxConfig *SandboxConfig) { + if len(sandboxConfig.Containers) > 0 { + // These values are required by remote hypervisor + for _, a := range []string{cri.SandboxName, crio.SandboxName} { + if value, ok := sandboxConfig.Containers[0].Annotations[a]; ok { + sandboxConfig.HypervisorConfig.SandboxName = value + } + } + + for _, a := range []string{cri.SandboxNamespace, crio.Namespace} { + if value, ok := sandboxConfig.Containers[0].Annotations[a]; ok { + sandboxConfig.HypervisorConfig.SandboxNamespace = value + } + } + } +} + func (s *Sandbox) createResourceController() error { var err error cgroupPath := "" diff --git a/src/runtime/virtcontainers/types/sandbox.go b/src/runtime/virtcontainers/types/sandbox.go index 5149b0423297..29c909c977fb 100644 --- a/src/runtime/virtcontainers/types/sandbox.go +++ b/src/runtime/virtcontainers/types/sandbox.go @@ -7,6 +7,7 @@ package types import ( "fmt" + "net" "os" "strings" @@ -37,6 +38,7 @@ const ( HybridVSockScheme = "hvsock" MockHybridVSockScheme = "mock" VSockScheme = "vsock" + RemoteSockScheme = "remote" ) // SandboxState is a sandbox state structure @@ -210,6 +212,16 @@ func (s *HybridVSock) String() string { return fmt.Sprintf("%s://%s:%d", HybridVSockScheme, s.UdsPath, s.Port) } +type RemoteSock struct { + Conn net.Conn + SandboxID string + TunnelSocketPath string +} + +func (s *RemoteSock) String() string { + return fmt.Sprintf("%s://%s", RemoteSockScheme, s.TunnelSocketPath) +} + // MockHybridVSock defines a mock hybrid vsocket for tests only. type MockHybridVSock struct { UdsPath string diff --git a/src/runtime/virtcontainers/types/sandbox_test.go b/src/runtime/virtcontainers/types/sandbox_test.go index 05075e449592..54e2e78c494f 100644 --- a/src/runtime/virtcontainers/types/sandbox_test.go +++ b/src/runtime/virtcontainers/types/sandbox_test.go @@ -6,6 +6,8 @@ package types import ( + "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -163,6 +165,78 @@ func TestVolumesStringSuccessful(t *testing.T) { assert.Equal(t, result, expected) } +func TestStringFromVSock(t *testing.T) { + assert := assert.New(t) + + dir := t.TempDir() + + contextID := uint64(16187) + port := uint32(1024) + vsockFilename := filepath.Join(dir, "vsock") + + vsockFile, err := os.Create(vsockFilename) + assert.NoError(err) + defer vsockFile.Close() + + vsock := VSock{ + ContextID: contextID, + Port: port, + VhostFd: vsockFile, + } + + expected := "vsock://16187:1024" + + assert.Equal(vsock.String(), expected) +} + +func TestStringFromHybridVSock(t *testing.T) { + assert := assert.New(t) + + udsPath := "udspath" + contextID := uint64(16187) + port := uint32(1024) + + sock := HybridVSock{ + UdsPath: udsPath, + ContextID: contextID, + Port: port, + } + + expected := "hvsock://udspath:1024" + + assert.Equal(sock.String(), expected) +} + +func TestStringFromRemoteSock(t *testing.T) { + assert := assert.New(t) + + sandboxID := "sandboxID" + tunnelSockerPath := "tunnelSocketPath" + + sock := RemoteSock{ + SandboxID: sandboxID, + TunnelSocketPath: tunnelSockerPath, + } + + expected := "remote://tunnelSocketPath" + + assert.Equal(sock.String(), expected) +} + +func TestStringFromMockHybridVSock(t *testing.T) { + assert := assert.New(t) + + udsPath := "udspath" + + sock := MockHybridVSock{ + UdsPath: udsPath, + } + + expected := "mock://udspath" + + assert.Equal(sock.String(), expected) +} + func TestSocketsSetSuccessful(t *testing.T) { sockets := &Sockets{}