runtime: Support the remote hypervisor type

This patch adds the support of the remote hypervisor type.
Shim opens a Unix domain socket specified in the config file,
and sends TTPRC requests to a external process to control
sandbox VMs.

Fixes kata-containers#4482

Co-authored-by: Pradipta Banerjee <[email protected]>
Co-authored-by: stevenhorsman <[email protected]>
Signed-off-by: Yohei Ueda <[email protected]>
(based on commit f9278f2)

src/runtime/cmd/kata-runtime/kata-check_amd64.go

@@ -323,6 +323,8 @@ func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
 		return kvmIsUsable()
 	case vc.AcrnHypervisor:
 		return acrnIsUsable()
+	case vc.RemoteHypervisor:
+		return nil
 	case vc.MockHypervisor:
 		return nil
 	default:

src/runtime/cmd/kata-runtime/kata-check_arm64.go

@@ -86,6 +86,9 @@ func checkKVMExtensions() error {
 }
 
 func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+	if hypervisorType == "remote" {
+		return nil
+	}
 	if err := kvmIsUsable(); err != nil {
 		return err
 	}

src/runtime/cmd/kata-runtime/kata-check_ppc64le.go

@@ -61,6 +61,9 @@ func setCPUtype(hypervisorType vc.HypervisorType) error {
 }
 
 func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+	if hypervisorType == "remote" {
+		return nil
+	}
 	return kvmIsUsable()
 }
 

src/runtime/cmd/kata-runtime/kata-check_s390x.go

@@ -55,6 +55,9 @@ func kvmIsUsable() error {
 }
 
 func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+	if hypervisorType == "remote" {
+		return nil
+	}
 	return kvmIsUsable()
 }
 

src/runtime/pkg/katautils/config.go

@@ -53,6 +53,7 @@ const (
 	acrnHypervisorTableType        = "acrn"
 	dragonballHypervisorTableType  = "dragonball"
 	stratovirtHypervisorTableType  = "stratovirt"
+	remoteHypervisorTableType      = "remote"
 
 	// the maximum amount of PCI bridges that can be cold plugged in a VM
 	maxPCIBridges uint32 = 5
@@ -105,6 +106,7 @@ type hypervisor struct {
 	GuestMemoryDumpPath            string                    `toml:"guest_memory_dump_path"`
 	SeccompSandbox                 string                    `toml:"seccompsandbox"`
 	BlockDeviceAIO                 string                    `toml:"block_device_aio"`
+	RemoteHypervisorSocket         string                    `toml:"remote_hypervisor_socket"`
 	HypervisorPathList             []string                  `toml:"valid_hypervisor_paths"`
 	JailerPathList                 []string                  `toml:"valid_jailer_paths"`
 	CtlPathList                    []string                  `toml:"valid_ctlpaths"`
@@ -134,6 +136,7 @@ type hypervisor struct {
 	MemSlots                       uint32                    `toml:"memory_slots"`
 	DefaultBridges                 uint32                    `toml:"default_bridges"`
 	Msize9p                        uint32                    `toml:"msize_9p"`
+	RemoteHypervisorTimeout        uint32                    `toml:"remote_hypervisor_timeout"`
 	NumVCPUs                       float32                   `toml:"default_vcpus"`
 	BlockDeviceCacheSet            bool                      `toml:"block_device_cache_set"`
 	BlockDeviceCacheDirect         bool                      `toml:"block_device_cache_direct"`
@@ -1242,6 +1245,14 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 	}, nil
 }
 
+func newRemoteHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
+
+	return vc.HypervisorConfig{
+		RemoteHypervisorSocket:  h.RemoteHypervisorSocket,
+		RemoteHypervisorTimeout: h.RemoteHypervisorTimeout,
+	}, nil
+}
+
 func newFactoryConfig(f factory) (oci.FactoryConfig, error) {
 	if f.TemplatePath == "" {
 		f.TemplatePath = defaultTemplatePath
@@ -1281,6 +1292,9 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi
 		case stratovirtHypervisorTableType:
 			config.HypervisorType = vc.StratovirtHypervisor
 			hConfig, err = newStratovirtHypervisorConfig(hypervisor)
+		case remoteHypervisorTableType:
+			config.HypervisorType = vc.RemoteHypervisor
+			hConfig, err = newRemoteHypervisorConfig(hypervisor)
 		default:
 			err = fmt.Errorf("%s: %+q", errInvalidHypervisorPrefix, k)
 		}
@@ -1882,6 +1896,11 @@ func checkFactoryConfig(config oci.RuntimeConfig) error {
 // checkHypervisorConfig performs basic "sanity checks" on the hypervisor
 // config.
 func checkHypervisorConfig(config vc.HypervisorConfig) error {
+
+	if config.RemoteHypervisorSocket != "" {
+		return nil
+	}
+
 	type image struct {
 		path   string
 		initrd bool

src/runtime/pkg/katautils/config_test.go

@@ -1564,6 +1564,17 @@ func TestCheckHypervisorConfig(t *testing.T) {
 		// reset logger
 		kataUtilsLogger.Logger.Out = savedOut
 	}
+
+	// Check remote hypervisor doesn't error with missing unnescessary config
+	remoteConfig := vc.HypervisorConfig{
+		RemoteHypervisorSocket: "dummy_socket",
+		ImagePath:              "",
+		InitrdPath:             "",
+		MemorySize:             0,
+	}
+
+	err := checkHypervisorConfig(remoteConfig)
+	assert.NoError(err, "remote hypervisor config")
 }
 
 func TestCheckNetNsConfig(t *testing.T) {

src/runtime/protocols/hypervisor/hypervisor.proto

@@ -1,3 +1,6 @@
+// (C) Copyright IBM Corp. 2022.
+// SPDX-License-Identifier: Apache-2.0
+
 syntax = "proto3";
 
 option go_package = "./";

src/runtime/virtcontainers/hypervisor.go

@@ -57,6 +57,9 @@ const (
 	// VirtFrameworkHypervisor is the Darwin Virtualization.framework hypervisor
 	VirtframeworkHypervisor HypervisorType = "virtframework"
 
+	// RemoteHypervisor is the Remote hypervisor.
+	RemoteHypervisor HypervisorType = "remote"
+
 	// MockHypervisor is a mock hypervisor for testing purposes
 	MockHypervisor HypervisorType = "mock"
 
@@ -240,6 +243,9 @@ func (hType *HypervisorType) Set(value string) error {
 	case "virtframework":
 		*hType = VirtframeworkHypervisor
 		return nil
+	case "remote":
+		*hType = RemoteHypervisor
+		return nil
 	case "mock":
 		*hType = MockHypervisor
 		return nil
@@ -261,6 +267,8 @@ func (hType *HypervisorType) String() string {
 		return string(ClhHypervisor)
 	case StratovirtHypervisor:
 		return string(StratovirtHypervisor)
+	case RemoteHypervisor:
+		return string(RemoteHypervisor)
 	case MockHypervisor:
 		return string(MockHypervisor)
 	default:
@@ -455,6 +463,15 @@ type HypervisorConfig struct {
 	// BlockiDeviceAIO specifies the I/O API to be used.
 	BlockDeviceAIO string
 
+	// The socket to connect to the remote hypervisor implementation on
+	RemoteHypervisorSocket string
+
+	// The name of the sandbox (pod)
+	SandboxName string
+
+	// The name of the namespace of the sandbox (pod)
+	SandboxNamespace string
+
 	// The user maps to the uid.
 	User string
 
@@ -563,6 +580,9 @@ type HypervisorConfig struct {
 	// Group ID.
 	Gid uint32
 
+	// Timeout for actions e.g. startVM for the remote hypervisor
+	RemoteHypervisorTimeout uint32
+
 	// BlockDeviceCacheSet specifies cache-related options will be set to block devices or not.
 	BlockDeviceCacheSet bool
 

src/runtime/virtcontainers/hypervisor_config_darwin.go

@@ -11,6 +11,10 @@ import (
 
 func validateHypervisorConfig(conf *HypervisorConfig) error {
 
+	if conf.RemoteHypervisorSocket != "" {
+		return nil
+	}
+
 	if conf.KernelPath == "" {
 		return fmt.Errorf("Missing kernel path")
 	}

src/runtime/virtcontainers/hypervisor_config_linux.go

@@ -13,6 +13,10 @@ import (
 
 func validateHypervisorConfig(conf *HypervisorConfig) error {
 
+	if conf.RemoteHypervisorSocket != "" {
+		return nil
+	}
+
 	if conf.KernelPath == "" {
 		return fmt.Errorf("Missing kernel path")
 	}

src/runtime/virtcontainers/hypervisor_config_test.go

@@ -28,3 +28,12 @@ func TestHypervisorConfigNoKernelPath(t *testing.T) {
 
 	testHypervisorConfigValid(t, hypervisorConfig, false)
 }
+
+func TestRemoteHypervisorConfigNoKernelPath(t *testing.T) {
+	hypervisorConfig := &HypervisorConfig{
+		RemoteHypervisorSocket: "dummy_socket",
+		KernelPath:             "",
+	}
+
+	testHypervisorConfigValid(t, hypervisorConfig, true)
+}

src/runtime/virtcontainers/hypervisor_linux.go

@@ -40,6 +40,8 @@ func NewHypervisor(hType HypervisorType) (Hypervisor, error) {
 		return &stratovirt{}, nil
 	case DragonballHypervisor:
 		return &mockHypervisor{}, nil
+	case RemoteHypervisor:
+		return &remoteHypervisor{}, nil
 	case MockHypervisor:
 		return &mockHypervisor{}, nil
 	default:

src/runtime/virtcontainers/hypervisor_test.go

@@ -7,11 +7,12 @@ package virtcontainers
 
 import (
 	"fmt"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
-	"github.com/stretchr/testify/assert"
 	"os"
 	"strings"
 	"testing"
+
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestGetKernelRootParams(t *testing.T) {
@@ -186,6 +187,10 @@ func TestSetMockHypervisorType(t *testing.T) {
 	testSetHypervisorType(t, "mock", MockHypervisor)
 }
 
+func TestSetRemoteHypervisorType(t *testing.T) {
+	testSetHypervisorType(t, "remote", RemoteHypervisor)
+}
+
 func TestSetUnknownHypervisorType(t *testing.T) {
 	var hypervisorType HypervisorType
 	assert := assert.New(t)
@@ -207,6 +212,11 @@ func TestStringFromQemuHypervisorType(t *testing.T) {
 	testStringFromHypervisorType(t, hypervisorType, "qemu")
 }
 
+func TestStringFromRemoteHypervisorType(t *testing.T) {
+	hypervisorType := RemoteHypervisor
+	testStringFromHypervisorType(t, hypervisorType, "remote")
+}
+
 func TestStringFromMockHypervisorType(t *testing.T) {
 	hypervisorType := MockHypervisor
 	testStringFromHypervisorType(t, hypervisorType, "mock")
@@ -224,6 +234,12 @@ func testNewHypervisorFromHypervisorType(t *testing.T, hypervisorType Hypervisor
 	assert.Exactly(hy, expected)
 }
 
+func TestNewHypervisorFromRemoteHypervisorType(t *testing.T) {
+	hypervisorType := RemoteHypervisor
+	expectedHypervisor := &remoteHypervisor{}
+	testNewHypervisorFromHypervisorType(t, hypervisorType, expectedHypervisor)
+}
+
 func TestNewHypervisorFromMockHypervisorType(t *testing.T) {
 	hypervisorType := MockHypervisor
 	expectedHypervisor := &mockHypervisor{}

src/runtime/virtcontainers/kata_agent.go

@@ -78,9 +78,13 @@ const (
 	defaultSeLinuxContainerType = "container_t"
 )
 
+type customRequestTimeoutKeyType struct{}
+
 var (
 	checkRequestTimeout           = 30 * time.Second
 	defaultRequestTimeout         = 60 * time.Second
+	remoteRequestTimeout          = 300 * time.Second
+	customRequestTimeoutKey       = customRequestTimeoutKeyType(struct{}{})
 	errorMissingOCISpec           = errors.New("Missing OCI specification")
 	defaultKataHostSharedDir      = "/run/kata-containers/shared/sandboxes/"
 	defaultKataGuestSharedDir     = "/run/kata-containers/shared/containers/"
@@ -376,6 +380,8 @@ func (k *kataAgent) agentURL() (string, error) {
 		return s.String(), nil
 	case types.HybridVSock:
 		return s.String(), nil
+	case types.RemoteSock:
+		return s.String(), nil
 	case types.MockHybridVSock:
 		return s.String(), nil
 	default:
@@ -426,6 +432,7 @@ func (k *kataAgent) configure(ctx context.Context, h Hypervisor, id, sharePath s
 		if err != nil {
 			return err
 		}
+	case types.RemoteSock:
 	case types.MockHybridVSock:
 	default:
 		return types.ErrInvalidConfigType
@@ -745,37 +752,43 @@ func (k *kataAgent) startSandbox(ctx context.Context, sandbox *Sandbox) error {
 		return err
 	}
 
-	// Check grpc server is serving
-	if err = k.check(ctx); err != nil {
-		return err
-	}
+	var kmodules []*grpc.KernelModule
 
-	// If a Policy has been specified, send it to the agent.
-	if len(sandbox.config.AgentConfig.Policy) > 0 {
-		if err := sandbox.agent.setPolicy(ctx, sandbox.config.AgentConfig.Policy); err != nil {
+	if sandbox.config.HypervisorType == RemoteHypervisor {
+		ctx = context.WithValue(ctx, customRequestTimeoutKey, remoteRequestTimeout)
+	} else {
+		// Check grpc server is serving
+		if err = k.check(ctx); err != nil {
 			return err
 		}
-	}
 
-	// Setup network interfaces and routes
-	interfaces, routes, neighs, err := generateVCNetworkStructures(ctx, sandbox.network)
-	if err != nil {
-		return err
-	}
-	if err = k.updateInterfaces(ctx, interfaces); err != nil {
-		return err
-	}
-	if _, err = k.updateRoutes(ctx, routes); err != nil {
-		return err
-	}
-	if err = k.addARPNeighbors(ctx, neighs); err != nil {
-		return err
+		// If a Policy has been specified, send it to the agent.
+		if len(sandbox.config.AgentConfig.Policy) > 0 {
+			if err := sandbox.agent.setPolicy(ctx, sandbox.config.AgentConfig.Policy); err != nil {
+				return err
+			}
+		}
+
+		// Setup network interfaces and routes
+		interfaces, routes, neighs, err := generateVCNetworkStructures(ctx, sandbox.network)
+		if err != nil {
+			return err
+		}
+		if err = k.updateInterfaces(ctx, interfaces); err != nil {
+			return err
+		}
+		if _, err = k.updateRoutes(ctx, routes); err != nil {
+			return err
+		}
+		if err = k.addARPNeighbors(ctx, neighs); err != nil {
+			return err
+		}
+
+		kmodules = setupKernelModules(k.kmodules)
 	}
 
 	storages := setupStorages(ctx, sandbox)
 
-	kmodules := setupKernelModules(k.kmodules)
-
 	req := &grpc.CreateSandboxRequest{
 		Hostname:      hostname,
 		Dns:           dns,
@@ -2104,7 +2117,12 @@ func (k *kataAgent) getReqContext(ctx context.Context, reqName string) (newCtx c
 	case grpcCheckRequest:
 		newCtx, cancel = context.WithTimeout(ctx, checkRequestTimeout)
 	default:
-		newCtx, cancel = context.WithTimeout(ctx, defaultRequestTimeout)
+		var requestTimeout = defaultRequestTimeout
+
+		if timeout, ok := ctx.Value(customRequestTimeoutKey).(time.Duration); ok {
+			requestTimeout = timeout
+		}
+		newCtx, cancel = context.WithTimeout(ctx, requestTimeout)
 	}
 
 	return newCtx, cancel