From 1996539698427ee18c8bb1c64180cf2892fa0253 Mon Sep 17 00:00:00 2001 From: Hamza El-Saawy Date: Wed, 11 Sep 2024 13:58:59 -0400 Subject: [PATCH] Update uvmboot WCOW, remove globals Update WCOW uvmboot command to work, and make functionality more inline with LCOW's. Specifically, add the `fwd-std*` and `output-handling` flags, and share them, along with the `exec` flag definition, between the two. Add a `no-cmd` flag to skip prepending `cmd /c` to the exec command and allow passing in a command line exactly. Fix naming conflicts, where `c` was either a `*cli.Context` or `*cmd.Cmd`, the former is now consistently `cCtx` (which matches usage elsewhere). Rename `cmd` variables to prevent overshadowing module import. Replace global variables with calls to `cCtx.*` flag-access functions to avoid global (mutatable) state. Signed-off-by: Hamza El-Saawy --- internal/tools/uvmboot/lcow.go | 142 ++++++++----------- internal/tools/uvmboot/main.go | 86 +++++++++--- internal/tools/uvmboot/wcow.go | 248 ++++++++++++++++++++++----------- 3 files changed, 286 insertions(+), 190 deletions(-) diff --git a/internal/tools/uvmboot/lcow.go b/internal/tools/uvmboot/lcow.go index 5779c4aa93..40e338440c 100644 --- a/internal/tools/uvmboot/lcow.go +++ b/internal/tools/uvmboot/lcow.go @@ -22,11 +22,9 @@ const ( consolePipeArgName = "console-pipe" kernelDirectArgName = "kernel-direct" kernelFileArgName = "kernel-file" - forwardStdoutArgName = "fwd-stdout" - forwardStderrArgName = "fwd-stderr" - outputHandlingArgName = "output-handling" kernelArgsArgName = "kernel-args" rootFSTypeArgName = "root-fs-type" + disableTimeSyncArgName = "disable-time-sync" vpMemMaxCountArgName = "vpmem-max-count" vpMemMaxSizeArgName = "vpmem-max-size" scsiMountsArgName = "mount-scsi" @@ -37,15 +35,12 @@ const ( securityPolicyEnforcerArgName = "security-policy-enforcer" ) -var ( - lcowUseTerminal bool - lcowDisableTimeSync bool -) - var lcowCommand = cli.Command{ Name: "lcow", Usage: "Boot an LCOW UVM", - Flags: []cli.Flag{ + CustomHelpTemplate: cli.CommandHelpTemplate + "EXAMPLES:\n" + + ` .\uvmboot.exe -gcs lcow -boot-files-path "C:\ContainerPlat\LinuxBootFiles" -root-fs-type vhd -t -exec "/bin/bash"`, + Flags: append(commonUVMFlags, cli.StringFlag{ Name: kernelArgsArgName, Value: "", @@ -76,9 +71,8 @@ var lcowCommand = cli.Command{ Usage: "The kernel `file` to use; either 'kernel' or 'vmlinux'. (default: 'kernel')", }, cli.BoolFlag{ - Name: "disable-time-sync", - Usage: "Disable the time synchronization service", - Destination: &lcowDisableTimeSync, + Name: disableTimeSyncArgName, + Usage: "Disable the time synchronization service", }, cli.StringFlag{ Name: securityPolicyArgName, @@ -93,31 +87,10 @@ var lcowCommand = cli.Command{ Name: securityHardwareFlag, Usage: "Use VMGS file to run on secure hardware. ('root-fs-type' must be set to 'none')", }, - cli.StringFlag{ - Name: execCommandLineArgName, - Usage: "Command to execute in the UVM.", - }, - cli.BoolFlag{ - Name: forwardStdoutArgName, - Usage: "Whether stdout from the process in the UVM should be forwarded", - }, - cli.BoolFlag{ - Name: forwardStderrArgName, - Usage: "Whether stderr from the process in the UVM should be forwarded", - }, - cli.StringFlag{ - Name: outputHandlingArgName, - Usage: "Controls how output from UVM is handled. Use 'stdout' to print all output to stdout", - }, cli.StringFlag{ Name: consolePipeArgName, Usage: "Named pipe for serial console output (which will be enabled)", }, - cli.BoolFlag{ - Name: "tty,t", - Usage: "create the process in the UVM with a TTY enabled", - Destination: &lcowUseTerminal, - }, cli.StringSliceFlag{ Name: scsiMountsArgName, Usage: "List of VHDs to SCSI mount into the UVM. Use repeat instances to add multiple. " + @@ -134,58 +107,53 @@ var lcowCommand = cli.Command{ Name: vpmemMountsArgName, Usage: "List of VHDs to VPMem mount into the UVM. Use repeat instances to add multiple. ", }, - }, - Action: func(c *cli.Context) error { - runMany(c, func(id string) error { + ), + Action: func(cCtx *cli.Context) error { + runMany(cCtx, func(id string) error { ctx := context.Background() - options, err := createLCOWOptions(ctx, c, id) + options, err := createLCOWOptions(ctx, cCtx, id) if err != nil { return err } - return runLCOW(ctx, options, c) + return runLCOW(ctx, cCtx, options) }) return nil }, } -func init() { - lcowCommand.CustomHelpTemplate = cli.CommandHelpTemplate + "EXAMPLES:\n" + - `.\uvmboot.exe -gcs lcow -boot-files-path "C:\ContainerPlat\LinuxBootFiles" -root-fs-type vhd -t -exec "/bin/bash"` -} - -func createLCOWOptions(ctx context.Context, c *cli.Context, id string) (*uvm.OptionsLCOW, error) { +func createLCOWOptions(ctx context.Context, cCtx *cli.Context, id string) (*uvm.OptionsLCOW, error) { options := uvm.NewDefaultOptionsLCOW(id, "") - setGlobalOptions(c, options.Options) + setGlobalOptions(cCtx, options.Options) // boot - if c.IsSet(bootFilesPathArgName) { - options.UpdateBootFilesPath(ctx, c.String(bootFilesPathArgName)) + if cCtx.IsSet(bootFilesPathArgName) { + options.UpdateBootFilesPath(ctx, cCtx.String(bootFilesPathArgName)) } // kernel - if c.IsSet(kernelDirectArgName) { - options.KernelDirect = c.Bool(kernelDirectArgName) + if cCtx.IsSet(kernelDirectArgName) { + options.KernelDirect = cCtx.Bool(kernelDirectArgName) } - if c.IsSet(kernelFileArgName) { - switch strings.ToLower(c.String(kernelFileArgName)) { + if cCtx.IsSet(kernelFileArgName) { + switch strings.ToLower(cCtx.String(kernelFileArgName)) { case uvm.KernelFile: options.KernelFile = uvm.KernelFile case uvm.UncompressedKernelFile: options.KernelFile = uvm.UncompressedKernelFile default: - return nil, unrecognizedError(c.String(kernelFileArgName), kernelFileArgName) + return nil, unrecognizedError(cCtx.String(kernelFileArgName), kernelFileArgName) } } - if c.IsSet(kernelArgsArgName) { - options.KernelBootOptions = c.String(kernelArgsArgName) + if cCtx.IsSet(kernelArgsArgName) { + options.KernelBootOptions = cCtx.String(kernelArgsArgName) } // rootfs - if c.IsSet(rootFSTypeArgName) { - switch strings.ToLower(c.String(rootFSTypeArgName)) { + if cCtx.IsSet(rootFSTypeArgName) { + switch strings.ToLower(cCtx.String(rootFSTypeArgName)) { case "initrd": options.RootFSFile = uvm.InitrdFile options.PreferredRootFSType = uvm.PreferredRootFSTypeInitRd @@ -196,31 +164,31 @@ func createLCOWOptions(ctx context.Context, c *cli.Context, id string) (*uvm.Opt options.RootFSFile = "" options.PreferredRootFSType = uvm.PreferredRootFSTypeNA default: - return nil, unrecognizedError(c.String(rootFSTypeArgName), rootFSTypeArgName) + return nil, unrecognizedError(cCtx.String(rootFSTypeArgName), rootFSTypeArgName) } } - if c.IsSet(vpMemMaxCountArgName) { - options.VPMemDeviceCount = uint32(c.Uint(vpMemMaxCountArgName)) + if cCtx.IsSet(vpMemMaxCountArgName) { + options.VPMemDeviceCount = uint32(cCtx.Uint(vpMemMaxCountArgName)) } - if c.IsSet(vpMemMaxSizeArgName) { - options.VPMemSizeBytes = c.Uint64(vpMemMaxSizeArgName) * memory.MiB // convert from MB to bytes + if cCtx.IsSet(vpMemMaxSizeArgName) { + options.VPMemSizeBytes = cCtx.Uint64(vpMemMaxSizeArgName) * memory.MiB // convert from MB to bytes } // GCS - options.UseGuestConnection = useGCS - if !useGCS { - if c.IsSet(execCommandLineArgName) { - options.ExecCommandLine = c.String(execCommandLineArgName) + options.UseGuestConnection = cCtx.GlobalBool(useGCSArgName) + if !options.UseGuestConnection { + if cCtx.IsSet(execCommandLineArgName) { + options.ExecCommandLine = cCtx.String(execCommandLineArgName) } - if c.IsSet(forwardStdoutArgName) { - options.ForwardStdout = c.Bool(forwardStdoutArgName) + if cCtx.IsSet(forwardStdoutArgName) { + options.ForwardStdout = cCtx.Bool(forwardStdoutArgName) } - if c.IsSet(forwardStderrArgName) { - options.ForwardStderr = c.Bool(forwardStderrArgName) + if cCtx.IsSet(forwardStderrArgName) { + options.ForwardStderr = cCtx.Bool(forwardStderrArgName) } - if c.IsSet(outputHandlingArgName) { - switch strings.ToLower(c.String(outputHandlingArgName)) { + if cCtx.IsSet(outputHandlingArgName) { + switch strings.ToLower(cCtx.String(outputHandlingArgName)) { case "stdout": options.OutputHandlerCreator = func(*uvm.Options) uvm.OutputHandler { return func(r io.Reader) { @@ -228,27 +196,27 @@ func createLCOWOptions(ctx context.Context, c *cli.Context, id string) (*uvm.Opt } } default: - return nil, unrecognizedError(c.String(outputHandlingArgName), outputHandlingArgName) + return nil, unrecognizedError(cCtx.String(outputHandlingArgName), outputHandlingArgName) } } } - if c.IsSet(consolePipeArgName) { - options.ConsolePipe = c.String(consolePipeArgName) + if cCtx.IsSet(consolePipeArgName) { + options.ConsolePipe = cCtx.String(consolePipeArgName) } // general settings - if lcowDisableTimeSync { - options.DisableTimeSyncService = true + if cCtx.IsSet(disableTimeSyncArgName) { + options.DisableTimeSyncService = cCtx.Bool(disableTimeSyncArgName) } // empty policy string defaults to open door - if c.IsSet(securityPolicyArgName) { - options.SecurityPolicy = c.String(securityPolicyArgName) + if cCtx.IsSet(securityPolicyArgName) { + options.SecurityPolicy = cCtx.String(securityPolicyArgName) } - if c.IsSet(securityPolicyEnforcerArgName) { - options.SecurityPolicyEnforcer = c.String(securityPolicyEnforcerArgName) + if cCtx.IsSet(securityPolicyEnforcerArgName) { + options.SecurityPolicyEnforcer = cCtx.String(securityPolicyEnforcerArgName) } - if c.IsSet(securityHardwareFlag) { + if cCtx.IsSet(securityHardwareFlag) { options.GuestStateFile = uvm.GuestStateFile options.SecurityPolicyEnabled = true options.AllowOvercommit = false @@ -257,7 +225,7 @@ func createLCOWOptions(ctx context.Context, c *cli.Context, id string) (*uvm.Opt return options, nil } -func runLCOW(ctx context.Context, options *uvm.OptionsLCOW, c *cli.Context) error { +func runLCOW(ctx context.Context, cCtx *cli.Context, options *uvm.OptionsLCOW) error { vm, err := uvm.CreateLCOW(ctx, options) if err != nil { return err @@ -270,20 +238,20 @@ func runLCOW(ctx context.Context, options *uvm.OptionsLCOW, c *cli.Context) erro return err } - if err := mountSCSI(ctx, c, vm); err != nil { + if err := mountSCSI(ctx, cCtx, vm); err != nil { return err } - if err := shareFiles(ctx, c, vm); err != nil { + if err := shareFiles(ctx, cCtx, vm); err != nil { return err } - if err := mountVPMem(ctx, c, vm); err != nil { + if err := mountVPMem(ctx, cCtx, vm); err != nil { return err } if options.UseGuestConnection { - if err := execViaGCS(ctx, vm, c); err != nil { + if err := execViaGCS(ctx, vm, cCtx); err != nil { return err } _ = vm.Terminate(ctx) @@ -298,7 +266,7 @@ func runLCOW(ctx context.Context, options *uvm.OptionsLCOW, c *cli.Context) erro func execViaGCS(ctx context.Context, vm *uvm.UtilityVM, cCtx *cli.Context) error { c := cmd.CommandContext(ctx, vm, "sh", "-c", cCtx.String(execCommandLineArgName)) c.Log = log.L.Dup() - if lcowUseTerminal { + if cCtx.Bool(useTerminalArgName) { c.Spec.Terminal = true c.Stdin = os.Stdin c.Stdout = os.Stdout diff --git a/internal/tools/uvmboot/main.go b/internal/tools/uvmboot/main.go index 891012ec77..297203d3b9 100644 --- a/internal/tools/uvmboot/main.go +++ b/internal/tools/uvmboot/main.go @@ -4,18 +4,20 @@ package main import ( "fmt" - "log" "os" "sync" "time" "github.com/sirupsen/logrus" "github.com/urfave/cli" + "go.opencensus.io/trace" + "github.com/Microsoft/hcsshim/internal/oc" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/internal/winapi" ) +// Global flag names. const ( cpusArgName = "cpus" memoryArgName = "memory" @@ -24,18 +26,48 @@ const ( measureArgName = "measure" parallelArgName = "parallel" countArgName = "count" + useGCSArgName = "gcs" +) +// Shared command flag names. +const ( execCommandLineArgName = "exec" + forwardStdoutArgName = "fwd-stdout" + forwardStderrArgName = "fwd-stderr" + outputHandlingArgName = "output-handling" + useTerminalArgName = "tty" ) -var ( - debug bool - useGCS bool -) +// Shared command flags. +var commonUVMFlags = []cli.Flag{ + cli.StringFlag{ + Name: execCommandLineArgName, + Usage: "Command to execute in the UVM.", + }, + cli.BoolFlag{ + Name: forwardStdoutArgName, + Usage: "Whether stdout from the process in the UVM should be forwarded", + }, + cli.BoolFlag{ + Name: forwardStderrArgName, + Usage: "Whether stderr from the process in the UVM should be forwarded", + }, + cli.StringFlag{ + Name: outputHandlingArgName, + Usage: "Controls how output from UVM is handled. Use 'stdout' to print all output to stdout", + }, + cli.BoolFlag{ + Name: useTerminalArgName + ",t", + Usage: "create the process in the UVM with a TTY enabled", + }, +} type uvmRunFunc func(string) error func main() { + var debugLogs bool + var traceLogs bool + app := cli.NewApp() app.Name = "uvmboot" app.Usage = "Boot a utility VM" @@ -73,13 +105,17 @@ func main() { }, cli.BoolFlag{ Name: "debug", - Usage: "Enable debug information", - Destination: &debug, + Usage: "Enable debug logs", + Destination: &debugLogs, }, cli.BoolFlag{ - Name: "gcs", - Usage: "Launch the GCS and perform requested operations via its RPC interface", - Destination: &useGCS, + Name: "trace", + Usage: "Enable trace logs (implies debug logs)", + Destination: &traceLogs, + }, + cli.BoolFlag{ + Name: useGCSArgName, + Usage: "Launch the GCS and perform requested operations via its RPC interface. Currently LCOW only", }, } @@ -88,22 +124,35 @@ func main() { wcowCommand, } - app.Before = func(c *cli.Context) error { + app.Before = func(cCtx *cli.Context) error { if !winapi.IsElevated() { - log.Fatal(c.App.Name + " must be run in an elevated context") + return fmt.Errorf(cCtx.App.Name + " must be run in an elevated context") } - if debug { - logrus.SetLevel(logrus.DebugLevel) - } else { - logrus.SetLevel(logrus.WarnLevel) + // configure logging/tracing + trace.ApplyConfig(trace.Config{DefaultSampler: oc.DefaultSampler}) + trace.RegisterExporter(&oc.LogrusExporter{}) + + logrus.SetFormatter(&logrus.TextFormatter{FullTimestamp: true}) + + lvl := logrus.WarnLevel + if traceLogs { + if debugLogs { + logrus.Warn(`"debug" and "trace" flags are mutually exclusive`) + } + lvl = logrus.TraceLevel + } else if debugLogs { + lvl = logrus.DebugLevel } + logrus.SetLevel(lvl) + logrus.WithField("args", fmt.Sprintf("%#+v", os.Args)).Tracef("running %s command", cCtx.App.Name) return nil } if err := app.Run(os.Args); err != nil { - logrus.Fatalf("%v\n", err) + fmt.Fprintln(app.ErrWriter, err) + os.Exit(1) } } @@ -122,7 +171,8 @@ func setGlobalOptions(c *cli.Context, options *uvm.Options) { } } -// todo: add a context here to propagate cancel/timeouts to runFunc uvm +// TODO: add a context here to propagate cancel/timeouts to runFunc uvm +// TODO: [runMany] can theoretically call runFunc multiple times on the same goroutine and starve others, fix that func runMany(c *cli.Context, runFunc uvmRunFunc) { parallelCount := c.GlobalInt(parallelArgName) diff --git a/internal/tools/uvmboot/wcow.go b/internal/tools/uvmboot/wcow.go index 37e0eeedc8..0b123207e5 100644 --- a/internal/tools/uvmboot/wcow.go +++ b/internal/tools/uvmboot/wcow.go @@ -10,120 +10,170 @@ import ( "os/exec" "path/filepath" "strings" + "time" "github.com/containerd/console" + "github.com/sirupsen/logrus" "github.com/urfave/cli" "github.com/Microsoft/hcsshim/internal/cmd" "github.com/Microsoft/hcsshim/internal/layers" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/uvm" + "github.com/Microsoft/hcsshim/internal/wclayer" ) -var ( - wcowDockerImage string - wcowCommandLine string - wcowImage string - wcowUseTerminal bool +type cleanupFn func(context.Context) + +const ( + wcowDockerImageArgName = "docker-image" + wcowImagePathArgName = "image" + wcowNoCMDPrependArgName = "no-cmd" ) var wcowCommand = cli.Command{ Name: "wcow", Usage: "boot a WCOW UVM", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "exec", - Usage: "Command to execute in the UVM.", - Destination: &wcowCommandLine, - }, + Flags: append(commonUVMFlags, cli.StringFlag{ - Name: "docker-image", - Usage: "Docker image to use for the UVM image", - Destination: &wcowDockerImage, + Name: wcowDockerImageArgName, + Usage: "Docker `image` to use for the UVM image", }, + // TODO: make this a StringSliceFlag, and allow passing in an array of layers cli.StringFlag{ - Name: "image", - Usage: "Path for the UVM image", - Destination: &wcowImage, + Name: wcowImagePathArgName, + Usage: "Path for the UVM boot `image`", }, cli.BoolFlag{ - Name: "tty,t", - Usage: "create the process in the UVM with a TTY enabled", - Destination: &wcowUseTerminal, + Name: wcowNoCMDPrependArgName, + Usage: "Don't prepend 'cmd /c' to the exec command", }, - }, - Action: func(c *cli.Context) error { - runMany(c, func(id string) error { - options := uvm.NewDefaultOptionsWCOW(id, "") - setGlobalOptions(c, options.Options) - var layerFolders []string - if wcowImage != "" { - layer, err := filepath.Abs(wcowImage) - if err != nil { - return err - } - layerFolders = []string{layer} - } else { - if wcowDockerImage == "" { - wcowDockerImage = "mcr.microsoft.com/windows/nanoserver:1809" - } - var err error - layerFolders, err = getLayers(wcowDockerImage) - if err != nil { - return err + ), + Action: func(cCtx *cli.Context) error { + runMany(cCtx, func(id string) error { + ctx := context.Background() + + options, cleanup, err := createWCOWOptions(ctx, cCtx, id) + defer func() { // schedule the cleanup first, since it may be non-nil regardless of the error + if cleanup == nil { + return } - } - tempDir, err := os.MkdirTemp("", "uvmboot") + cleanup(ctx) + }() if err != nil { return err } - defer os.RemoveAll(tempDir) - layerFolders = append(layerFolders, tempDir) - options.BootFiles, err = layers.GetWCOWUVMBootFilesFromLayers(context.TODO(), nil, layerFolders) - if err != nil { - return err - } - vm, err := uvm.CreateWCOW(context.TODO(), options) - if err != nil { - return err - } - defer vm.Close() - if err := vm.Start(context.TODO()); err != nil { - return err - } - if wcowCommandLine != "" { - cmd := cmd.Command(vm, "cmd.exe", "/c", wcowCommandLine) - cmd.Spec.User.Username = `NT AUTHORITY\SYSTEM` - cmd.Log = log.L.Dup() - if wcowUseTerminal { - cmd.Spec.Terminal = true - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - con, err := console.ConsoleFromFile(os.Stdin) - if err == nil { - err = con.SetRaw() - if err != nil { - return err - } - defer func() { - _ = con.Reset() - }() - } - } else { - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stdout - } - err = cmd.Run() + + return runWCOW(ctx, cCtx, options) + }) + + return nil + }, +} + +func createWCOWOptions(ctx context.Context, cCtx *cli.Context, id string) (*uvm.OptionsWCOW, cleanupFn, error) { + options := uvm.NewDefaultOptionsWCOW(id, "") + setGlobalOptions(cCtx, options.Options) + + var layerFolders []string + if wcowImage := cCtx.String(wcowImagePathArgName); wcowImage != "" { + layer, err := filepath.Abs(wcowImage) + if err != nil { + return nil, nil, err + } + layerFolders = []string{layer} + } else { + wcowDockerImage := cCtx.String(wcowDockerImageArgName) + if wcowDockerImage == "" { + wcowDockerImage = "mcr.microsoft.com/windows/nanoserver:1809" + } + var err error + layerFolders, err = getLayers(wcowDockerImage) + if err != nil { + return nil, nil, err + } + } + + tempDir, err := os.MkdirTemp("", "uvmboot") + if err != nil { + return nil, nil, err + } + cleanup := func(ctx context.Context) { + if err := destroyLayer(ctx, tempDir); err != nil { + log.G(ctx).WithFields(logrus.Fields{ + logrus.ErrorKey: err, + "directory": tempDir, + }).Warn("could not destroy scratch directory") + } + } + + layerFolders = append(layerFolders, tempDir) + options.BootFiles, err = layers.GetWCOWUVMBootFilesFromLayers(ctx, nil, layerFolders) + if err != nil { + cleanup(ctx) + return nil, nil, err + } + return options, cleanup, nil +} + +func runWCOW(ctx context.Context, cCtx *cli.Context, options *uvm.OptionsWCOW) error { + vm, err := uvm.CreateWCOW(ctx, options) + if err != nil { + return err + } + defer func() { + _ = vm.CloseCtx(ctx) + }() + + if err := vm.Start(ctx); err != nil { + return err + } + + if commandLine := cCtx.String(execCommandLineArgName); commandLine != "" { + logrus.WithField("command", commandLine).Debug("creating exec command") + var c *cmd.Cmd + if cCtx.Bool(wcowNoCMDPrependArgName) { + // Cmd on Windows host doesn't use arg array, except when escapping them to create [c.Spec.CommandLine] + // we can play fast and loose with the arguments themselves if we are providing the CommandLine directly + c = cmd.CommandContext(ctx, vm, commandLine) + c.Spec.Args = nil + c.Spec.CommandLine = commandLine + } else { + c = cmd.CommandContext(ctx, vm, "cmd.exe", "/c", commandLine) + } + c.Spec.User.Username = `NT AUTHORITY\SYSTEM` + c.Log = log.L.Dup() + if cCtx.Bool(useTerminalArgName) { + c.Spec.Terminal = true + c.Stdin = os.Stdin + c.Stdout = os.Stdout + con, err := console.ConsoleFromFile(os.Stdin) + if err == nil { + err = con.SetRaw() if err != nil { return err } + defer func() { + _ = con.Reset() + }() } - _ = vm.Terminate(context.TODO()) - _ = vm.Wait() - return vm.ExitError() - }) - return nil - }, + } else if cCtx.String(outputHandlingArgName) == "stdout" { + if cCtx.Bool(forwardStdoutArgName) { + c.Stdout = os.Stdout + } + if cCtx.Bool(forwardStderrArgName) { + c.Stderr = os.Stdout + } + } + err = c.Run() + if err != nil { + return err + } + } + + _ = vm.Terminate(ctx) + _ = vm.Wait() + return vm.ExitError() } func getLayers(imageName string) ([]string, error) { @@ -153,3 +203,31 @@ func getLayerChain(layerFolder string) ([]string, error) { } return layerChain, nil } + +// TODO: move DestroyLayer out of "github.com/Microsoft/hcsshim/test/internal/util" and use it here +func destroyLayer(ctx context.Context, p string) (err error) { + // check if the path exists + if _, err := os.Stat(p); os.IsNotExist(err) { + return nil + } + + repeat := func(f func() error, n int, d time.Duration) (err error) { + if n < 1 { + n = 1 + } + + err = f() + for i := 1; i < n; i++ { + if err == nil { + break + } + + time.Sleep(d) + err = f() + } + + return err + } + + return repeat(func() error { return wclayer.DestroyLayer(ctx, p) }, 3, time.Millisecond) +}