From bde2e39244f2a69467fc48ec75619fdf5fea2a2b Mon Sep 17 00:00:00 2001 From: Chris Gunn Date: Tue, 14 Nov 2023 16:06:07 -0800 Subject: [PATCH] Toolkit: Add retry to safemount.Close(). (#6762) --- toolkit/tools/internal/safemount/main_test.go | 44 ++++++ toolkit/tools/internal/safemount/safemount.go | 10 +- .../internal/safemount/safemount_test.go | 135 ++++++++++++++++++ 3 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 toolkit/tools/internal/safemount/main_test.go create mode 100644 toolkit/tools/internal/safemount/safemount_test.go diff --git a/toolkit/tools/internal/safemount/main_test.go b/toolkit/tools/internal/safemount/main_test.go new file mode 100644 index 00000000000..5804f54b54f --- /dev/null +++ b/toolkit/tools/internal/safemount/main_test.go @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package safemount + +import ( + "os" + "path/filepath" + "testing" + + "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/logger" +) + +var ( + tmpDir string + workingDir string +) + +func TestMain(m *testing.M) { + var err error + + logger.InitStderrLog() + + workingDir, err = os.Getwd() + if err != nil { + logger.Log.Panicf("Failed to get working directory, error: %s", err) + } + + tmpDir = filepath.Join(workingDir, "_tmp") + + err = os.MkdirAll(tmpDir, os.ModePerm) + if err != nil { + logger.Log.Panicf("Failed to create tmp directory, error: %s", err) + } + + retVal := m.Run() + + err = os.RemoveAll(tmpDir) + if err != nil { + logger.Log.Warnf("Failed to cleanup tmp dir (%s). Error: %s", tmpDir, err) + } + + os.Exit(retVal) +} diff --git a/toolkit/tools/internal/safemount/safemount.go b/toolkit/tools/internal/safemount/safemount.go index c45a7130092..5c3289f942b 100644 --- a/toolkit/tools/internal/safemount/safemount.go +++ b/toolkit/tools/internal/safemount/safemount.go @@ -7,8 +7,10 @@ package safemount import ( "fmt" "os" + "time" "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/logger" + "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/retry" "golang.org/x/sys/unix" ) @@ -92,7 +94,13 @@ func (m *Mount) close(async bool) error { if m.isMounted { if !async { logger.Log.Debugf("Unmounting (%s)", m.target) - err = unix.Unmount(m.target, 0) + _, err = retry.RunWithExpBackoff( + func() error { + logger.Log.Debugf("Trying to unmount (%s)", m.target) + umountErr := unix.Unmount(m.target, 0) + return umountErr + }, + 3, time.Second, 2.0, nil) if err != nil { return fmt.Errorf("failed to unmount (%s):\n%w", m.target, err) } diff --git a/toolkit/tools/internal/safemount/safemount_test.go b/toolkit/tools/internal/safemount/safemount_test.go new file mode 100644 index 00000000000..2084b05212e --- /dev/null +++ b/toolkit/tools/internal/safemount/safemount_test.go @@ -0,0 +1,135 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package safemount + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/microsoft/CBL-Mariner/toolkit/tools/imagegen/configuration" + "github.com/microsoft/CBL-Mariner/toolkit/tools/imagegen/diskutils" + "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/buildpipeline" + "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/file" + "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/safeloopback" + "github.com/moby/sys/mountinfo" + "github.com/stretchr/testify/assert" +) + +const ( + RetryDuration = 3 * time.Second +) + +func TestResourceBusy(t *testing.T) { + if testing.Short() { + t.Skip("Short mode enabled") + } + + if !buildpipeline.IsRegularBuild() { + t.Skip("loopback block device not available") + } + + if os.Geteuid() != 0 { + t.Skip("Test must be run as root because it uses loopback devices") + } + + buildDir := filepath.Join(tmpDir, "TestResourceBusy") + err := os.MkdirAll(buildDir, 0o770) + if !assert.NoError(t, err, "failed to test temp directory (%s)", buildDir) { + return + } + + diskConfig := configuration.Disk{ + PartitionTableType: configuration.PartitionTableTypeGpt, + MaxSize: 4096, + Partitions: []configuration.Partition{ + { + ID: "a", + Start: 1, + End: 0, + FsType: "ext4", + }, + }, + } + + // Create raw disk image file. + rawDisk, err := diskutils.CreateEmptyDisk(buildDir, "disk.raw", diskConfig.MaxSize) + assert.NoError(t, err, "failed to create empty disk file (%s)", buildDir) + + // Connect raw disk image file. + loopback, err := safeloopback.NewLoopback(rawDisk) + if !assert.NoError(t, err, "failed to mount raw disk as a loopback device (%s)", rawDisk) { + return + } + defer loopback.Close() + + // Set up partitions. + _, _, _, _, err = diskutils.CreatePartitions(loopback.DevicePath(), diskConfig, + configuration.RootEncryption{}, configuration.ReadOnlyVerityRoot{}) + if !assert.NoError(t, err, "failed to create partitions on disk", loopback.DevicePath()) { + return + } + + // Mount the partition. + partitionDevPath := loopback.DevicePath() + "p1" + partitionMountPath := filepath.Join(buildDir, "mount") + + mount, err := NewMount(partitionDevPath, partitionMountPath, "ext4", 0, "", true) + if !assert.NoError(t, err, "failed to mount partition", partitionDevPath, partitionMountPath) { + return + } + defer mount.Close() + + // Check that the mount exists. + exists, err := file.PathExists(partitionMountPath) + if !assert.NoError(t, err, "failed to check if mount directory exists") { + return + } + if !assert.Equal(t, true, exists, "mount directory doesn't exist") { + return + } + + isMounted, err := mountinfo.Mounted(partitionMountPath) + if !assert.NoError(t, err, "failed to check if directory is not a mount point") { + return + } + if !assert.Equal(t, true, isMounted, "directory is not a mount point") { + return + } + + // Open a file. + fileOnPartitionPath := filepath.Join(partitionMountPath, "test") + + fileOnPartition, err := os.OpenFile(fileOnPartitionPath, os.O_RDWR|os.O_CREATE, 0) + if !assert.NoErrorf(t, err, "failed to open file", fileOnPartitionPath) { + return + } + defer fileOnPartition.Close() + + // Try to close the mount. + startTime := time.Now() + err = mount.CleanClose() + endTime := time.Now() + + assert.Error(t, err) + assert.ErrorContains(t, err, "busy") + + // Sanity check that the retries were attempted. + assert.LessOrEqual(t, RetryDuration, endTime.Sub(startTime)) + + // Close the file. + fileOnPartition.Close() + + // Try to close the mount again. + err = mount.CleanClose() + assert.NoError(t, err, "failed to close the mount") + + // Make sure directory is deleted. + exists, err = file.PathExists(partitionMountPath) + if !assert.NoError(t, err, "failed to check if mount still directory exists") { + return + } + assert.Equal(t, false, exists, "mount directory still exists") +}